diff --git a/advanced_topics/08_fslpy.ipynb b/advanced_topics/08_fslpy.ipynb index 637779b92680b92ca997778801e22efb77eefb64..15761daf766d2f362fbb6f4aee31e3759d15deb3 100644 --- a/advanced_topics/08_fslpy.ipynb +++ b/advanced_topics/08_fslpy.ipynb @@ -32,7 +32,11 @@ " * [In-memory images](#in-memory-images)\n", " * [Loading outputs into Python](#loading-outputs-into-python)\n", " * [The `fslmaths` wrapper](#the-fslmaths-wrapper)\n", - "* [The `filetree`](#the-filetree)\n", + "* [The `FileTree`](#the-filetree)\n", + " * [Describing your data](#describing-your-data)\n", + " * [Using the `FileTree`](#using-the-filetree)\n", + " * [Building a processing pipeline with `FileTree`](#building-a-processing-pipeline-with-filetree)\n", + " * [The `FileTreeQuery`](#the-filetreequery)\n", "* [Calling shell commands](#calling-shell-commands)\n", " * [The `runfsl` function](#the-runfsl-function)\n", " * [Submitting to the cluster](#submitting-to-the-cluster)\n", @@ -129,13 +133,24 @@ "metadata": {}, "outputs": [], "source": [ - "import shlex\n", - "import IPython.display as display\n", - "from fsleyes.render import main\n", - "\n", "def render(cmdline):\n", + "\n", + " import shlex\n", + " import IPython.display as display\n", + "\n", " prefix = '-of screenshot.png -hl -c 2 '\n", - " main(shlex.split(prefix + cmdline))\n", + "\n", + " try:\n", + " from fsleyes.render import main\n", + " main(shlex.split(prefix + cmdline))\n", + "\n", + " except ImportError:\n", + " # fall-back for macOS - we have to run\n", + " # FSLeyes render in a separate process\n", + " from fsl.utils.run import runfsl\n", + " prefix = 'render ' + prefix\n", + " runfsl(prefix + cmdline, env={})\n", + "\n", " return display.Image('screenshot.png')" ] }, @@ -381,7 +396,7 @@ " class uses `dcm2niix` to load NIfTI images contained within a DICOM\n", " directory<sup>*</sup>.\n", "* The\n", - " [`fsl.data.mghimahe.MGHImage`](https://users.fmrib.ox.ac.uk/~paulmc/fsleyes/fslpy/latest/fsl.data.mghimage.html)\n", + " [`fsl.data.mghimage.MGHImage`](https://users.fmrib.ox.ac.uk/~paulmc/fsleyes/fslpy/latest/fsl.data.mghimage.html)\n", " class can be used too load `.mgh`/`.mgz` images (they are converted into\n", " NIfTI images).\n", "* The\n", @@ -407,11 +422,12 @@ " and\n", " [`fsl.data.vtk`](https://users.fmrib.ox.ac.uk/~paulmc/fsleyes/fslpy/latest/fsl.data.vtk.html)\n", " modules contain functionality form loading surface data from GIfTI,\n", - " freesurfer, and VTK files respectively.\n", + " freesurfer, and ASCII VTK files respectively.\n", "\n", "\n", - "> <sup>*</sup>You must make sure that `dcm2niix` is installed on your system\n", - "> in order to use this class.\n", + "> <sup>*</sup>You must make sure that\n", + "> [`dcm2niix`](https://github.com/rordenlab/dcm2niix/) is installed on your\n", + "> system in order to use this class.\n", "\n", "\n", "<a class=\"anchor\" id=\"nifti-coordinate-systems\"></a>\n", @@ -592,6 +608,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "> Below we will use the\n", + "> [`fsl.transform.flirt.fromFlirt`](https://users.fmrib.ox.ac.uk/~paulmc/fsleyes/fslpy/latest/fsl.transform.flirt.html#fsl.transform.flirt.fromFlirt)\n", + "> function, which does all of the above for us.\n", + "\n", + "\n", "So we've now got some voxel coordinates from our functional data, and an\n", "affine to transform into MNI world coordinates. The rest is easy:" ] @@ -791,6 +812,10 @@ "> # \"twod\" applies the -2D flag\n", "> flirt('source.nii.gz', 'ref.nii.gz', omat='src2ref.mat', twod=True)\n", "> ```\n", + ">\n", + "> Some of the `fsl.wrappers` functions also support aliases which may make\n", + "> your code more readable. For example, when calling `bet`, you can use either\n", + "> `m=True` or `mask=True` to apply the `-m` command line flag.\n", "\n", "\n", "<a class=\"anchor\" id=\"in-memory-images\"></a>\n", @@ -830,7 +855,7 @@ "\n", "\n", "By using the special `fsl.wrappers.LOAD` symbol, you can also have any output\n", - "files produced by the tool automatically loaded:" + "files produced by the tool automatically loaded into memory for you:" ] }, { @@ -840,7 +865,11 @@ "outputs": [], "source": [ "cropped = Image('bighead_cropped')\n", - "betted = bet(cropped, LOAD)['output']\n", + "\n", + "# The loaded result is called \"output\",\n", + "# because that is the name of the\n", + "# argument in the bet wrapper function.\n", + "betted = bet(cropped, LOAD).output\n", "\n", "fig = ortho(cropped.data, (80, 112, 85), cmap=plt.cm.gray)\n", "fig = ortho(betted .data, (80, 112, 85), cmap=plt.cm.inferno, fig=fig)" @@ -851,8 +880,7 @@ "metadata": {}, "source": [ "You can use the `LOAD` symbol for any output argument - any output files which\n", - "are loaded will be returned in a dictionary, with the argument name used as\n", - "the key:" + "are loaded will be available through the return value of the wrapper function:" ] }, { @@ -869,9 +897,10 @@ "\n", "aligned = flirt(tstat1, std2mm, applyxfm=True, init=func2std, out=LOAD)\n", "\n", - "print(aligned)\n", - "\n", - "aligned = aligned['out'].data\n", + "# Here the resampled tstat image\n", + "# is called \"out\", because that\n", + "# is the name of the flirt argument.\n", + "aligned = aligned.out.data\n", "aligned[aligned < 1] = 0\n", "\n", "fig = ortho(std2mm .data, (45, 54, 45), cmap=plt.cm.gray)\n", @@ -884,7 +913,7 @@ "source": [ "For tools like `bet` and `fast`, which expect an output *prefix* or\n", "*basename*, you can just set the prefix to `LOAD` - all output files with that\n", - "prefix will be available in the returned dictionary:" + "prefix will be available in the object that is returned:" ] }, { @@ -896,11 +925,9 @@ "img = Image('bighead_cropped')\n", "betted = bet(img, LOAD, f=0.3, m=True)\n", "\n", - "print(betted)\n", - "\n", - "fig = ortho(img .data, (80, 112, 85), cmap=plt.cm.gray)\n", - "fig = ortho(betted['output'] .data, (80, 112, 85), cmap=plt.cm.inferno, fig=fig)\n", - "fig = ortho(betted['output_mask'].data, (80, 112, 85), cmap=plt.cm.summer, fig=fig, alpha=0.5)" + "fig = ortho(img .data, (80, 112, 85), cmap=plt.cm.gray)\n", + "fig = ortho(betted.output .data, (80, 112, 85), cmap=plt.cm.inferno, fig=fig)\n", + "fig = ortho(betted.output_mask.data, (80, 112, 85), cmap=plt.cm.summer, fig=fig, alpha=0.5)" ] }, { @@ -966,12 +993,280 @@ "metadata": {}, "source": [ "<a class=\"anchor\" id=\"the-filetree\"></a>\n", - "## The `filetree`\n", + "## The `FileTree`\n", + "\n", + "\n", + "The\n", + "[`fsl.utils.filetree`](https://users.fmrib.ox.ac.uk/~paulmc/fsleyes/fslpy/latest/fsl.utils.filetree.html)\n", + "library provides functionality which allows you to work with *structured data\n", + "directories*, such as HCP or BIDS datasets. You can use `filetree` for both\n", + "reading and for creating datasets.\n", + "\n", + "\n", + "This practical gives a very brief introduction to the `filetree` library -\n", + "refer to the [full\n", + "documentation](https://users.fmrib.ox.ac.uk/~paulmc/fsleyes/fslpy/latest/fsl.utils.filetree.html)\n", + "to get a feel for how powerful it can be.\n", + "\n", + "\n", + "<a class=\"anchor\" id=\"describing-your-data\"></a>\n", + "### Describing your data\n", + "\n", + "\n", + "To introduce `filetree`, we'll begin with a small example. Imagine that we\n", + "have a dataset which looks like this:\n", + "\n", + "\n", + "> ```\n", + "> mydata\n", + "> ├── sub_A\n", + "> │  ├── ses_1\n", + "> │  │  └── T1w.nii.gz\n", + "> │  ├── ses_2\n", + "> │  │  └── T1w.nii.gz\n", + "> │  └── T2w.nii.gz\n", + "> ├── sub_B\n", + "> │  ├── ses_1\n", + "> │  │  └── T1w.nii.gz\n", + "> │  ├── ses_2\n", + "> │  │  └── T1w.nii.gz\n", + "> │  └── T2w.nii.gz\n", + "> └── sub_C\n", + "> ├── ses_1\n", + "> │  └── T1w.nii.gz\n", + "> ├── ses_2\n", + "> │  └── T1w.nii.gz\n", + "> └── T2w.nii.gz\n", + "> ```\n", + "\n", + "\n", + "(Run the code cell below to create a dummy data set with the above structure):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "for sub in A B C; do\n", + " subdir=mydata/sub_$sub/\n", + " mkdir -p $subdir\n", + " cp $FSLDIR/data/standard/MNI152_T1_2mm.nii.gz $subdir/T2w.nii.gz\n", + " for ses in 1 2; do\n", + " sesdir=$subdir/ses_$ses/\n", + " mkdir $sesdir\n", + " cp $FSLDIR/data/standard/MNI152_T1_2mm.nii.gz $sesdir/T1w.nii.gz\n", + " done\n", + "done" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To use `filetree` with this dataset, we must first describe its structure - we\n", + "do this by creating a `.tree` file:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%writefile mydata.tree\n", + "sub_{subject}\n", + " T2w.nii.gz\n", + " ses_{session}\n", + " T1w.nii.gz" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "A `.tree` file is simply a description of the structure of your data\n", + "directory - it describes the *file types* (also known as *templates*) which\n", + "are present in the dataset (`T1w` and `T2w`), and the *variables* which are\n", + "implicitly present in the structure of the dataset (`subject` and `session`).\n", + "\n", + "\n", + "<a class=\"anchor\" id=\"using-the-filetree\"></a>\n", + "### Using the `FileTree`\n", + "\n", + "\n", + "Now that we have a `.tree` file which describe our data, we can create a\n", + "`FileTree` to work with it:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from fsl.utils.filetree import FileTree\n", + "\n", + "tree = FileTree.read('mydata.tree', 'mydata')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can list all of the T1 images via the `FileTree.get_all` method. The\n", + "`glob_vars='all'` option tells the `FileTree` to fill in the `T1w` template\n", + "with all possible combinations of variables:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for t1file in tree.get_all('T1w', glob_vars='all'):\n", + " fvars = tree.extract_variables('T1w', t1file)\n", + " print(t1file, fvars)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `FileTree.update` method allows you to \"fill in\" variable values; it\n", + "returns a new `FileTree` object which can be used on a selection of the\n", + "data set:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "treeA = tree.update(subject='A')\n", + "for t1file in treeA.get_all('T1w', glob_vars='all'):\n", + " fvars = treeA.extract_variables('T1w', t1file)\n", + " print(t1file, fvars)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "<a class=\"anchor\" id=\"building-a-processing-pipeline-with-filetree\"></a>\n", + "### Building a processing pipeline with `FileTree`\n", "\n", "\n", - "todo\n", + "Let's say we want to run BET on all of our T1 images. Let's start by modifying\n", + "our `.tree` definition to include the BET outputs:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%writefile mydata.tree\n", + "sub_{subject}\n", + " T2w.nii.gz\n", + " ses_{session}\n", + " T1w.nii.gz\n", + " T1w_brain.nii.gz\n", + " T1w_brain_mask.nii.gz" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can use the `FileTree` to generate the relevant file names for us.\n", + "Here we'll use the `FileTree.get_all_trees` method to create a sub-tree for\n", + "each subject and each session:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from fsl.wrappers import bet\n", + "tree = FileTree.read('mydata.tree', 'mydata')\n", + "for subtree in tree.get_all_trees('T1w', glob_vars='all'):\n", + " t1file = subtree.get('T1w')\n", + " t1brain = subtree.get('T1w_brain')\n", + " print('Running BET: {} -> {} ...'.format(t1file, t1brain))\n", + " bet(t1file, t1brain, mask=True)\n", + "print('Done!')\n", + "\n", + "example = tree.update(subject='A', session='1')\n", + "render('{} {} -ot mask -ol -w 2 -mc 0 1 0'.format(\n", + " example.get('T1w'),\n", + " example.get('T1w_brain_mask')))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "<a class=\"anchor\" id=\"the-filetreequery\"></a>\n", + "### The `FileTreeQuery`\n", "\n", "\n", + "The `filetree` module contains another class called the\n", + "[`FileTreeQuery`](https://users.fmrib.ox.ac.uk/~paulmc/fsleyes/fslpy/latest/fsl.utils.filetree.query.html),\n", + "which provides an interface that is more convenient if you are reading data\n", + "from large datasets with many different file types and variables.\n", + "\n", + "\n", + "When you create a `FileTreeQuery`, it scans the entire data directory and\n", + "identifies all of the values that are present for each variable defined in the\n", + "`.tree` file:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from fsl.utils.filetree import FileTreeQuery\n", + "tree = FileTree.read('mydata.tree', 'mydata')\n", + "query = FileTreeQuery(tree)\n", + "print('T1w variables:', query.variables('T1w'))\n", + "print('T2w variables:', query.variables('T2w'))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `FileTreeQuery.query` method will return the paths to all existing files\n", + "which match a set of variable values:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print('All T1w images for subject A')\n", + "for template in query.templates:\n", + " print(' ', template)\n", + " for file in query.query(template, subject='A'):\n", + " print(file)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ "<a class=\"anchor\" id=\"calling-shell-commands\"></a>\n", "## Calling shell commands\n", "\n", @@ -1487,8 +1782,8 @@ "frontal = lblatlas.get(name='Frontal Pole').data\n", "frontal = np.ma.masked_where(frontal < 1, frontal)\n", "\n", - "fig = ortho(std2mm, (45, 54, 45), cmap=plt.cm.gray)\n", - "fig = ortho(frontal, (45, 54, 45), cmap=plt.cm.winter, fig=fig)" + "fig = ortho(std2mm.data, (45, 54, 45), cmap=plt.cm.gray)\n", + "fig = ortho(frontal, (45, 54, 45), cmap=plt.cm.winter, fig=fig)" ] }, { @@ -1510,8 +1805,8 @@ "frontal = probatlas.get(name='Frontal Pole')\n", "frontal = np.ma.masked_where(frontal < 1, frontal)\n", "\n", - "fig = ortho(std2mm, (45, 54, 45), cmap=plt.cm.gray)\n", - "fig = ortho(frontal, (45, 54, 45), cmap=plt.cm.inferno, fig=fig)" + "fig = ortho(std2mm.data, (45, 54, 45), cmap=plt.cm.gray)\n", + "fig = ortho(frontal, (45, 54, 45), cmap=plt.cm.inferno, fig=fig)" ] }, { diff --git a/advanced_topics/08_fslpy.md b/advanced_topics/08_fslpy.md index 01074b5140523b0965741dc8a003a3ce3fb19877..437a3d3620fe87e5da1f45b8732ab43dad0b9c17 100644 --- a/advanced_topics/08_fslpy.md +++ b/advanced_topics/08_fslpy.md @@ -26,7 +26,11 @@ perform analyses and image processing in conjunction with FSL. * [In-memory images](#in-memory-images) * [Loading outputs into Python](#loading-outputs-into-python) * [The `fslmaths` wrapper](#the-fslmaths-wrapper) -* [The `filetree`](#the-filetree) +* [The `FileTree`](#the-filetree) + * [Describing your data](#describing-your-data) + * [Using the `FileTree`](#using-the-filetree) + * [Building a processing pipeline with `FileTree`](#building-a-processing-pipeline-with-filetree) + * [The `FileTreeQuery`](#the-filetreequery) * [Calling shell commands](#calling-shell-commands) * [The `runfsl` function](#the-runfsl-function) * [Submitting to the cluster](#submitting-to-the-cluster) @@ -616,6 +620,10 @@ render('bighead_cropped -b 40 ' > # "twod" applies the -2D flag > flirt('source.nii.gz', 'ref.nii.gz', omat='src2ref.mat', twod=True) > ``` +> +> Some of the `fsl.wrappers` functions also support aliases which may make +> your code more readable. For example, when calling `bet`, you can use either +> `m=True` or `mask=True` to apply the `-m` command line flag. <a class="anchor" id="in-memory-images"></a> @@ -745,8 +753,206 @@ fig = ortho(erodedbrain.data, (80, 112, 85), cmap=plt.cm.inferno, fig=fig) <a class="anchor" id="the-filetree"></a> -## The `filetree` +## The `FileTree` + + +The +[`fsl.utils.filetree`](https://users.fmrib.ox.ac.uk/~paulmc/fsleyes/fslpy/latest/fsl.utils.filetree.html) +library provides functionality which allows you to work with *structured data +directories*, such as HCP or BIDS datasets. You can use `filetree` for both +reading and for creating datasets. + + +This practical gives a very brief introduction to the `filetree` library - +refer to the [full +documentation](https://users.fmrib.ox.ac.uk/~paulmc/fsleyes/fslpy/latest/fsl.utils.filetree.html) +to get a feel for how powerful it can be. + + +<a class="anchor" id="describing-your-data"></a> +### Describing your data + + +To introduce `filetree`, we'll begin with a small example. Imagine that we +have a dataset which looks like this: + + +> ``` +> mydata +> ├── sub_A +> │  ├── ses_1 +> │  │  └── T1w.nii.gz +> │  ├── ses_2 +> │  │  └── T1w.nii.gz +> │  └── T2w.nii.gz +> ├── sub_B +> │  ├── ses_1 +> │  │  └── T1w.nii.gz +> │  ├── ses_2 +> │  │  └── T1w.nii.gz +> │  └── T2w.nii.gz +> └── sub_C +> ├── ses_1 +> │  └── T1w.nii.gz +> ├── ses_2 +> │  └── T1w.nii.gz +> └── T2w.nii.gz +> ``` + + +(Run the code cell below to create a dummy data set with the above structure): + + +``` +%%bash +for sub in A B C; do + subdir=mydata/sub_$sub/ + mkdir -p $subdir + cp $FSLDIR/data/standard/MNI152_T1_2mm.nii.gz $subdir/T2w.nii.gz + for ses in 1 2; do + sesdir=$subdir/ses_$ses/ + mkdir $sesdir + cp $FSLDIR/data/standard/MNI152_T1_2mm.nii.gz $sesdir/T1w.nii.gz + done +done +``` + + +To use `filetree` with this dataset, we must first describe its structure - we +do this by creating a `.tree` file: + + +``` +%%writefile mydata.tree +sub_{subject} + T2w.nii.gz + ses_{session} + T1w.nii.gz +``` + + +A `.tree` file is simply a description of the structure of your data +directory - it describes the *file types* (also known as *templates*) which +are present in the dataset (`T1w` and `T2w`), and the *variables* which are +implicitly present in the structure of the dataset (`subject` and `session`). + + +<a class="anchor" id="using-the-filetree"></a> +### Using the `FileTree` + + +Now that we have a `.tree` file which describe our data, we can create a +`FileTree` to work with it: + + +``` +from fsl.utils.filetree import FileTree + +tree = FileTree.read('mydata.tree', 'mydata') +``` + +We can list all of the T1 images via the `FileTree.get_all` method. The +`glob_vars='all'` option tells the `FileTree` to fill in the `T1w` template +with all possible combinations of variables: + + +``` +for t1file in tree.get_all('T1w', glob_vars='all'): + fvars = tree.extract_variables('T1w', t1file) + print(t1file, fvars) +``` + + +The `FileTree.update` method allows you to "fill in" variable values; it +returns a new `FileTree` object which can be used on a selection of the +data set: + + +``` +treeA = tree.update(subject='A') +for t1file in treeA.get_all('T1w', glob_vars='all'): + fvars = treeA.extract_variables('T1w', t1file) + print(t1file, fvars) +``` + + +<a class="anchor" id="building-a-processing-pipeline-with-filetree"></a> +### Building a processing pipeline with `FileTree` + + +Let's say we want to run BET on all of our T1 images. Let's start by modifying +our `.tree` definition to include the BET outputs: + + +``` +%%writefile mydata.tree +sub_{subject} + T2w.nii.gz + ses_{session} + T1w.nii.gz + T1w_brain.nii.gz + T1w_brain_mask.nii.gz +``` + + +Now we can use the `FileTree` to generate the relevant file names for us. +Here we'll use the `FileTree.get_all_trees` method to create a sub-tree for +each subject and each session: + + +``` +from fsl.wrappers import bet +tree = FileTree.read('mydata.tree', 'mydata') +for subtree in tree.get_all_trees('T1w', glob_vars='all'): + t1file = subtree.get('T1w') + t1brain = subtree.get('T1w_brain') + print('Running BET: {} -> {} ...'.format(t1file, t1brain)) + bet(t1file, t1brain, mask=True) +print('Done!') + +example = tree.update(subject='A', session='1') +render('{} {} -ot mask -ol -w 2 -mc 0 1 0'.format( + example.get('T1w'), + example.get('T1w_brain_mask'))) + +``` + + +<a class="anchor" id="the-filetreequery"></a> +### The `FileTreeQuery` + + +The `filetree` module contains another class called the +[`FileTreeQuery`](https://users.fmrib.ox.ac.uk/~paulmc/fsleyes/fslpy/latest/fsl.utils.filetree.query.html), +which provides an interface that is more convenient if you are reading data +from large datasets with many different file types and variables. + +When you create a `FileTreeQuery`, it scans the entire data directory and +identifies all of the values that are present for each variable defined in the +`.tree` file: + + +``` +from fsl.utils.filetree import FileTreeQuery +tree = FileTree.read('mydata.tree', 'mydata') +query = FileTreeQuery(tree) +print('T1w variables:', query.variables('T1w')) +print('T2w variables:', query.variables('T2w')) +``` + + +The `FileTreeQuery.query` method will return the paths to all existing files +which match a set of variable values: + + +``` +print('All T1w images for subject A') +for template in query.templates: + print(' ', template) + for file in query.query(template, subject='A'): + print(file) +``` <a class="anchor" id="calling-shell-commands"></a>