Merge branch 'enh_filetree' into 'master'

Added filetree to fslpy

See merge request fsl/fslpy!69

fsl/utils/filetree/__init__.py

+"""mc_filetree - Easy format to define intput/output files in a python pipeline"""
+
+__author__ = 'Michiel Cottaar <Michiel.Cottaar@ndcn.ox.ac.uk>'
+
+from .filetree import FileTree, register_tree, MissingVariable
+from .parse import tree_directories

fsl/utils/filetree/filetree.py

+from pathlib import Path, PurePath
+from typing import Tuple, Optional, Dict, Any, Set
+from copy import deepcopy
+from . import parse
+import pickle
+import os.path as op
+from . import utils
+
+
+class MissingVariable(KeyError):
+    """
+    Returned when the variables of a tree or its parents do not contain a given variable
+    """
+    pass
+
+
+class FileTree(object):
+    """
+    Contains the input/output filename tree
+
+    Properties
+    - ``templates``: dictionary mapping short names to filename templates
+    - ``variables``: dictionary mapping variables in the templates to specific values (variables set to None are explicitly unset)
+    - ``sub_trees``: filename trees describing specific sub-directories
+    - ``parent``: parent FileTree, of which this sub-tree is a sub-directory
+    """
+    def __init__(self,
+                 templates: Dict[str, str],
+                 variables: Dict[str, Any],
+                 sub_trees: Dict[str, "FileTree"]=None,
+                 parent: Optional["FileTree"]=None):
+        """
+        Creates a new filename tree.
+        """
+        self.templates = templates
+        self.variables = variables
+        if sub_trees is None:
+            sub_trees = {}
+        self.sub_trees = sub_trees
+        self._parent = parent
+
+    @property
+    def parent(self, ):
+        return self._parent
+
+    @property
+    def all_variables(self, ):
+        """
+        All tree variables including those from the parent tree
+        """
+        if self.parent is None:
+            return self.variables
+        res = self.parent.all_variables
+        res.update(self.variables)
+        return res
+
+    def get_variable(self, name: str, default=None) -> str:
+        """
+        Gets a variable used to fill out the template
+
+        :param name: variable name
+        :param default: default variables (if not set an error is raised for a missing variable)
+        :return: value of the variable
+        """
+        variables = self.all_variables
+        if name in variables and variables[name] is not None:
+            return variables[name]
+        if default is None:
+            raise MissingVariable('Variable {} not found in sub-tree or parents'.format(name))
+        return default
+
+    def _get_template_tree(self, short_name: str) -> Tuple["FileTree", str]:
+        """
+        Retrieves template text from this tree, parent tree or sub_tree
+
+        :param short_name: filename reference name
+        :return: tuple with the containing tree and the template text
+        """
+        if '/' in short_name:
+            sub_tree, sub_name = short_name.split('/', maxsplit=1)
+            if sub_tree == '..':
+                if self.parent is None:
+                    raise KeyError("Tried to access the parent of the top-level tree")
+                return self.parent._get_template_tree(sub_name)
+            return self.sub_trees[sub_tree]._get_template_tree(sub_name)
+        return self, self.templates[short_name]
+
+    def get_template(self, short_name: str) -> Tuple[str, Dict[str, str]]:
+        """
+        Returns the sub-tree that defines a given short name
+
+        - '/' characters in short_name refer to sub-trees
+        - '../' characters in short_name refer to parents
+
+        For example:
+        - eddy/output refers to the output in the eddy sub_tree (i.e. self.sub_trees['eddy'].templates['output']
+        - ../other/name refers to the 'other' sub-tree of the parent tree (i.e., self.parent.sub_trees['other'].templates['name']
+
+        :param short_name: name of the template
+        :return: tuple with the template and the variables corresponding to the template
+        """
+        tree, text = self._get_template_tree(short_name)
+        return text, tree.all_variables
+
+    def template_variables(self, short_name: Optional[str]=None, optional=True, required=True) -> Set[str]:
+        """
+        Returns the variables needed to define a template
+
+        :param short_name: name of the template (defaults to all)
+        :param optional: if set to False don't include the optional variables
+        :param required: if set to False don't include the required variables
+        :return: set of variable names
+        """
+        if not optional and not required:
+            return set()
+        if short_name is None:
+            all_vars = set()
+            required_vars = set()
+            for short_name in self.templates.keys():
+                all_vars.update(self.template_variables(short_name))
+                if required or optional:
+                    required_vars.update(self.template_variables(short_name, optional=False))
+            for sub_tree in self.sub_trees.values():
+                all_vars.update(sub_tree.template_variables())
+                if required or optional:
+                    required_vars.update(sub_tree.template_variables(optional=False))
+            if optional and required:
+                return all_vars
+            if required:
+                return required_vars
+            if optional:
+                return all_vars.difference(required_vars)
+        else:
+            _, text = self._get_template_tree(short_name)
+            all_vars = set(utils.find_variables(text))
+            if optional and required:
+                return all_vars
+            opt_vars = set(utils.optional_variables(text))
+            if optional:
+                return opt_vars
+            if required:
+                return all_vars.difference(opt_vars)
+
+    def get(self, short_name, make_dir=False) -> str:
+        """
+        Gets a full filename based on its short name
+
+        :param short_name: identifier in the tree
+        :param make_dir: if True make sure that the directory leading to this file exists
+        :return: full filename
+        """
+        text, variables = self.get_template(short_name)
+        res = Path(utils.resolve(text, variables))
+        if make_dir:
+            res.parents[0].mkdir(parents=True, exist_ok=True)
+        return str(res)
+
+    def get_all(self, short_name: str, glob_vars=()) -> Tuple[str]:
+        """
+        Gets all existing directory/file names matching a specific pattern
+
+        :param short_name: short name of the path template
+        :param glob_vars: sequence of undefined variables that can take any possible values when looking for matches on the disk
+        Any defined variables in `glob_vars` will be ignored.
+        If glob_vars is set to 'all', all undefined variables will be used to look up matches
+        :return: sorted sequence of paths
+        """
+        text, variables = self.get_template(short_name)
+        return tuple(utils.get_all(text, variables, glob_vars=glob_vars))
+
+    def get_all_vars(self, short_name: str, glob_vars=()) -> Tuple[Dict[str, str]]:
+        """
+        Gets all the parameters that generate existing filenames
+
+        :param short_name: short name of the path template
+        :param glob_vars: sequence of undefined variables that can take any possible values when looking for matches on the disk
+        Any defined variables in `glob_vars` will be ignored.
+        If glob_vars is set to 'all', all undefined variables will be used to look up matches
+        :return: sequence of dictionaries with the variables settings used to generate each filename
+        """
+        return tuple(self.extract_variables(short_name, fn) for fn in self.get_all(short_name, glob_vars=glob_vars))
+
+    def update(self, **variables) -> "FileTree":
+        """
+        Creates a new filetree with updated variables
+
+        :arg variables: new values for the variables
+        Setting variables to None will explicitly unset them
+        """
+        new_tree = deepcopy(self)
+        new_tree.variables.update(variables)
+        return new_tree
+
+    def extract_variables(self, short_name: str, filename: str) -> Dict[str, str]:
+        """
+        Extracts the variables from the given filename
+
+        :param short_name: short name of the path template
+        :param filename: filename matching the template
+        :return: variables needed to get to the given filename
+        Variables with None value are optional variables in the template that were not used
+        """
+        text, _ = self.get_template(short_name)
+        return utils.extract_variables(text, filename, self.variables)
+
+    def save_pickle(self, filename):
+        """
+        Saves the Filetree to a pickle file
+
+        :param filename: filename to store the file tree (usually ending with .pck)
+        """
+        with open(filename, 'wb') as f:
+            pickle.dump(self, f)
+
+    @classmethod
+    def load_pickle(cls, filename):
+        """
+        Loads the Filetree from a pickle file
+
+        :param filename: filename produced from Filetree.save
+        :return: stored Filetree
+        """
+        with open(filename, 'rb') as f:
+            res = pickle.load(f)
+        if not isinstance(res, cls):
+            raise IOError("Pickle file did not contain %s object" % cls)
+        return res
+
+    def exists(self, short_names, on_disk=False, error=False, glob_vars=()):
+        """
+        Checks whether the short_names are defined in the tree (and optional exist on the disk)
+
+        :param short_names: list of expected short names to exist in the tree
+        :param on_disk: if True checks whether the files exist on disk
+        :param error: if True raises a helpful error when the check fails
+        :param glob_vars: sequence of undefined variables that can take any possible values when looking for matches on the disk
+        If `glob_vars` contains any defined variables, it will be ignored.
+        :return: True if short names exist and optionally exist on disk (False otherwise)
+        :raise:
+        - ValueError if error is set and the tree is incomplete
+        - IOError if error is set and any files are missing from the disk
+        """
+        if isinstance(short_names, str):
+            short_names = (short_names, )
+
+        def single_test(short_name):
+            try:
+                self._get_template_tree(short_name)
+            except KeyError:
+                return True
+            return False
+
+        missing = tuple(name for name in short_names if single_test(name))
+        if len(missing) > 0:
+            if error:
+                raise ValueError("Provided Filetree is missing file definitions for {}".format(missing))
+            return False
+        if on_disk:
+            try:
+                missing = tuple(name for name in short_names if len(self.get_all(name, glob_vars=glob_vars)) == 0)
+            except KeyError:
+                if error:
+                    raise
+                return False
+            if len(missing) > 0:
+                if error:
+                    raise IOError("Failed to find any files existing for {}".format(missing))
+                return False
+        return True
+
+    @classmethod
+    def read(cls, tree_name: str, directory='.', **variables) -> "FileTree":
+        """
+        Reads a FileTree from a specific file
+
+        The return type is ``cls`` unless the tree_name has been previously registered
+        The return type of any sub-tree is FileTree unless the tree_name has been previously registered
+
+        :param tree_name: file containing the filename tree.
+        Can provide the filename of a tree file or the name for a tree in the ``filetree.tree_directories``.
+        :param directory: parent directory of the full tree (defaults to current directory)
+        :param variables: variable settings
+        :return: dictionary from specifier to filename
+        """
+        if op.exists(tree_name):
+            filename = tree_name
+        elif op.exists(tree_name + '.tree'):
+            filename = tree_name + '.tree'
+        else:
+            filename = parse.search_tree(tree_name)
+        filename = Path(filename)
+
+        templates = {}
+        nspaces_level = []
+        sub_trees = {}
+
+        file_variables = {}
+
+        with open(str(filename), 'r') as f:
+            for full_line in f:
+                # ignore anything behind the first #-character
+                line = full_line.split('#')[0]
+
+                if len(line.strip()) == 0:
+                    continue
+
+                if line.strip()[:2] == '->':
+                    nspaces = line.index('->')
+
+                    if len(nspaces_level) == 0:
+                        sub_dir = directory
+                    elif nspaces > nspaces_level[-1]:
+                        sub_dir = current_filename
+                    elif nspaces < nspaces_level[-1]:
+                        if nspaces not in nspaces_level:
+                            raise ValueError('line %s dropped to a non-existent level' % line)
+                        new_level = nspaces_level.index(nspaces)
+                        current_filename = current_filename.parents[len(nspaces_level) - new_level - 1] / filename
+                        nspaces_level = nspaces_level[:new_level + 1]
+                        sub_dir = current_filename.parents[0]
+                    else:
+                        sub_dir = current_filename.parents[0]
+
+                    _, sub_tree, short_name = parse.read_subtree_line(line, sub_dir)
+                    if short_name in sub_trees:
+                        raise ValueError("Name of sub_tree {short_name} used multiple times in {tree_name}.tree".format(**locals()))
+
+                    sub_trees[short_name] = sub_tree
+                elif '=' in line:
+                    key, value = line.split('=')
+                    if len(key.split()) != 1:
+                        raise ValueError("Variable assignment could not be parsed: {line}".format(**locals()))
+                    file_variables[key.strip()] = value.strip()
+                else:
+                    nspaces, filename, short_name = parse.read_line(line)
+                    if short_name in templates:
+                        raise ValueError("Name of directory/file {short_name} used multiple times in {tree_name}.tree".format(**locals()))
+
+                    if len(nspaces_level) == 0:
+                        current_filename = PurePath(directory) / filename
+                        nspaces_level.append(nspaces)
+                    elif nspaces > nspaces_level[-1]:
+                        # increase the level
+                        current_filename = current_filename / filename
+                        nspaces_level.append(nspaces)
+                    elif nspaces < nspaces_level[-1]:
+                        # decreased the level
+                        if nspaces not in nspaces_level:
+                            raise ValueError('line %s dropped to a non-existent level' % full_line)
+                        new_level = nspaces_level.index(nspaces)
+                        current_filename = current_filename.parents[len(nspaces_level) - new_level - 1] / filename
+                        nspaces_level = nspaces_level[:new_level + 1]
+                    else:
+                        current_filename = current_filename.parents[0] / filename
+                    templates[short_name] = str(current_filename)
+
+        file_variables.update(variables)
+        res = get_registered(tree_name, cls)(templates, variables=file_variables, sub_trees=sub_trees)
+        for tree in sub_trees.values():
+            tree._parent = res
+        return res
+
+
+_registered_subtypes = {}
+
+
+def register_tree(name: str, tree_subtype: type):
+    """
+    Registers a tree_subtype under name
+
+    Loading a tree with given name will lead to the `tree_subtype` rather than FileTree to be returned
+
+    :param name: name of tree filename
+    :param tree_subtype: tree subtype
+    """
+    global _registered_subtypes
+    if not issubclass(tree_subtype, FileTree):
+        raise ValueError("Only sub-classes of FileTree can be registered")
+    _registered_subtypes[name] = tree_subtype
+
+
+def get_registered(name, default=FileTree) -> type:
+    """
+    Get the previously registered subtype for ``name``
+
+    :param name: name of the sub-tree
+    :param default: type to return if the name has not been registered
+    :return: FileTree or sub-type thereof
+    """
+    if name in _registered_subtypes:
+        return _registered_subtypes[name]
+    name = op.split(name)[1]
+    if name in _registered_subtypes:
+        return _registered_subtypes[name]
+    while name.endswith('.tree'):
+        name = name[:-5]
+        if name in _registered_subtypes:
+            return _registered_subtypes[name]
+    return default

fsl/utils/filetree/parse.py

+import os.path as op
+from . import filetree
+from pathlib import PurePath
+from typing import Tuple
+import re
+
+
+tree_directories = ['.', op.join(op.split(__file__)[0], 'trees')]
+
+
+def search_tree(name: str) -> str:
+    """
+    Searches for the file defining the specific tree
+
+    Iteratively searches through the directories in ``tree_directories`` till a file named ${name}.tree is found
+
+    :param name: Name of the tree
+    :return: path to the file defining the tree
+    """
+    for directory in tree_directories:
+        filename = op.join(directory, name)
+        if op.exists(filename):
+            return filename
+        elif op.exists(filename + '.tree'):
+            return filename + '.tree'
+    raise ValueError("No file tree found for %s" % name)
+
+
+def read_line(line: str) -> Tuple[int, PurePath, str]:
+    """
+    Parses line from the tree file
+
+    :param line: input line from a *.tree file
+    :return: Tuple with:
+    - number of spaces in front of the name
+    - name of the file or the sub_tree
+    - short name of the file
+    """
+    if line.strip()[:1] == '->':
+        return read_subtree_line(line)
+    match = re.match(r'^(\s*)(\S*)\s*\((\S*)\)\s*$', line)
+    if match is not None:
+        gr = match.groups()
+        return len(gr[0]), PurePath(gr[1]), gr[2]
+    match = re.match(r'^(\s*)(\S*)\s*$', line)
+    if match is not None:
+        gr = match.groups()
+        short_name = gr[1].split('.')[0]
+        return len(gr[0]), PurePath(gr[1]), short_name
+    raise ValueError('Unrecognized line %s' % line)
+
+
+def read_subtree_line(line: str, directory: str) -> Tuple[int, "filetree.FileTree", str]:
+    """
+    Parses the line defining a sub_tree
+
+    :param line: input line from a *.tree file
+    :param directory: containing directory
+    :return: Tuple with
+    - number of spaces in front of the name
+    - sub_tree
+    - short name of the sub_tree
+    """
+    match = re.match(r'^(\s*)->\s*(\S*)(.*)\((\S*)\)', line)
+    if match is None:
+        raise ValueError("Sub-tree line could not be parsed: {}".format(line.strip()))
+    spaces, type_name, variables_str, short_name = match.groups()
+
+    variables = {}
+    if len(variables_str.strip()) != 0:
+        for single_variable in variables_str.split(','):
+            key, value = single_variable.split('=')
+            variables[key.strip()] = value.strip()
+
+    sub_tree = filetree.FileTree.read(type_name, directory, **variables)
+    return len(spaces), sub_tree, short_name

fsl/utils/filetree/trees/BedpostX.tree

+bvals
+bvecs
+commands.txt
+dyads1.nii.gz
+dyads1_dispersion.nii.gz
+dyads2.nii.gz
+dyads2_dispersion.nii.gz
+dyads2_thr0.05.nii.gz (dyads2_thr0.05)
+dyads2_thr0.05_modf2.nii.gz (dyads2_thr0.05_modf2)
+dyads3.nii.gz
+dyads3_dispersion.nii.gz
+dyads3_thr0.05.nii.gz (dyads3_thr0.05)
+dyads3_thr0.05_modf3.nii.gz (dyads3_thr0.05_modf3)
+mean_Rsamples.nii.gz
+mean_S0samples.nii.gz
+mean_d_stdsamples.nii.gz
+mean_dsamples.nii.gz
+mean_f1samples.nii.gz
+mean_f2samples.nii.gz
+mean_f3samples.nii.gz
+mean_fsumsamples.nii.gz
+mean_ph1samples.nii.gz
+mean_ph2samples.nii.gz
+mean_ph3samples.nii.gz
+mean_tausamples.nii.gz
+mean_th1samples.nii.gz
+mean_th2samples.nii.gz
+mean_th3samples.nii.gz
+merged_f1samples.nii.gz
+merged_f2samples.nii.gz
+merged_f3samples.nii.gz
+merged_ph1samples.nii.gz
+merged_ph2samples.nii.gz
+merged_ph3samples.nii.gz
+merged_th1samples.nii.gz
+merged_th2samples.nii.gz
+merged_th3samples.nii.gz
+nodif_brain_mask.nii.gz
+xfms
+    eye.mat

fsl/utils/filetree/trees/Diffusion.tree

+data.nii.gz
+nodif_brain_mask.nii.gz
+bvals
+bvecs
+grad_dev.nii.gz

fsl/utils/filetree/trees/HCP_Surface.tree

+{subject}.{hemi}.sulc.{space}.shape.gii (sulc_gifti)
+{subject}.{hemi}.atlasroi.{space}.shape.gii (atlasroi_gifti)
+{subject}.{hemi}.thickness.{space}.shape.gii (thick_gifti)
+{subject}.{hemi}.curvature.{space}.shape.gii (curv_gifti)
+{subject}.{hemi}.white.{space}.surf.gii (white)
+{subject}.{hemi}.midthickness.{space}.surf.gii (mid)
+{subject}.{hemi}.pial.{space}.surf.gii (pial)
+{subject}.{hemi}.inflated.{space}.surf.gii (inflated)
+{subject}.{hemi}.very_inflated.{space}.surf.gii (very_inflated)
+{subject}.{hemi}.white_MSMAll.{space}.surf.gii (white_msm)
+{subject}.{hemi}.midthickness_MSMAll.{space}.surf.gii (mid_msm)
+{subject}.{hemi}.pial_MSMAll.{space}.surf.gii (pial_msm)
+{subject}.{hemi}.inflated_MSMAll.{space}.surf.gii (inflated_msm)
+{subject}.{hemi}.very_inflated_MSMAll.{space}.surf.gii (very_inflated_msm)
+{subject}.{hemi}.sphere.{space}.surf.gii (sphere)
+{subject}.{hemi}.sphere.reg.native.surf.gii (sphere_reg)
+{subject}.{hemi}.sphere.reg.reg_LR.native.surf.gii (sphere_reg_LR)
+{subject}.{hemi}.sphere.MSMSulc.native.surf.gii (sphere_MSMSulc)
+{subject}.{hemi}.sphere.MSMAll.native.surf.gii (sphere_msm)
+{subject}.{space}.wb.spec (spec)
+{subject}.sulc.{space}.dscalar.nii (sulc_cifti)
+{subject}.thickness.{space}.dscalar.nii (thick_cifti)
+{subject}.curvature.{space}.dscalar.nii (curv_cifti)
+{subject}.sulc_MSMAll.{space}.dscalar.nii (sulc_msm_cifti)
+{subject}.thickness_MSMAll.{space}.dscalar.nii (thick_msm_cifti)
+{subject}.curvature_MSMAll.{space}.dscalar.nii (curv_msm_cifti)

fsl/utils/filetree/trees/HCP_directory.tree

+{subject} (directory)
+    MNINonLinear
+        ->HCP_Surface space=164k_fs_LR (mni_164k)
+        BiasField.nii.gz
+        Native (mni_native)
+            ->HCP_Surface space=native (mni_native)
+        ROIs
+        T1w.nii.gz
+        T1w_restore.2.nii.gz (T1w_restore_2)
+        T1w_restore.nii.gz
+        T1w_restore_brain.nii.gz
+        T2w.nii.gz
+        T2w_restore.2.nii.gz (T2w_restore_2)
+        T2w_restore.nii.gz
+        T2w_restore_brain.nii.gz
+        aparc+aseg.nii.gz (mni_aparc)
+        aparc.a2009s+aseg.nii.gz (min_aparc_a2009s)
+        brainmask_fs.nii.gz (mni_brainmask)
+        fsaverage_LR32k (mni_32k)
+            ->HCP_Surface space=32k_fs_LR (mni_32k)
+        ribbon.nii.gz (mni_ribbon)
+        wmparc.nii.gz (mni_wmparc)
+        xfms
+            NonlinearRegJacobians.nii.gz
+            acpc_dc2standard.nii.gz
+            standard2acpc_dc.nii.gz
+    T1w (acpc_dc)
+        Diffusion
+            ->Diffusion (Diffusion)
+        Diffusion.bedpostX (bedpost)
+            ->BedpostX (bedpost)
+        {subject}_3T.csv
+        BiasField_acpc_dc.nii.gz
+        Native (T1w_native)
+            ->HCP_Surface space=native (T1w_native)
+        T1wDividedByT2w.nii.gz
+        T1wDividedByT2w_ribbon.nii.gz
+        T1w_acpc_dc.nii.gz
+        T1w_acpc_dc_restore.nii.gz
+        T1w_acpc_dc_restore_brain.nii.gz
+        T1w_acpc_dc_restore_1.25.nii.gz (T1w_diffusion)
+        T2w_acpc_dc.nii.gz
+        T2w_acpc_dc_restore.nii.gz
+        T2w_acpc_dc_restore_brain.nii.gz
+        aparc+aseg.nii.gz (T1w_aparc)
+        aparc.a2009s+aseg.nii.gz (T1w_apard_a2009s)
+        brainmask_fs.nii.gz (T1w_brainmask)
+        fsaverage_LR32k (T1w_32k)
+            ->HCP_Surface space=32k_fs_LR (T1w_32k)
+        ribbon.nii.gz (T1w_ribbon)
+        wmparc.nii.gz (T1w_wmparc)
+    release-notes
+

fsl/utils/filetree/trees/ProbtrackX.tree

+basename=fdt
+
+probtrackx.log (log_cmd)
+{basename}.log (log_settings)
+waytotal
+
+# single seed tracking
+coords = {x}_{y}_{z}   # default format for coords
+{basename}_{coords}.nii.gz (single_path) # --opd output
+{basename}_{coords}_length.nii.gz (single_path_length)  # --ompl output
+{basename}_{coords}_alt.nii.gz (single_alt_path)  # --opd/--fopd output
+{basename}_{coords}_alt_length.nii.gz (single_alt_path_length)  # --ompl/--fopd output
+
+# output paths
+{basename}.nii.gz (path)  # --opd output
+{basename}_length.nii.gz (path_length)  # --ompl output
+
+{basename}_alt.nii.gz (alt_path)  # --opd/--fopd output
+{basename}_alt_length.nii.gz (alt_path_length)  # --ompl/--fopd output
+
+# target mask connectivity
+seeds_to_{target}.nii.gz (seed2target)  # --os2t/--targetmasks output
+seeds_{seed_id}_to_{target}.nii.gz (multi_seed2target) # --os2t/--targetmasks with multiple seeds
+target_paths_{target}.nii.gz (target_path)  # --otargetpaths
+
+# ROIxROI connectivity (--network)
+fdt_network_matrix (network)
+tmpnetmaskfile (network_masks)
+
+# matrix output
+# matrix 1, 2, or 3
+coords_for_fdt_matrix{mat_id} (mat_seed_coord)  # matrix 1, 2, or3
+fdt_matrix{mat_id}.dot (mat)  # matrix 1, 2, or 3
+lookup_tractspace_fdt_matrix{mat_id}.nii.gz (mat_target_space)  # only matrix 2
+tract_space_coords_for_fdt_matrix{mat_id} (mat_target_coord) # matrix 2 or 3
+
+# matrix4
+fdt_matrix4_{part}.mtx (mat4)
+lookup_tractspace_fdt_matrix4.nii.gz (mat4_space)
+tract_space_coords_for_fdt_matrix4 (mat4_target_coord)
+
+
+# other
+{basename}_localdir.nii.gz (localdir)  # --opathdir
+
+saved_paths.txt  # --savepaths

fsl/utils/filetree/trees/bet.tree

+{name}.nii.gz (input)
+{name}_brain.nii.gz (output)
+{name}_brain_mask.nii.gz (mask)
\ No newline at end of file

fsl/utils/filetree/trees/bids_raw.tree

+ext=.nii.gz
+
+dataset_description.json
+participants.tsv
+README
+CHANGES
+sub-{participant}
+    [ses-{session}]
+        sub-{participant}_sessions.tsv (sessions_tsv)
+        anat (anat_dir)
+            sub-{participant}[_ses-{session}][_acq-{acq}][_rec-{rec}][_run-{run_index}]_{modality}{ext} (anat_image)
+        func (func_dir)
+            sub-{participant}[_ses-{session}]_task-{task}[_acq-{acq}][_rec-{rec}][_run-{run_index}]_bold{ext}  (task_image)
+            sub-{participant}[_ses-{session}]_task-{task}[_acq-{acq}][_rec-{rec}][_run-{run_index}]_sbref{ext} (task_sbref)
+            sub-{participant}[_ses-{session}]_task-{task}[_acq-{acq}][_rec-{rec}][_run-{run_index}]_events.tsv  (task_events)
+            sub-{participant}[_ses-{session}]_task-{task}[_acq-{acq}][_rec-{rec}][_run-{run_index}][_recording-{recording}]_physio{ext} (task_physio)
+            sub-{participant}[_ses-{session}]_task-{task}[_acq-{acq}][_rec-{rec}][_run-{run_index}][_recording-{recording}]_stim{ext} (task_stim)
+        dwi (dwi_dir)
+            sub-{participant}[_ses-{session}][_acq-{acq}][_run-{run_index}]_dwi{ext} (dwi_image)
+            sub-{participant}[_ses-{session}][_acq-{acq}][_run-{run_index}]_dwi.bval (bval)
+            sub-{participant}[_ses-{session}][_acq-{acq}][_run-{run_index}]_dwi.bvec (bvec)
+        fmap (fmap_dir)
+            sub-{participant}[_ses-{session}][_acq-{acq}][_run-{run_index}]_phasediff{ext} (fmap_phasediff)
+            sub-{participant}[_ses-{session}][_acq-{acq}][_run-{run_index}]_magnitude{ext} (fmap_mag)
+            sub-{participant}[_ses-{session}][_acq-{acq}][_run-{run_index}]_magnitude1{ext} (fmap_mag1)
+            sub-{participant}[_ses-{session}][_acq-{acq}][_run-{run_index}]_magnitude2{ext} (fmap_mag2)
+            sub-{participant}[_ses-{session}][_acq-{acq}][_run-{run_index}]_phase1{ext} (fmap_phase1)
+            sub-{participant}[_ses-{session}][_acq-{acq}][_run-{run_index}]_phase2{ext} (fmap_phase2)
+            sub-{participant}[_ses-{session}][_acq-{acq}][_run-{run_index}]_fieldmap{ext} (fmap)
+            sub-{participant}[_ses-{session}][_acq-{acq}]_dir-{dir}[_run-{run_index}]_epi{ext} (fmap_epi)

fsl/utils/filetree/trees/dti.tree

+basename = dti
+
+{basename} (basename)
+{basename}_MD.nii.gz (MD)
+{basename}_FA.nii.gz (FA)
+{basename}_MO.nii.gz (MO)
+{basename}_V1.nii.gz (V1)
+{basename}_V2.nii.gz (V2)
+{basename}_V3.nii.gz (V3)
+{basename}_L1.nii.gz (L1)
+{basename}_L2.nii.gz (L2)
+{basename}_L3.nii.gz (L3)
+
+{basename}_MK.nii.gz (MK)

fsl/utils/filetree/trees/eddy.tree

+basename = eddy_output
+
+{basename} (basename)
+{basename}.eddy_movement_rms (movement_rms)
+{basename}.eddy_outlier_map (outlier_map)
+{basename}.eddy_outlier_n_sqr_stdev_map (outlier_n_sqr_stdev_map)
+{basename}.eddy_outlier_n_stdev_map (outlier_n_stdev_map)
+{basename}.eddy_outlier_report (outlier_report)
+{basename}.eddy_parameters (paramaters)
+{basename}.eddy_post_eddy_shell_PE_translation_parameters (shell_PE_translation_parameters)
+{basename}.eddy_post_eddy_shell_alignment_parameters (shell_alignment_parameters)
+{basename}.eddy_restricted_movement_rms (restricted_movement_rms)
+{basename}.eddy_rotated_bvecs (rotated_bvecs)
+{basename}.nii.gz (image)

fsl/utils/filetree/trees/epi_reg.tree

+{basename} (basename)
+{basename}.mat (transform)
+{basename}.nii.gz (output)
+{basename}_fast_wmedge.nii.gz (wmedge)
+{basename}_fast_wmseg.nii.gz (wmseg)
+{basename}_init.mat (init_transform)
\ No newline at end of file

fsl/utils/filetree/trees/fast.tree

+{basename} (basename)
+{basename}.nii.gz (input)
+{basename}_mixeltype.nii.gz (mixeltype)
+{basename}_pve_{tissue_idx}.nii.gz (pve)
+{basename}_pveseg.nii.gz (pveseg)
+{basename}_seg.nii.gz (seg)
+
+# for T1-weighted images and 3 classes:
+{basename}_pve_0.nii.gz (csf_pve)
+{basename}_pve_1.nii.gz (gm_pve)
+{basename}_pve_2.nii.gz (wm_pve)

fsl/utils/filetree/trees/freesurfer.tree

+{subject} (directory)
+    orig (struct_dir)
+        001.mgz (vol1)
+        002.mgz (vol2)
+        T2raw.mgz
+        FLAIRraw.mgz
+        rawavg.mgz
+        orig.mgz
+        orig_nu.mgz
+        transforms
+            talairach.auto.xfm (talairach_auto)
+            talairach.xfm (talairach_xfm)
+            talairach_avi.log
+            talairach_with_skull.lta
+            talairach.lta (talairach_lta)
+            talairach.m3z (talairach_m3z)
+            T2raw.lta (T2raw_lta)
+        nu.mgz
+        T1.mgz
+        brainmask.auto.mgz (brainmask_auto)
+        brainmask.mgz
+        norm.mgz
+        brain.mgz
+        brain.finalsurfs.mgz (final_surfs)
+        wm.seg.mgz (wm_seg)
+        wm.asegedit.mgz (wm_asegedit)
+        wm.mgz
+        filled.mgz
+        filled-pretess255.mgz
+        filled-pretess127.mgz
+        ribbon.mgz (vol_ribbon)
+        wmparc.mgz
+
+        T2.prenorm.mgz (T2_prenorm)
+        T2.norm.mgz (T2_norm)
+        T2.mgz
+
+        {hemi}h.orig.nofix (orig_nofix)
+        {hemi}h.smoothwm.nofix (smoothwm_nofix)
+        {hemi}h.inflated.nofix (inflated_nofix)
+        {hemi}h.qsphere.nofix (qsphere_nofix)
+        {hemi}h.orig (white)
+        {hemi}h.inflated (inflated)
+        {hemi}h.sphere (sphere)
+        {hemi}h.sphere.reg (sphere_reg)
+        {hemi}h.white.preaparc (white_preaparc)
+        {hemi}h.curv (curv)
+        {hemi}h.sulc (sulc)
+        {hemi}h.area (area)
+        {hemi}h.jacobian_white (jacobian_white)
+        {hemi}h.avg_curv (avg_curv)
+        {hemi}h.cortex.label (cortex_label)
+        {hemi}h.smoothwm (smoothwm)
+        {hemi}h.qsphere (qsphere)
+
+        {hemi}h.pial (pial)
+        {hemi}h.woT2.pial (woT2_pial)
+        {hemi}h.curv.pial (curv_pial)
+        {hemi}h.area.pial (area_pial)
+        {hemi}h.thickness (thickness)
+        {hemi}h.ribbon.mgz (surf_ribbon)
+
+        aseg.auto_noCCseg.mgz (aseg_auto_noCCseg)
+        aseg.auto.mgz (aseg_auto)
+        aseg.presurf.mgz (aseg_presurf)
+        aparc+aseg.mgz (aparc_aseg)
+        aparc.a2009s+aseg.mgz (aparc_a2009s_aseg)
+        aseg.mgz
+
+        label/
+            {hemi}h.aparc.annot (aparc.annot)
+
+        stats/
+            aseg.stats (aseg_stats)
+            wmparc.stats (wmparc_stats)
+

fsl/utils/filetree/trees/tbss.tree

+FA
+    {subject}_FA.nii.gz (eroded)
+    {subject}_FA_mask.nii.gz (mask)
+    {subject}_FA_to_target.mat (affine2std)
+    {subject}_FA_to_target.nii.gz (eroded_std)
+    {subject}_FA_to_target_warp.msf (warp2std_msf)
+    {subject}_FA_to_target_warp.nii.gz (warp2std)
+    {subject}_FA_to_target_warp_inv.nii.gz (warp2std_inv)
+    slicesdir
+        index.html (eroded_html)
+origdata
+    {subject}.nii.gz (input)
+stats
+    all_FA.nii.gz
+    all_FA_skeletonised.nii.gz
+    mean_FA.nii.gz
+    mean_FA_mask.nii.gz
+    mean_FA_skeleton.nii.gz
+    mean_FA_skeleton_mask.nii.gz
+    mean_FA_skeleton_mask_dst.nii.gz
+    thresh.txt

fsl/utils/filetree/trees/topup.tree

+basename = topup_output
+
+{basename} (basename)
+{basename}_fieldcoef.nii.gz (fieldcoef)
+{basename}_movpar.txt (movement)

fsl/utils/filetree/utils.py

+import re
+import itertools
+import glob
+from . import filetree
+
+
+def resolve(template, variables):
+    """
+    Resolves the template given a set of variables
+
+    :param template: template
+    :param variables: mapping of variable names to values
+    :return: cleaned string
+    """
+    filled = fill_known(template, variables)
+    filename = resolve_optionals(filled)
+    remaining = find_variables(filename)
+    if len(remaining) > 0:
+        raise filetree.MissingVariable('Variables %s not defined' % set(remaining))
+    return filename
+
+
+def get_all(template, variables, glob_vars=()):
+    """
+    Gets all variables matching the templates given the variables
+
+    :param template: template
+    :param variables: (incomplete) mapping of variable names to values
+    :param glob_vars: sequence of undefined variables that can take any possible values when looking for matches on the disk
+    If `glob_vars` contains any defined variables, it will be ignored.
+    :return: sequence of filenames
+    """
+    filled = fill_known(template, variables)
+    remaining = set(find_variables(filled))
+    optional = optional_variables(filled)
+    res = set()
+    if glob_vars == 'all':
+        glob_vars = remaining
+    glob_vars = set(glob_vars).difference(variables.keys())
+
+    undefined_vars = remaining.difference(glob_vars).difference(optional)
+    if len(undefined_vars) > 0:
+        raise KeyError("Required variables {} were not defined".format(undefined_vars))
+
+    for keep in itertools.product(*[(True, False) for _ in optional.intersection(glob_vars)]):
+        sub_variables = {var: '*' for k, var in zip(keep, optional) if k}
+        for var in remaining.difference(optional).intersection(glob_vars):
+            sub_variables[var] = '*'
+        sub_filled = fill_known(filled, sub_variables)
+
+        pattern = resolve_optionals(sub_filled)
+        assert len(find_variables(pattern)) == 0
+
+        for filename in glob.glob(pattern):
+            try:
+                extract_variables(filled, filename)
+            except ValueError:
+                continue
+            res.add(filename)
+    return sorted(res)
+
+
+def fill_known(template, variables):
+    """
+    Fills in the known variables filling the other variables with {<variable_name>}
+
+    :param template: template
+    :param variables: mapping of variable names to values (ignoring any None)
+    :return: cleaned string
+    """
+    prev = ''
+    while prev != template:
+        prev = template
+        settings = {}
+        for name in set(find_variables(template)):
+            if name in variables and variables[name] is not None:
+                settings[name] = variables[name]
+            else:
+                settings[name] = '{' + name + '}'
+        template = template.format(**settings)
+    return template
+
+
+def resolve_optionals(text):
+    """
+    Resolves the optional sections
+
+    :param text: template after filling in the known variables
+    :return: cleaned string
+    """
+    def resolve_single_optional(part):
+        if len(part) == 0:
+            return part
+        if part[0] != '[' or part[-1] != ']':
+            return part
+        elif len(find_variables(part)) == 0:
+            return part[1:-1]
+        else:
+            return ''
+
+    res = [resolve_single_optional(text) for text in re.split('(\[.*?\])', text)]
+    return ''.join(res)
+
+
+def find_variables(template):
+    """
+    Finds all the variables in the template
+
+    :param template: full template
+    :return: sequence of variables
+    """
+    return tuple(var.split(':')[0] for var in re.findall("\{(.*?)\}", template))
+
+
+def optional_variables(template):
+    """
+    Finds the variables that can be skipped
+
+    :param template: full template
+    :return: set of variables that are only present in optional parts of the string
+    """
+    include = set()
+    exclude = set()
+    for text in re.split('(\[.*?\])', template):
+        if len(text) == 0:
+            continue
+        variables = find_variables(text)
+        if text[0] == '[' and text[-1] == ']':
+            include.update(variables)
+        else:
+            exclude.update(variables)
+    return include.difference(exclude)
+
+
+def extract_variables(template, filename, known_vars=None):
+    """
+    Extracts the variable values from the filename
+
+    :param template: template matching the given filename
+    :param filename: filename
+    :param known_vars: already known variables
+    :return: dictionary from variable names to string representations (unused variables set to None)
+    """
+    if known_vars is None:
+        known_vars = {}
+    template = fill_known(template, known_vars)
+    while '//' in filename:
+        filename = filename.replace('//', '/')
+    remaining = set(find_variables(template))
+    optional = optional_variables(template)
+    for keep in itertools.product(*[(True, False) for _ in optional]):
+        sub_re = resolve_optionals(fill_known(
+                template,
+                dict(
+                        **{var: '(\S+)' for k, var in zip(keep, optional) if k},
+                        **{var: '(\S+)' for var in remaining.difference(optional)}
+                )
+        ))
+        while '//' in sub_re:
+            sub_re = sub_re.replace('//', '/')
+        if re.match(sub_re, filename) is None:
+            continue
+
+        extracted_value = {}
+        kept_vars = [var for var in find_variables(template)
+                     if var not in optional or keep[list(optional).index(var)]]
+        for var, value in zip(kept_vars, re.match(sub_re, filename).groups()):
+            if var in extracted_value:
+                if value != extracted_value[var]:
+                    raise ValueError('Multiple values found for {}'.format(var))
+            else:
+                extracted_value[var] = value
+        for name in find_variables(template):
+            if name not in extracted_value:
+                extracted_value[name] = None
+        extracted_value.update(known_vars)
+        return extracted_value
+    raise ValueError("{} did not match {}".format(filename, template))

setup.py

@@ -111,6 +111,7 @@ setup(
 
     install_requires=install_requires,
     extras_require=extra_requires,
+    package_data={'fsl': ['utils/filetree/trees/*']},
 
     test_suite='tests',
 

tests/test_filetree/custom.tree

+parent
+    [opt_layer_{opt}]
+        sub_file.nii.gz
+        opt_file_{opt}.nii.gz (opt_file)
\ No newline at end of file