diff --git a/fsl/utils/filetree/__init__.py b/fsl/utils/filetree/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..b5ffec3ef7d33eb05b20f6e0b2a4ae2b2b26f538
--- /dev/null
+++ b/fsl/utils/filetree/__init__.py
@@ -0,0 +1,6 @@
+"""mc_filetree - Easy format to define intput/output files in a python pipeline"""
+
+__author__ = 'Michiel Cottaar <Michiel.Cottaar@ndcn.ox.ac.uk>'
+
+from .filetree import FileTree, register_tree, MissingVariable
+from .parse import tree_directories
diff --git a/fsl/utils/filetree/filetree.py b/fsl/utils/filetree/filetree.py
new file mode 100644
index 0000000000000000000000000000000000000000..10afe1e2a344382f801c9e8251f38e3532b320a3
--- /dev/null
+++ b/fsl/utils/filetree/filetree.py
@@ -0,0 +1,399 @@
+from pathlib import Path, PurePath
+from typing import Tuple, Optional, Dict, Any, Set
+from copy import deepcopy
+from . import parse
+import pickle
+import os.path as op
+from . import utils
+
+
+class MissingVariable(KeyError):
+ """
+ Returned when the variables of a tree or its parents do not contain a given variable
+ """
+ pass
+
+
+class FileTree(object):
+ """
+ Contains the input/output filename tree
+
+ Properties
+ - ``templates``: dictionary mapping short names to filename templates
+ - ``variables``: dictionary mapping variables in the templates to specific values (variables set to None are explicitly unset)
+ - ``sub_trees``: filename trees describing specific sub-directories
+ - ``parent``: parent FileTree, of which this sub-tree is a sub-directory
+ """
+ def __init__(self,
+ templates: Dict[str, str],
+ variables: Dict[str, Any],
+ sub_trees: Dict[str, "FileTree"]=None,
+ parent: Optional["FileTree"]=None):
+ """
+ Creates a new filename tree.
+ """
+ self.templates = templates
+ self.variables = variables
+ if sub_trees is None:
+ sub_trees = {}
+ self.sub_trees = sub_trees
+ self._parent = parent
+
+ @property
+ def parent(self, ):
+ return self._parent
+
+ @property
+ def all_variables(self, ):
+ """
+ All tree variables including those from the parent tree
+ """
+ if self.parent is None:
+ return self.variables
+ res = self.parent.all_variables
+ res.update(self.variables)
+ return res
+
+ def get_variable(self, name: str, default=None) -> str:
+ """
+ Gets a variable used to fill out the template
+
+ :param name: variable name
+ :param default: default variables (if not set an error is raised for a missing variable)
+ :return: value of the variable
+ """
+ variables = self.all_variables
+ if name in variables and variables[name] is not None:
+ return variables[name]
+ if default is None:
+ raise MissingVariable('Variable {} not found in sub-tree or parents'.format(name))
+ return default
+
+ def _get_template_tree(self, short_name: str) -> Tuple["FileTree", str]:
+ """
+ Retrieves template text from this tree, parent tree or sub_tree
+
+ :param short_name: filename reference name
+ :return: tuple with the containing tree and the template text
+ """
+ if '/' in short_name:
+ sub_tree, sub_name = short_name.split('/', maxsplit=1)
+ if sub_tree == '..':
+ if self.parent is None:
+ raise KeyError("Tried to access the parent of the top-level tree")
+ return self.parent._get_template_tree(sub_name)
+ return self.sub_trees[sub_tree]._get_template_tree(sub_name)
+ return self, self.templates[short_name]
+
+ def get_template(self, short_name: str) -> Tuple[str, Dict[str, str]]:
+ """
+ Returns the sub-tree that defines a given short name
+
+ - '/' characters in short_name refer to sub-trees
+ - '../' characters in short_name refer to parents
+
+ For example:
+ - eddy/output refers to the output in the eddy sub_tree (i.e. self.sub_trees['eddy'].templates['output']
+ - ../other/name refers to the 'other' sub-tree of the parent tree (i.e., self.parent.sub_trees['other'].templates['name']
+
+ :param short_name: name of the template
+ :return: tuple with the template and the variables corresponding to the template
+ """
+ tree, text = self._get_template_tree(short_name)
+ return text, tree.all_variables
+
+ def template_variables(self, short_name: Optional[str]=None, optional=True, required=True) -> Set[str]:
+ """
+ Returns the variables needed to define a template
+
+ :param short_name: name of the template (defaults to all)
+ :param optional: if set to False don't include the optional variables
+ :param required: if set to False don't include the required variables
+ :return: set of variable names
+ """
+ if not optional and not required:
+ return set()
+ if short_name is None:
+ all_vars = set()
+ required_vars = set()
+ for short_name in self.templates.keys():
+ all_vars.update(self.template_variables(short_name))
+ if required or optional:
+ required_vars.update(self.template_variables(short_name, optional=False))
+ for sub_tree in self.sub_trees.values():
+ all_vars.update(sub_tree.template_variables())
+ if required or optional:
+ required_vars.update(sub_tree.template_variables(optional=False))
+ if optional and required:
+ return all_vars
+ if required:
+ return required_vars
+ if optional:
+ return all_vars.difference(required_vars)
+ else:
+ _, text = self._get_template_tree(short_name)
+ all_vars = set(utils.find_variables(text))
+ if optional and required:
+ return all_vars
+ opt_vars = set(utils.optional_variables(text))
+ if optional:
+ return opt_vars
+ if required:
+ return all_vars.difference(opt_vars)
+
+ def get(self, short_name, make_dir=False) -> str:
+ """
+ Gets a full filename based on its short name
+
+ :param short_name: identifier in the tree
+ :param make_dir: if True make sure that the directory leading to this file exists
+ :return: full filename
+ """
+ text, variables = self.get_template(short_name)
+ res = Path(utils.resolve(text, variables))
+ if make_dir:
+ res.parents[0].mkdir(parents=True, exist_ok=True)
+ return str(res)
+
+ def get_all(self, short_name: str, glob_vars=()) -> Tuple[str]:
+ """
+ Gets all existing directory/file names matching a specific pattern
+
+ :param short_name: short name of the path template
+ :param glob_vars: sequence of undefined variables that can take any possible values when looking for matches on the disk
+ Any defined variables in `glob_vars` will be ignored.
+ If glob_vars is set to 'all', all undefined variables will be used to look up matches
+ :return: sorted sequence of paths
+ """
+ text, variables = self.get_template(short_name)
+ return tuple(utils.get_all(text, variables, glob_vars=glob_vars))
+
+ def get_all_vars(self, short_name: str, glob_vars=()) -> Tuple[Dict[str, str]]:
+ """
+ Gets all the parameters that generate existing filenames
+
+ :param short_name: short name of the path template
+ :param glob_vars: sequence of undefined variables that can take any possible values when looking for matches on the disk
+ Any defined variables in `glob_vars` will be ignored.
+ If glob_vars is set to 'all', all undefined variables will be used to look up matches
+ :return: sequence of dictionaries with the variables settings used to generate each filename
+ """
+ return tuple(self.extract_variables(short_name, fn) for fn in self.get_all(short_name, glob_vars=glob_vars))
+
+ def update(self, **variables) -> "FileTree":
+ """
+ Creates a new filetree with updated variables
+
+ :arg variables: new values for the variables
+ Setting variables to None will explicitly unset them
+ """
+ new_tree = deepcopy(self)
+ new_tree.variables.update(variables)
+ return new_tree
+
+ def extract_variables(self, short_name: str, filename: str) -> Dict[str, str]:
+ """
+ Extracts the variables from the given filename
+
+ :param short_name: short name of the path template
+ :param filename: filename matching the template
+ :return: variables needed to get to the given filename
+ Variables with None value are optional variables in the template that were not used
+ """
+ text, _ = self.get_template(short_name)
+ return utils.extract_variables(text, filename, self.variables)
+
+ def save_pickle(self, filename):
+ """
+ Saves the Filetree to a pickle file
+
+ :param filename: filename to store the file tree (usually ending with .pck)
+ """
+ with open(filename, 'wb') as f:
+ pickle.dump(self, f)
+
+ @classmethod
+ def load_pickle(cls, filename):
+ """
+ Loads the Filetree from a pickle file
+
+ :param filename: filename produced from Filetree.save
+ :return: stored Filetree
+ """
+ with open(filename, 'rb') as f:
+ res = pickle.load(f)
+ if not isinstance(res, cls):
+ raise IOError("Pickle file did not contain %s object" % cls)
+ return res
+
+ def exists(self, short_names, on_disk=False, error=False, glob_vars=()):
+ """
+ Checks whether the short_names are defined in the tree (and optional exist on the disk)
+
+ :param short_names: list of expected short names to exist in the tree
+ :param on_disk: if True checks whether the files exist on disk
+ :param error: if True raises a helpful error when the check fails
+ :param glob_vars: sequence of undefined variables that can take any possible values when looking for matches on the disk
+ If `glob_vars` contains any defined variables, it will be ignored.
+ :return: True if short names exist and optionally exist on disk (False otherwise)
+ :raise:
+ - ValueError if error is set and the tree is incomplete
+ - IOError if error is set and any files are missing from the disk
+ """
+ if isinstance(short_names, str):
+ short_names = (short_names, )
+
+ def single_test(short_name):
+ try:
+ self._get_template_tree(short_name)
+ except KeyError:
+ return True
+ return False
+
+ missing = tuple(name for name in short_names if single_test(name))
+ if len(missing) > 0:
+ if error:
+ raise ValueError("Provided Filetree is missing file definitions for {}".format(missing))
+ return False
+ if on_disk:
+ try:
+ missing = tuple(name for name in short_names if len(self.get_all(name, glob_vars=glob_vars)) == 0)
+ except KeyError:
+ if error:
+ raise
+ return False
+ if len(missing) > 0:
+ if error:
+ raise IOError("Failed to find any files existing for {}".format(missing))
+ return False
+ return True
+
+ @classmethod
+ def read(cls, tree_name: str, directory='.', **variables) -> "FileTree":
+ """
+ Reads a FileTree from a specific file
+
+ The return type is ``cls`` unless the tree_name has been previously registered
+ The return type of any sub-tree is FileTree unless the tree_name has been previously registered
+
+ :param tree_name: file containing the filename tree.
+ Can provide the filename of a tree file or the name for a tree in the ``filetree.tree_directories``.
+ :param directory: parent directory of the full tree (defaults to current directory)
+ :param variables: variable settings
+ :return: dictionary from specifier to filename
+ """
+ if op.exists(tree_name):
+ filename = tree_name
+ elif op.exists(tree_name + '.tree'):
+ filename = tree_name + '.tree'
+ else:
+ filename = parse.search_tree(tree_name)
+ filename = Path(filename)
+
+ templates = {}
+ nspaces_level = []
+ sub_trees = {}
+
+ file_variables = {}
+
+ with open(str(filename), 'r') as f:
+ for full_line in f:
+ # ignore anything behind the first #-character
+ line = full_line.split('#')[0]
+
+ if len(line.strip()) == 0:
+ continue
+
+ if line.strip()[:2] == '->':
+ nspaces = line.index('->')
+
+ if len(nspaces_level) == 0:
+ sub_dir = directory
+ elif nspaces > nspaces_level[-1]:
+ sub_dir = current_filename
+ elif nspaces < nspaces_level[-1]:
+ if nspaces not in nspaces_level:
+ raise ValueError('line %s dropped to a non-existent level' % line)
+ new_level = nspaces_level.index(nspaces)
+ current_filename = current_filename.parents[len(nspaces_level) - new_level - 1] / filename
+ nspaces_level = nspaces_level[:new_level + 1]
+ sub_dir = current_filename.parents[0]
+ else:
+ sub_dir = current_filename.parents[0]
+
+ _, sub_tree, short_name = parse.read_subtree_line(line, sub_dir)
+ if short_name in sub_trees:
+ raise ValueError("Name of sub_tree {short_name} used multiple times in {tree_name}.tree".format(**locals()))
+
+ sub_trees[short_name] = sub_tree
+ elif '=' in line:
+ key, value = line.split('=')
+ if len(key.split()) != 1:
+ raise ValueError("Variable assignment could not be parsed: {line}".format(**locals()))
+ file_variables[key.strip()] = value.strip()
+ else:
+ nspaces, filename, short_name = parse.read_line(line)
+ if short_name in templates:
+ raise ValueError("Name of directory/file {short_name} used multiple times in {tree_name}.tree".format(**locals()))
+
+ if len(nspaces_level) == 0:
+ current_filename = PurePath(directory) / filename
+ nspaces_level.append(nspaces)
+ elif nspaces > nspaces_level[-1]:
+ # increase the level
+ current_filename = current_filename / filename
+ nspaces_level.append(nspaces)
+ elif nspaces < nspaces_level[-1]:
+ # decreased the level
+ if nspaces not in nspaces_level:
+ raise ValueError('line %s dropped to a non-existent level' % full_line)
+ new_level = nspaces_level.index(nspaces)
+ current_filename = current_filename.parents[len(nspaces_level) - new_level - 1] / filename
+ nspaces_level = nspaces_level[:new_level + 1]
+ else:
+ current_filename = current_filename.parents[0] / filename
+ templates[short_name] = str(current_filename)
+
+ file_variables.update(variables)
+ res = get_registered(tree_name, cls)(templates, variables=file_variables, sub_trees=sub_trees)
+ for tree in sub_trees.values():
+ tree._parent = res
+ return res
+
+
+_registered_subtypes = {}
+
+
+def register_tree(name: str, tree_subtype: type):
+ """
+ Registers a tree_subtype under name
+
+ Loading a tree with given name will lead to the `tree_subtype` rather than FileTree to be returned
+
+ :param name: name of tree filename
+ :param tree_subtype: tree subtype
+ """
+ global _registered_subtypes
+ if not issubclass(tree_subtype, FileTree):
+ raise ValueError("Only sub-classes of FileTree can be registered")
+ _registered_subtypes[name] = tree_subtype
+
+
+def get_registered(name, default=FileTree) -> type:
+ """
+ Get the previously registered subtype for ``name``
+
+ :param name: name of the sub-tree
+ :param default: type to return if the name has not been registered
+ :return: FileTree or sub-type thereof
+ """
+ if name in _registered_subtypes:
+ return _registered_subtypes[name]
+ name = op.split(name)[1]
+ if name in _registered_subtypes:
+ return _registered_subtypes[name]
+ while name.endswith('.tree'):
+ name = name[:-5]
+ if name in _registered_subtypes:
+ return _registered_subtypes[name]
+ return default
diff --git a/fsl/utils/filetree/parse.py b/fsl/utils/filetree/parse.py
new file mode 100644
index 0000000000000000000000000000000000000000..2e5195361b2fe1b51bb0dcb60219949adaff9a50
--- /dev/null
+++ b/fsl/utils/filetree/parse.py
@@ -0,0 +1,76 @@
+import os.path as op
+from . import filetree
+from pathlib import PurePath
+from typing import Tuple
+import re
+
+
+tree_directories = ['.', op.join(op.split(__file__)[0], 'trees')]
+
+
+def search_tree(name: str) -> str:
+ """
+ Searches for the file defining the specific tree
+
+ Iteratively searches through the directories in ``tree_directories`` till a file named ${name}.tree is found
+
+ :param name: Name of the tree
+ :return: path to the file defining the tree
+ """
+ for directory in tree_directories:
+ filename = op.join(directory, name)
+ if op.exists(filename):
+ return filename
+ elif op.exists(filename + '.tree'):
+ return filename + '.tree'
+ raise ValueError("No file tree found for %s" % name)
+
+
+def read_line(line: str) -> Tuple[int, PurePath, str]:
+ """
+ Parses line from the tree file
+
+ :param line: input line from a *.tree file
+ :return: Tuple with:
+ - number of spaces in front of the name
+ - name of the file or the sub_tree
+ - short name of the file
+ """
+ if line.strip()[:1] == '->':
+ return read_subtree_line(line)
+ match = re.match(r'^(\s*)(\S*)\s*\((\S*)\)\s*$', line)
+ if match is not None:
+ gr = match.groups()
+ return len(gr[0]), PurePath(gr[1]), gr[2]
+ match = re.match(r'^(\s*)(\S*)\s*$', line)
+ if match is not None:
+ gr = match.groups()
+ short_name = gr[1].split('.')[0]
+ return len(gr[0]), PurePath(gr[1]), short_name
+ raise ValueError('Unrecognized line %s' % line)
+
+
+def read_subtree_line(line: str, directory: str) -> Tuple[int, "filetree.FileTree", str]:
+ """
+ Parses the line defining a sub_tree
+
+ :param line: input line from a *.tree file
+ :param directory: containing directory
+ :return: Tuple with
+ - number of spaces in front of the name
+ - sub_tree
+ - short name of the sub_tree
+ """
+ match = re.match(r'^(\s*)->\s*(\S*)(.*)\((\S*)\)', line)
+ if match is None:
+ raise ValueError("Sub-tree line could not be parsed: {}".format(line.strip()))
+ spaces, type_name, variables_str, short_name = match.groups()
+
+ variables = {}
+ if len(variables_str.strip()) != 0:
+ for single_variable in variables_str.split(','):
+ key, value = single_variable.split('=')
+ variables[key.strip()] = value.strip()
+
+ sub_tree = filetree.FileTree.read(type_name, directory, **variables)
+ return len(spaces), sub_tree, short_name
diff --git a/fsl/utils/filetree/trees/BedpostX.tree b/fsl/utils/filetree/trees/BedpostX.tree
new file mode 100644
index 0000000000000000000000000000000000000000..4bc58e42b8802334ddfe0635ea72875b76cdd2f4
--- /dev/null
+++ b/fsl/utils/filetree/trees/BedpostX.tree
@@ -0,0 +1,40 @@
+bvals
+bvecs
+commands.txt
+dyads1.nii.gz
+dyads1_dispersion.nii.gz
+dyads2.nii.gz
+dyads2_dispersion.nii.gz
+dyads2_thr0.05.nii.gz (dyads2_thr0.05)
+dyads2_thr0.05_modf2.nii.gz (dyads2_thr0.05_modf2)
+dyads3.nii.gz
+dyads3_dispersion.nii.gz
+dyads3_thr0.05.nii.gz (dyads3_thr0.05)
+dyads3_thr0.05_modf3.nii.gz (dyads3_thr0.05_modf3)
+mean_Rsamples.nii.gz
+mean_S0samples.nii.gz
+mean_d_stdsamples.nii.gz
+mean_dsamples.nii.gz
+mean_f1samples.nii.gz
+mean_f2samples.nii.gz
+mean_f3samples.nii.gz
+mean_fsumsamples.nii.gz
+mean_ph1samples.nii.gz
+mean_ph2samples.nii.gz
+mean_ph3samples.nii.gz
+mean_tausamples.nii.gz
+mean_th1samples.nii.gz
+mean_th2samples.nii.gz
+mean_th3samples.nii.gz
+merged_f1samples.nii.gz
+merged_f2samples.nii.gz
+merged_f3samples.nii.gz
+merged_ph1samples.nii.gz
+merged_ph2samples.nii.gz
+merged_ph3samples.nii.gz
+merged_th1samples.nii.gz
+merged_th2samples.nii.gz
+merged_th3samples.nii.gz
+nodif_brain_mask.nii.gz
+xfms
+ eye.mat
diff --git a/fsl/utils/filetree/trees/Diffusion.tree b/fsl/utils/filetree/trees/Diffusion.tree
new file mode 100644
index 0000000000000000000000000000000000000000..76c3ee16cf3f4456ab43159d2116dce763ba1a20
--- /dev/null
+++ b/fsl/utils/filetree/trees/Diffusion.tree
@@ -0,0 +1,5 @@
+data.nii.gz
+nodif_brain_mask.nii.gz
+bvals
+bvecs
+grad_dev.nii.gz
diff --git a/fsl/utils/filetree/trees/HCP_Surface.tree b/fsl/utils/filetree/trees/HCP_Surface.tree
new file mode 100644
index 0000000000000000000000000000000000000000..7850b8db7ff488c38674ab6e6d9d5514258fe980
--- /dev/null
+++ b/fsl/utils/filetree/trees/HCP_Surface.tree
@@ -0,0 +1,26 @@
+{subject}.{hemi}.sulc.{space}.shape.gii (sulc_gifti)
+{subject}.{hemi}.atlasroi.{space}.shape.gii (atlasroi_gifti)
+{subject}.{hemi}.thickness.{space}.shape.gii (thick_gifti)
+{subject}.{hemi}.curvature.{space}.shape.gii (curv_gifti)
+{subject}.{hemi}.white.{space}.surf.gii (white)
+{subject}.{hemi}.midthickness.{space}.surf.gii (mid)
+{subject}.{hemi}.pial.{space}.surf.gii (pial)
+{subject}.{hemi}.inflated.{space}.surf.gii (inflated)
+{subject}.{hemi}.very_inflated.{space}.surf.gii (very_inflated)
+{subject}.{hemi}.white_MSMAll.{space}.surf.gii (white_msm)
+{subject}.{hemi}.midthickness_MSMAll.{space}.surf.gii (mid_msm)
+{subject}.{hemi}.pial_MSMAll.{space}.surf.gii (pial_msm)
+{subject}.{hemi}.inflated_MSMAll.{space}.surf.gii (inflated_msm)
+{subject}.{hemi}.very_inflated_MSMAll.{space}.surf.gii (very_inflated_msm)
+{subject}.{hemi}.sphere.{space}.surf.gii (sphere)
+{subject}.{hemi}.sphere.reg.native.surf.gii (sphere_reg)
+{subject}.{hemi}.sphere.reg.reg_LR.native.surf.gii (sphere_reg_LR)
+{subject}.{hemi}.sphere.MSMSulc.native.surf.gii (sphere_MSMSulc)
+{subject}.{hemi}.sphere.MSMAll.native.surf.gii (sphere_msm)
+{subject}.{space}.wb.spec (spec)
+{subject}.sulc.{space}.dscalar.nii (sulc_cifti)
+{subject}.thickness.{space}.dscalar.nii (thick_cifti)
+{subject}.curvature.{space}.dscalar.nii (curv_cifti)
+{subject}.sulc_MSMAll.{space}.dscalar.nii (sulc_msm_cifti)
+{subject}.thickness_MSMAll.{space}.dscalar.nii (thick_msm_cifti)
+{subject}.curvature_MSMAll.{space}.dscalar.nii (curv_msm_cifti)
diff --git a/fsl/utils/filetree/trees/HCP_directory.tree b/fsl/utils/filetree/trees/HCP_directory.tree
new file mode 100644
index 0000000000000000000000000000000000000000..11c1ec5eb00767c802dd98daede9f5fe2a193123
--- /dev/null
+++ b/fsl/utils/filetree/trees/HCP_directory.tree
@@ -0,0 +1,53 @@
+{subject} (directory)
+ MNINonLinear
+ ->HCP_Surface space=164k_fs_LR (mni_164k)
+ BiasField.nii.gz
+ Native (mni_native)
+ ->HCP_Surface space=native (mni_native)
+ ROIs
+ T1w.nii.gz
+ T1w_restore.2.nii.gz (T1w_restore_2)
+ T1w_restore.nii.gz
+ T1w_restore_brain.nii.gz
+ T2w.nii.gz
+ T2w_restore.2.nii.gz (T2w_restore_2)
+ T2w_restore.nii.gz
+ T2w_restore_brain.nii.gz
+ aparc+aseg.nii.gz (mni_aparc)
+ aparc.a2009s+aseg.nii.gz (min_aparc_a2009s)
+ brainmask_fs.nii.gz (mni_brainmask)
+ fsaverage_LR32k (mni_32k)
+ ->HCP_Surface space=32k_fs_LR (mni_32k)
+ ribbon.nii.gz (mni_ribbon)
+ wmparc.nii.gz (mni_wmparc)
+ xfms
+ NonlinearRegJacobians.nii.gz
+ acpc_dc2standard.nii.gz
+ standard2acpc_dc.nii.gz
+ T1w (acpc_dc)
+ Diffusion
+ ->Diffusion (Diffusion)
+ Diffusion.bedpostX (bedpost)
+ ->BedpostX (bedpost)
+ {subject}_3T.csv
+ BiasField_acpc_dc.nii.gz
+ Native (T1w_native)
+ ->HCP_Surface space=native (T1w_native)
+ T1wDividedByT2w.nii.gz
+ T1wDividedByT2w_ribbon.nii.gz
+ T1w_acpc_dc.nii.gz
+ T1w_acpc_dc_restore.nii.gz
+ T1w_acpc_dc_restore_brain.nii.gz
+ T1w_acpc_dc_restore_1.25.nii.gz (T1w_diffusion)
+ T2w_acpc_dc.nii.gz
+ T2w_acpc_dc_restore.nii.gz
+ T2w_acpc_dc_restore_brain.nii.gz
+ aparc+aseg.nii.gz (T1w_aparc)
+ aparc.a2009s+aseg.nii.gz (T1w_apard_a2009s)
+ brainmask_fs.nii.gz (T1w_brainmask)
+ fsaverage_LR32k (T1w_32k)
+ ->HCP_Surface space=32k_fs_LR (T1w_32k)
+ ribbon.nii.gz (T1w_ribbon)
+ wmparc.nii.gz (T1w_wmparc)
+ release-notes
+
diff --git a/fsl/utils/filetree/trees/ProbtrackX.tree b/fsl/utils/filetree/trees/ProbtrackX.tree
new file mode 100644
index 0000000000000000000000000000000000000000..38db04c26930365b9046b8171ba28bfbd116ff39
--- /dev/null
+++ b/fsl/utils/filetree/trees/ProbtrackX.tree
@@ -0,0 +1,46 @@
+basename=fdt
+
+probtrackx.log (log_cmd)
+{basename}.log (log_settings)
+waytotal
+
+# single seed tracking
+coords = {x}_{y}_{z} # default format for coords
+{basename}_{coords}.nii.gz (single_path) # --opd output
+{basename}_{coords}_length.nii.gz (single_path_length) # --ompl output
+{basename}_{coords}_alt.nii.gz (single_alt_path) # --opd/--fopd output
+{basename}_{coords}_alt_length.nii.gz (single_alt_path_length) # --ompl/--fopd output
+
+# output paths
+{basename}.nii.gz (path) # --opd output
+{basename}_length.nii.gz (path_length) # --ompl output
+
+{basename}_alt.nii.gz (alt_path) # --opd/--fopd output
+{basename}_alt_length.nii.gz (alt_path_length) # --ompl/--fopd output
+
+# target mask connectivity
+seeds_to_{target}.nii.gz (seed2target) # --os2t/--targetmasks output
+seeds_{seed_id}_to_{target}.nii.gz (multi_seed2target) # --os2t/--targetmasks with multiple seeds
+target_paths_{target}.nii.gz (target_path) # --otargetpaths
+
+# ROIxROI connectivity (--network)
+fdt_network_matrix (network)
+tmpnetmaskfile (network_masks)
+
+# matrix output
+# matrix 1, 2, or 3
+coords_for_fdt_matrix{mat_id} (mat_seed_coord) # matrix 1, 2, or3
+fdt_matrix{mat_id}.dot (mat) # matrix 1, 2, or 3
+lookup_tractspace_fdt_matrix{mat_id}.nii.gz (mat_target_space) # only matrix 2
+tract_space_coords_for_fdt_matrix{mat_id} (mat_target_coord) # matrix 2 or 3
+
+# matrix4
+fdt_matrix4_{part}.mtx (mat4)
+lookup_tractspace_fdt_matrix4.nii.gz (mat4_space)
+tract_space_coords_for_fdt_matrix4 (mat4_target_coord)
+
+
+# other
+{basename}_localdir.nii.gz (localdir) # --opathdir
+
+saved_paths.txt # --savepaths
diff --git a/fsl/utils/filetree/trees/bet.tree b/fsl/utils/filetree/trees/bet.tree
new file mode 100644
index 0000000000000000000000000000000000000000..b2c8f1b23dd05e9efbd88f199443867bb01752b0
--- /dev/null
+++ b/fsl/utils/filetree/trees/bet.tree
@@ -0,0 +1,3 @@
+{name}.nii.gz (input)
+{name}_brain.nii.gz (output)
+{name}_brain_mask.nii.gz (mask)
\ No newline at end of file
diff --git a/fsl/utils/filetree/trees/bids_raw.tree b/fsl/utils/filetree/trees/bids_raw.tree
new file mode 100644
index 0000000000000000000000000000000000000000..d07f190047a01670e77b74475f7caeadf5df22a1
--- /dev/null
+++ b/fsl/utils/filetree/trees/bids_raw.tree
@@ -0,0 +1,30 @@
+ext=.nii.gz
+
+dataset_description.json
+participants.tsv
+README
+CHANGES
+sub-{participant}
+ [ses-{session}]
+ sub-{participant}_sessions.tsv (sessions_tsv)
+ anat (anat_dir)
+ sub-{participant}[_ses-{session}][_acq-{acq}][_rec-{rec}][_run-{run_index}]_{modality}{ext} (anat_image)
+ func (func_dir)
+ sub-{participant}[_ses-{session}]_task-{task}[_acq-{acq}][_rec-{rec}][_run-{run_index}]_bold{ext} (task_image)
+ sub-{participant}[_ses-{session}]_task-{task}[_acq-{acq}][_rec-{rec}][_run-{run_index}]_sbref{ext} (task_sbref)
+ sub-{participant}[_ses-{session}]_task-{task}[_acq-{acq}][_rec-{rec}][_run-{run_index}]_events.tsv (task_events)
+ sub-{participant}[_ses-{session}]_task-{task}[_acq-{acq}][_rec-{rec}][_run-{run_index}][_recording-{recording}]_physio{ext} (task_physio)
+ sub-{participant}[_ses-{session}]_task-{task}[_acq-{acq}][_rec-{rec}][_run-{run_index}][_recording-{recording}]_stim{ext} (task_stim)
+ dwi (dwi_dir)
+ sub-{participant}[_ses-{session}][_acq-{acq}][_run-{run_index}]_dwi{ext} (dwi_image)
+ sub-{participant}[_ses-{session}][_acq-{acq}][_run-{run_index}]_dwi.bval (bval)
+ sub-{participant}[_ses-{session}][_acq-{acq}][_run-{run_index}]_dwi.bvec (bvec)
+ fmap (fmap_dir)
+ sub-{participant}[_ses-{session}][_acq-{acq}][_run-{run_index}]_phasediff{ext} (fmap_phasediff)
+ sub-{participant}[_ses-{session}][_acq-{acq}][_run-{run_index}]_magnitude{ext} (fmap_mag)
+ sub-{participant}[_ses-{session}][_acq-{acq}][_run-{run_index}]_magnitude1{ext} (fmap_mag1)
+ sub-{participant}[_ses-{session}][_acq-{acq}][_run-{run_index}]_magnitude2{ext} (fmap_mag2)
+ sub-{participant}[_ses-{session}][_acq-{acq}][_run-{run_index}]_phase1{ext} (fmap_phase1)
+ sub-{participant}[_ses-{session}][_acq-{acq}][_run-{run_index}]_phase2{ext} (fmap_phase2)
+ sub-{participant}[_ses-{session}][_acq-{acq}][_run-{run_index}]_fieldmap{ext} (fmap)
+ sub-{participant}[_ses-{session}][_acq-{acq}]_dir-{dir}[_run-{run_index}]_epi{ext} (fmap_epi)
diff --git a/fsl/utils/filetree/trees/dti.tree b/fsl/utils/filetree/trees/dti.tree
new file mode 100644
index 0000000000000000000000000000000000000000..bac4c29ae37e26302a4c30c2ab9ca0ba2d5efc1d
--- /dev/null
+++ b/fsl/utils/filetree/trees/dti.tree
@@ -0,0 +1,14 @@
+basename = dti
+
+{basename} (basename)
+{basename}_MD.nii.gz (MD)
+{basename}_FA.nii.gz (FA)
+{basename}_MO.nii.gz (MO)
+{basename}_V1.nii.gz (V1)
+{basename}_V2.nii.gz (V2)
+{basename}_V3.nii.gz (V3)
+{basename}_L1.nii.gz (L1)
+{basename}_L2.nii.gz (L2)
+{basename}_L3.nii.gz (L3)
+
+{basename}_MK.nii.gz (MK)
diff --git a/fsl/utils/filetree/trees/eddy.tree b/fsl/utils/filetree/trees/eddy.tree
new file mode 100644
index 0000000000000000000000000000000000000000..323abfd78a65ec65761ef7886d091105a0896b1a
--- /dev/null
+++ b/fsl/utils/filetree/trees/eddy.tree
@@ -0,0 +1,14 @@
+basename = eddy_output
+
+{basename} (basename)
+{basename}.eddy_movement_rms (movement_rms)
+{basename}.eddy_outlier_map (outlier_map)
+{basename}.eddy_outlier_n_sqr_stdev_map (outlier_n_sqr_stdev_map)
+{basename}.eddy_outlier_n_stdev_map (outlier_n_stdev_map)
+{basename}.eddy_outlier_report (outlier_report)
+{basename}.eddy_parameters (paramaters)
+{basename}.eddy_post_eddy_shell_PE_translation_parameters (shell_PE_translation_parameters)
+{basename}.eddy_post_eddy_shell_alignment_parameters (shell_alignment_parameters)
+{basename}.eddy_restricted_movement_rms (restricted_movement_rms)
+{basename}.eddy_rotated_bvecs (rotated_bvecs)
+{basename}.nii.gz (image)
diff --git a/fsl/utils/filetree/trees/epi_reg.tree b/fsl/utils/filetree/trees/epi_reg.tree
new file mode 100644
index 0000000000000000000000000000000000000000..729d87859e38f688e3f9462adb3c7da4243dfa80
--- /dev/null
+++ b/fsl/utils/filetree/trees/epi_reg.tree
@@ -0,0 +1,6 @@
+{basename} (basename)
+{basename}.mat (transform)
+{basename}.nii.gz (output)
+{basename}_fast_wmedge.nii.gz (wmedge)
+{basename}_fast_wmseg.nii.gz (wmseg)
+{basename}_init.mat (init_transform)
\ No newline at end of file
diff --git a/fsl/utils/filetree/trees/fast.tree b/fsl/utils/filetree/trees/fast.tree
new file mode 100644
index 0000000000000000000000000000000000000000..f39d407484924f7358ad95bed359b79d370e1e4f
--- /dev/null
+++ b/fsl/utils/filetree/trees/fast.tree
@@ -0,0 +1,11 @@
+{basename} (basename)
+{basename}.nii.gz (input)
+{basename}_mixeltype.nii.gz (mixeltype)
+{basename}_pve_{tissue_idx}.nii.gz (pve)
+{basename}_pveseg.nii.gz (pveseg)
+{basename}_seg.nii.gz (seg)
+
+# for T1-weighted images and 3 classes:
+{basename}_pve_0.nii.gz (csf_pve)
+{basename}_pve_1.nii.gz (gm_pve)
+{basename}_pve_2.nii.gz (wm_pve)
diff --git a/fsl/utils/filetree/trees/freesurfer.tree b/fsl/utils/filetree/trees/freesurfer.tree
new file mode 100644
index 0000000000000000000000000000000000000000..8a20ac2110861bd31205235bfee3ce0715e40240
--- /dev/null
+++ b/fsl/utils/filetree/trees/freesurfer.tree
@@ -0,0 +1,76 @@
+{subject} (directory)
+ orig (struct_dir)
+ 001.mgz (vol1)
+ 002.mgz (vol2)
+ T2raw.mgz
+ FLAIRraw.mgz
+ rawavg.mgz
+ orig.mgz
+ orig_nu.mgz
+ transforms
+ talairach.auto.xfm (talairach_auto)
+ talairach.xfm (talairach_xfm)
+ talairach_avi.log
+ talairach_with_skull.lta
+ talairach.lta (talairach_lta)
+ talairach.m3z (talairach_m3z)
+ T2raw.lta (T2raw_lta)
+ nu.mgz
+ T1.mgz
+ brainmask.auto.mgz (brainmask_auto)
+ brainmask.mgz
+ norm.mgz
+ brain.mgz
+ brain.finalsurfs.mgz (final_surfs)
+ wm.seg.mgz (wm_seg)
+ wm.asegedit.mgz (wm_asegedit)
+ wm.mgz
+ filled.mgz
+ filled-pretess255.mgz
+ filled-pretess127.mgz
+ ribbon.mgz (vol_ribbon)
+ wmparc.mgz
+
+ T2.prenorm.mgz (T2_prenorm)
+ T2.norm.mgz (T2_norm)
+ T2.mgz
+
+ {hemi}h.orig.nofix (orig_nofix)
+ {hemi}h.smoothwm.nofix (smoothwm_nofix)
+ {hemi}h.inflated.nofix (inflated_nofix)
+ {hemi}h.qsphere.nofix (qsphere_nofix)
+ {hemi}h.orig (white)
+ {hemi}h.inflated (inflated)
+ {hemi}h.sphere (sphere)
+ {hemi}h.sphere.reg (sphere_reg)
+ {hemi}h.white.preaparc (white_preaparc)
+ {hemi}h.curv (curv)
+ {hemi}h.sulc (sulc)
+ {hemi}h.area (area)
+ {hemi}h.jacobian_white (jacobian_white)
+ {hemi}h.avg_curv (avg_curv)
+ {hemi}h.cortex.label (cortex_label)
+ {hemi}h.smoothwm (smoothwm)
+ {hemi}h.qsphere (qsphere)
+
+ {hemi}h.pial (pial)
+ {hemi}h.woT2.pial (woT2_pial)
+ {hemi}h.curv.pial (curv_pial)
+ {hemi}h.area.pial (area_pial)
+ {hemi}h.thickness (thickness)
+ {hemi}h.ribbon.mgz (surf_ribbon)
+
+ aseg.auto_noCCseg.mgz (aseg_auto_noCCseg)
+ aseg.auto.mgz (aseg_auto)
+ aseg.presurf.mgz (aseg_presurf)
+ aparc+aseg.mgz (aparc_aseg)
+ aparc.a2009s+aseg.mgz (aparc_a2009s_aseg)
+ aseg.mgz
+
+ label/
+ {hemi}h.aparc.annot (aparc.annot)
+
+ stats/
+ aseg.stats (aseg_stats)
+ wmparc.stats (wmparc_stats)
+
diff --git a/fsl/utils/filetree/trees/tbss.tree b/fsl/utils/filetree/trees/tbss.tree
new file mode 100644
index 0000000000000000000000000000000000000000..4e89f0d43e390a05fa7ad5b6b1a7e4229b954a52
--- /dev/null
+++ b/fsl/utils/filetree/trees/tbss.tree
@@ -0,0 +1,21 @@
+FA
+ {subject}_FA.nii.gz (eroded)
+ {subject}_FA_mask.nii.gz (mask)
+ {subject}_FA_to_target.mat (affine2std)
+ {subject}_FA_to_target.nii.gz (eroded_std)
+ {subject}_FA_to_target_warp.msf (warp2std_msf)
+ {subject}_FA_to_target_warp.nii.gz (warp2std)
+ {subject}_FA_to_target_warp_inv.nii.gz (warp2std_inv)
+ slicesdir
+ index.html (eroded_html)
+origdata
+ {subject}.nii.gz (input)
+stats
+ all_FA.nii.gz
+ all_FA_skeletonised.nii.gz
+ mean_FA.nii.gz
+ mean_FA_mask.nii.gz
+ mean_FA_skeleton.nii.gz
+ mean_FA_skeleton_mask.nii.gz
+ mean_FA_skeleton_mask_dst.nii.gz
+ thresh.txt
diff --git a/fsl/utils/filetree/trees/topup.tree b/fsl/utils/filetree/trees/topup.tree
new file mode 100644
index 0000000000000000000000000000000000000000..9a6702dd65a3da0b36e8d012d9e75585340f6c50
--- /dev/null
+++ b/fsl/utils/filetree/trees/topup.tree
@@ -0,0 +1,5 @@
+basename = topup_output
+
+{basename} (basename)
+{basename}_fieldcoef.nii.gz (fieldcoef)
+{basename}_movpar.txt (movement)
diff --git a/fsl/utils/filetree/utils.py b/fsl/utils/filetree/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..f78046fff8f6878309225b55ae22d6c8dab519ff
--- /dev/null
+++ b/fsl/utils/filetree/utils.py
@@ -0,0 +1,178 @@
+import re
+import itertools
+import glob
+from . import filetree
+
+
+def resolve(template, variables):
+ """
+ Resolves the template given a set of variables
+
+ :param template: template
+ :param variables: mapping of variable names to values
+ :return: cleaned string
+ """
+ filled = fill_known(template, variables)
+ filename = resolve_optionals(filled)
+ remaining = find_variables(filename)
+ if len(remaining) > 0:
+ raise filetree.MissingVariable('Variables %s not defined' % set(remaining))
+ return filename
+
+
+def get_all(template, variables, glob_vars=()):
+ """
+ Gets all variables matching the templates given the variables
+
+ :param template: template
+ :param variables: (incomplete) mapping of variable names to values
+ :param glob_vars: sequence of undefined variables that can take any possible values when looking for matches on the disk
+ If `glob_vars` contains any defined variables, it will be ignored.
+ :return: sequence of filenames
+ """
+ filled = fill_known(template, variables)
+ remaining = set(find_variables(filled))
+ optional = optional_variables(filled)
+ res = set()
+ if glob_vars == 'all':
+ glob_vars = remaining
+ glob_vars = set(glob_vars).difference(variables.keys())
+
+ undefined_vars = remaining.difference(glob_vars).difference(optional)
+ if len(undefined_vars) > 0:
+ raise KeyError("Required variables {} were not defined".format(undefined_vars))
+
+ for keep in itertools.product(*[(True, False) for _ in optional.intersection(glob_vars)]):
+ sub_variables = {var: '*' for k, var in zip(keep, optional) if k}
+ for var in remaining.difference(optional).intersection(glob_vars):
+ sub_variables[var] = '*'
+ sub_filled = fill_known(filled, sub_variables)
+
+ pattern = resolve_optionals(sub_filled)
+ assert len(find_variables(pattern)) == 0
+
+ for filename in glob.glob(pattern):
+ try:
+ extract_variables(filled, filename)
+ except ValueError:
+ continue
+ res.add(filename)
+ return sorted(res)
+
+
+def fill_known(template, variables):
+ """
+ Fills in the known variables filling the other variables with {<variable_name>}
+
+ :param template: template
+ :param variables: mapping of variable names to values (ignoring any None)
+ :return: cleaned string
+ """
+ prev = ''
+ while prev != template:
+ prev = template
+ settings = {}
+ for name in set(find_variables(template)):
+ if name in variables and variables[name] is not None:
+ settings[name] = variables[name]
+ else:
+ settings[name] = '{' + name + '}'
+ template = template.format(**settings)
+ return template
+
+
+def resolve_optionals(text):
+ """
+ Resolves the optional sections
+
+ :param text: template after filling in the known variables
+ :return: cleaned string
+ """
+ def resolve_single_optional(part):
+ if len(part) == 0:
+ return part
+ if part[0] != '[' or part[-1] != ']':
+ return part
+ elif len(find_variables(part)) == 0:
+ return part[1:-1]
+ else:
+ return ''
+
+ res = [resolve_single_optional(text) for text in re.split('(\[.*?\])', text)]
+ return ''.join(res)
+
+
+def find_variables(template):
+ """
+ Finds all the variables in the template
+
+ :param template: full template
+ :return: sequence of variables
+ """
+ return tuple(var.split(':')[0] for var in re.findall("\{(.*?)\}", template))
+
+
+def optional_variables(template):
+ """
+ Finds the variables that can be skipped
+
+ :param template: full template
+ :return: set of variables that are only present in optional parts of the string
+ """
+ include = set()
+ exclude = set()
+ for text in re.split('(\[.*?\])', template):
+ if len(text) == 0:
+ continue
+ variables = find_variables(text)
+ if text[0] == '[' and text[-1] == ']':
+ include.update(variables)
+ else:
+ exclude.update(variables)
+ return include.difference(exclude)
+
+
+def extract_variables(template, filename, known_vars=None):
+ """
+ Extracts the variable values from the filename
+
+ :param template: template matching the given filename
+ :param filename: filename
+ :param known_vars: already known variables
+ :return: dictionary from variable names to string representations (unused variables set to None)
+ """
+ if known_vars is None:
+ known_vars = {}
+ template = fill_known(template, known_vars)
+ while '//' in filename:
+ filename = filename.replace('//', '/')
+ remaining = set(find_variables(template))
+ optional = optional_variables(template)
+ for keep in itertools.product(*[(True, False) for _ in optional]):
+ sub_re = resolve_optionals(fill_known(
+ template,
+ dict(
+ **{var: '(\S+)' for k, var in zip(keep, optional) if k},
+ **{var: '(\S+)' for var in remaining.difference(optional)}
+ )
+ ))
+ while '//' in sub_re:
+ sub_re = sub_re.replace('//', '/')
+ if re.match(sub_re, filename) is None:
+ continue
+
+ extracted_value = {}
+ kept_vars = [var for var in find_variables(template)
+ if var not in optional or keep[list(optional).index(var)]]
+ for var, value in zip(kept_vars, re.match(sub_re, filename).groups()):
+ if var in extracted_value:
+ if value != extracted_value[var]:
+ raise ValueError('Multiple values found for {}'.format(var))
+ else:
+ extracted_value[var] = value
+ for name in find_variables(template):
+ if name not in extracted_value:
+ extracted_value[name] = None
+ extracted_value.update(known_vars)
+ return extracted_value
+ raise ValueError("{} did not match {}".format(filename, template))
diff --git a/setup.py b/setup.py
index 63a6da5b801304ac8f3867c7ab3832eb198aac5e..1383c0e1228c3093d0de4433853e5f1adf2ec26d 100644
--- a/setup.py
+++ b/setup.py
@@ -111,6 +111,7 @@ setup(
install_requires=install_requires,
extras_require=extra_requires,
+ package_data={'fsl': ['utils/filetree/trees/*']},
test_suite='tests',
diff --git a/tests/test_filetree/custom.tree b/tests/test_filetree/custom.tree
new file mode 100644
index 0000000000000000000000000000000000000000..ee08e30a03d47f0433f46fe24e3c234e6c654f41
--- /dev/null
+++ b/tests/test_filetree/custom.tree
@@ -0,0 +1,4 @@
+parent
+ [opt_layer_{opt}]
+ sub_file.nii.gz
+ opt_file_{opt}.nii.gz (opt_file)
\ No newline at end of file
diff --git a/tests/test_filetree/extract_vars.tree b/tests/test_filetree/extract_vars.tree
new file mode 100644
index 0000000000000000000000000000000000000000..18209f62f75293a46c07ea7b988c348a3b7c518a
--- /dev/null
+++ b/tests/test_filetree/extract_vars.tree
@@ -0,0 +1,4 @@
+parent
+ opt_layer_test
+ {p1}_{p2}.nii.gz (fn)
+
diff --git a/tests/test_filetree/format.tree b/tests/test_filetree/format.tree
new file mode 100644
index 0000000000000000000000000000000000000000..3162cde68003e4b8cf44663905624a9c4eb31915
--- /dev/null
+++ b/tests/test_filetree/format.tree
@@ -0,0 +1,3 @@
+{var:.0f} (int)
+{var:.1f} (f1)
+{var:.2f} (f2)
diff --git a/tests/test_filetree/parent.tree b/tests/test_filetree/parent.tree
new file mode 100644
index 0000000000000000000000000000000000000000..2558ed21630812025ec0d9f1d4bdbb0ae9bd42ce
--- /dev/null
+++ b/tests/test_filetree/parent.tree
@@ -0,0 +1,7 @@
+dir1
+ ->eddy basename=1 (sub1)
+ dir2
+ ->eddy basename=2 (sub2)
+ ->eddy basename=1b (sub1b)
+->eddy basename=0 (sub0)
+->eddy basename=subvar_{subvar} (subvar)
diff --git a/tests/test_filetree/parent/opt_file_test.nii.gz b/tests/test_filetree/parent/opt_file_test.nii.gz
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/tests/test_filetree/parent/opt_layer_test/opt_file_test.nii.gz b/tests/test_filetree/parent/opt_layer_test/opt_file_test.nii.gz
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/tests/test_filetree/parent/opt_layer_test/opt_file_test2.nii.gz b/tests/test_filetree/parent/opt_layer_test/opt_file_test2.nii.gz
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/tests/test_filetree/parent/opt_layer_test/sub_file.nii.gz b/tests/test_filetree/parent/opt_layer_test/sub_file.nii.gz
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/tests/test_filetree/parent/sub_file.nii.gz b/tests/test_filetree/parent/sub_file.nii.gz
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/tests/test_filetree/test_read.py b/tests/test_filetree/test_read.py
new file mode 100644
index 0000000000000000000000000000000000000000..b27984477218cd69f757199948865c2a06e6f8d1
--- /dev/null
+++ b/tests/test_filetree/test_read.py
@@ -0,0 +1,148 @@
+# Sample Test passing with nose and pytest
+from fsl.utils import filetree
+from pathlib import PurePath
+import os.path as op
+import pytest
+from glob import glob
+
+
+def same_path(p1, p2):
+ assert PurePath(p1) == PurePath(p2)
+
+
+def test_simple_tree():
+ tree = filetree.FileTree.read('eddy')
+ assert tree.variables['basename'] == 'eddy_output'
+ same_path(tree.get('basename'), './eddy_output')
+ same_path(tree.get('image'), './eddy_output.nii.gz')
+
+ tree = filetree.FileTree.read('eddy.tree', basename='out')
+ same_path(tree.get('basename'), './out')
+ same_path(tree.update(basename='test').get('basename'), './test')
+ same_path(tree.get('basename'), './out')
+
+ with pytest.raises(ValueError):
+ filetree.FileTree.read('non_existing')
+
+
+def test_complicated_tree():
+ tree = filetree.FileTree.read('HCP_directory').update(subject='100307')
+
+ same_path(tree.get('T1w_acpc_dc'), '100307/T1w/T1w_acpc_dc.nii.gz')
+
+ L_white = '100307/T1w/fsaverage_LR32k/100307.L.white.32k_fs_LR.surf.gii'
+ same_path(tree.update(hemi='L').get('T1w_32k/white'), L_white)
+ same_path(tree.sub_trees['T1w_32k'].update(hemi='L').get('white'), L_white)
+
+ assert tree.exists(('T1w_32k/white', ))
+ assert tree.exists('T1w_32k/white')
+ assert not tree.exists(('T1w_32k/white_misspelled', ))
+ assert not tree.exists(('T1w_32k/white', 'T1w_32k/white_misspelled', ))
+ assert not tree.exists(('T1w_32k_err/white', ))
+ assert not tree.exists(('../test'))
+ with pytest.raises(ValueError):
+ assert not tree.exists(('../test'), error=True)
+ with pytest.raises(ValueError):
+ tree.exists(('T1w_32k_err/white', ), error=True)
+ assert tree.exists(('T1w_32k/white', ), error=True)
+
+
+def test_parent_tree():
+ directory = op.split(__file__)[0]
+ tree = filetree.FileTree.read(op.join(directory, 'parent.tree'))
+ same_path(tree.get('sub0/basename'), '0')
+ same_path(tree.get('sub1/basename'), 'dir1/1')
+ same_path(tree.get('sub1b/basename'), 'dir1/1b')
+ same_path(tree.get('sub2/basename'), 'dir1/dir2/2')
+ same_path(tree.update(subvar='grot').get('subvar/basename'), 'subvar_grot')
+ with pytest.raises(KeyError):
+ tree.get('sub_var/basename')
+
+
+def test_custom_tree():
+ directory = op.split(__file__)[0]
+ tree = filetree.FileTree.read(op.join(directory, 'custom.tree'), directory=directory)
+ same_path(tree.get('sub_file'), op.join(directory, 'parent/sub_file.nii.gz'))
+ same_path(tree.update(opt='test').get('sub_file'), op.join(directory, 'parent/opt_layer_test/sub_file.nii.gz'))
+
+ with pytest.raises(KeyError):
+ tree.get('opt_file')
+ same_path(tree.update(opt='test').get('opt_file'), op.join(directory, 'parent/opt_layer_test/opt_file_test.nii.gz'))
+
+ assert len(tree.update(opt='test').get_all('sub_file')) == 1
+ assert len(tree.update(opt='test').get_all_vars('sub_file')) == 1
+ assert len(tree.update(opt='test2').get_all('sub_file')) == 0
+ assert len(tree.update(opt='test2').get_all_vars('sub_file')) == 0
+ assert len(tree.get_all('sub_file', glob_vars=['opt'])) == 2
+ assert len(tree.get_all('sub_file', glob_vars='all')) == 2
+ assert len(tree.get_all('sub_file')) == 1
+ assert len(tree.update(opt=None).get_all('sub_file')) == 1
+ assert len(tree.update(opt=None).get_all('sub_file', glob_vars=['opt'])) == 1
+ assert len(tree.update(opt=None).get_all('sub_file', glob_vars='all')) == 1
+
+ for fn, settings in zip(tree.get_all('sub_file', glob_vars='all'),
+ tree.get_all_vars('sub_file', glob_vars='all')):
+ same_path(fn, tree.update(**settings).get('sub_file'))
+
+ assert len(tree.update(opt='test2').get_all('opt_file')) == 0
+ assert len(tree.update(opt='test').get_all('opt_file')) == 1
+ with pytest.raises(KeyError):
+ tree.get_all('opt_file')
+ assert len(tree.get_all('opt_file', glob_vars=['opt'])) == 1
+
+ for vars in ({'opt': None}, {'opt': 'test'}):
+ filename = tree.update(**vars).get('sub_file')
+ assert vars == tree.extract_variables('sub_file', filename)
+ assert {'opt': None} == tree.extract_variables('sub_file', tree.get('sub_file'))
+
+ assert tree.exists(('sub_file', 'opt_file'), error=True, on_disk=True, glob_vars=['opt'])
+ assert tree.exists(('sub_file', 'opt_file'), on_disk=True, glob_vars=['opt'])
+ assert not tree.exists(('sub_file', 'opt_file'), error=False, on_disk=True)
+ with pytest.raises(KeyError):
+ assert tree.exists(('sub_file', 'opt_file'), error=True, on_disk=True)
+ assert not tree.update(opt='test2').exists(('sub_file', 'opt_file'), on_disk=True)
+ with pytest.raises(IOError):
+ tree.update(opt='test2').exists(('sub_file', 'opt_file'), on_disk=True, error=True)
+
+ assert tree.template_variables() == {'opt'}
+ assert tree.template_variables(optional=False) == {'opt'}
+ assert tree.template_variables(required=False) == set()
+ assert tree.template_variables(required=False, optional=False) == set()
+
+ assert tree.template_variables('sub_file') == {'opt'}
+ assert tree.template_variables('sub_file', required=False) == {'opt'}
+ assert tree.template_variables('sub_file', optional=False) == set()
+ assert tree.template_variables('sub_file', optional=False, required=False) == set()
+
+ assert tree.template_variables('opt_file') == {'opt'}
+ assert tree.template_variables('opt_file', required=False) == set()
+ assert tree.template_variables('opt_file', optional=False) == {'opt'}
+ assert tree.template_variables('opt_file', optional=False, required=False) == set()
+
+
+def test_format():
+ directory = op.split(__file__)[0]
+ tree = filetree.FileTree.read(op.join(directory, 'format.tree'), var=1.23)
+ same_path(tree.get('int'), '1')
+ same_path(tree.get('f1'), '1.2')
+ same_path(tree.get('f2'), '1.23')
+
+
+def test_read_all():
+ for directory in filetree.tree_directories:
+ if directory != '.':
+ for filename in glob(op.join(directory, '*.tree')):
+ filetree.FileTree.read(filename)
+
+
+def test_extract_vars_but():
+ """
+ Reproduces a bug where the already provided variables are ignored
+ """
+ directory = op.split(__file__)[0]
+ tree = filetree.FileTree.read(op.join(directory, 'extract_vars.tree'))
+ fn = 'parent/opt_layer_test/opt_file_test.nii.gz'
+ assert {'p1': 'opt_file', 'p2': 'test'} == tree.extract_variables('fn', fn)
+ assert {'p1': 'opt_file', 'p2': 'test'} == tree.update(p1='opt_file').extract_variables('fn', fn)
+ assert {'p1': 'opt', 'p2': 'file_test'} == tree.update(p1='opt').extract_variables('fn', fn)
+ assert {'p1': 'opt_{p3}', 'p2': 'test', 'p3': 'file'} == tree.update(p1='opt_{p3}').extract_variables('fn', fn)
diff --git a/tests/test_filetree/test_registration.py b/tests/test_filetree/test_registration.py
new file mode 100644
index 0000000000000000000000000000000000000000..aa7171ee81d30accb2f55068a387a214557a8965
--- /dev/null
+++ b/tests/test_filetree/test_registration.py
@@ -0,0 +1,73 @@
+from fsl.utils.filetree import register_tree, FileTree
+import os.path as op
+
+
+class SubFileTree(FileTree):
+ pass
+
+
+def test_register_parent():
+ directory = op.split(__file__)[0]
+ filename = op.join(directory, 'parent.tree')
+
+ # call from sub-type
+ tree = SubFileTree.read(filename)
+ assert isinstance(tree, FileTree)
+ assert isinstance(tree, SubFileTree)
+ for child in tree.sub_trees.values():
+ assert isinstance(child, FileTree)
+ assert not isinstance(child, SubFileTree)
+
+ # call from FileTree
+ tree = FileTree.read(filename)
+ assert isinstance(tree, FileTree)
+ assert not isinstance(tree, SubFileTree)
+ for child in tree.sub_trees.values():
+ assert isinstance(child, FileTree)
+ assert not isinstance(child, SubFileTree)
+
+ # register + call from FileTree
+ register_tree('parent', SubFileTree)
+ tree = FileTree.read(filename)
+ assert isinstance(tree, FileTree)
+ assert isinstance(tree, SubFileTree)
+ for child in tree.sub_trees.values():
+ assert isinstance(child, FileTree)
+ assert not isinstance(child, SubFileTree)
+
+ # register + call from SubFileTree
+ register_tree('parent', FileTree)
+ tree = SubFileTree.read(filename)
+ assert isinstance(tree, FileTree)
+ assert not isinstance(tree, SubFileTree)
+ for child in tree.sub_trees.values():
+ assert isinstance(child, FileTree)
+ assert not isinstance(child, SubFileTree)
+
+
+def test_children():
+ directory = op.split(__file__)[0]
+ filename = op.join(directory, 'parent.tree')
+
+ tree = SubFileTree.read(filename)
+ assert isinstance(tree, FileTree)
+ assert not isinstance(tree, SubFileTree)
+ for child in tree.sub_trees.values():
+ assert isinstance(child, FileTree)
+ assert not isinstance(child, SubFileTree)
+
+ register_tree('eddy', SubFileTree)
+ tree = SubFileTree.read(filename)
+ assert isinstance(tree, FileTree)
+ assert not isinstance(tree, SubFileTree)
+ for child in tree.sub_trees.values():
+ assert isinstance(child, FileTree)
+ assert isinstance(child, SubFileTree)
+
+ register_tree('eddy', FileTree)
+ tree = SubFileTree.read(filename)
+ assert isinstance(tree, FileTree)
+ assert not isinstance(tree, SubFileTree)
+ for child in tree.sub_trees.values():
+ assert isinstance(child, FileTree)
+ assert not isinstance(child, SubFileTree)
diff --git a/tests/test_filetree/test_template.py b/tests/test_filetree/test_template.py
new file mode 100644
index 0000000000000000000000000000000000000000..96ecc18f00adb1add2b191c8d3744460f1c02bc6
--- /dev/null
+++ b/tests/test_filetree/test_template.py
@@ -0,0 +1,23 @@
+from fsl.utils.filetree import utils
+import pytest
+
+
+def test_variables():
+ assert ('var', 'other_var', 'var') == tuple(utils.find_variables('some{var}_{other_var}_{var}'))
+ assert ('var', 'other_var', 'var') == tuple(utils.find_variables('some{var}_[{other_var}_{var}]'))
+ assert {'other_var'} == utils.optional_variables('some{var}_[{other_var}_{var}]')
+
+
+def test_get_variables():
+ assert {'var': 'test'} == utils.extract_variables('{var}', 'test')
+ assert {'var': 'test'} == utils.extract_variables('2{var}', '2test')
+ assert {'var': 'test', 'other_var': None} == utils.extract_variables('{var}[_{other_var}]', 'test')
+ assert {'var': 'test', 'other_var': 'foo'} == utils.extract_variables('{var}[_{other_var}]', 'test_foo')
+ assert {'var': 'test', 'other_var': 'foo'} == utils.extract_variables('{var}[_{other_var}]_{var}', 'test_foo_test')
+ assert {'var': 'test', 'other_var': None} == utils.extract_variables('{var}[_{other_var}]_{var}', 'test_test')
+ with pytest.raises(ValueError):
+ utils.extract_variables('{var}[_{other_var}]_{var}', 'test_foo')
+ with pytest.raises(ValueError):
+ utils.extract_variables('{var}[_{other_var}]_{var}', 'test_foo_bar')
+ with pytest.raises(ValueError):
+ utils.extract_variables('bar{var}[_{other_var}]_{var}', 'test')