Merge branch 'bf/filetree' into 'master'

Bf/filetree

See merge request fsl/fslpy!203

CHANGELOG.rst

@@ -2,6 +2,21 @@ This document contains the ``fslpy`` release history in reverse chronological
 order.
 
 
+
+2.8.4 (Monday 2nd March 2020)
+-----------------------------
+
+
+Added
+^^^^^
+
+
+* Added a new ``partial_fill`` option to :meth:`.FileTree.read`, which
+  effectively eliminates any variables which only have one value. This was
+  added to accommodate some behavioural changes that were introduced in 2.8.2.
+
+
+
 2.8.3 (Friday 28th February 2020)
 ---------------------------------
 

fsl/utils/filetree/filetree.py

@@ -28,11 +28,11 @@ class FileTree(object):
     - ``name``: descriptive name of the tree
     """
     def __init__(self,
-                 templates: Dict[str, str],
-                 variables: Dict[str, Any],
-                 sub_trees: Dict[str, "FileTree"] = None,
-                 parent:    Optional["FileTree"] = None,
-                 name:      str = None):
+                 templates:    Dict[str, str],
+                 variables:    Dict[str, Any],
+                 sub_trees:    Dict[str, "FileTree"] = None,
+                 parent:       Optional["FileTree"] = None,
+                 name:         str = None):
         """
         Creates a new filename tree.
         """
@@ -51,7 +51,6 @@ class FileTree(object):
         """
         return self._parent
 
-
     @property
     def name(self, ):
         """
@@ -59,7 +58,6 @@ class FileTree(object):
         """
         return self._name
 
-
     @property
     def all_variables(self, ):
         """
@@ -328,8 +326,35 @@ class FileTree(object):
             return False
         return True
 
+    def partial_fill(self, ) -> "FileTree":
+        """
+        Fills in known variables into the templates
+
+        :return: The resulting tree will have empty `variables` dictionaries and updated templates
+        """
+        new_tree = deepcopy(self)
+        to_update = new_tree
+        while to_update.parent is not None:
+            to_update = to_update.parent
+        to_update._update_partial_fill()
+        return new_tree
+
+    def _update_partial_fill(self, ):
+        """
+        Helper function for `partial_fill` that updates the templates in place
+        """
+        new_templates = {}
+        for short_name in self.templates:
+            template, variables = self.get_template(short_name)
+            new_templates[short_name] = str(utils.Template.parse(template).fill_known(variables))
+        self.templates = new_templates
+
+        for tree in self.sub_trees.values():
+            tree._update_partial_fill()
+        self.variables = {}
+
     @classmethod
-    def read(cls, tree_name: str, directory='.', **variables) -> "FileTree":
+    def read(cls, tree_name: str, directory='.', partial_fill=True, **variables) -> "FileTree":
         """
         Reads a FileTree from a specific file
 
@@ -339,6 +364,7 @@ class FileTree(object):
         :param tree_name: file containing the filename tree.
             Can provide the filename of a tree file or the name for a tree in the ``filetree.tree_directories``.
         :param directory: parent directory of the full tree (defaults to current directory)
+        :param partial_fill: By default any known `variables` are filled into the `template` immediately
         :param variables: variable settings
         :return: dictionary from specifier to filename
         """
@@ -422,6 +448,8 @@ class FileTree(object):
         res = get_registered(tree_name, cls)(templates, variables=file_variables, sub_trees=sub_trees, name=tree_name)
         for tree in sub_trees.values():
             tree._parent = res
+        if partial_fill:
+            res = res.partial_fill()
         return res
 
 

fsl/utils/filetree/query.py

@@ -373,7 +373,7 @@ def scan(tree : FileTree) -> List[Match]:
 
             filename = tree.update(**variables).get(template)
 
-            if not op.isfile(tree.update(**variables).get(template)):
+            if not op.isfile(filename):
                 continue
 
             matches.append(Match(filename, template, tree, variables))

fsl/utils/filetree/utils.py

@@ -132,6 +132,9 @@ class Template:
     Splits a template into its constituent parts
     """
     def __init__(self, parts: Sequence[Part]):
+        if isinstance(parts, str):
+            raise ValueError("Input to Template should be a sequence of parts; " +
+                             "did you mean to call `Template.parse` instead?")
         self.parts = tuple(parts)
 
     @classmethod

tests/test_filetree/test_query.py

@@ -396,6 +396,193 @@ def test_query_subtree():
             op.join('subj-03', 'surf', 'R.white.gii')]
 
 
+def test_query_variable_partial_set():
+    tree1 = tw.dedent("""
+    subj-{participant}
+        T1w.nii.gz (T1w)
+        native
+            ->surface space=native (surf_native)
+        standard
+            ->surface (surf_standard)
+    """)
+    tree2 = tw.dedent("""
+    {hemi}.{space}.gii (surface)
+    """)
+
+    files = [
+        op.join('subj-01', 'T1w.nii.gz'),
+        op.join('subj-01', 'native', 'L.native.gii'),
+        op.join('subj-01', 'native', 'R.native.gii'),
+        op.join('subj-01', 'standard',    'L.mni.gii'),
+        op.join('subj-01', 'standard',    'R.mni.gii'),
+        op.join('subj-01', 'standard',    'L.freesurfer.gii'),
+        op.join('subj-01', 'standard',    'R.freesurfer.gii'),
+        op.join('subj-02', 'T1w.nii.gz'),
+        op.join('subj-02', 'native', 'L.native.gii'),
+        op.join('subj-02', 'native', 'R.native.gii'),
+        op.join('subj-02', 'standard',    'L.mni.gii'),
+        op.join('subj-02', 'standard',    'R.mni.gii'),
+        op.join('subj-02', 'standard',    'L.freesurfer.gii'),
+        op.join('subj-02', 'standard',    'R.freesurfer.gii'),
+        op.join('subj-03', 'T1w.nii.gz'),
+        op.join('subj-03', 'native', 'L.native.gii'),
+        op.join('subj-03', 'native', 'R.native.gii'),
+        op.join('subj-03', 'standard',    'L.mni.gii'),
+        op.join('subj-03', 'standard',    'R.mni.gii')]
+
+    with testdir(files):
+        with open('tree1.tree',   'wt') as f: f.write(tree1)
+        with open('surface.tree', 'wt') as f: f.write(tree2)
+
+        tree = filetree.FileTree.read('tree1.tree', '.')
+        query = filetree.FileTreeQuery(tree)
+
+        assert sorted(query.templates) == ['T1w',
+                                           'surf_native/surface',
+                                           'surf_standard/surface']
+
+        qvars = query.variables()
+        assert sorted(qvars.keys()) == ['hemi', 'participant', 'space']
+        assert qvars['hemi']        == ['L', 'R']
+        assert qvars['participant'] == ['01', '02', '03']
+        assert qvars['space'] == ['freesurfer', 'mni']
+
+        qvars = query.variables('T1w')
+        assert sorted(qvars.keys()) == ['participant']
+        assert qvars['participant'] == ['01', '02', '03']
+
+        qvars = query.variables('surf_native/surface')
+        assert sorted(qvars.keys()) == ['hemi', 'participant']
+        assert qvars['hemi']        == ['L', 'R']
+        assert qvars['participant'] == ['01', '02', '03']
+
+        qvars = query.variables('surf_standard/surface')
+        assert sorted(qvars.keys()) == ['hemi', 'participant', 'space']
+        assert qvars['hemi']        == ['L', 'R']
+        assert qvars['participant'] == ['01', '02', '03']
+        assert qvars['space'] == ['freesurfer', 'mni']
+
+        got = query.query('T1w')
+        assert [m.filename for m in sorted(got)] == [
+            op.join('subj-01', 'T1w.nii.gz'),
+            op.join('subj-02', 'T1w.nii.gz'),
+            op.join('subj-03', 'T1w.nii.gz')]
+
+        got = query.query('T1w', participant='01')
+        assert [m.filename for m in sorted(got)] == [
+            op.join('subj-01', 'T1w.nii.gz')]
+
+        got = query.query('surf_native/surface')
+        assert [m.filename for m in sorted(got)] == [
+            op.join('subj-01', 'native', 'L.native.gii'),
+            op.join('subj-01', 'native', 'R.native.gii'),
+            op.join('subj-02', 'native', 'L.native.gii'),
+            op.join('subj-02', 'native', 'R.native.gii'),
+            op.join('subj-03', 'native', 'L.native.gii'),
+            op.join('subj-03', 'native', 'R.native.gii')]
+
+        got = query.query('surf_native/surface', hemi='L')
+        assert [m.filename for m in sorted(got)] == [
+            op.join('subj-01', 'native', 'L.native.gii'),
+            op.join('subj-02', 'native', 'L.native.gii'),
+            op.join('subj-03', 'native', 'L.native.gii')]
+
+        got = query.query('surf_standard/surface', hemi='L')
+        assert [m.filename for m in sorted(got)] == [
+            op.join('subj-01', 'standard', 'L.freesurfer.gii'),
+            op.join('subj-01', 'standard', 'L.mni.gii'),
+            op.join('subj-02', 'standard', 'L.freesurfer.gii'),
+            op.join('subj-02', 'standard', 'L.mni.gii'),
+            # subj03/standard/L.freesurfer.gii was skipped when creating files
+            op.join('subj-03', 'standard', 'L.mni.gii')]
+
+
+def test_query_multi_subtree():
+    tree1 = tw.dedent("""
+    subj-{participant}
+        T1w.nii.gz (T1w)
+        native
+            ->surface space=native (surf_native)
+        mni
+            ->surface space=mni (surf_mni)
+    """)
+    tree2 = tw.dedent("""
+    {hemi}.{space}.gii (surface)
+    """)
+
+    files = [
+        op.join('subj-01', 'T1w.nii.gz'),
+        op.join('subj-01', 'native', 'L.native.gii'),
+        op.join('subj-01', 'native', 'R.native.gii'),
+        op.join('subj-01', 'mni',    'L.mni.gii'),
+        op.join('subj-01', 'mni',    'R.mni.gii'),
+        op.join('subj-02', 'T1w.nii.gz'),
+        op.join('subj-02', 'native', 'L.native.gii'),
+        op.join('subj-02', 'native', 'R.native.gii'),
+        op.join('subj-02', 'mni',    'L.mni.gii'),
+        op.join('subj-02', 'mni',    'R.mni.gii'),
+        op.join('subj-03', 'T1w.nii.gz'),
+        op.join('subj-03', 'native', 'L.native.gii'),
+        op.join('subj-03', 'native', 'R.native.gii'),
+        op.join('subj-03', 'mni',    'L.mni.gii'),
+        op.join('subj-03', 'mni',    'R.mni.gii')]
+
+    with testdir(files):
+        with open('tree1.tree',   'wt') as f: f.write(tree1)
+        with open('surface.tree', 'wt') as f: f.write(tree2)
+
+        tree = filetree.FileTree.read('tree1.tree', '.')
+        query = filetree.FileTreeQuery(tree)
+
+        assert sorted(query.templates) == ['T1w',
+                                           'surf_mni/surface',
+                                           'surf_native/surface']
+
+        qvars = query.variables()
+        assert sorted(qvars.keys()) == ['hemi', 'participant']
+        assert qvars['hemi']        == ['L', 'R']
+        assert qvars['participant'] == ['01', '02', '03']
+
+        qvars = query.variables('T1w')
+        assert sorted(qvars.keys()) == ['participant']
+        assert qvars['participant'] == ['01', '02', '03']
+
+        qvars = query.variables('surf_mni/surface')
+        assert sorted(qvars.keys()) == ['hemi', 'participant']
+        assert qvars['hemi']        == ['L', 'R']
+        assert qvars['participant'] == ['01', '02', '03']
+
+        qvars = query.variables('surf_native/surface')
+        assert sorted(qvars.keys()) == ['hemi', 'participant']
+        assert qvars['hemi']        == ['L', 'R']
+        assert qvars['participant'] == ['01', '02', '03']
+
+        got = query.query('T1w')
+        assert [m.filename for m in sorted(got)] == [
+            op.join('subj-01', 'T1w.nii.gz'),
+            op.join('subj-02', 'T1w.nii.gz'),
+            op.join('subj-03', 'T1w.nii.gz')]
+
+        got = query.query('T1w', participant='01')
+        assert [m.filename for m in sorted(got)] == [
+            op.join('subj-01', 'T1w.nii.gz')]
+
+        got = query.query('surf_mni/surface')
+        assert [m.filename for m in sorted(got)] == [
+            op.join('subj-01', 'mni', 'L.mni.gii'),
+            op.join('subj-01', 'mni', 'R.mni.gii'),
+            op.join('subj-02', 'mni', 'L.mni.gii'),
+            op.join('subj-02', 'mni', 'R.mni.gii'),
+            op.join('subj-03', 'mni', 'L.mni.gii'),
+            op.join('subj-03', 'mni', 'R.mni.gii')]
+
+        got = query.query('surf_native/surface', hemi='L')
+        assert [m.filename for m in sorted(got)] == [
+            op.join('subj-01', 'native', 'L.native.gii'),
+            op.join('subj-02', 'native', 'L.native.gii'),
+            op.join('subj-03', 'native', 'L.native.gii')]
+
+
 def test_scan():
 
     with _test_data():

tests/test_filetree/test_read.py

@@ -13,18 +13,25 @@ def same_path(p1, p2):
 
 def test_simple_tree():
     tree = filetree.FileTree.read('eddy')
-    assert tree.variables['basename'] == 'eddy_output'
+    assert 'basename' not in tree.variables
+
     same_path(tree.get('basename'), './eddy_output')
     same_path(tree.get('image'), './eddy_output.nii.gz')
 
     tree = filetree.FileTree.read('eddy.tree', basename='out')
-    same_path(tree.get('basename'), './out')
-    same_path(tree.update(basename='test').get('basename'), './test')
+    assert 'basename' not in tree.variables
     same_path(tree.get('basename'), './out')
 
     with pytest.raises(ValueError):
         filetree.FileTree.read('non_existing')
 
+    # without partial_fill
+    tree_no_fill = filetree.FileTree.read('eddy', partial_fill=False)
+    tree_no_fill.variables['basename'] == 'eddy_output'
+    same_path(tree_no_fill.get('basename'), './eddy_output')
+    same_path(tree_no_fill.update(basename='test').get('basename'), './test')
+    same_path(tree_no_fill.get('basename'), './eddy_output')
+
 
 def test_complicated_tree():
     tree = filetree.FileTree.read('HCP_directory').update(subject='100307')