ENH: Added partial filling of variables into the templates

This is now the new default behaviour when loading a new file

This change is not backwards compatible!

fsl/utils/filetree/filetree.py

@@ -28,11 +28,11 @@ class FileTree(object):
     - ``name``: descriptive name of the tree
     """
     def __init__(self,
-                 templates: Dict[str, str],
-                 variables: Dict[str, Any],
-                 sub_trees: Dict[str, "FileTree"] = None,
-                 parent:    Optional["FileTree"] = None,
-                 name:      str = None):
+                 templates:    Dict[str, str],
+                 variables:    Dict[str, Any],
+                 sub_trees:    Dict[str, "FileTree"] = None,
+                 parent:       Optional["FileTree"] = None,
+                 name:         str = None):
         """
         Creates a new filename tree.
         """
@@ -51,7 +51,6 @@ class FileTree(object):
         """
         return self._parent
 
-
     @property
     def name(self, ):
         """
@@ -59,7 +58,6 @@ class FileTree(object):
         """
         return self._name
 
-
     @property
     def all_variables(self, ):
         """
@@ -328,8 +326,35 @@ class FileTree(object):
             return False
         return True
 
+    def partial_fill(self, ) -> "FileTree":
+        """
+        Fills in known variables into the templates
+
+        :return: The resulting tree will have empty `variables` dictionaries and updated templates
+        """
+        new_tree = deepcopy(self)
+        to_update = new_tree
+        while to_update.parent is not None:
+            to_update = to_update.parent
+        to_update._update_partial_fill()
+        return new_tree
+
+    def _update_partial_fill(self, ):
+        """
+        Helper function for `partial_fill` that updates the templates in place
+        """
+        new_templates = {}
+        for short_name in self.templates:
+            template, variables = self.get_template(short_name)
+            new_templates[short_name] = str(utils.Template.parse(template).fill_known(variables))
+        self.templates = new_templates
+
+        for tree in self.sub_trees.values():
+            tree._update_partial_fill()
+        self.variables = {}
+
     @classmethod
-    def read(cls, tree_name: str, directory='.', **variables) -> "FileTree":
+    def read(cls, tree_name: str, directory='.', partial_fill=True, **variables) -> "FileTree":
         """
         Reads a FileTree from a specific file
 
@@ -339,6 +364,7 @@ class FileTree(object):
         :param tree_name: file containing the filename tree.
             Can provide the filename of a tree file or the name for a tree in the ``filetree.tree_directories``.
         :param directory: parent directory of the full tree (defaults to current directory)
+        :param partial_fill: By default any known `variables` are filled into the `template` immediately
         :param variables: variable settings
         :return: dictionary from specifier to filename
         """
@@ -422,6 +448,8 @@ class FileTree(object):
         res = get_registered(tree_name, cls)(templates, variables=file_variables, sub_trees=sub_trees, name=tree_name)
         for tree in sub_trees.values():
             tree._parent = res
+        if partial_fill:
+            res = res.partial_fill()
         return res
 
 

fsl/utils/filetree/utils.py

@@ -132,6 +132,9 @@ class Template:
     Splits a template into its constituent parts
     """
     def __init__(self, parts: Sequence[Part]):
+        if isinstance(parts, str):
+            raise ValueError("Input to Template should be a sequence of parts; " +
+                             "did you mean to call `Template.parse` instead?")
         self.parts = tuple(parts)
 
     @classmethod

tests/test_filetree/test_query.py

@@ -396,6 +396,107 @@ def test_query_subtree():
             op.join('subj-03', 'surf', 'R.white.gii')]
 
 
+def test_query_variable_partial_set():
+    tree1 = tw.dedent("""
+    subj-{participant}
+        T1w.nii.gz (T1w)
+        native
+            ->surface space=native (surf_native)
+        standard
+            ->surface (surf_standard)
+    """)
+    tree2 = tw.dedent("""
+    {hemi}.{space}.gii (surface)
+    """)
+
+    files = [
+        op.join('subj-01', 'T1w.nii.gz'),
+        op.join('subj-01', 'native', 'L.native.gii'),
+        op.join('subj-01', 'native', 'R.native.gii'),
+        op.join('subj-01', 'standard',    'L.mni.gii'),
+        op.join('subj-01', 'standard',    'R.mni.gii'),
+        op.join('subj-01', 'standard',    'L.freesurfer.gii'),
+        op.join('subj-01', 'standard',    'R.freesurfer.gii'),
+        op.join('subj-02', 'T1w.nii.gz'),
+        op.join('subj-02', 'native', 'L.native.gii'),
+        op.join('subj-02', 'native', 'R.native.gii'),
+        op.join('subj-02', 'standard',    'L.mni.gii'),
+        op.join('subj-02', 'standard',    'R.mni.gii'),
+        op.join('subj-02', 'standard',    'L.freesurfer.gii'),
+        op.join('subj-02', 'standard',    'R.freesurfer.gii'),
+        op.join('subj-03', 'T1w.nii.gz'),
+        op.join('subj-03', 'native', 'L.native.gii'),
+        op.join('subj-03', 'native', 'R.native.gii'),
+        op.join('subj-03', 'standard',    'L.mni.gii'),
+        op.join('subj-03', 'standard',    'R.mni.gii')]
+
+    with testdir(files):
+        with open('tree1.tree',   'wt') as f: f.write(tree1)
+        with open('surface.tree', 'wt') as f: f.write(tree2)
+
+        tree = filetree.FileTree.read('tree1.tree', '.')
+        query = filetree.FileTreeQuery(tree)
+
+        assert sorted(query.templates) == ['T1w',
+                                           'surf_native/surface',
+                                           'surf_standard/surface']
+
+        qvars = query.variables()
+        assert sorted(qvars.keys()) == ['hemi', 'participant', 'space']
+        assert qvars['hemi']        == ['L', 'R']
+        assert qvars['participant'] == ['01', '02', '03']
+        assert qvars['space'] == ['freesurfer', 'mni']
+
+        qvars = query.variables('T1w')
+        assert sorted(qvars.keys()) == ['participant']
+        assert qvars['participant'] == ['01', '02', '03']
+
+        qvars = query.variables('surf_native/surface')
+        assert sorted(qvars.keys()) == ['hemi', 'participant']
+        assert qvars['hemi']        == ['L', 'R']
+        assert qvars['participant'] == ['01', '02', '03']
+
+        qvars = query.variables('surf_standard/surface')
+        assert sorted(qvars.keys()) == ['hemi', 'participant', 'space']
+        assert qvars['hemi']        == ['L', 'R']
+        assert qvars['participant'] == ['01', '02', '03']
+        assert qvars['space'] == ['freesurfer', 'mni']
+
+        got = query.query('T1w')
+        assert [m.filename for m in sorted(got)] == [
+            op.join('subj-01', 'T1w.nii.gz'),
+            op.join('subj-02', 'T1w.nii.gz'),
+            op.join('subj-03', 'T1w.nii.gz')]
+
+        got = query.query('T1w', participant='01')
+        assert [m.filename for m in sorted(got)] == [
+            op.join('subj-01', 'T1w.nii.gz')]
+
+        got = query.query('surf_native/surface')
+        assert [m.filename for m in sorted(got)] == [
+            op.join('subj-01', 'native', 'L.native.gii'),
+            op.join('subj-01', 'native', 'R.native.gii'),
+            op.join('subj-02', 'native', 'L.native.gii'),
+            op.join('subj-02', 'native', 'R.native.gii'),
+            op.join('subj-03', 'native', 'L.native.gii'),
+            op.join('subj-03', 'native', 'R.native.gii')]
+
+        got = query.query('surf_native/surface', hemi='L')
+        assert [m.filename for m in sorted(got)] == [
+            op.join('subj-01', 'native', 'L.native.gii'),
+            op.join('subj-02', 'native', 'L.native.gii'),
+            op.join('subj-03', 'native', 'L.native.gii')]
+
+        got = query.query('surf_standard/surface', hemi='L')
+        assert [m.filename for m in sorted(got)] == [
+            op.join('subj-01', 'standard', 'L.freesurfer.gii'),
+            op.join('subj-01', 'standard', 'L.mni.gii'),
+            op.join('subj-02', 'standard', 'L.freesurfer.gii'),
+            op.join('subj-02', 'standard', 'L.mni.gii'),
+            # subj03/standard/L.freesurfer.gii was skipped when creating files
+            op.join('subj-03', 'standard', 'L.mni.gii')]
+
+
 def test_query_multi_subtree():
     tree1 = tw.dedent("""
     subj-{participant}
@@ -438,26 +539,23 @@ def test_query_multi_subtree():
                                            'surf_native/surface']
 
         qvars = query.variables()
-        assert sorted(qvars.keys()) == ['hemi', 'participant', 'space']
+        assert sorted(qvars.keys()) == ['hemi', 'participant']
         assert qvars['hemi']        == ['L', 'R']
         assert qvars['participant'] == ['01', '02', '03']
-        assert qvars['space']       == ['mni', 'native']
 
         qvars = query.variables('T1w')
         assert sorted(qvars.keys()) == ['participant']
         assert qvars['participant'] == ['01', '02', '03']
 
         qvars = query.variables('surf_mni/surface')
-        assert sorted(qvars.keys()) == ['hemi', 'participant', 'space']
+        assert sorted(qvars.keys()) == ['hemi', 'participant']
         assert qvars['hemi']        == ['L', 'R']
         assert qvars['participant'] == ['01', '02', '03']
-        assert qvars['space']       == ['mni', 'native']
 
         qvars = query.variables('surf_native/surface')
-        assert sorted(qvars.keys()) == ['hemi', 'participant', 'space']
+        assert sorted(qvars.keys()) == ['hemi', 'participant']
         assert qvars['hemi']        == ['L', 'R']
         assert qvars['participant'] == ['01', '02', '03']
-        assert qvars['space']       == ['mni', 'native']
 
         got = query.query('T1w')
         assert [m.filename for m in sorted(got)] == [
@@ -484,15 +582,6 @@ def test_query_multi_subtree():
             op.join('subj-02', 'native', 'L.native.gii'),
             op.join('subj-03', 'native', 'L.native.gii')]
 
-        got = query.query('surf_native/surface', space='mni')
-        assert [m.filename for m in sorted(got)] == []
-
-        got = query.query('surf_native/surface', space='native', hemi='R')
-        assert [m.filename for m in sorted(got)] == [
-            op.join('subj-01', 'native', 'R.native.gii'),
-            op.join('subj-02', 'native', 'R.native.gii'),
-            op.join('subj-03', 'native', 'R.native.gii')]
-
 
 def test_scan():
 

tests/test_filetree/test_read.py

@@ -13,18 +13,25 @@ def same_path(p1, p2):
 
 def test_simple_tree():
     tree = filetree.FileTree.read('eddy')
-    assert tree.variables['basename'] == 'eddy_output'
+    assert 'basename' not in tree.variables
+
     same_path(tree.get('basename'), './eddy_output')
     same_path(tree.get('image'), './eddy_output.nii.gz')
 
     tree = filetree.FileTree.read('eddy.tree', basename='out')
-    same_path(tree.get('basename'), './out')
-    same_path(tree.update(basename='test').get('basename'), './test')
+    assert 'basename' not in tree.variables
     same_path(tree.get('basename'), './out')
 
     with pytest.raises(ValueError):
         filetree.FileTree.read('non_existing')
 
+    # without partial_fill
+    tree_no_fill = filetree.FileTree.read('eddy', partial_fill=False)
+    tree_no_fill.variables['basename'] == 'eddy_output'
+    same_path(tree_no_fill.get('basename'), './eddy_output')
+    same_path(tree_no_fill.update(basename='test').get('basename'), './test')
+    same_path(tree_no_fill.get('basename'), './eddy_output')
+
 
 def test_complicated_tree():
     tree = filetree.FileTree.read('HCP_directory').update(subject='100307')