Skip to content

GitLab

Commit f9bde009 authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

Merge branch 'bf/filetree' into 'master' 

Bf/filetree

See merge request fsl/fslpy!154
parents 287207e5 ac5f5d3d
Pipeline #4124 passed with stages
in 15 minutes and 47 seconds
Hide whitespace changes
Inline Side-by-side
doc/contributing.rst
View file @ f9bde009
......@@ -144,6 +144,7 @@ for a list of error codes):
- E302: expected 2 blank lines, found 0
- E303: too many blank lines (3)
- E701: multiple statements on one line (colon)
- W504: line break after binary operator
The ``pylint`` tool can be *very* opinionated about how you write your code,
......@@ -162,7 +163,7 @@ refactoring and convention messages, and a few select warnings (type ``pylint
To check code with ``flake8`` and ``pylint``, I use the following commands::
flake8 --ignore=E127,E201,E203,E221,E222,E241,E271,E272,E301,E302,E303,E701 fsl
flake8 --ignore=E127,E201,E203,E221,E222,E241,E271,E272,E301,E302,E303,E701,W504 fsl
pylint --extension-pkg-whitelist=numpy,wx \
--generated-members=np.int8,np.uint8,np.int16,np.uint16,np.int32,np.uint32,np.int64,np.uint64,np.float32,np.float64,np.float128,wx.PyDeadObjectError \
--disable=R,C,W0511,W0703,W1202 fsl
fsl/utils/filetree/query.py
View file @ f9bde009
......@@ -6,7 +6,7 @@
# Author: Michiel Cottaar <michiel.cottaar@.ndcn.ox.ac.uk>
#
"""This module contains the :class:`FileTreeQuery` class, which can be used to
search for files in a directory described by a `.FileTree`. A
search for files in a directory described by a :class:`.FileTree`. A
``FileTreeQuery`` object returns :class:`Match` objects which each represent a
file that is described by the ``FileTree``, and which is present in the
directory.
......@@ -22,8 +22,9 @@ defined in this module:
"""
import logging
import collections
import logging
import collections
import functools as ft
import os.path as op
from typing import Dict, List, Tuple
......@@ -121,6 +122,12 @@ class FileTreeQuery(object):
tvarlens = [len(allvars[v]) for v in tvars]
# "Scalar" match objects - templates
# which have no variables, and for
# which zero or one file is present
if len(tvarlens) == 0:
tvarlens = 1
# An ND array for this short
# name. Each element is a
# Match object, or nan.
......@@ -142,10 +149,13 @@ class FileTreeQuery(object):
tvaridxs = varidxs[ match.full_name]
tarr = matcharrays[ match.full_name]
idx = []
for var in tvars:
val = match.variables[var]
idx.append(tvaridxs[var][val])
if len(match.variables) == 0:
idx = [0]
else:
for var in tvars:
val = match.variables[var]
idx.append(tvaridxs[var][val])
tarr[tuple(idx)] = match
......@@ -253,6 +263,7 @@ class FileTreeQuery(object):
else: return [m for m in result.flat if isinstance(m, Match)]
@ft.total_ordering
class Match(object):
"""A ``Match`` object represents a file with a name matching a template in
a ``FileTree``. The :func:`scan` function and :meth:`FileTree.query`
......@@ -338,10 +349,6 @@ class Match(object):
return isinstance(other, Match) and self.filename < other.filename
def __le__(self, other):
return isinstance(other, Match) and self.filename <= other.filename
def __repr__(self):
"""Returns a string representation of this ``Match``. """
return 'Match({}: {})'.format(self.full_name, self.filename)
......@@ -397,9 +404,13 @@ def allVariables(
containing the variables which are relevant to each template.
"""
allvars = collections.defaultdict(set)
alltemplates = collections.defaultdict(set)
alltemplates = {}
for m in matches:
if m.full_name not in alltemplates:
alltemplates[m.full_name] = set()
for var, val in m.variables.items():
allvars[ var] .add(val)
alltemplates[m.full_name].add(var)
......@@ -411,7 +422,7 @@ def allVariables(
allvars = {var : list(sorted(vals, key=key))
for var, vals in allvars.items()}
alltemplates = {sn : list(sorted(vars))
for sn, vars in alltemplates.items()}
alltemplates = {tn : list(sorted(vars))
for tn, vars in alltemplates.items()}
return allvars, alltemplates
setup.cfg
View file @ f9bde009
......@@ -22,4 +22,4 @@ testpaths = tests
addopts = -v --niters=50 --cov=fsl -m "not longtest"
[flake8]
ignore = E127,E201,E203,E221,E222,E241,E271,E272,E301,E302,E303,E701
\ No newline at end of file
ignore = E127,E201,E203,E221,E222,E241,E271,E272,E301,E302,E303,E701,W504
\ No newline at end of file
tests/test_filetree/test_query.py
View file @ f9bde009
......@@ -25,6 +25,7 @@ subj-{participant}
T1w.nii.gz (T1w)
T2w.nii.gz (T2w)
{hemi}.{surf}.gii (surface)
scalar_file.txt (scalar)
""".strip()
_subjs = ['01', '02', '03']
......@@ -36,7 +37,7 @@ _surfs = ['midthickness', 'pial', 'white']
@contextlib.contextmanager
def _test_data():
files = []
files = ['scalar_file.txt']
for subj, ses in it.product(_subjs, _sess):
sesdir = op.join('subj-{}'.format(subj), 'ses-{}'.format(ses))
......@@ -60,6 +61,12 @@ def _expected_matches(template, tree, **kwargs):
surfs = kwargs.get('surf', _surfs)
hemis = kwargs.get('hemi', _hemis)
if template == 'scalar':
matches.append(ftquery.Match('scalar_file.txt',
template,
tree,
{}))
for subj, ses in it.product(subjs, sess):
sesdir = op.join('subj-{}'.format(subj), 'ses-{}'.format(ses))
......@@ -100,7 +107,10 @@ def _run_and_check_query(query, template, asarray=False, **vars):
else:
snvars = query.variables(template)
assert len(snvars) == len(gotmatches.shape)
if len(snvars) == 0:
assert gotmatches.shape == (1,)
else:
assert len(snvars) == len(gotmatches.shape)
for i, var in enumerate(sorted(snvars.keys())):
if var not in vars or vars[var] == '*':
......@@ -124,30 +134,32 @@ def test_query_properties():
tree = filetree.FileTree.read('_test_tree.tree', '.')
query = filetree.FileTreeQuery(tree)
assert sorted(query.axes('scalar')) == []
assert sorted(query.axes('T1w')) == ['participant', 'session']
assert sorted(query.axes('T2w')) == ['participant', 'session']
assert sorted(query.axes('surface')) == ['hemi',
'participant',
'session',
'surf']
assert sorted(query.templates) == ['T1w', 'T2w', 'surface']
assert query.variables('T1w') == {'participant' : ['01', '02', '03'],
'session' : ['1', '2']}
assert query.variables('T2w') == {'participant' : ['01', '02', '03'],
'session' : ['1', '2']}
assert query.variables('surface') == {'participant' : ['01', '02', '03'],
'session' : ['1', '2'],
'surf' : ['midthickness',
'pial',
'white'],
'hemi' : ['L', 'R']}
assert query.variables() == {'participant' : ['01', '02', '03'],
'session' : ['1', '2'],
'surf' : ['midthickness',
'pial',
'white'],
'hemi' : ['L', 'R']}
assert sorted(query.templates) == ['T1w', 'T2w', 'scalar', 'surface']
assert query.variables('scalar') == {}
assert query.variables('T1w') == {'participant' : ['01', '02', '03'],
'session' : ['1', '2']}
assert query.variables('T2w') == {'participant' : ['01', '02', '03'],
'session' : ['1', '2']}
assert query.variables('surface') == {'participant' : ['01', '02', '03'],
'session' : ['1', '2'],
'surf' : ['midthickness',
'pial',
'white'],
'hemi' : ['L', 'R']}
assert query.variables() == {'participant' : ['01', '02', '03'],
'session' : ['1', '2'],
'surf' : ['midthickness',
'pial',
'white'],
'hemi' : ['L', 'R']}
def test_query():
......@@ -155,6 +167,7 @@ def test_query():
tree = filetree.FileTree.read('_test_tree.tree', '.')
query = filetree.FileTreeQuery(tree)
_run_and_check_query(query, 'scalar')
_run_and_check_query(query, 'T1w')
_run_and_check_query(query, 'T1w', participant='01')
_run_and_check_query(query, 'T1w', session='2')
......@@ -269,6 +282,7 @@ def test_query_asarray():
tree = filetree.FileTree.read('_test_tree.tree', '.')
query = filetree.FileTreeQuery(tree)
_run_and_check_query(query, 'scalar', asarray=True)
_run_and_check_query(query, 'T1w', asarray=True)
_run_and_check_query(query, 'T1w', asarray=True, participant='01')
_run_and_check_query(query, 'T1w', asarray=True, session='2')
......@@ -388,7 +402,10 @@ def test_scan():
tree = filetree.FileTree.read('_test_tree.tree', '.')
gotmatches = ftquery.scan(tree)
expmatches = []
expmatches = [ftquery.Match('scalar_file.txt',
'scalar',
tree,
{})]
for subj, ses in it.product(_subjs, _sess):
......@@ -416,6 +433,8 @@ def test_scan():
assert len(gotmatches) == len(expmatches)
for got, exp in zip(sorted(gotmatches), sorted(expmatches)):
assert got == exp
assert str(got) == str(exp)
assert got.filename == exp.filename
assert got.template == exp.template
assert got.variables == exp.variables
......@@ -433,6 +452,7 @@ def test_allVariables():
'surf' : _surfs,
'hemi' : _hemis}
expsnames = {
'scalar' : [],
'T1w' : ['participant', 'session'],
'T2w' : ['participant', 'session'],
'surface' : ['hemi', 'participant', 'session', 'surf']}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment