Commit 3f4bbb1a authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

Merge branch 'mnt/v3.3/recover' into 'v3.3'

Mnt/v3.3/recover

See merge request fsl/fslpy!259
parents 3d4a345d cbeca106
Pipeline #5680 failed with stages
in 77 minutes and 11 seconds
This document contains the ``fslpy`` release history in reverse chronological This document contains the ``fslpy`` release history in reverse chronological
order. order.
3.3.3 (Wednesday 13th October 2020)
-----------------------------------
Changed
^^^^^^^
* The :func:`.fileOrImage` (and related) decorators will not manipulate the
return value of a decorated function if an argument ``cmdonly=True`` is
passed. This is so that wrapper functions will directly return the command
that would be executed when ``cmdonly=True``.
3.3.2 (Tuesday 12th October 2020)
---------------------------------
Changed
^^^^^^^
* Most :func:`.wrapper` functions now accept an argument called ``cmdonly``
which, if ``True``, will cause the generated command-line call to be
returned, instead of executed.
3.3.1 (Thursday 8th October 2020) 3.3.1 (Thursday 8th October 2020)
--------------------------------- ---------------------------------
......
...@@ -151,6 +151,10 @@ def run(*args, **kwargs): ...@@ -151,6 +151,10 @@ def run(*args, **kwargs):
the :func:`.fslsub.submit` function. May also be a the :func:`.fslsub.submit` function. May also be a
dictionary containing arguments to that function. dictionary containing arguments to that function.
:arg cmdonly: Defaults to ``False``. If ``True``, the command is not
executed, but rather is returned directly, as a list of
arguments.
:arg log: Must be passed as a keyword argument. An optional ``dict`` :arg log: Must be passed as a keyword argument. An optional ``dict``
which may be used to redirect the command's standard output which may be used to redirect the command's standard output
and error. The following keys are recognised: and error. The following keys are recognised:
...@@ -181,6 +185,7 @@ def run(*args, **kwargs): ...@@ -181,6 +185,7 @@ def run(*args, **kwargs):
returnStderr = kwargs.pop('stderr', False) returnStderr = kwargs.pop('stderr', False)
returnExitcode = kwargs.pop('exitcode', False) returnExitcode = kwargs.pop('exitcode', False)
submit = kwargs.pop('submit', {}) submit = kwargs.pop('submit', {})
cmdonly = kwargs.pop('cmdonly', False)
log = kwargs.pop('log', None) log = kwargs.pop('log', None)
args = prepareArgs(args) args = prepareArgs(args)
...@@ -207,6 +212,9 @@ def run(*args, **kwargs): ...@@ -207,6 +212,9 @@ def run(*args, **kwargs):
raise ValueError('submit must be a mapping containing ' raise ValueError('submit must be a mapping containing '
'options for fsl.utils.fslsub.submit') 'options for fsl.utils.fslsub.submit')
if cmdonly:
return args
if DRY_RUN: if DRY_RUN:
return _dryrun( return _dryrun(
submit, returnStdout, returnStderr, returnExitcode, *args) submit, returnStdout, returnStderr, returnExitcode, *args)
......
...@@ -47,7 +47,7 @@ import re ...@@ -47,7 +47,7 @@ import re
import string import string
__version__ = '3.3.1' __version__ = '3.3.3'
"""Current version number, as a string. """ """Current version number, as a string. """
......
...@@ -149,46 +149,73 @@ def _unwrap(func): ...@@ -149,46 +149,73 @@ def _unwrap(func):
return func return func
def cmdwrapper(func): def genxwrapper(func, runner):
"""This decorator can be used on functions which generate a command line. """This function is used by :func:`cmdwrapper` and :func:`fslwrapper`.
It will pass the return value of the function to the It is not intended to be used in any other circumstances.
:func:`fsl.utils.run.run` function in a standardised manner.
This function generates a wrapper function which calls ``func`` to
generate a command-line call, and then uses ``runner`` to invoke that
command.
``func`` is assumed to be a wrapper function which generates a command-
line. ``runner`` is assumed to be Either :func:`.run.run` or
:func:`.run.runfsl`.
The generated wrapper function will pass all of its arguments to ``func``,
and will then pass the generated command-line to ``runner``, returning
whatever is returned.
The following keyword arguments will be intercepted by the wrapper
function, and will *not* be passed to ``func``:
- ``stdout``: Passed to ``runner``. Defaults to ``True``.
- ``stderr``: Passed to ``runner``. Defaults to ``True``.
- ``exitcode``: Passed to ``runner``. Defaults to ``False``.
- ``submit``: Passed to ``runner``. Defaults to ``None``.
- ``log``: Passed to ``runner``. Defaults to ``{'tee':True}``.
- ``cmdonly``: Passed to ``runner``. Defaults to ``False``.
:arg func: A function which generates a command line.
:arg runner: Either :func:`.run.run` or :func:`.run.runfsl`.
""" """
def wrapper(*args, **kwargs): def wrapper(*args, **kwargs):
stdout = kwargs.pop('stdout', True) stdout = kwargs.pop('stdout', True)
stderr = kwargs.pop('stderr', True) stderr = kwargs.pop('stderr', True)
exitcode = kwargs.pop('exitcode', False) exitcode = kwargs.pop('exitcode', False)
submit = kwargs.pop('submit', None) submit = kwargs.pop('submit', None)
cmdonly = kwargs.pop('cmdonly', False)
log = kwargs.pop('log', {'tee' : True}) log = kwargs.pop('log', {'tee' : True})
cmd = func(*args, **kwargs) cmd = func(*args, **kwargs)
return run.run(cmd,
stderr=stderr, return runner(cmd,
log=log, stderr=stderr,
submit=submit, log=log,
stdout=stdout, submit=submit,
exitcode=exitcode) cmdonly=cmdonly,
stdout=stdout,
exitcode=exitcode)
return _update_wrapper(wrapper, func) return _update_wrapper(wrapper, func)
def cmdwrapper(func):
"""This decorator can be used on functions which generate a command line.
It will pass the return value of the function to the
:func:`fsl.utils.run.run` function in a standardised manner.
See the :func:`genxwrapper` function for details.
"""
return genxwrapper(func, run.run)
def fslwrapper(func): def fslwrapper(func):
"""This decorator can be used on functions which generate a FSL command """This decorator can be used on functions which generate a FSL command
line. It will pass the return value of the function to the line. It will pass the return value of the function to the
:func:`fsl.utils.run.runfsl` function in a standardised manner. :func:`fsl.utils.run.runfsl` function in a standardised manner.
See the :func:`genxwrapper` function for details.
""" """
def wrapper(*args, **kwargs): return genxwrapper(func, run.runfsl)
stdout = kwargs.pop('stdout', True)
stderr = kwargs.pop('stderr', True)
exitcode = kwargs.pop('exitcode', False)
submit = kwargs.pop('submit', None)
log = kwargs.pop('log', {'tee' : True})
cmd = func(*args, **kwargs)
return run.runfsl(cmd,
stderr=stderr,
log=log,
submit=submit,
stdout=stdout,
exitcode=exitcode)
return _update_wrapper(wrapper, func)
SHOW_IF_TRUE = object() SHOW_IF_TRUE = object()
...@@ -455,25 +482,27 @@ class FileOrThing(object): ...@@ -455,25 +482,27 @@ class FileOrThing(object):
the dictionary. the dictionary.
**Cluster submission** **Exceptions**
The above description holds in all situations, except when an argument
called ``submit`` is passed, and is set to a value which evaluates to
``True``. In this case, the ``FileOrThing`` decorator will pass all
arguments straight through to the decorated function, and will return its
return value unchanged.
The above description holds in all situations, except when arguments called
``submit`` and/or ``cmdonly`` are passed, and are set to values which
evaluate to ``True``. In this case, the ``FileOrThing`` decorator will pass
all arguments straight through to the decorated function, and will return
its return value unchanged.
This is because most functions that are decorated with the This is because most functions that are decorated with the
:func:`fileOrImage` or :func:`fileOrArray` decorators will invoke a call :func:`fileOrImage` or :func:`fileOrArray` decorators will invoke a call
to :func:`.run.run` or :func:`.runfsl`, where a value of ``submit=True`` to :func:`.run.run` or :func:`.runfsl`, where:
will cause the command to be executed asynchronously on a cluster
platform.
- a value of ``submit=True`` will cause the command to be executed
asynchronously on a cluster platform.
- a value of ``cmdonly=True`` will cause the command to *not* be executed,
but instead the command that would have been executed is returned.
A :exc:`ValueError` will be raised if the decorated function is called A :exc:`ValueError` will be raised if the decorated function is called
with ``submit=True``, and with any in-memory objects or ``LOAD`` symbols. with ``submit=True`` and/or ``cmdonly=True``, and with any in-memory
objects or ``LOAD`` symbols.
**Example** **Example**
...@@ -657,9 +686,11 @@ class FileOrThing(object): ...@@ -657,9 +686,11 @@ class FileOrThing(object):
# Special case - if fsl.utils.run[fsl] is # Special case - if fsl.utils.run[fsl] is
# being decorated (e.g. via cmdwrapper/ # being decorated (e.g. via cmdwrapper/
# fslwrapper), and submit=True, this call # fslwrapper), and submit=True or
# will ultimately submit the job to the # cmdonly=True, this call will ultimately
# cluster, and will return immediately. # submit the job to the cluster, or will
# return the command that would have been
# executed, and will return immediately.
# #
# We error if we are given any in-memory # We error if we are given any in-memory
# things, or LOAD symbols. # things, or LOAD symbols.
...@@ -667,7 +698,8 @@ class FileOrThing(object): ...@@ -667,7 +698,8 @@ class FileOrThing(object):
# n.b. testing values to be strings could # n.b. testing values to be strings could
# interfere with the fileOrText decorator. # interfere with the fileOrText decorator.
# Possible solution is to use pathlib? # Possible solution is to use pathlib?
if kwargs.get('submit', False): if kwargs.get('submit', False) or \
kwargs.get('cmdonly', False):
allargs = {**dict(zip(argnames, args)), **kwargs} allargs = {**dict(zip(argnames, args)), **kwargs}
for name, val in allargs.items(): for name, val in allargs.items():
if (name in self.__things) and \ if (name in self.__things) and \
......
...@@ -179,6 +179,13 @@ def test_run_passthrough(): ...@@ -179,6 +179,13 @@ def test_run_passthrough():
assert run.run('./script.sh', env=env) == expstdout assert run.run('./script.sh', env=env) == expstdout
def test_cmdonly():
assert run.run('script.sh', cmdonly=True) == ['script.sh']
assert run.run('script.sh 1 2 3', cmdonly=True) == ['script.sh', '1', '2', '3']
assert run.run(['script.sh'], cmdonly=True) == ['script.sh']
assert run.run(['script.sh', '1'], cmdonly=True) == ['script.sh', '1']
def test_dryrun(): def test_dryrun():
test_script = textwrap.dedent(""" test_script = textwrap.dedent("""
......
...@@ -714,13 +714,15 @@ def test_fileOrThing_chained_outprefix(): ...@@ -714,13 +714,15 @@ def test_fileOrThing_chained_outprefix():
assert np.all(res['out_array'] == exparr) assert np.all(res['out_array'] == exparr)
def test_fileOrThing_submit(): def test_fileOrThing_submit_cmdonly():
@wutils.fileOrImage('input', 'output') @wutils.fileOrImage('input', 'output')
def func(input, output, submit=False): def func(input, output, submit=False, cmdonly=False):
if submit: if submit:
return 'submitted!' return 'submitted!'
if cmdonly:
return 'cmdonly!'
img = nib.load(input) img = nib.load(input)
img = nib.nifti1.Nifti1Image(np.asanyarray(img.dataobj) * 2, np.eye(4)) img = nib.nifti1.Nifti1Image(np.asanyarray(img.dataobj) * 2, np.eye(4))
...@@ -735,7 +737,8 @@ def test_fileOrThing_submit(): ...@@ -735,7 +737,8 @@ def test_fileOrThing_submit():
result = func(img, wutils.LOAD) result = func(img, wutils.LOAD)
assert np.all(np.asanyarray(result['output'].dataobj) == exp) assert np.all(np.asanyarray(result['output'].dataobj) == exp)
assert func('input.nii.gz', 'output.nii.gz', submit=True) == 'submitted!' assert func('input.nii.gz', 'output.nii.gz', submit=True) == 'submitted!'
assert func('input.nii.gz', 'output.nii.gz', cmdonly=True) == 'cmdonly!'
with pytest.raises(ValueError): with pytest.raises(ValueError):
func(img, wutils.LOAD, submit=True) func(img, wutils.LOAD, submit=True)
...@@ -753,15 +756,20 @@ def test_cmdwrapper(): ...@@ -753,15 +756,20 @@ def test_cmdwrapper():
with run.dryrun(): with run.dryrun():
assert func(1, 2)[0] == 'func 1 2' assert func(1, 2)[0] == 'func 1 2'
assert func(1, 2, cmdonly=True) == ['func', '1', '2']
def test_fslwrapper(): def test_fslwrapper():
@wutils.fslwrapper @wutils.fslwrapper
def func(a, b): def func(a, b):
return ['func', str(a), str(b)] return ['func', str(a), str(b)]
with run.dryrun(), mockFSLDIR(bin=('func',)) as fsldir: with mockFSLDIR(bin=('func',)) as fsldir:
expected = '{} 1 2'.format(op.join(fsldir, 'bin', 'func')) expected = '{} 1 2'.format(op.join(fsldir, 'bin', 'func'))
assert func(1, 2)[0] == expected with run.dryrun():
assert func(1, 2)[0] == expected
assert func(1, 2, cmdonly=True) == list(shlex.split(expected))
_test_script = textwrap.dedent(""" _test_script = textwrap.dedent("""
...@@ -835,3 +843,21 @@ def test_fslwrapper_submit(): ...@@ -835,3 +843,21 @@ def test_fslwrapper_submit():
assert stdout.strip() == 'test_script running: 1 2' assert stdout.strip() == 'test_script running: 1 2'
assert stderr.strip() == experr assert stderr.strip() == experr
@pytest.mark.unixtest
def test_cmdwrapper_fileorthing_cmdonly():
test_func = wutils.fileOrImage('a')(wutils.cmdwrapper(_test_script_func))
newpath = op.pathsep.join(('.', os.environ['PATH']))
with tempdir.tempdir(), \
mock.patch.dict(os.environ, {'PATH' : newpath}):
with open('test_script', 'wt') as f:
f.write(_test_script)
os.chmod('test_script', 0o755)
ran = test_func('1', '2')
cmd = test_func('1', '2', cmdonly=True)
assert ran.stdout[0].strip() == 'test_script running: 1 2'
assert cmd == ['test_script', '1', '2']
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment