Commit 3f4bbb1a authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

Merge branch 'mnt/v3.3/recover' into 'v3.3'

Mnt/v3.3/recover

See merge request fsl/fslpy!259
parents 3d4a345d cbeca106
Pipeline #5680 failed with stages
in 77 minutes and 11 seconds
This document contains the ``fslpy`` release history in reverse chronological
order.
3.3.3 (Wednesday 13th October 2020)
-----------------------------------
Changed
^^^^^^^
* The :func:`.fileOrImage` (and related) decorators will not manipulate the
return value of a decorated function if an argument ``cmdonly=True`` is
passed. This is so that wrapper functions will directly return the command
that would be executed when ``cmdonly=True``.
3.3.2 (Tuesday 12th October 2020)
---------------------------------
Changed
^^^^^^^
* Most :func:`.wrapper` functions now accept an argument called ``cmdonly``
which, if ``True``, will cause the generated command-line call to be
returned, instead of executed.
3.3.1 (Thursday 8th October 2020)
---------------------------------
......
......@@ -151,6 +151,10 @@ def run(*args, **kwargs):
the :func:`.fslsub.submit` function. May also be a
dictionary containing arguments to that function.
:arg cmdonly: Defaults to ``False``. If ``True``, the command is not
executed, but rather is returned directly, as a list of
arguments.
:arg log: Must be passed as a keyword argument. An optional ``dict``
which may be used to redirect the command's standard output
and error. The following keys are recognised:
......@@ -181,6 +185,7 @@ def run(*args, **kwargs):
returnStderr = kwargs.pop('stderr', False)
returnExitcode = kwargs.pop('exitcode', False)
submit = kwargs.pop('submit', {})
cmdonly = kwargs.pop('cmdonly', False)
log = kwargs.pop('log', None)
args = prepareArgs(args)
......@@ -207,6 +212,9 @@ def run(*args, **kwargs):
raise ValueError('submit must be a mapping containing '
'options for fsl.utils.fslsub.submit')
if cmdonly:
return args
if DRY_RUN:
return _dryrun(
submit, returnStdout, returnStderr, returnExitcode, *args)
......
......@@ -47,7 +47,7 @@ import re
import string
__version__ = '3.3.1'
__version__ = '3.3.3'
"""Current version number, as a string. """
......
......@@ -149,46 +149,73 @@ def _unwrap(func):
return func
def cmdwrapper(func):
"""This decorator can be used on functions which generate a command line.
It will pass the return value of the function to the
:func:`fsl.utils.run.run` function in a standardised manner.
def genxwrapper(func, runner):
"""This function is used by :func:`cmdwrapper` and :func:`fslwrapper`.
It is not intended to be used in any other circumstances.
This function generates a wrapper function which calls ``func`` to
generate a command-line call, and then uses ``runner`` to invoke that
command.
``func`` is assumed to be a wrapper function which generates a command-
line. ``runner`` is assumed to be Either :func:`.run.run` or
:func:`.run.runfsl`.
The generated wrapper function will pass all of its arguments to ``func``,
and will then pass the generated command-line to ``runner``, returning
whatever is returned.
The following keyword arguments will be intercepted by the wrapper
function, and will *not* be passed to ``func``:
- ``stdout``: Passed to ``runner``. Defaults to ``True``.
- ``stderr``: Passed to ``runner``. Defaults to ``True``.
- ``exitcode``: Passed to ``runner``. Defaults to ``False``.
- ``submit``: Passed to ``runner``. Defaults to ``None``.
- ``log``: Passed to ``runner``. Defaults to ``{'tee':True}``.
- ``cmdonly``: Passed to ``runner``. Defaults to ``False``.
:arg func: A function which generates a command line.
:arg runner: Either :func:`.run.run` or :func:`.run.runfsl`.
"""
def wrapper(*args, **kwargs):
stdout = kwargs.pop('stdout', True)
stderr = kwargs.pop('stderr', True)
exitcode = kwargs.pop('exitcode', False)
submit = kwargs.pop('submit', None)
cmdonly = kwargs.pop('cmdonly', False)
log = kwargs.pop('log', {'tee' : True})
cmd = func(*args, **kwargs)
return run.run(cmd,
stderr=stderr,
log=log,
submit=submit,
stdout=stdout,
exitcode=exitcode)
return runner(cmd,
stderr=stderr,
log=log,
submit=submit,
cmdonly=cmdonly,
stdout=stdout,
exitcode=exitcode)
return _update_wrapper(wrapper, func)
def cmdwrapper(func):
"""This decorator can be used on functions which generate a command line.
It will pass the return value of the function to the
:func:`fsl.utils.run.run` function in a standardised manner.
See the :func:`genxwrapper` function for details.
"""
return genxwrapper(func, run.run)
def fslwrapper(func):
"""This decorator can be used on functions which generate a FSL command
line. It will pass the return value of the function to the
:func:`fsl.utils.run.runfsl` function in a standardised manner.
See the :func:`genxwrapper` function for details.
"""
def wrapper(*args, **kwargs):
stdout = kwargs.pop('stdout', True)
stderr = kwargs.pop('stderr', True)
exitcode = kwargs.pop('exitcode', False)
submit = kwargs.pop('submit', None)
log = kwargs.pop('log', {'tee' : True})
cmd = func(*args, **kwargs)
return run.runfsl(cmd,
stderr=stderr,
log=log,
submit=submit,
stdout=stdout,
exitcode=exitcode)
return _update_wrapper(wrapper, func)
return genxwrapper(func, run.runfsl)
SHOW_IF_TRUE = object()
......@@ -455,25 +482,27 @@ class FileOrThing(object):
the dictionary.
**Cluster submission**
**Exceptions**
The above description holds in all situations, except when an argument
called ``submit`` is passed, and is set to a value which evaluates to
``True``. In this case, the ``FileOrThing`` decorator will pass all
arguments straight through to the decorated function, and will return its
return value unchanged.
The above description holds in all situations, except when arguments called
``submit`` and/or ``cmdonly`` are passed, and are set to values which
evaluate to ``True``. In this case, the ``FileOrThing`` decorator will pass
all arguments straight through to the decorated function, and will return
its return value unchanged.
This is because most functions that are decorated with the
:func:`fileOrImage` or :func:`fileOrArray` decorators will invoke a call
to :func:`.run.run` or :func:`.runfsl`, where a value of ``submit=True``
will cause the command to be executed asynchronously on a cluster
platform.
to :func:`.run.run` or :func:`.runfsl`, where:
- a value of ``submit=True`` will cause the command to be executed
asynchronously on a cluster platform.
- a value of ``cmdonly=True`` will cause the command to *not* be executed,
but instead the command that would have been executed is returned.
A :exc:`ValueError` will be raised if the decorated function is called
with ``submit=True``, and with any in-memory objects or ``LOAD`` symbols.
with ``submit=True`` and/or ``cmdonly=True``, and with any in-memory
objects or ``LOAD`` symbols.
**Example**
......@@ -657,9 +686,11 @@ class FileOrThing(object):
# Special case - if fsl.utils.run[fsl] is
# being decorated (e.g. via cmdwrapper/
# fslwrapper), and submit=True, this call
# will ultimately submit the job to the
# cluster, and will return immediately.
# fslwrapper), and submit=True or
# cmdonly=True, this call will ultimately
# submit the job to the cluster, or will
# return the command that would have been
# executed, and will return immediately.
#
# We error if we are given any in-memory
# things, or LOAD symbols.
......@@ -667,7 +698,8 @@ class FileOrThing(object):
# n.b. testing values to be strings could
# interfere with the fileOrText decorator.
# Possible solution is to use pathlib?
if kwargs.get('submit', False):
if kwargs.get('submit', False) or \
kwargs.get('cmdonly', False):
allargs = {**dict(zip(argnames, args)), **kwargs}
for name, val in allargs.items():
if (name in self.__things) and \
......
......@@ -179,6 +179,13 @@ def test_run_passthrough():
assert run.run('./script.sh', env=env) == expstdout
def test_cmdonly():
assert run.run('script.sh', cmdonly=True) == ['script.sh']
assert run.run('script.sh 1 2 3', cmdonly=True) == ['script.sh', '1', '2', '3']
assert run.run(['script.sh'], cmdonly=True) == ['script.sh']
assert run.run(['script.sh', '1'], cmdonly=True) == ['script.sh', '1']
def test_dryrun():
test_script = textwrap.dedent("""
......
......@@ -714,13 +714,15 @@ def test_fileOrThing_chained_outprefix():
assert np.all(res['out_array'] == exparr)
def test_fileOrThing_submit():
def test_fileOrThing_submit_cmdonly():
@wutils.fileOrImage('input', 'output')
def func(input, output, submit=False):
def func(input, output, submit=False, cmdonly=False):
if submit:
return 'submitted!'
if cmdonly:
return 'cmdonly!'
img = nib.load(input)
img = nib.nifti1.Nifti1Image(np.asanyarray(img.dataobj) * 2, np.eye(4))
......@@ -735,7 +737,8 @@ def test_fileOrThing_submit():
result = func(img, wutils.LOAD)
assert np.all(np.asanyarray(result['output'].dataobj) == exp)
assert func('input.nii.gz', 'output.nii.gz', submit=True) == 'submitted!'
assert func('input.nii.gz', 'output.nii.gz', submit=True) == 'submitted!'
assert func('input.nii.gz', 'output.nii.gz', cmdonly=True) == 'cmdonly!'
with pytest.raises(ValueError):
func(img, wutils.LOAD, submit=True)
......@@ -753,15 +756,20 @@ def test_cmdwrapper():
with run.dryrun():
assert func(1, 2)[0] == 'func 1 2'
assert func(1, 2, cmdonly=True) == ['func', '1', '2']
def test_fslwrapper():
@wutils.fslwrapper
def func(a, b):
return ['func', str(a), str(b)]
with run.dryrun(), mockFSLDIR(bin=('func',)) as fsldir:
with mockFSLDIR(bin=('func',)) as fsldir:
expected = '{} 1 2'.format(op.join(fsldir, 'bin', 'func'))
assert func(1, 2)[0] == expected
with run.dryrun():
assert func(1, 2)[0] == expected
assert func(1, 2, cmdonly=True) == list(shlex.split(expected))
_test_script = textwrap.dedent("""
......@@ -835,3 +843,21 @@ def test_fslwrapper_submit():
assert stdout.strip() == 'test_script running: 1 2'
assert stderr.strip() == experr
@pytest.mark.unixtest
def test_cmdwrapper_fileorthing_cmdonly():
test_func = wutils.fileOrImage('a')(wutils.cmdwrapper(_test_script_func))
newpath = op.pathsep.join(('.', os.environ['PATH']))
with tempdir.tempdir(), \
mock.patch.dict(os.environ, {'PATH' : newpath}):
with open('test_script', 'wt') as f:
f.write(_test_script)
os.chmod('test_script', 0o755)
ran = test_func('1', '2')
cmd = test_func('1', '2', cmdonly=True)
assert ran.stdout[0].strip() == 'test_script running: 1 2'
assert cmd == ['test_script', '1', '2']
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment