Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
FSL
fslpy
Commits
3f4bbb1a
Commit
3f4bbb1a
authored
Oct 20, 2020
by
Paul McCarthy
🚵
Browse files
Merge branch 'mnt/v3.3/recover' into 'v3.3'
Mnt/v3.3/recover See merge request fsl/fslpy!259
parents
3d4a345d
cbeca106
Pipeline
#5680
failed with stages
in 77 minutes and 11 seconds
Changes
6
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
CHANGELOG.rst
View file @
3f4bbb1a
This document contains the ``fslpy`` release history in reverse chronological
order.
3.3.3 (Wednesday 13th October 2020)
-----------------------------------
Changed
^^^^^^^
* The :func:`.fileOrImage` (and related) decorators will not manipulate the
return value of a decorated function if an argument ``cmdonly=True`` is
passed. This is so that wrapper functions will directly return the command
that would be executed when ``cmdonly=True``.
3.3.2 (Tuesday 12th October 2020)
---------------------------------
Changed
^^^^^^^
* Most :func:`.wrapper` functions now accept an argument called ``cmdonly``
which, if ``True``, will cause the generated command-line call to be
returned, instead of executed.
3.3.1 (Thursday 8th October 2020)
---------------------------------
...
...
fsl/utils/run.py
View file @
3f4bbb1a
...
...
@@ -151,6 +151,10 @@ def run(*args, **kwargs):
the :func:`.fslsub.submit` function. May also be a
dictionary containing arguments to that function.
:arg cmdonly: Defaults to ``False``. If ``True``, the command is not
executed, but rather is returned directly, as a list of
arguments.
:arg log: Must be passed as a keyword argument. An optional ``dict``
which may be used to redirect the command's standard output
and error. The following keys are recognised:
...
...
@@ -181,6 +185,7 @@ def run(*args, **kwargs):
returnStderr
=
kwargs
.
pop
(
'stderr'
,
False
)
returnExitcode
=
kwargs
.
pop
(
'exitcode'
,
False
)
submit
=
kwargs
.
pop
(
'submit'
,
{})
cmdonly
=
kwargs
.
pop
(
'cmdonly'
,
False
)
log
=
kwargs
.
pop
(
'log'
,
None
)
args
=
prepareArgs
(
args
)
...
...
@@ -207,6 +212,9 @@ def run(*args, **kwargs):
raise
ValueError
(
'submit must be a mapping containing '
'options for fsl.utils.fslsub.submit'
)
if
cmdonly
:
return
args
if
DRY_RUN
:
return
_dryrun
(
submit
,
returnStdout
,
returnStderr
,
returnExitcode
,
*
args
)
...
...
fsl/version.py
View file @
3f4bbb1a
...
...
@@ -47,7 +47,7 @@ import re
import
string
__version__
=
'3.3.
1
'
__version__
=
'3.3.
3
'
"""Current version number, as a string. """
...
...
fsl/wrappers/wrapperutils.py
View file @
3f4bbb1a
...
...
@@ -149,46 +149,73 @@ def _unwrap(func):
return
func
def
cmdwrapper
(
func
):
"""This decorator can be used on functions which generate a command line.
It will pass the return value of the function to the
:func:`fsl.utils.run.run` function in a standardised manner.
def
genxwrapper
(
func
,
runner
):
"""This function is used by :func:`cmdwrapper` and :func:`fslwrapper`.
It is not intended to be used in any other circumstances.
This function generates a wrapper function which calls ``func`` to
generate a command-line call, and then uses ``runner`` to invoke that
command.
``func`` is assumed to be a wrapper function which generates a command-
line. ``runner`` is assumed to be Either :func:`.run.run` or
:func:`.run.runfsl`.
The generated wrapper function will pass all of its arguments to ``func``,
and will then pass the generated command-line to ``runner``, returning
whatever is returned.
The following keyword arguments will be intercepted by the wrapper
function, and will *not* be passed to ``func``:
- ``stdout``: Passed to ``runner``. Defaults to ``True``.
- ``stderr``: Passed to ``runner``. Defaults to ``True``.
- ``exitcode``: Passed to ``runner``. Defaults to ``False``.
- ``submit``: Passed to ``runner``. Defaults to ``None``.
- ``log``: Passed to ``runner``. Defaults to ``{'tee':True}``.
- ``cmdonly``: Passed to ``runner``. Defaults to ``False``.
:arg func: A function which generates a command line.
:arg runner: Either :func:`.run.run` or :func:`.run.runfsl`.
"""
def
wrapper
(
*
args
,
**
kwargs
):
stdout
=
kwargs
.
pop
(
'stdout'
,
True
)
stderr
=
kwargs
.
pop
(
'stderr'
,
True
)
exitcode
=
kwargs
.
pop
(
'exitcode'
,
False
)
submit
=
kwargs
.
pop
(
'submit'
,
None
)
cmdonly
=
kwargs
.
pop
(
'cmdonly'
,
False
)
log
=
kwargs
.
pop
(
'log'
,
{
'tee'
:
True
})
cmd
=
func
(
*
args
,
**
kwargs
)
return
run
.
run
(
cmd
,
stderr
=
stderr
,
log
=
log
,
submit
=
submit
,
stdout
=
stdout
,
exitcode
=
exitcode
)
return
runner
(
cmd
,
stderr
=
stderr
,
log
=
log
,
submit
=
submit
,
cmdonly
=
cmdonly
,
stdout
=
stdout
,
exitcode
=
exitcode
)
return
_update_wrapper
(
wrapper
,
func
)
def
cmdwrapper
(
func
):
"""This decorator can be used on functions which generate a command line.
It will pass the return value of the function to the
:func:`fsl.utils.run.run` function in a standardised manner.
See the :func:`genxwrapper` function for details.
"""
return
genxwrapper
(
func
,
run
.
run
)
def
fslwrapper
(
func
):
"""This decorator can be used on functions which generate a FSL command
line. It will pass the return value of the function to the
:func:`fsl.utils.run.runfsl` function in a standardised manner.
See the :func:`genxwrapper` function for details.
"""
def
wrapper
(
*
args
,
**
kwargs
):
stdout
=
kwargs
.
pop
(
'stdout'
,
True
)
stderr
=
kwargs
.
pop
(
'stderr'
,
True
)
exitcode
=
kwargs
.
pop
(
'exitcode'
,
False
)
submit
=
kwargs
.
pop
(
'submit'
,
None
)
log
=
kwargs
.
pop
(
'log'
,
{
'tee'
:
True
})
cmd
=
func
(
*
args
,
**
kwargs
)
return
run
.
runfsl
(
cmd
,
stderr
=
stderr
,
log
=
log
,
submit
=
submit
,
stdout
=
stdout
,
exitcode
=
exitcode
)
return
_update_wrapper
(
wrapper
,
func
)
return
genxwrapper
(
func
,
run
.
runfsl
)
SHOW_IF_TRUE
=
object
()
...
...
@@ -455,25 +482,27 @@ class FileOrThing(object):
the dictionary.
**Cluster submission**
**Exceptions**
The above description holds in all situations, except when an argument
called ``submit`` is passed, and is set to a value which evaluates to
``True``. In this case, the ``FileOrThing`` decorator will pass all
arguments straight through to the decorated function, and will return its
return value unchanged.
The above description holds in all situations, except when arguments called
``submit`` and/or ``cmdonly`` are passed, and are set to values which
evaluate to ``True``. In this case, the ``FileOrThing`` decorator will pass
all arguments straight through to the decorated function, and will return
its return value unchanged.
This is because most functions that are decorated with the
:func:`fileOrImage` or :func:`fileOrArray` decorators will invoke a call
to :func:`.run.run` or :func:`.runfsl`, where a value of ``submit=True``
will cause the command to be executed asynchronously on a cluster
platform.
to :func:`.run.run` or :func:`.runfsl`, where:
- a value of ``submit=True`` will cause the command to be executed
asynchronously on a cluster platform.
- a value of ``cmdonly=True`` will cause the command to *not* be executed,
but instead the command that would have been executed is returned.
A :exc:`ValueError` will be raised if the decorated function is called
with ``submit=True``, and with any in-memory objects or ``LOAD`` symbols.
with ``submit=True`` and/or ``cmdonly=True``, and with any in-memory
objects or ``LOAD`` symbols.
**Example**
...
...
@@ -657,9 +686,11 @@ class FileOrThing(object):
# Special case - if fsl.utils.run[fsl] is
# being decorated (e.g. via cmdwrapper/
# fslwrapper), and submit=True, this call
# will ultimately submit the job to the
# cluster, and will return immediately.
# fslwrapper), and submit=True or
# cmdonly=True, this call will ultimately
# submit the job to the cluster, or will
# return the command that would have been
# executed, and will return immediately.
#
# We error if we are given any in-memory
# things, or LOAD symbols.
...
...
@@ -667,7 +698,8 @@ class FileOrThing(object):
# n.b. testing values to be strings could
# interfere with the fileOrText decorator.
# Possible solution is to use pathlib?
if
kwargs
.
get
(
'submit'
,
False
):
if
kwargs
.
get
(
'submit'
,
False
)
or
\
kwargs
.
get
(
'cmdonly'
,
False
):
allargs
=
{
**
dict
(
zip
(
argnames
,
args
)),
**
kwargs
}
for
name
,
val
in
allargs
.
items
():
if
(
name
in
self
.
__things
)
and
\
...
...
tests/test_run.py
View file @
3f4bbb1a
...
...
@@ -179,6 +179,13 @@ def test_run_passthrough():
assert
run
.
run
(
'./script.sh'
,
env
=
env
)
==
expstdout
def
test_cmdonly
():
assert
run
.
run
(
'script.sh'
,
cmdonly
=
True
)
==
[
'script.sh'
]
assert
run
.
run
(
'script.sh 1 2 3'
,
cmdonly
=
True
)
==
[
'script.sh'
,
'1'
,
'2'
,
'3'
]
assert
run
.
run
([
'script.sh'
],
cmdonly
=
True
)
==
[
'script.sh'
]
assert
run
.
run
([
'script.sh'
,
'1'
],
cmdonly
=
True
)
==
[
'script.sh'
,
'1'
]
def
test_dryrun
():
test_script
=
textwrap
.
dedent
(
"""
...
...
tests/test_wrappers/test_wrapperutils.py
View file @
3f4bbb1a
...
...
@@ -714,13 +714,15 @@ def test_fileOrThing_chained_outprefix():
assert
np
.
all
(
res
[
'out_array'
]
==
exparr
)
def
test_fileOrThing_submit
():
def
test_fileOrThing_submit
_cmdonly
():
@
wutils
.
fileOrImage
(
'input'
,
'output'
)
def
func
(
input
,
output
,
submit
=
False
):
def
func
(
input
,
output
,
submit
=
False
,
cmdonly
=
False
):
if
submit
:
return
'submitted!'
if
cmdonly
:
return
'cmdonly!'
img
=
nib
.
load
(
input
)
img
=
nib
.
nifti1
.
Nifti1Image
(
np
.
asanyarray
(
img
.
dataobj
)
*
2
,
np
.
eye
(
4
))
...
...
@@ -735,7 +737,8 @@ def test_fileOrThing_submit():
result
=
func
(
img
,
wutils
.
LOAD
)
assert
np
.
all
(
np
.
asanyarray
(
result
[
'output'
].
dataobj
)
==
exp
)
assert
func
(
'input.nii.gz'
,
'output.nii.gz'
,
submit
=
True
)
==
'submitted!'
assert
func
(
'input.nii.gz'
,
'output.nii.gz'
,
submit
=
True
)
==
'submitted!'
assert
func
(
'input.nii.gz'
,
'output.nii.gz'
,
cmdonly
=
True
)
==
'cmdonly!'
with
pytest
.
raises
(
ValueError
):
func
(
img
,
wutils
.
LOAD
,
submit
=
True
)
...
...
@@ -753,15 +756,20 @@ def test_cmdwrapper():
with
run
.
dryrun
():
assert
func
(
1
,
2
)[
0
]
==
'func 1 2'
assert
func
(
1
,
2
,
cmdonly
=
True
)
==
[
'func'
,
'1'
,
'2'
]
def
test_fslwrapper
():
@
wutils
.
fslwrapper
def
func
(
a
,
b
):
return
[
'func'
,
str
(
a
),
str
(
b
)]
with
run
.
dryrun
(),
mockFSLDIR
(
bin
=
(
'func'
,))
as
fsldir
:
with
mockFSLDIR
(
bin
=
(
'func'
,))
as
fsldir
:
expected
=
'{} 1 2'
.
format
(
op
.
join
(
fsldir
,
'bin'
,
'func'
))
assert
func
(
1
,
2
)[
0
]
==
expected
with
run
.
dryrun
():
assert
func
(
1
,
2
)[
0
]
==
expected
assert
func
(
1
,
2
,
cmdonly
=
True
)
==
list
(
shlex
.
split
(
expected
))
_test_script
=
textwrap
.
dedent
(
"""
...
...
@@ -835,3 +843,21 @@ def test_fslwrapper_submit():
assert
stdout
.
strip
()
==
'test_script running: 1 2'
assert
stderr
.
strip
()
==
experr
@
pytest
.
mark
.
unixtest
def
test_cmdwrapper_fileorthing_cmdonly
():
test_func
=
wutils
.
fileOrImage
(
'a'
)(
wutils
.
cmdwrapper
(
_test_script_func
))
newpath
=
op
.
pathsep
.
join
((
'.'
,
os
.
environ
[
'PATH'
]))
with
tempdir
.
tempdir
(),
\
mock
.
patch
.
dict
(
os
.
environ
,
{
'PATH'
:
newpath
}):
with
open
(
'test_script'
,
'wt'
)
as
f
:
f
.
write
(
_test_script
)
os
.
chmod
(
'test_script'
,
0o755
)
ran
=
test_func
(
'1'
,
'2'
)
cmd
=
test_func
(
'1'
,
'2'
,
cmdonly
=
True
)
assert
ran
.
stdout
[
0
].
strip
()
==
'test_script running: 1 2'
assert
cmd
==
[
'test_script'
,
'1'
,
'2'
]
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment