Commit f3d34686 authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

Merge branch 'rf/fslsub' into 'master'

Add fsl.wrappers.fsl_sub, deprecate fsl.utils.fslsub.submit

Closes #376 and #373

See merge request fsl/fslpy!309
parents 714a7f79 d53aae7c
...@@ -2,6 +2,47 @@ This document contains the ``fslpy`` release history in reverse chronological ...@@ -2,6 +2,47 @@ This document contains the ``fslpy`` release history in reverse chronological
order. order.
3.7.0 (Under development)
-------------------------
Added
^^^^^
* New :mod:`fsl.wrappers.fsl_sub` wrapper function for the ``fsl_sub``
command.
Changed
^^^^^^^
* The default behaviour of the :func:`fsl.utils.run.run` function (and hence
that of all :mod:`fsl.wrappers` functions) has been changed so that the
standard output and error of the called command is now forwarded to the
calling Python process, in addition to being returned from ``run`` as
strings. In other words, the default behaviour of ``run('cmd')``, is now
equivalent to ``run('cmd', log={"tee":True})``. The previous default
behaviour can be achieved with ``run('cmd', log={"tee":False})``.
* The :func:`fsl.utils.run.run` and :func:`fsl.utils.run.runfsl` functions
(and hence all :mod:`fsl.wrappers` functions) have been modified to use
``fsl.wrappers.fsl_sub`` instead of ``fsl.utils.fslsub.submit``. This is an
internal change which should not affect the usage of the ``run``, ``runfsl``
or wrapper functions.
Deprecated
^^^^^^^^^^
* :class:`fsl.utils.fslsub.SubmitParams` and :func:`fsl.utils.fslsub.submit`
have been deprecated in favour of using the ``fsl.wrappers.fsl_sub`` wrapper
function.
* The :func:`fsl.utils.fslsub.info` function has been deprecated in favour of
using the ``fsl_sub.report`` function, from the separate `fsl_sub
<https://git.fmrib.ox.ac.uk/fsl/fsl_sub>`_ Python library.
3.6.4 (Tuesday 3rd August 2021) 3.6.4 (Tuesday 3rd August 2021)
------------------------------- -------------------------------
......
``fsl.wrappers.epi_reg``
========================
.. automodule:: fsl.wrappers.epi_reg
:members:
:undoc-members:
:show-inheritance:
``fsl.wrappers.fsl_sub``
========================
.. automodule:: fsl.wrappers.fsl_sub
:members:
:undoc-members:
:show-inheritance:
...@@ -6,12 +6,14 @@ ...@@ -6,12 +6,14 @@
fsl.wrappers.bet fsl.wrappers.bet
fsl.wrappers.eddy fsl.wrappers.eddy
fsl.wrappers.epi_reg
fsl.wrappers.fast fsl.wrappers.fast
fsl.wrappers.flirt fsl.wrappers.flirt
fsl.wrappers.fnirt fsl.wrappers.fnirt
fsl.wrappers.fsl_anat
fsl.wrappers.fsl_sub
fsl.wrappers.fslmaths fsl.wrappers.fslmaths
fsl.wrappers.fslstats fsl.wrappers.fslstats
fsl.wrappers.fsl_anat
fsl.wrappers.fugue fsl.wrappers.fugue
fsl.wrappers.melodic fsl.wrappers.melodic
fsl.wrappers.misc fsl.wrappers.misc
......
...@@ -53,13 +53,19 @@ import warnings ...@@ -53,13 +53,19 @@ import warnings
import os import os
import fsl.utils.deprecated as deprecated
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@dataclass @dataclass
class SubmitParams(object): class SubmitParams(object):
""" """Represents the fsl_sub parameters
Represents the fsl_sub parameters
The ``SubmitParams`` class is deprecated - you should use
:mod:`fsl.wrappers.fsl_sub` instead, or use the ``fsl_sub`` Python
library, which is installed as part of FSL.
Any command line script can be submitted by the parameters by calling the `SubmitParams` object: Any command line script can be submitted by the parameters by calling the `SubmitParams` object:
...@@ -152,6 +158,8 @@ class SubmitParams(object): ...@@ -152,6 +158,8 @@ class SubmitParams(object):
def __str__(self): def __str__(self):
return 'SubmitParams({})'.format(" ".join(self.as_flags())) return 'SubmitParams({})'.format(" ".join(self.as_flags()))
@deprecated.deprecated('3.7.0', '4.0.0',
'Use fsl.wrappers.fsl_sub instead')
def __call__(self, *command, **kwargs): def __call__(self, *command, **kwargs):
""" """
Submits the command to the cluster. Submits the command to the cluster.
...@@ -254,6 +262,10 @@ def submit(*command, **kwargs): ...@@ -254,6 +262,10 @@ def submit(*command, **kwargs):
""" """
Submits a given command to the cluster Submits a given command to the cluster
The ``submit`` function is deprecated - you should use
:mod:`fsl.wrappers.fsl_sub` instead, or use the ``fsl_sub`` Python
library, which is available in FSL 6.0.5 and newer.
You can pass the command and arguments as a single string, or as a regular or unpacked sequence. You can pass the command and arguments as a single string, or as a regular or unpacked sequence.
:arg command: string or regular/unpacked sequence of strings with the job command :arg command: string or regular/unpacked sequence of strings with the job command
...@@ -288,9 +300,14 @@ def submit(*command, **kwargs): ...@@ -288,9 +300,14 @@ def submit(*command, **kwargs):
return SubmitParams(**kwargs)(*command) return SubmitParams(**kwargs)(*command)
@deprecated.deprecated('3.7.0', '4.0.0', 'Use fsl_sub.report instead')
def info(job_ids) -> Dict[str, Optional[Dict[str, str]]]: def info(job_ids) -> Dict[str, Optional[Dict[str, str]]]:
"""Gets information on a given job id """Gets information on a given job id
The ``info`` function is deprecated - you should use the
``fsl_sub.report`` function from the ``fsl_sub`` Python library, which
is available in FSL 6.0.5 and newer.
Uses `qstat -j <job_ids>` Uses `qstat -j <job_ids>`
:arg job_ids: string with job id or (nested) sequence with jobs :arg job_ids: string with job id or (nested) sequence with jobs
......
...@@ -31,10 +31,10 @@ import os.path as op ...@@ -31,10 +31,10 @@ import os.path as op
import os import os
from fsl.utils.platform import platform as fslplatform from fsl.utils.platform import platform as fslplatform
import fsl.utils.fslsub as fslsub
import fsl.utils.tempdir as tempdir import fsl.utils.tempdir as tempdir
import fsl.utils.path as fslpath import fsl.utils.path as fslpath
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
...@@ -52,17 +52,16 @@ class FSLNotPresent(Exception): ...@@ -52,17 +52,16 @@ class FSLNotPresent(Exception):
"""Error raised by the :func:`runfsl` function when ``$FSLDIR`` cannot """Error raised by the :func:`runfsl` function when ``$FSLDIR`` cannot
be found. be found.
""" """
pass
@contextlib.contextmanager @contextlib.contextmanager
def dryrun(*args): def dryrun(*_):
"""Context manager which causes all calls to :func:`run` to be logged but """Context manager which causes all calls to :func:`run` to be logged but
not executed. See the :data:`DRY_RUN` flag. not executed. See the :data:`DRY_RUN` flag.
The returned standard output will be equal to ``' '.join(args)``. The returned standard output will be equal to ``' '.join(args)``.
""" """
global DRY_RUN global DRY_RUN # pylint: disable=global-statement
oldval = DRY_RUN oldval = DRY_RUN
DRY_RUN = True DRY_RUN = True
...@@ -146,19 +145,21 @@ def run(*args, **kwargs): ...@@ -146,19 +145,21 @@ def run(*args, **kwargs):
:arg submit: Must be passed as a keyword argument. Defaults to ``None``. :arg submit: Must be passed as a keyword argument. Defaults to ``None``.
If ``True``, the command is submitted as a cluster job via If ``True``, the command is submitted as a cluster job via
the :func:`.fslsub.submit` function. May also be a the :mod:`fsl.wrappers.fsl_sub` function. May also be a
dictionary containing arguments to that function. dictionary containing arguments to that function.
:arg cmdonly: Defaults to ``False``. If ``True``, the command is not :arg cmdonly: Defaults to ``False``. If ``True``, the command is not
executed, but rather is returned directly, as a list of executed, but rather is returned directly, as a list of
arguments. arguments.
:arg log: Must be passed as a keyword argument. An optional ``dict`` :arg log: Must be passed as a keyword argument. Defaults to
which may be used to redirect the command's standard output ``{'tee' : True}``. An optional ``dict`` which may be used
and error. The following keys are recognised: to redirect the command's standard output and error. The
following keys are recognised:
- tee: If ``True``, the command's standard output/error - tee: If ``True`` (the default), the command's
streams are forwarded to this processes streams. standard output/error streams are forwarded to
this processes streams.
- stdout: Optional file-like object to which the command's - stdout: Optional file-like object to which the command's
standard output stream can be forwarded. standard output stream can be forwarded.
...@@ -173,10 +174,10 @@ def run(*args, **kwargs): ...@@ -173,10 +174,10 @@ def run(*args, **kwargs):
object (via :func:`_realrun`), unless ``submit=True``, in which case they object (via :func:`_realrun`), unless ``submit=True``, in which case they
are passed through to the :func:`.fslsub.submit` function. are passed through to the :func:`.fslsub.submit` function.
:returns: If ``submit`` is provided, the return value of :returns: If ``submit`` is provided, the ID of the submitted job is
:func:`.fslsub` is returned. Otherwise returns a single returned as a string. Otherwise returns a single value or a
value or a tuple, based on the based on the ``stdout``, tuple, based on the based on the ``stdout``, ``stderr``, and
``stderr``, and ``exitcode`` arguments. ``exitcode`` arguments.
""" """
returnStdout = kwargs.pop('stdout', True) returnStdout = kwargs.pop('stdout', True)
...@@ -184,16 +185,16 @@ def run(*args, **kwargs): ...@@ -184,16 +185,16 @@ def run(*args, **kwargs):
returnExitcode = kwargs.pop('exitcode', False) returnExitcode = kwargs.pop('exitcode', False)
submit = kwargs.pop('submit', {}) submit = kwargs.pop('submit', {})
cmdonly = kwargs.pop('cmdonly', False) cmdonly = kwargs.pop('cmdonly', False)
log = kwargs.pop('log', None) logg = kwargs.pop('log', None)
args = prepareArgs(args) args = prepareArgs(args)
if log is None: if logg is None:
log = {} logg = {}
tee = log.get('tee', False) tee = logg.get('tee', True)
logStdout = log.get('stdout', None) logStdout = logg.get('stdout', None)
logStderr = log.get('stderr', None) logStderr = logg.get('stderr', None)
logCmd = log.get('cmd', None) logCmd = logg.get('cmd', None)
if not bool(submit): if not bool(submit):
submit = None submit = None
...@@ -217,9 +218,12 @@ def run(*args, **kwargs): ...@@ -217,9 +218,12 @@ def run(*args, **kwargs):
return _dryrun( return _dryrun(
submit, returnStdout, returnStderr, returnExitcode, *args) submit, returnStdout, returnStderr, returnExitcode, *args)
# submit - delegate to fslsub # submit - delegate to fsl_sub. This will induce a nested
# call back to this run function, which is a bit confusing,
# but harmless, as we've popped the "submit" arg above.
if submit is not None: if submit is not None:
return fslsub.submit(' '.join(args), **submit, **kwargs) from fsl.wrappers import fsl_sub # pylint: disable=import-outside-toplevel # noqa: E501
return fsl_sub(*args, **submit, **kwargs)[0].strip()
# Run directly - delegate to _realrun # Run directly - delegate to _realrun
stdout, stderr, exitcode = _realrun( stdout, stderr, exitcode = _realrun(
...@@ -391,25 +395,32 @@ def runfsl(*args, **kwargs): ...@@ -391,25 +395,32 @@ def runfsl(*args, **kwargs):
def wslcmd(cmdpath, *args): def wslcmd(cmdpath, *args):
"""Convert a command + arguments into an equivalent set of arguments that
will run the command under Windows Subsystem for Linux
:param cmdpath: Fully qualified path to the command. This is essentially
a WSL path not a Windows one since FSLDIR is specified
as a WSL path, however it may have backslashes as path
separators due to previous use of ``os.path.join``
:param args: Sequence of command arguments (the first of which is the
unqualified command name)
:return: If ``cmdpath`` exists and is executable in WSL, return a
sequence of command arguments which when executed will run the
command in WSL. Windows paths in the argument list will be
converted to WSL paths. If ``cmdpath`` was not executable in
WSL, returns None
""" """
Convert a command + arguments into an equivalent set of arguments that will run the command # Check if command exists in WSL (remembering
under Windows Subsystem for Linux # that the command path may include FSLDIR
# which is a Windows path)
:param cmdpath: Fully qualified path to the command. This is essentially a WSL path not a Windows
one since FSLDIR is specified as a WSL path, however it may have backslashes
as path separators due to previous use of ``os.path.join``
:param args: Sequence of command arguments (the first of which is the unqualified command name)
:return: If ``cmdpath`` exists and is executable in WSL, return a sequence of command arguments
which when executed will run the command in WSL. Windows paths in the argument list will
be converted to WSL paths. If ``cmdpath`` was not executable in WSL, returns None
"""
# Check if command exists in WSL (remembering that the command path may include FSLDIR which
# is a Windows path)
cmdpath = fslpath.wslpath(cmdpath) cmdpath = fslpath.wslpath(cmdpath)
_stdout, _stderr, retcode = _realrun(False, None, None, None, "wsl", "test", "-x", cmdpath) _stdout, _stderr, retcode = _realrun(
False, None, None, None, "wsl", "test", "-x", cmdpath)
if retcode == 0: if retcode == 0:
# Form a new argument list and convert any Windows paths in it into WSL paths # Form a new argument list and convert
# any Windows paths in it into WSL paths
wslargs = [fslpath.wslpath(arg) for arg in args] wslargs = [fslpath.wslpath(arg) for arg in args]
wslargs[0] = cmdpath wslargs[0] = cmdpath
local_fsldir = fslpath.wslpath(fslplatform.fsldir) local_fsldir = fslpath.wslpath(fslplatform.fsldir)
...@@ -417,8 +428,10 @@ def wslcmd(cmdpath, *args): ...@@ -417,8 +428,10 @@ def wslcmd(cmdpath, *args):
local_fsldevdir = fslpath.wslpath(fslplatform.fsldevdir) local_fsldevdir = fslpath.wslpath(fslplatform.fsldevdir)
else: else:
local_fsldevdir = None local_fsldevdir = None
# Prepend important environment variables - note that it seems we cannot # Prepend important environment variables -
# use WSLENV for this due to its insistance on path mapping. FIXME FSLDEVDIR? # note that it seems we cannot use WSLENV
# for this due to its insistance on path
# mapping. FIXME FSLDEVDIR?
local_path = "$PATH" local_path = "$PATH"
if local_fsldevdir: if local_fsldevdir:
local_path += ":%s/bin" % local_fsldevdir local_path += ":%s/bin" % local_fsldevdir
...@@ -438,13 +451,17 @@ def wslcmd(cmdpath, *args): ...@@ -438,13 +451,17 @@ def wslcmd(cmdpath, *args):
def hold(job_ids, hold_filename=None): def hold(job_ids, hold_filename=None):
""" """Waits until all jobs have finished
Waits until all jobs have finished
:param job_ids: possibly nested sequence of job ids. The job ids
themselves should be strings.
:param job_ids: possibly nested sequence of job ids. The job ids themselves should be strings. :param hold_filename: filename to use as a hold file. The
:param hold_filename: filename to use as a hold file. containing directory should exist, but the
The containing directory should exist, but the file itself should not. file itself should not. Defaults to a
Defaults to a ./.<random characters>.hold in the current directory. ./.<random characters>.hold in the current
:return: only returns when all the jobs have finished directory. :return: only returns when all
the jobs have finished
""" """
import fsl.utils.fslsub as fslsub # pylint: disable=import-outside-toplevel # noqa: E501
fslsub.hold(job_ids, hold_filename) fslsub.hold(job_ids, hold_filename)
#!/usr/bin/env python #!/usr/bin/env python
# #
# pylint: disable=unused-import
# flake8: noqa: F401
#
# __init__.py - Wrappers for FSL command-line tools. # __init__.py - Wrappers for FSL command-line tools.
# #
# Author: Paul McCarthy <pauldmccarthy@gmail.com> # Author: Paul McCarthy <pauldmccarthy@gmail.com>
...@@ -79,36 +82,37 @@ decorators. ...@@ -79,36 +82,37 @@ decorators.
""" """
from .wrapperutils import (LOAD,) # noqa from fsl.wrappers.wrapperutils import (LOAD,)
from .bet import (bet, # noqa from fsl.wrappers.bet import (bet,
robustfov) robustfov)
from .eddy import (eddy_cuda, # noqa from fsl.wrappers.eddy import (eddy_cuda,
topup, topup,
applytopup) applytopup)
from .fast import (fast,) # noqa from fsl.wrappers.fast import (fast,)
from .fsl_anat import (fsl_anat,) # noqa from fsl.wrappers.fsl_anat import (fsl_anat,)
from .flirt import (flirt, # noqa from fsl.wrappers.fsl_sub import (fsl_sub,)
from fsl.wrappers.flirt import (flirt,
invxfm, invxfm,
applyxfm, applyxfm,
applyxfm4D, applyxfm4D,
concatxfm, concatxfm,
mcflirt) mcflirt)
from .fnirt import (fnirt, # noqa from fsl.wrappers.fnirt import (fnirt,
applywarp, applywarp,
invwarp, invwarp,
convertwarp) convertwarp)
from .fslmaths import (fslmaths,) # noqa from fsl.wrappers.fslmaths import (fslmaths,)
from .fslstats import (fslstats,) # noqa from fsl.wrappers.fslstats import (fslstats,)
from .fugue import (fugue, # noqa from fsl.wrappers.fugue import (fugue,
prelude, prelude,
sigloss, sigloss,
fsl_prepare_fieldmap) fsl_prepare_fieldmap)
from .melodic import (melodic, # noqa from fsl.wrappers.melodic import (melodic,
fsl_regfilt) fsl_regfilt)
from .misc import (fslreorient2std, # noqa from fsl.wrappers.misc import (fslreorient2std,
fslroi, fslroi,
slicer, slicer,
cluster, cluster,
gps) gps)
from .epi_reg import epi_reg from fsl.wrappers.epi_reg import epi_reg
from . import tbss # noqa from fsl.wrappers import tbss
#!/usr/bin/env python
#
# fsl_sub.py - Wrapper for the fsl_sub command.
#
# Author: Paul McCarthy <pauldmccarthy@gmail.com>
#
"""This module provides the :func:`fsl_sub` function, a wrapper for the FSL
`fsl_sub <https://git.fmrib.ox.ac.uk/fsl/fsl_sub>`_ command.
"""
from . import wrapperutils as wutils
@wutils.fslwrapper
def fsl_sub(*args, **kwargs):
"""Wrapper for the ``fsl_sub`` command.
"""
cmd = ['fsl_sub']
cmd += wutils.applyArgStyle('--', singlechar_args=True, **kwargs)
cmd += list(args)
return cmd
...@@ -441,7 +441,7 @@ file should be loaded into memory and returned as a Python object. ...@@ -441,7 +441,7 @@ file should be loaded into memory and returned as a Python object.
""" """
class FileOrThing(object): class FileOrThing:
"""Decorator which ensures that certain arguments which are passed into the """Decorator which ensures that certain arguments which are passed into the
decorated function are always passed as file names. Both positional and decorated function are always passed as file names. Both positional and
keyword arguments can be specified. keyword arguments can be specified.
......
...@@ -69,7 +69,7 @@ def touch(fname): ...@@ -69,7 +69,7 @@ def touch(fname):
pass pass
class CaptureStdout(object): class CaptureStdout:
"""Context manager which captures stdout and stderr. """ """Context manager which captures stdout and stderr. """
def __init__(self): def __init__(self):
......
...@@ -119,12 +119,23 @@ def test_run_tee(): ...@@ -119,12 +119,23 @@ def test_run_tee():
capture = CaptureStdout() capture = CaptureStdout()
# default behaviour is for tee=True
with capture: with capture:
stdout = run.run('./script.sh 1 2 3', log={'tee' : True}) stdout = run.run('./script.sh 1 2 3')
assert stdout == expstdout
assert capture.stdout == expstdout
with capture.reset():
stdout = run.run('./script.sh 1 2 3', log={'tee' : True})
assert stdout == expstdout assert stdout == expstdout
assert capture.stdout == expstdout assert capture.stdout == expstdout
# disable forwarding
with capture.reset():
stdout = run.run('./script.sh 1 2 3', log={'tee' : False})
assert stdout == expstdout
assert capture.stdout == ''
with capture.reset(): with capture.reset():
stdout, stderr = run.run('./script.sh 1 2 3', stderr=True, stdout, stderr = run.run('./script.sh 1 2 3', stderr=True,
log={'tee' : True}) log={'tee' : True})
...@@ -268,9 +279,9 @@ def test_runfsl(): ...@@ -268,9 +279,9 @@ def test_runfsl():
run.FSL_PREFIX = None run.FSL_PREFIX = None
def mock_submit(cmd, **kwargs): def mock_fsl_sub(*cmd, **kwargs):
if isinstance(cmd, str): if len(cmd) == 1 and isinstance(cmd[0], str):
name = cmd.split()[0] name = cmd[0].split()[0]
else: else:
name = cmd[0] name = cmd[0]
...@@ -286,7 +297,7 @@ def mock_submit(cmd, **kwargs): ...@@ -286,7 +297,7 @@ def mock_submit(cmd, **kwargs):
for k in sorted(kwargs.keys()): for k in sorted(kwargs.keys()):
f.write('{}: {}\n'.format(k, kwargs[k])) f.write('{}: {}\n'.format(k, kwargs[k]))
return jid return (jid, '')
def test_run_submit(): def test_run_submit():
...@@ -304,7 +315,7 @@ def test_run_submit(): ...@@ -304,7 +315,7 @@ def test_run_submit():
with tempdir.tempdir(), \ with tempdir.tempdir(), \
mockFSLDIR(), \ mockFSLDIR(), \
mock.patch('fsl.utils.fslsub.submit', mock_submit):