Commit d4c36b80 authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

Merge branch 'rf/wrapper_updates' into 'master'

Rf/wrapper updates

See merge request fsl/fslpy!54
parents 0ee48df0 d9a43461
Pipeline #2071 passed with stages
in 13 minutes and 46 seconds
......@@ -2,6 +2,42 @@ This document contains the ``fslpy`` release history in reverse chronological
order.
1.9.0 (Under development)
-------------------------
Added
^^^^^
* New :meth:`.Image.data` property method, for easy access to image data
as a ``numpy`` array.
* New ``log`` option to the :func:`.run.run` function, allowing more
fine-grained control over sub-process output streams.
* New :meth:`.Platform.fsldevdir` property, allowing the ``$FSLDEVDIR``
environment variable to be queried/changed.
Changed
^^^^^^^
* :meth:`.Image.ndims` has been renamed to :meth:`.Image.ndim`, to align
more closely with ``numpy`` naming conventions.
* The ``err`` and ``ret`` parameters to the :func:`.run.run` function have
been renamed to ``stderr`` and ``exitcode`` respectively.
* The :func:`.runfsl` function will give priority to the ``$FSLDEVDIR``
environment variable if it is set.
Deprecated
^^^^^^^^^^
* :meth:`.Image.ndims`.
* The ``err`` and ``ret`` parameters to :func:`.run.run`.
1.8.1 (Friday May 11th 2018)
----------------------------
......
......@@ -491,7 +491,7 @@ class Nifti(notifier.Notifier, meta.Meta):
@property
def ndims(self):
def ndim(self):
"""Returns the number of dimensions in this image. This number may not
match the number of dimensions specified in the NIFTI header, as
trailing dimensions of length 1 are ignored. But it is guaranteed to be
......@@ -500,6 +500,15 @@ class Nifti(notifier.Notifier, meta.Meta):
return len(self.__shape)
@property
@deprecation.deprecated(deprecated_in='1.9.0',
removed_in='2.0.0',
details='Use ndim instead')
def ndims(self):
"""Deprecated - use :mod::meth:``ndim`` instead. """
return self.ndim()
@deprecation.deprecated(deprecated_in='1.1.0',
removed_in='2.0.0',
details='Use ndims instead')
......@@ -1006,6 +1015,17 @@ class Image(Nifti):
return self.__nibImage
@property
def data(self):
"""Returns the image data as a ``numpy`` array.
.. warning:: Calling this method will cause the entire image to be
loaded into memory.
"""
self.__imageWrapper.loadData()
return self[:]
@property
def saveState(self):
"""Returns ``True`` if this ``Image`` has been saved to disk, ``False``
......
......@@ -77,7 +77,7 @@ class MelodicImage(fslimage.Image):
dataImage = fslimage.Image(dataFile,
loadData=False,
calcRange=False)
if dataImage.ndims >= 4:
if dataImage.ndim >= 4:
self.__tr = dataImage.pixdim[3]
......
......@@ -88,7 +88,6 @@ def isWidgetAlive(widget):
import wx
if platform.wxFlavour == platform.WX_PHOENIX:
excType = RuntimeError
elif platform.wxFlavour == platform.WX_PYTHON:
......@@ -116,6 +115,7 @@ class Platform(notifier.Notifier):
os
frozen
fsldir
fsldevdir
haveGui
canHaveGui
inSSHSession
......@@ -146,8 +146,9 @@ class Platform(notifier.Notifier):
self.__glRenderer = None
self.__glIsSoftware = None
self.__fslVersion = None
self.__fsldir = None
self.fsldir = os.environ.get('FSLDIR', None)
# initialise fsldir - see fsldir.setter
self.fsldir = self.fsldir
# Determine if a display is available. We do
# this once at init (instead of on-demand in
......@@ -284,7 +285,13 @@ class Platform(notifier.Notifier):
any registered listeners are notified via the
:class:`.Notifier` interface.
"""
return self.__fsldir
return os.environ.get('FSLDIR', None)
@property
def fsldevdir(self):
"""The FSL development directory location. """
return os.environ.get('FSLDEVDIR', None)
@fsldir.setter
......@@ -301,9 +308,9 @@ class Platform(notifier.Notifier):
elif not op.exists(value): value = None
elif not op.isdir(value): value = None
self.__fsldir = value
if value is not None:
if value is None:
os.environ.pop('FSLDIR', None)
else:
os.environ['FSLDIR'] = value
# Set the FSL version field if we can
......@@ -316,6 +323,26 @@ class Platform(notifier.Notifier):
self.notify(value=value)
@fsldevdir.setter
def fsldevdir(self, value):
"""Changes the value of the :attr:`fsldevdir` property, and notifies
any registered listeners.
"""
if value is not None:
value = value.strip()
if value is None: pass
elif value == '': value = None
elif not op.exists(value): value = None
elif not op.isdir(value): value = None
if value is None:
os.environ.pop('FSLDEVDIR', None)
else:
os.environ['FSLDEVDIR'] = value
@property
def fslVersion(self):
"""Returns the FSL version as a string, e.g. ``'5.0.9'``. Returns
......
......@@ -16,7 +16,10 @@
"""
import sys
import logging
import warnings
import threading
import contextlib
import collections
import subprocess as sp
......@@ -26,6 +29,7 @@ import six
from fsl.utils.platform import platform as fslplatform
import fsl.utils.fslsub as fslsub
import fsl.utils.tempdir as tempdir
log = logging.getLogger(__name__)
......@@ -37,6 +41,10 @@ execute them.
"""
FSL_PREFIX = None
"""Global override for the FSL executable location used by :func:`runfsl`. """
class FSLNotPresent(Exception):
"""Error raised by the :func:`runfsl` function when ``$FSLDIR`` cannot
be found.
......@@ -80,6 +88,36 @@ def _prepareArgs(args):
return list(args)
real_stdout = sys.stdout
def _forwardStream(in_, *outs):
"""Creates and starts a daemon thread which forwards the given input stream
to one or more output streams. Used by the :func:`run` function to redirect
a command's standard output/error streams to more than one destination.
It is necessary to read the process stdout/ stderr on separate threads to
avoid deadlocks.
:arg in_: Input stream
:arg outs: Output stream(s)
:returns: The thread that has been started.
"""
# not all file-likes have a mode attribute -
# if not present, assume a string stream
omodes = [getattr(o, 'mode', 'w') for o in outs]
def realForward():
for line in in_:
for i, o in enumerate(outs):
if 'b' in omodes[i]: o.write(line)
else: o.write(line.decode('utf-8'))
t = threading.Thread(target=realForward)
t.daemon = True
t.start()
return t
def run(*args, **kwargs):
"""Call a command and return its output. You can pass the command and
arguments as a single string, or as a regular or unpacked sequence.
......@@ -90,39 +128,79 @@ def run(*args, **kwargs):
An exception is raised if the command returns a non-zero exit code, unless
the ``ret`` option is set to ``True``.
:arg submit: Must be passed as a keyword argument. Defaults to ``None``.
Accepted values are ``True`` or a
If ``True``, the command is submitted as a cluster job via
the :func:`.fslsub.submit` function. May also be a
dictionary containing arguments to that function.
:arg err: Must be passed as a keyword argument. Defaults to
``False``. If ``True``, standard error is captured and
returned. Ignored if ``submit`` is specified.
:arg ret: Must be passed as a keyword argument. Defaults to ``False``.
If ``True``, and the command's return code is non-0, an
exception is not raised. Ignored if ``submit`` is specified.
:returns: If ``submit`` is provided, the cluster job ID is returned.
Otherwise if ``err is False and ret is False`` (the default)
a string containing the command's standard output. is
returned. Or, if ``err is True`` and/or ``ret is True``, a
tuple containing the standard output, standard error (if
``err``), and return code (if ``ret``).
:arg stdout: Must be passed as a keyword argument. Defaults to ``True``.
If ``True``, standard output is captured and returned.
Ignored if ``submit`` is specified.
:arg stderr: Must be passed as a keyword argument. Defaults to ``False``.
If ``True``, standard error is captured and returned.
Ignored if ``submit`` is specified.
:arg exitcode: Must be passed as a keyword argument. Defaults to ``False``.
If ``True``, and the command's return code is non-0, an
exception is not raised. Ignored if ``submit`` is
specified.
:arg err: Deprecated - use ``stderr`` instead.
:arg ret: Deprecated - use ``exitcode`` instead.
:arg submit: Must be passed as a keyword argument. Defaults to ``None``.
If ``True``, the command is submitted as a cluster job via
the :func:`.fslsub.submit` function. May also be a
dictionary containing arguments to that function.
:arg log: Must be passed as a keyword argument. An optional ``dict``
which may be used to redirect the command's standard output
and error. The following keys are recognised:
- tee: If ``True``, the command's standard output/error
streams are forwarded to this processes streams.
- stdout: Optional file-like object to which the command's
standard output stream can be forwarded.
- stderr: Optional file-like object to which the command's
standard error stream can be forwarded.
- cmd: If ``True``, the command itself is logged to the
standard output stream(s).
:returns: If ``submit`` is provided, the return value of
:func:`.fslsub` is returned. Otherwise returns a single
value or a tuple, based on the based on the ``stdout``,
``stderr``, and ``exitcode`` arguments.
"""
err = kwargs.get('err', False)
ret = kwargs.get('ret', False)
submit = kwargs.get('submit', None)
args = _prepareArgs(args)
if 'err' in kwargs:
warnings.warn('err is deprecated and will be removed '
'in fslpy 2.0.0 - use stderr instead',
DeprecationWarning)
kwargs['stderr'] = kwargs.get('stderr', kwargs['err'])
if 'ret' in kwargs:
warnings.warn('ret is deprecated and will be removed '
'in fslpy 2.0.0 - use exitcode instead',
DeprecationWarning)
kwargs['exitcode'] = kwargs.get('exitcode', kwargs['ret'])
returnStdout = kwargs.get('stdout', True)
returnStderr = kwargs.get('stderr', False)
returnExitcode = kwargs.get('exitcode', False)
submit = kwargs.get('submit', {})
log = kwargs.get('log', {})
tee = log .get('tee', False)
logStdout = log .get('stdout', None)
logStderr = log .get('stderr', None)
logCmd = log .get('cmd', False)
args = _prepareArgs(args)
if not bool(submit):
submit = None
if submit is not None:
err = False
ret = False
returnStdout = False
returnStderr = False
returnExitcode = False
if submit is True:
submit = dict()
......@@ -132,54 +210,150 @@ def run(*args, **kwargs):
'options for fsl.utils.fslsub.submit')
if DRY_RUN:
log.debug('dryrun: {}'.format(' '.join(args)))
else:
log.debug('run: {}'.format(' '.join(args)))
return _dryrun(
submit, returnStdout, returnStderr, returnExitcode, *args)
if DRY_RUN:
stderr = ''
if submit is None:
stdout = ' '.join(args)
else:
stdout = '[submit] ' + ' '.join(args)
elif submit is not None:
# submit - delegate to fslsub
if submit is not None:
return fslsub.submit(' '.join(args), **submit)
else:
proc = sp.Popen(args, stdout=sp.PIPE, stderr=sp.PIPE)
stdout, stderr = proc.communicate()
retcode = proc.returncode
# Run directly - delegate to _realrun
stdout, stderr, exitcode = _realrun(
tee, logStdout, logStderr, logCmd, *args)
if not returnExitcode and (exitcode != 0):
raise RuntimeError('{} returned non-zero exit code: {}'.format(
args[0], exitcode))
stdout = stdout.decode('utf-8').strip()
stderr = stderr.decode('utf-8').strip()
results = []
if returnStdout: results.append(stdout)
if returnStderr: results.append(stderr)
if returnExitcode: results.append(exitcode)
log.debug('stdout: {}'.format(stdout))
log.debug('stderr: {}'.format(stderr))
if len(results) == 1: return results[0]
else: return tuple(results)
def _dryrun(submit, returnStdout, returnStderr, returnExitcode, *args):
"""Used by the :func:`run` function when the :attr:`DRY_RUN` flag is
active.
"""
if not ret and (retcode != 0):
raise RuntimeError('{} returned non-zero exit code: {}'.format(
args[0], retcode))
if submit:
return ('0',)
results = [stdout]
results = []
stderr = ''
stdout = ' '.join(args)
if err: results.append(stderr)
if ret: results.append(retcode)
if returnStdout: results.append(stdout)
if returnStderr: results.append(stderr)
if returnExitcode: results.append(0)
if len(results) == 1: return results[0]
else: return tuple(results)
def _realrun(tee, logStdout, logStderr, logCmd, *args):
"""Used by :func:`run`. Runs the given command and manages its standard
output and error streams.
:arg tee: If ``True``, the command's standard output and error
streams are forwarded to this process' standard output/
error.
:arg logStdout: Optional file-like object to which the command's standard
output stream can be forwarded.
:arg logStderr: Optional file-like object to which the command's standard
error stream can be forwarded.
:arg logCmd: If ``True``, the command itself is logged to the standard
output stream(s).
:arg args: Command to run
:returns: A tuple containing:
- the command's standard output as a string.
- the command's standard error as a string.
- the command's exit code.
"""
proc = sp.Popen(args, stdout=sp.PIPE, stderr=sp.PIPE)
with tempdir.tempdir(changeto=False) as td:
# We always direct the command's stdout/
# stderr to two temporary files
stdoutf = op.join(td, 'stdout')
stderrf = op.join(td, 'stderr')
with open(stdoutf, 'wb') as stdout, \
open(stderrf, 'wb') as stderr: # noqa
outstreams = [stdout]
errstreams = [stderr]
# if tee, we duplicate the command's
# stdout/stderr to this process'
# stdout/stderr
if tee:
outstreams.append(sys.stdout)
errstreams.append(sys.stderr)
# And we also duplicate to caller-
# provided streams if they're given.
if logStdout is not None: outstreams.append(logStdout)
if logStderr is not None: errstreams.append(logStderr)
# log the command to
# stdout if requested
if logCmd:
cmd = ' '.join(args) + '\n'
for o in outstreams:
if 'b' in getattr(o, 'mode', 'w'):
o.write(cmd.encode('utf-8'))
else:
o.write(cmd)
stdoutt = _forwardStream(proc.stdout, *outstreams)
stderrt = _forwardStream(proc.stderr, *errstreams)
# Wait until the forwarding threads
# have finished cleanly, and the
# command has terminated.
stdoutt.join()
stderrt.join()
proc.communicate()
# Read in the command's stdout/stderr
with open(stdoutf, 'rb') as f: stdout = f.read()
with open(stderrf, 'rb') as f: stderr = f.read()
exitcode = proc.returncode
stdout = stdout.decode('utf-8')
stderr = stderr.decode('utf-8')
return stdout, stderr, exitcode
def runfsl(*args, **kwargs):
"""Call a FSL command and return its output. This function simply prepends
``$FSLDIR/bin/`` to the command before passing it to :func:`run`.
"""
if fslplatform.fsldir is None:
prefix = None
if FSL_PREFIX is not None:
prefix = FSL_PREFIX
elif fslplatform.fsldevdir is not None:
prefix = op.join(fslplatform.fsldevdir, 'bin')
elif fslplatform.fsldir is not None:
prefix = op.join(fslplatform.fsldir, 'bin')
else:
raise FSLNotPresent('$FSLDIR is not set - FSL cannot be found!')
args = _prepareArgs(args)
args[0] = op.join(fslplatform.fsldir, 'bin', args[0])
args[0] = op.join(prefix, args[0])
return run(*args, **kwargs)
......
......@@ -39,8 +39,19 @@ def bet(input, output, **kwargs):
valmap = {
'm' : wutils.SHOW_IF_TRUE,
'R' : wutils.SHOW_IF_TRUE,
'o' : wutils.SHOW_IF_TRUE,
's' : wutils.SHOW_IF_TRUE,
'n' : wutils.HIDE_IF_TRUE,
't' : wutils.SHOW_IF_TRUE,
'e' : wutils.SHOW_IF_TRUE,
'R' : wutils.SHOW_IF_TRUE,
'S' : wutils.SHOW_IF_TRUE,
'B' : wutils.SHOW_IF_TRUE,
'Z' : wutils.SHOW_IF_TRUE,
'F' : wutils.SHOW_IF_TRUE,
'A' : wutils.SHOW_IF_TRUE,
'v' : wutils.SHOW_IF_TRUE,
'd' : wutils.SHOW_IF_TRUE,
}
cmd = ['bet', input, output]
......
......@@ -142,8 +142,10 @@ def cmdwrapper(func):
"""
def wrapper(*args, **kwargs):
submit = kwargs.pop('submit', None)
cmd = func(*args, **kwargs)
return run.run(cmd, err=True, submit=submit)
stderr = kwargs.pop('stderr', True)
log = kwargs.pop('log', {'tee' : True})
cmd = func(*args, **kwargs)
return run.run(cmd, stderr=stderr, log=log, submit=submit)
return _update_wrapper(wrapper, func)
......@@ -154,8 +156,10 @@ def fslwrapper(func):
"""
def wrapper(*args, **kwargs):
submit = kwargs.pop('submit', None)
cmd = func(*args, **kwargs)
return run.runfsl(cmd, err=True, submit=submit)
stderr = kwargs.pop('stderr', True)
log = kwargs.pop('log', {'tee' : True})
cmd = func(*args, **kwargs)
return run.runfsl(cmd, stderr=stderr, log=log, submit=submit)
return _update_wrapper(wrapper, func)
......
......@@ -36,7 +36,8 @@ logging.getLogger().setLevel(logging.WARNING)
@contextlib.contextmanager
def mockFSLDIR():
oldval = fslplatform.fsldir
oldfsldir = fslplatform.fsldir
oldfsldevdir = fslplatform.fsldevdir
try:
with tempdir() as td:
......@@ -44,13 +45,15 @@ def mockFSLDIR():
bindir = op.join(fsldir, 'bin')
os.makedirs(bindir)
fslplatform.fsldir = fsldir
fslplatform.fsldevdir = None
path = op.pathsep.join((bindir, os.environ['PATH']))
with mock.patch.dict(os.environ, {'PATH': path}):
yield fsldir
finally:
fslplatform.fsldir = oldval
fslplatform.fsldir = oldfsldir
fslplatform.fsldevdir = oldfsldevdir
def touch(fname):
......@@ -67,6 +70,9 @@ class CaptureStdout(object):
def reset(self):
self.__mock_stdout = StringIO('')
self.__mock_stderr = StringIO('')
self.__mock_stdout.mode = 'w'
self.__mock_stderr.mode = 'w'
return self
def __enter__(self):
self.__real_stdout = sys.stdout
......
......@@ -285,11 +285,12 @@ def _test_Image_atts(imgtype):
i = fslimage.Image(path)
assert tuple(i.shape) == tuple(expdims)
assert tuple(i.data.shape) == tuple(expdims)
assert tuple(i.pixdim) == tuple(exppixdims)
assert tuple(i.nibImage.shape) == tuple(dims)
assert tuple(i.nibImage.header.get_zooms()) == tuple(pixdims)
assert i.ndims == expndims
assert i.ndim == expndims
assert i.dtype == dtype
assert i.name == op.basename(path)
assert i.dataSource == fslpath.addExt(path,
......@@ -859,8 +860,9 @@ def _test_Image_5D(imgtype):
img = fslimage.Image(path)