From 4026cbba710aeba0904b9d14c1bac9360f500f6f Mon Sep 17 00:00:00 2001
From: Paul McCarthy <pauldmccarthy@gmail.com>
Date: Mon, 16 Mar 2020 15:13:41 +0000
Subject: [PATCH] RF: FileOrThing passes-through if submit=True, and errors if
in-memory or LOAD is used in conjunction with submit=True
---
fsl/wrappers/wrapperutils.py | 64 ++++++++++++++++++++++++++++++------
1 file changed, 54 insertions(+), 10 deletions(-)
diff --git a/fsl/wrappers/wrapperutils.py b/fsl/wrappers/wrapperutils.py
index 81fb86826..41ed391e2 100644
--- a/fsl/wrappers/wrapperutils.py
+++ b/fsl/wrappers/wrapperutils.py
@@ -154,12 +154,12 @@ def cmdwrapper(func):
:func:`fsl.utils.run.run` function in a standardised manner.
"""
def wrapper(*args, **kwargs):
- stdout = kwargs.pop('stdout', True)
- stderr = kwargs.pop('stderr', True)
+ stdout = kwargs.pop('stdout', True)
+ stderr = kwargs.pop('stderr', True)
exitcode = kwargs.pop('exitcode', False)
- submit = kwargs.pop('submit', None)
- log = kwargs.pop('log', {'tee' : True})
- cmd = func(*args, **kwargs)
+ submit = kwargs.pop('submit', None)
+ log = kwargs.pop('log', {'tee' : True})
+ cmd = func(*args, **kwargs)
return run.run(cmd,
stderr=stderr,
log=log,
@@ -175,12 +175,12 @@ def fslwrapper(func):
:func:`fsl.utils.run.runfsl` function in a standardised manner.
"""
def wrapper(*args, **kwargs):
- stdout = kwargs.pop('stdout', True)
- stderr = kwargs.pop('stderr', True)
+ stdout = kwargs.pop('stdout', True)
+ stderr = kwargs.pop('stderr', True)
exitcode = kwargs.pop('exitcode', False)
- submit = kwargs.pop('submit', None)
- log = kwargs.pop('log', {'tee' : True})
- cmd = func(*args, **kwargs)
+ submit = kwargs.pop('submit', None)
+ log = kwargs.pop('log', {'tee' : True})
+ cmd = func(*args, **kwargs)
return run.runfsl(cmd,
stderr=stderr,
log=log,
@@ -452,6 +452,25 @@ class _FileOrThing(object):
generated by the function will not be present in the dictionary.
+ **Cluster submission**
+
+
+ The above description holds in all situations, except when an argument
+ called ``submit`` is passed, and is set to ``True``. In this case, the
+ ``_FileOrThing`` decorator will pass all arguments straight through to the
+ decorated function, and will return its return value unchanged.
+
+
+ This is because most functions that are decorated with the
+ :func:`fileOrImage` or :func:`fileOrArray` decorators will invoke a call
+ to :func:`.run` or :func:`.runfsl`, where a value of ``submit=True`` will
+ cause the command to be executed asynchronously on a cluster platform.
+
+
+ A :exc:`ValueError` will be raised if the decorated function is called
+ with ``submit=True``, and with any in-memory objects or ``LOAD`` symbols.
+
+
**Example**
@@ -528,9 +547,13 @@ class _FileOrThing(object):
The decorated function's actual return value is accessible via the
:meth:`output` property.
"""
+
+
def __init__(self, output):
+ super().__init__()
self.__output = output
+
@property
def output(self):
"""Access the return value of the decorated function. """
@@ -604,6 +627,27 @@ class _FileOrThing(object):
func = self.__func
argnames = namedPositionals(func, args)
+ # Special case - if fsl.utils.run[fsl] is
+ # being decorated (e.g. via cmdwrapper/
+ # fslwrapper), and submit=True, this call
+ # will ultimately submit the job to the
+ # cluster, and will return immediately.
+ #
+ # We error if we are given any in-memory
+ # things, or LOAD symbols.
+ #
+ # n.b. testing values to be strings could
+ # interfere with the fileOrText decorator.
+ # Possible solution is to use pathlib?
+ if kwargs.get('submit', False):
+ allargs = {**dict(zip(argnames, args)), **kwargs}
+ for name, val in allargs.items():
+ if (name in self.__things) and \
+ (not isinstance(val, six.string_types)):
+ raise ValueError('Cannot use in-memory objects '
+ 'or LOAD with submit=True!')
+ return func(*args, **kwargs)
+
# If this _FileOrThing is being called
# by another _FileOrThing don't create
# another working directory. We do this
--
GitLab