wrapperutils.py 34.5 KB
Newer Older
1
2
#!/usr/bin/env python
#
Paul McCarthy's avatar
Paul McCarthy committed
3
4
# wrapperutils.py - Functions and decorators used by the FSL wrapper
# functions.
5
6
7
#
# Author: Paul McCarthy <pauldmccarthy@gmail.com>
#
Paul McCarthy's avatar
Paul McCarthy committed
8
9
10
11
"""This module contains functions and decorators used by the FSL wrapper
functions.


12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
The :func:`cmdwrapper` and :func:`fslwrapper` functions are conenience
decorators which allow you to write your wrapper function such that it simply
generates the command-line needed to respectively run a standard shell
command or a FSL command. For example::


    @fslwrapper
    def fslreorient2std(input, output):
        return ['fslreorient2std', input, output]


When this ``fslreorient2std`` function is called, the ``fslwrapper`` decorator
will take care of invoking the command in a standardised way.


Paul McCarthy's avatar
Paul McCarthy committed
27
The :func:`applyArgStyle` function can be used to automatically convert
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
keyword arguments into command-line arguments, based on a set of standard
patterns. For example::


    @fslwrapper
    def flirt(src, ref, **kwargs):
        cmd  = ['flirt', '-in', src, '-ref', ref]
        return cmd + applyArgStyle('-=', **kwargs)


The :func:`fileOrImage` and :func:`fileOrArray` functions can be used to
decorate a wrapper function such that in-memory ``nibabel`` images or Numpy
arrays can be passed in as arguments - they will be automatically saved out to
files, and then the file names passed into the wrapper function. For exmaple::


    @fileOrImage('src', 'ref')
    @fslwrapper
    def flirt(src, ref, **kwargs):
        cmd  = ['flirt', '-in', src, '-ref', ref]
        return cmd + applyArgStyle('-=', **kwargs)


Now this ``flirt`` function can be called either with file names, or
``nibabel`` images.


55
56
57
58
59
60
61
.. note:: Because the :func:`fileOrImage` and :func:`fileOrArray` decorators
          manipulate the return value of the decorated function, they should
          be applied *after* any other decorators. Furthermore, if you need to
          apply both a ``fileOrImage`` and ``fileOrArray`` decorator to a
          function, they should be grouped together, e.g.::

              @fileOrImage('a', 'b')
Paul McCarthy's avatar
Paul McCarthy committed
62
              @fileOrArray('c', 'd')
63
64
65
66
67
              @fslwrapper
              def func(**kwargs):
                  ...


68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
Command outputs can also be loaded back into memory by using the special
:data:`LOAD` value when calling a wrapper function. For example::


    @fileOrImage('src', 'ref', 'out')
    @fslwrapper
    def flirt(src, ref, **kwargs):
        cmd  = ['flirt', '-in', src, '-ref', ref]
        return cmd + applyArgStyle('-=', **kwargs)


If we set the ``out`` argument to ``LOAD``, the output image will be loaded
and returned::

    src     = nib.load('src.nii')
    ref     = nib.load('ref.nii')
    aligned = flirt(src, ref, out=LOAD)['out']
Paul McCarthy's avatar
Paul McCarthy committed
85
"""
86
87


88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
import itertools as it
import os.path   as op
import              os
import              re
import              sys
import              glob
import              random
import              string
import              fnmatch
import              inspect
import              logging
import              tempfile
import              warnings
import              functools
import              collections
103
104
105

import            six
import nibabel as nib
Paul McCarthy's avatar
Paul McCarthy committed
106
import numpy   as np
107

108
import fsl.utils.run     as run
109
110
import fsl.utils.path    as fslpath
import fsl.utils.tempdir as tempdir
111
112
113
import fsl.data.image    as fslimage


114
115
116
log = logging.getLogger(__name__)


Paul McCarthy's avatar
Paul McCarthy committed
117
118
119
120
121
def _update_wrapper(wrapper, wrapped, *args, **kwargs):
    """Replacement for the built-in ``functools.update_wrapper``. This
    implementation ensures that the wrapper function has an attribute
    called ``__wrapped__``, which refers to the ``wrapped`` function.

122
    This custom function is only needed in Python versions < 3.4.
Paul McCarthy's avatar
Paul McCarthy committed
123
124
125
126
127
    """

    wrapper = functools.update_wrapper(wrapper, wrapped, *args, **kwargs)

    # Python >= 3.4 does things right
Paul McCarthy's avatar
Paul McCarthy committed
128
    if (sys.version_info[0] * 10 + sys.version_info[1]) < 34:
Paul McCarthy's avatar
Paul McCarthy committed
129
130
        wrapper.__wrapped__ = wrapped
    return wrapper
131
132


Paul McCarthy's avatar
Paul McCarthy committed
133
134
135
136
137
138
def _unwrap(func):
    """Replacement for the built-in ``inspect.unwrap`` function, which
    is not present in Python versions prior to 3.4.
    """

    # Python >= 3.4 has an inspect.unwrap function
Paul McCarthy's avatar
Paul McCarthy committed
139
    if (sys.version_info[0] * 10 + sys.version_info[1]) >= 34:
Paul McCarthy's avatar
Paul McCarthy committed
140
141
142
143
144
145
146
147
148
        return inspect.unwrap(func)

    # Otherwise we follow the __wrapped__ chain ourselves
    if hasattr(func, '__wrapped__'):
        return _unwrap(func.__wrapped__)

    return func


149
150
151
152
153
154
def cmdwrapper(func):
    """This decorator can be used on functions which generate a command line.
    It will pass the return value of the function to the
    :func:`fsl.utils.run.run` function in a standardised manner.
    """
    def wrapper(*args, **kwargs):
155
        stdout = kwargs.pop('stdout', True)
156
        stderr = kwargs.pop('stderr', True)
157
        exitcode = kwargs.pop('exitcode', False)
158
        submit = kwargs.pop('submit', None)
159
160
        log    = kwargs.pop('log',    {'tee' : True})
        cmd    = func(*args, **kwargs)
161
        return run.run(cmd, stderr=stderr, log=log, submit=submit, stdout=stdout, exitcode=exitcode)
162
163
164
165
166
167
168
169
170
    return _update_wrapper(wrapper, func)


def fslwrapper(func):
    """This decorator can be used on functions which generate a FSL command
    line. It will pass the return value of the function to the
    :func:`fsl.utils.run.runfsl` function in a standardised manner.
    """
    def wrapper(*args, **kwargs):
171
        stdout = kwargs.pop('stdout', True)
172
        stderr = kwargs.pop('stderr', True)
173
        exitcode = kwargs.pop('exitcode', False)
174
        submit = kwargs.pop('submit', None)
175
176
        log    = kwargs.pop('log',    {'tee' : True})
        cmd    = func(*args, **kwargs)
177
        return run.runfsl(cmd, stderr=stderr, log=log, submit=submit, stdout=stdout, exitcode=exitcode)
178
179
180
    return _update_wrapper(wrapper, func)


Paul McCarthy's avatar
Paul McCarthy committed
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
SHOW_IF_TRUE = object()
"""Constant to be used in the ``valmap`` passed to the :func:`applyArgStyle`
function.

When a ``SHOW_IF_TRUE`` argument is ``True``, it is added to the generated
command line arguments.
"""


HIDE_IF_TRUE = object()
"""Constant to be used in the ``valmap`` passed to the :func:`applyArgStyle`
function.

When a ``HIDE_IF_TRUE`` argument is ``True``, it is suppressed from the
generated command line arguments.
"""
197
198


199
def applyArgStyle(style, valsep=None, argmap=None, valmap=None, singlechar_args=False, **kwargs):
Paul McCarthy's avatar
Paul McCarthy committed
200
201
202
203
    """Turns the given ``kwargs`` into command line options. This function
    is intended to be used to automatically generate command line options
    from arguments passed into a Python function.

204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
    The ``style`` and ``valsep`` arguments control how key-value pairs
    are converted into command-line options:


    =========  ==========  ===========================
    ``style``  ``valsep``  Result
    =========  ==========  ===========================
    ``'-'``    ' '         ``-name val1 val2 val3``
    ``'-'``    '"'         ``-name "val1 val2 val3"``
    ``'-'``    ','         ``-name val1,val2,val3``
    ``'--'``   ' '         ``--name val1 val2 val3``
    ``'--'``   '"'         ``--name "val1 val2 val3"``
    ``'--'``   ','         ``--name val1,val2,val3``
    ``'-='``   ' '         Not supported
    ``'-='``   '"'         ``-name="val1 val2 val3"``
    ``'-='``   ','         ``-name=val1,val2,val3``
    ``'--='``  ' '         Not supported
    ``'--='``  '"'         ``--name="val1 val2 val3"``
    ``'--='``  ','         ``--name=val1,val2,val3``
    =========  ==========  ===========================


Paul McCarthy's avatar
Paul McCarthy committed
226
    :arg style:  Controls how the ``kwargs`` are converted into command-line
227
228
229
230
231
                 options - must be one of ``'-'``, ``'--'``, ``'-='``, or
                 ``'--='``.

    :arg valsep: Controls how the values passed to command-line options
                 which expect multiple arguments are delimited - must be
232
233
                 one of ``' '``, ``','`` or ``'"'``. Defaults to ``' '``
                 if ``'=' not in style``, ``','`` otherwise.
Paul McCarthy's avatar
Paul McCarthy committed
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256

    :arg argmap: Dictionary of ``{kwarg-name : cli-name}`` mappings. This can
                 be used if you want to use different argument names in your
                 Python function for the command-line options.

    :arg valmap: Dictionary of ``{cli-name : value}`` mappings. This can be
                 used to define specific semantics for some command-line
                 options. Acceptable values for ``value`` are as follows

                  - :data:`SHOW_IF_TRUE` - if the argument is present, and
                    ``True`` in ``kwargs``, the command line option
                    will be added (without any arguments).

                  - :data:`HIDE_IF_TRUE` - if the argument is present, and
                    ``False`` in ``kwargs``, the command line option
                    will be added (without any arguments).

                  - Any other constant value. If the argument is present
                    in ``kwargs``, its command-line option will be
                    added, with the constant value as its argument.

                 The argument for any options not specified in the ``valmap``
                 will be converted into strings.
257

258
259
260
    :arg singlechar_args: If True, single character arguments always take a single
                          hyphen prefix (e.g. -h) regardless of the style

Paul McCarthy's avatar
Paul McCarthy committed
261
262
263
264
265
    :arg kwargs: Arguments to be converted into command-line options.

    :returns:    A list containing the generated command-line options.
    """

266
267
268
269
    if valsep is None:
        if '=' in style: valsep = ','
        else:            valsep = ' '

Paul McCarthy's avatar
Paul McCarthy committed
270
271
    if style not in ('-', '--', '-=', '--='):
        raise ValueError('Invalid style: {}'.format(style))
272
273
274
275
276
277
278
279
280
    if valsep not in (' ', ',', '"'):
        raise ValueError('Invalid valsep: {}'.format(valsep))

    # we don't handle the case where '=' in
    # style, and valsep == ' ', because no
    # sane CLI app would do this. Right?
    if '=' in style and valsep == ' ':
        raise ValueError('Incompatible style and valsep: s={} v={}'.format(
            style, valsep))
Paul McCarthy's avatar
Paul McCarthy committed
281
282
283
284
285

    if argmap is None: argmap = {}
    if valmap is None: valmap = {}

    def fmtarg(arg):
286
        if   style in ('-',  '-=') or (singlechar_args and len(arg) == 1):  arg =  '-{}'.format(arg)
287
288
289
        elif style in ('--', '--='): arg = '--{}'.format(arg)
        return arg

290
    # always returns a sequence
Paul McCarthy's avatar
Paul McCarthy committed
291
    def fmtval(val):
292
293
        if     isinstance(val, collections.Sequence) and \
           not isinstance(val, six.string_types):
294
295
296
297
298

            val = [str(v) for v in val]
            if   valsep == ' ': return val
            elif valsep == '"': return [' '   .join(val)]
            else:               return [valsep.join(val)]
299
        else:
300
301
302
303
304
305
306
307
            return [str(val)]

    # val is assumed to be a sequence
    def fmtargval(arg, val):
        # if '=' in style, val will
        # always be a single string
        if '=' in style: return ['{}={}'.format(arg, val[0])]
        else:            return [arg] + val
308
309
310
311
312

    args = []

    for k, v in kwargs.items():

313
314
        if v is None: continue

315
        k    = argmap.get(k, k)
Paul McCarthy's avatar
Paul McCarthy committed
316
317
318
        mapv = valmap.get(k, fmtval(v))
        k    = fmtarg(k)

319
320
321
322
        if mapv in (SHOW_IF_TRUE, HIDE_IF_TRUE):
            if (mapv is SHOW_IF_TRUE and     v) or \
               (mapv is HIDE_IF_TRUE and not v):
                args.append(k)
323
        else:
324
            args.extend(fmtargval(k, mapv))
325
326
327
328

    return args


329
def namedPositionals(func, args):
330
    """Given a function, and a sequence of positional arguments destined
331
    for that function, identifies the name for each positional argument.
332
    Variable positional arguments are given an automatic name.
333

334
335
    :arg func: Function which will accept ``args`` as positionals.
    :arg args: Tuple of positional arguments to be passed to ``func``.
336
    """
Paul McCarthy's avatar
Paul McCarthy committed
337

338
339
340
341
342
343
344
345
346
347
    # Current implementation will
    # result in naming collisions
    # for something like this:
    #
    # def func(args0, *args):
    #     ...
    # because of automatic vararg
    # naming. But who would write
    # a function like that anyway?

348
349
    # Remove any decorators
    # from the function
Paul McCarthy's avatar
Paul McCarthy committed
350
351
    func = _unwrap(func)

352
353
354
    # getargspec is the only way to
    # get the names of positional
    # arguments in Python 2.x.
Paul McCarthy's avatar
Paul McCarthy committed
355
    if sys.version_info[0] < 3:
356
357
358
        spec     = inspect.getargspec(func)
        argnames = spec.args
        varargs  = spec.varargs
Paul McCarthy's avatar
Paul McCarthy committed
359

360
361
    # But getargspec is deprecated
    # in python 3.x
Paul McCarthy's avatar
Paul McCarthy committed
362
363
364
365
366
367
    else:

        # getfullargspec is deprecated in
        # python 3.5, but not in python 3.6.
        with warnings.catch_warnings():
            warnings.filterwarnings('ignore', category=DeprecationWarning)
368
369
370
            spec     = inspect.getfullargspec(func)
            argnames = spec.args
            varargs  = spec.varargs
371

372
373
374
    # we only care about the arguments
    # that are being passed in
    argnames = argnames[:len(args)]
375

376
377
378
379
    # make up names for varargs
    nvarargs = len(args) - len(argnames)
    if varargs is not None and nvarargs > 0:
        argnames += ['{}{}'.format(varargs, i) for i in range(nvarargs)]
380

381
    return argnames
382
383


Paul McCarthy's avatar
Paul McCarthy committed
384
LOAD = object()
Paul McCarthy's avatar
Paul McCarthy committed
385
386
"""Constant used by the :class:`_FileOrThing` class to indicate that an output
file should be loaded into memory and returned as a Python object.
387
388
389
"""


Paul McCarthy's avatar
Paul McCarthy committed
390
391
392
393
394
395
396
397
398
399
class _FileOrThing(object):
    """Decorator which ensures that certain arguments which are passed into the
    decorated function are always passed as file names. Both positional and
    keyword arguments can be specified.


    The ``_FileOrThing`` class is not intended to be used directly - see the
    :func:`fileOrImage` and :func:`fileOrArray` decorator functions for more
    details.

400

Paul McCarthy's avatar
Paul McCarthy committed
401
402
403
404
405
406
407
408
409
410
411
412
    These decorators are intended for functions which wrap a command-line tool,
    i.e. where some inputs/outputs need to be specified as file names.


    **Inputs**


    Any arguments which are not of type ``Thing`` are passed through to the
    decorated function unmodified.  Arguments which are of type ``Thing`` are
    saved to a temporary file, and the name of that file is passed to the
    function.

413

Paul McCarthy's avatar
Paul McCarthy committed
414
    **Outputs**
415
416


Paul McCarthy's avatar
Paul McCarthy committed
417
    If an argument is given the special :data:`LOAD` value, it is assumed
Paul McCarthy's avatar
Paul McCarthy committed
418
419
420
    to be an output argument. In this case, it is replaced with a temporary
    file name then, after the function has completed, that file is loaded
    into memory, and the value returned (along with the function's output,
Paul McCarthy's avatar
Paul McCarthy committed
421
    and any other arguments with a value of ``LOAD``).
Paul McCarthy's avatar
Paul McCarthy committed
422
423
424
425
426
427


    **Return value**


    Functions decorated with a ``_FileOrThing`` decorator will always return a
428
429
430
431
432
    ``dict``-like object, where the function's actual return value is
    accessible via an attribute called `output`. All output arguments with a
    value of ``LOAD`` will be present as dictionary entries, with the keyword
    argument names used as keys. Any ``LOAD``ed output arguments which were not
    generated by the function will not be present in the dictionary.
Paul McCarthy's avatar
Paul McCarthy committed
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456


    **Example**


    As an example of using the ``fileOrArray`` decorator on a function
    which concatenates two files containing affine transformations, and
    saves the output to a file::

        # if atob, btoc, or output are passed
        # in as arrays, they are converted to
        # file names.
        @fileOrArray('atob', 'btoc', 'output')
        def concat(atob, btoc, output=None):

            # inputs are guaranteed to be files
            atob = np.loadtxt(atob)
            btoc = np.loadtxt(atoc)

            atoc = np.dot(btoc, atob)

            if output is not None:
                np.savetxt(output, atoc)

457
458
            return 'Done'

Paul McCarthy's avatar
Paul McCarthy committed
459
460
461
462

    Because we have decorated the ``concat`` function with :func:`fileToArray`,
    it can be called with either file names, or Numpy arrays::

463

Paul McCarthy's avatar
Paul McCarthy committed
464
465
        # All arguments are passed through
        # unmodified - the output will be
466
        # saved to a file called atoc.mat.
Paul McCarthy's avatar
Paul McCarthy committed
467
468
        concat('atob.txt', 'btoc.txt', 'atoc.mat')

469
470
471
472
473
474
475
476
477
        # The function's return value
        # is accessed via an attribute called
        # "output" on the dict
        assert concat('atob.txt', 'btoc.txt', 'atoc.mat').output == 'Done'

        # Outputs to be loaded into memory
        # are returned in a dictionary,
        # with argument names as keys.
        atoc = concat('atob.txt', 'btoc.txt', LOAD)['atoc']
Paul McCarthy's avatar
Paul McCarthy committed
478

479
480
481
482
483
484
485
486
487
        # In-memory inputs are saved to
        # temporary files, and those file
        # names are passed to the concat
        # function.
        atoc = concat(np.diag([2, 2, 2, 0]),
                      np.diag([3, 3, 3, 3]), LOAD)['atoc']


    **Using with other decorators**
Paul McCarthy's avatar
Paul McCarthy committed
488
489
490
491
492
493
494
495
496


    ``_FileOrThing`` decorators can be chained with other ``_FileOrThing``
    decorators, and other decorators.  When multiple ``_FileOrThing``
    decorators are used on a single function, the outputs from each decorator
    are merged together into a single dict-like object.


    ``_FileOrThing`` decorators can be used with any other decorators
497
498
    **as long as** they do not manipulate the return value, and as long as
    the ``_FileOrThing`` decorators are adjacent to each other.
499
500
501
    """


502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
    class _Results(dict):
        """A custom ``dict`` type used to return outputs from a function
        decorated with ``_FileOrThing``. All outputs are stored as dictionary
        items, with the argument name as key, and the output object (the
        "thing") as value.

        The decorated function's actual return value is accessible via the
        :meth:`output` property.
        """
        def __init__(self, output):
            self.__output = output

        @property
        def output(self):
            """Access the return value of the decorated function. """
            return self.__output


520
521
522
523
524
525
    def __init__(self,
                 func,
                 prepIn,
                 prepOut,
                 load,
                 removeExt,
Paul McCarthy's avatar
Paul McCarthy committed
526
527
                 *args,
                 **kwargs):
Paul McCarthy's avatar
Paul McCarthy committed
528
529
        """Initialise a ``_FileOrThing`` decorator.

530
531
532
533
        :arg func:      The function to be decorated.

        :arg prepIn:    Function which returns a file name to be used in
                        place of an input argument.
534

535
536
        :arg prepOut:   Function which generates a file name to use for
                        arguments that were set to :data:`LOAD`.
Paul McCarthy's avatar
Paul McCarthy committed
537

538
539
540
        :arg load:      Function which is called to load items for arguments
                        that were set to :data:`LOAD`. Must accept a file path
                        as its sole argument.
541

542
543
544
        :arg removeExt: Function which can remove a file extension from a file
                        path.

Paul McCarthy's avatar
Paul McCarthy committed
545
546
547
548
549
550
551
552
553
554
        :arg outprefix: Must be passed as a keyword argument. The name of a
                        positional or keyword argument to the function, which
                        specifies an output file name prefix.  All other
                        arguments with names that begin with this prefix may
                        be interpreted as things to ``LOAD``.

        All other positional arguments are interpreted as the names of the
        arguments to the function which will be handled by this
        ``_FileOrThing`` decorator. If not provided, *all* arguments passed to
        the function will be handled.
555

556
557
558
559
560
561

        The ``prepIn`` and ``prepOut`` functions must accept the following
        positional arguments:

          - A directory in which all temporary input/output files should be
            stored
Paul McCarthy's avatar
Paul McCarthy committed
562

563
564
565
566
          - The name of the keyword argument to be processed

          - The argument value that was passed in
        """
567
568
569
570
        self.__func      = func
        self.__prepIn    = prepIn
        self.__prepOut   = prepOut
        self.__load      = load
571
        self.__removeExt = removeExt
Paul McCarthy's avatar
Paul McCarthy committed
572
573
        self.__things    = args
        self.__outprefix = kwargs.get('outprefix', None)
Paul McCarthy's avatar
Paul McCarthy committed
574
575


576
    def __call__(self, *args, **kwargs):
577
        """Function which calls ``func``, ensuring that any arguments of
Paul McCarthy's avatar
Paul McCarthy committed
578
        type ``Thing`` are saved to temporary files, and any arguments
Paul McCarthy's avatar
Paul McCarthy committed
579
        with the value :data:`LOAD` are loaded and returned.
580

581
        All other arguments are passed through to ``func``.
582
        """
Paul McCarthy's avatar
Paul McCarthy committed
583

584
585
        func     = self.__func
        argnames = namedPositionals(func, args)
586

587
588
589
590
591
592
593
594
595
596
        # If this _FileOrThing is being called
        # by another _FileOrThing don't create
        # another working directory. We do this
        # sneakily, by setting an attribute on
        # the wrapped function which stores the
        # current working directory.
        wrapped     = _unwrap(func)
        fot_workdir = getattr(wrapped, '_fot_workdir', None)
        parent      = fot_workdir is None

597
        # Create a tempdir to store any temporary
Paul McCarthy's avatar
Paul McCarthy committed
598
        # input/output things, but don't change
599
600
        # into it, as file paths passed to the
        # function may be relative.
601
        with tempdir.tempdir(changeto=False, override=fot_workdir) as td:
602

603
604
            log.debug('Redirecting LOADed outputs to %s', td)

605
606
            # Replace any things with file names.
            # Also get a list of LOAD outputs
607
            args = self.__prepareArgs(parent, td, argnames, args, kwargs)
608
            args, kwargs, outprefix, outfiles, prefixes = args
609

610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
            # The prefix/patterns may be
            # overridden by a parent FoT
            outprefix = getattr(wrapped, '_fot_outprefix', outprefix)
            prefixes  = getattr(wrapped, '_fot_prefixes',  prefixes)

            # if there are any other FileOrThings
            # in the decorator chain, get them to
            # use our working directory, and
            # prefixes, instead of creating their
            # own.
            if parent:
                setattr(wrapped, '_fot_workdir',   td)
                setattr(wrapped, '_fot_outprefix', outprefix)
                setattr(wrapped, '_fot_prefixes',  prefixes)

625
            # Call the function
626
627
628
629
630
631
632
633
634
635
636
            try:
                result = func(*args, **kwargs)

            finally:
                # if we're the top-level FileOrThing
                # decorator, remove the attributes we
                # added above.
                if parent:
                    delattr(wrapped, '_fot_workdir')
                    delattr(wrapped, '_fot_outprefix')
                    delattr(wrapped, '_fot_prefixes')
637

638
639
            return self.__generateResult(
                td, result, outprefix, outfiles, prefixes)
640
641


642
    def __prepareArgs(self, parent, workdir, argnames, args, kwargs):
643
644
645
646
647
        """Prepares all input and output arguments to be passed to the
        decorated function. Any arguments with a value of :data:`LOAD` are
        passed to the ``prepOut`` function specified at :meth:`__init__`.
        All other arguments are passed through the ``prepIn`` function.

648
649
650
        :arg parent:  ``True`` if this ``_FileOrThing`` is the first in a
                      chain of ``_FileOrThing`` decorators.

651
652
        :arg workdir: Directory in which all temporary files should be stored.

653
654
655
        :arg args:    Positional arguments to be passed to the decorated
                      function.

656
657
658
659
        :arg kwargs:  Keyword arguments to be passed to the decorated function.

        :returns:     A tuple containing:

660
661
662
                        - An updated copy of ``args``.

                        - An updated copy of ``kwargs``.
663

664
665
666
667
668
669
670
                        - The output file prefix that was actually passed in
                          (it is subsequently modified so that prefixed outputs
                          are redirected to a temporary location). All prefixed
                          outputs that are not ``LOAD``ed should be moved into
                          this directory. ``None`` if there is no output
                          prefix.

671
672
                        - A dictionary of ``{ name : filename }`` mappings,
                          for all arguments with a value of ``LOAD``.
673

674
675
                        - A dictionary   ``{ filepat : replstr }`` paths, for
                          all output-prefix arguments with a value of ``LOAD``.
676
677
        """

678
679
680
681
        # These containers keep track
        # of output files which are to
        # be loaded into memory
        outfiles      = dict()
682
        prefixedFiles = dict()
683

684
685
686
        allargs  = {k : v for k, v in zip(argnames, args)}
        allargs.update(kwargs)

687
688
689
690
        # Has an output prefix been specified?
        prefix     = allargs.get(self.__outprefix, None)
        realPrefix = None

691
692
693
694
695
696
697
        # Prefixed outputs are only
        # managed by the parent
        # _FileOrthing in a chain of
        # FoT decorators.
        if not parent:
            prefix = None

698
699
700
701
702
703
704
705
706
707
        # If so, replace it with a new output
        # prefix which will redirect all output
        # to the temp dir.
        #
        # Importantly, here we assume that the
        # underlying function (and hence the
        # underlying command-line tool) will
        # accept an output prefix which contains
        # a directory path.
        if prefix is not None:
708
709
710
711
712
713
714
715
716
717

            # If prefix is set to LOAD,
            # all generated output files
            # should be loaded - we use a
            # randomly generated prefix,
            # and add it to prefixedFiles,
            # so that every file which
            # starts with it will be
            # loaded.
            if prefix is LOAD:
718
719
720
                prefix                = random.sample(string.ascii_letters, 10)
                prefix                = ''.join(prefix)
                prefixedFiles[prefix] = self.__outprefix
721

722
            realPrefix                = prefix
723
724
725
            fakePrefix                = op.join(workdir, prefix)
            allargs[self.__outprefix] = fakePrefix

726
727
            log.debug('Replacing output prefix: %s -> %s',
                      realPrefix, fakePrefix)
728

729
730
731
732
            # If the prefix specifies a
            # directory, make sure it
            # exists (remember that we're
            # in a temporary directory)
733
734
735
            pdir = op.dirname(fakePrefix)
            if pdir != '' and not op.exists(pdir):
                os.makedirs(pdir)
736

737
738
        if len(self.__things) > 0: things = self.__things
        else:                      things = allargs.keys()
739

740
        for name, val in list(allargs.items()):
741

742
743
744
745
746
            # don't process the
            # outprefix argument
            if name == self.__outprefix:
                continue

747
            # is this argument referring
748
            # to a prefixed output?
749
750
            isprefixed = (prefix is not None and
                          name.startswith(prefix))
751

752
            if not (isprefixed or name in things):
753
754
                continue

755
756
757
758
            # Prefixed output files may only
            # be given a value of LOAD
            if isprefixed and val is not LOAD:
                raise ValueError('Cannot specify name of prefixed file - the '
759
760
                                 'name is defined by the output prefix: '
                                 '{}'.format(name))
761

762
            if val is LOAD:
763

764
765
766
767
768
769
770
                # this argument refers to an output
                # that is generated from the output
                # prefix argument, and doesn't map
                # directly to an argument of the
                # function. So we don't pass it
                # through.
                if isprefixed:
771
                    prefixedFiles[name] = name
772
773
774
775
776
                    allargs.pop(name)

                # regular output-file argument
                else:
                    outfile = self.__prepOut(workdir, name, val)
777
                    outfiles[name] = outfile
778
                    allargs[ name] = outfile
Paul McCarthy's avatar
Paul McCarthy committed
779

780
            # Assumed to be an input file
781
            else:
782
783
                # sequences may be
                # accepted for inputs
784
785
786
787
788
789
790
791
792
                if isinstance(val, (list, tuple)):
                    infile = list(val)
                    for i, v in enumerate(val):
                        v = self.__prepIn(workdir, name, v)
                        if v is not None:
                            infile[i] = v

                else:
                    infile = self.__prepIn(workdir, name, val)
Paul McCarthy's avatar
Paul McCarthy committed
793

794
                if infile is not None:
795
796
                    allargs[name] = infile

797
798
799
        if realPrefix is not None and len(prefixedFiles) == 0:
            allargs[self.__outprefix] = realPrefix

800
801
        args   = [allargs.pop(k) for k in argnames]
        kwargs = allargs
802

803
        return args, kwargs, realPrefix, outfiles, prefixedFiles
804
805


806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
    def __generateResult(
            self, workdir, result, outprefix, outfiles, prefixes):
        """Loads function outputs and returns a :class:`_Results` object.

        Called by :meth:`__call__` after the decorated function has been
        called. Figures out what files should be loaded, and loads them into
        a ``_Results`` object.

        :arg workdir:   Directory which contains the function outputs.
        :arg result:    Function return value.
        :arg outprefix: Original output prefix that was passed into the
                        function (or ``None`` if one wasn't passed)
        :arg outfiles:  Dictionary containing output files to be loaded (see
                        :meth:`__prepareArgs`).
        :arg prefixes:  Dictionary containing output-prefix patterns to be
                        loaded (see :meth:`__prepareArgs`).

        :returns:       A ``_Results`` object containing all loaded outputs.
        """

        # make a _Results object to store
        # the output. If we are decorating
        # another _FileOrThing, the
        # results will get merged together
        # into a single _Results dict.
        if not isinstance(result, _FileOrThing._Results):
            result = _FileOrThing._Results(result)

        # Load the LOADed outputs
        for oname, ofile in outfiles.items():

            log.debug('Loading output %s: %s', oname, ofile)

            if op.exists(ofile): oval = self.__load(ofile)
            else:                oval = None

            result[oname] = oval

        # No output prefix - we're done
        if outprefix is None or len(prefixes) == 0:
            return result

        # Load or move output-prefixed files.
        # Find all files with a name that
        # matches the prefix that was passed
        # in (recursing into matching sub-
        # directories too).
        allPrefixed = glob.glob(op.join(workdir, '{}*'.format(outprefix)))
        allPrefixed = [fslpath.allFiles(f) if op.isdir(f) else [f]
                       for f in allPrefixed]

        for prefixed in it.chain(*allPrefixed):
            fullpath = prefixed
            prefixed = op.relpath(prefixed, workdir)
            for prefPat, prefName in prefixes.items():
                if not fnmatch.fnmatch(prefixed, '{}*'.format(prefPat)):
                    continue

                log.debug('Loading prefixed output %s [%s]: %s',
                          prefPat, prefName, prefixed)

                # if the load function returns
                # None, this file is probably
                # not of the correct type.
                fval = self.__load(fullpath)
                if fval is not None:
872
                    noext = self.__removeExt(prefixed)
Paul McCarthy's avatar
Paul McCarthy committed
873
                    prefPat  = prefPat.replace('\\', '\\\\')
874
                    noext = re.sub('^' + prefPat, prefName, noext)
875
876
877
                    # If there is already an item in result with the
                    # name (stripped of prefix), then instead store
                    # the result with the full prefixed name
878
879
880
881
882
                    if noext not in result:
                        result[noext] = fval
                    else:
                        withext = re.sub('^' + prefPat, prefName, prefixed)
                        result[withext] = fval
883
884
885
886
887
                    break

        return result


888
def fileOrImage(*args, **kwargs):
Paul McCarthy's avatar
Paul McCarthy committed
889
890
    """Decorator which can be used to ensure that any NIfTI images are saved
    to file, and output images can be loaded and returned as ``nibabel``
891
    image objects or :class:`.Image` objects.
Paul McCarthy's avatar
Paul McCarthy committed
892
893
    """

894
895
896
897
898
899
    # keep track of the input argument
    # types on each call, so we know
    # whether to return a fsl.Image or
    # a nibabel image
    intypes = []

900
    def prepIn(workdir, name, val):
Paul McCarthy's avatar
Paul McCarthy committed
901

902
        infile = None
Paul McCarthy's avatar
Paul McCarthy committed
903

904
905
906
907
908
        if isinstance(val, fslimage.Image):
            intypes.append(fslimage.Image)

        elif isinstance(val, nib.nifti1.Nifti1Image):
            intypes.append(nib.nifti1.Nifti1Image)
909
910
911
912

        if isinstance(val, fslimage.Image):
            val = val.nibImage

Paul McCarthy's avatar
Paul McCarthy committed
913
        if isinstance(val, nib.nifti1.Nifti1Image):
914
            infile = val.get_filename()
Paul McCarthy's avatar
Paul McCarthy committed
915
916
917

            # in-memory image - we have
            # to save it out to a file
918
919
            if infile is None:
                hd, infile = tempfile.mkstemp(fslimage.defaultExt())
Paul McCarthy's avatar
Paul McCarthy committed
920
                os.close(hd)
921
                val.to_filename(infile)
Paul McCarthy's avatar
Paul McCarthy committed
922

923
        return infile
Paul McCarthy's avatar
Paul McCarthy committed
924

925
926
    def prepOut(workdir, name, val):
        return op.join(workdir, '{}.nii.gz'.format(name))
Paul McCarthy's avatar
Paul McCarthy committed
927

928
    def load(path):
929
930
931
932

        if not fslimage.looksLikeImage(path):
            return None

Paul McCarthy's avatar
Paul McCarthy committed
933
934
        # create an independent in-memory
        # copy of the image file
935
        img = nib.load(path, mmap=False)
936
937
938
939
940
941
942
943
944
945
946
947
948
949

        # if any arguments were fsl images,
        # that takes precedence.
        if fslimage.Image in intypes:
            return fslimage.Image(img.get_data(), header=img.header)
        # but if all inputs were file names,
        # nibabel takes precedence
        elif nib.nifti1.Nifti1Image in intypes or len(intypes) == 0:
            return nib.nifti1.Nifti1Image(img.get_data(), None, img.header)

        # this function should not be called
        # under any other circumstances
        else:
            raise RuntimeError('Cannot handle type: {}'.format(intypes))
Paul McCarthy's avatar
Paul McCarthy committed
950

951
    def decorator(func):
952
953
954
955
956
957
958
        fot = _FileOrThing(func,
                           prepIn,
                           prepOut,
                           load,
                           fslimage.removeExt,
                           *args,
                           **kwargs)
959
960

        def wrapper(*args, **kwargs):
961
962
963
            result = fot(*args, **kwargs)
            intypes[:] = []
            return result
964
965
966
967

        return _update_wrapper(wrapper, func)

    return decorator
Paul McCarthy's avatar
Paul McCarthy committed
968
969


970
def fileOrArray(*args, **kwargs):
Paul McCarthy's avatar
Paul McCarthy committed
971
972
973
974
    """Decorator which can be used to ensure that any Numpy arrays are saved
    to text files, and output files can be loaded and returned as Numpy arrays.
    """

975
    def prepIn(workdir, name, val):
Paul McCarthy's avatar
Paul McCarthy committed
976

977
        infile = None
Paul McCarthy's avatar
Paul McCarthy committed
978
979

        if isinstance(val, np.ndarray):
980
            hd, infile = tempfile.mkstemp('.txt')
Paul McCarthy's avatar
Paul McCarthy committed
981
            os.close(hd)
982
            np.savetxt(infile, val, fmt='%0.18f')
Paul McCarthy's avatar
Paul McCarthy committed
983

984
        return infile
Paul McCarthy's avatar
Paul McCarthy committed
985

986
987
    def prepOut(workdir, name, val):
        return op.join(workdir, '{}.txt'.format(name))
Paul McCarthy's avatar
Paul McCarthy committed
988

989
990
991
    def load(path):
        try:              return np.loadtxt(path)
        except Exception: return None
Paul McCarthy's avatar
Paul McCarthy committed
992

993
    def decorator(func):
994
995
996
997
998
999
1000
        fot = _FileOrThing(func,
                           prepIn,
                           prepOut,
                           load,
                           fslpath.removeExt,
                           *args,
                           **kwargs)
1001
1002
1003
1004
1005
1006
1007

        def wrapper(*args, **kwargs):
            return fot(*args, **kwargs)

        return _update_wrapper(wrapper, func)

    return decorator