wrapperutils.py 41.2 KB
Newer Older
1
2
#!/usr/bin/env python
#
Paul McCarthy's avatar
Paul McCarthy committed
3
4
# wrapperutils.py - Functions and decorators used by the FSL wrapper
# functions.
5
6
#
# Author: Paul McCarthy <pauldmccarthy@gmail.com>
7
# Author: Martin Craig <martin.craig@eng.ox.ac.uk>
8
#
Paul McCarthy's avatar
Paul McCarthy committed
9
10
11
12
"""This module contains functions and decorators used by the FSL wrapper
functions.


Paul McCarthy's avatar
Paul McCarthy committed
13
The :func:`cmdwrapper` and :func:`fslwrapper` functions are convenience
14
15
16
17
18
19
20
21
22
23
24
25
26
27
decorators which allow you to write your wrapper function such that it simply
generates the command-line needed to respectively run a standard shell
command or a FSL command. For example::


    @fslwrapper
    def fslreorient2std(input, output):
        return ['fslreorient2std', input, output]


When this ``fslreorient2std`` function is called, the ``fslwrapper`` decorator
will take care of invoking the command in a standardised way.


Paul McCarthy's avatar
Paul McCarthy committed
28
The :func:`applyArgStyle` function can be used to automatically convert
29
30
31
32
33
34
35
36
37
38
39
40
41
keyword arguments into command-line arguments, based on a set of standard
patterns. For example::


    @fslwrapper
    def flirt(src, ref, **kwargs):
        cmd  = ['flirt', '-in', src, '-ref', ref]
        return cmd + applyArgStyle('-=', **kwargs)


The :func:`fileOrImage` and :func:`fileOrArray` functions can be used to
decorate a wrapper function such that in-memory ``nibabel`` images or Numpy
arrays can be passed in as arguments - they will be automatically saved out to
Paul McCarthy's avatar
Paul McCarthy committed
42
files, and then the file names passed into the wrapper function. For example::
43
44
45
46
47
48
49
50
51
52
53
54
55


    @fileOrImage('src', 'ref')
    @fslwrapper
    def flirt(src, ref, **kwargs):
        cmd  = ['flirt', '-in', src, '-ref', ref]
        return cmd + applyArgStyle('-=', **kwargs)


Now this ``flirt`` function can be called either with file names, or
``nibabel`` images.


56
57
58
59
60
61
62
.. note:: Because the :func:`fileOrImage` and :func:`fileOrArray` decorators
          manipulate the return value of the decorated function, they should
          be applied *after* any other decorators. Furthermore, if you need to
          apply both a ``fileOrImage`` and ``fileOrArray`` decorator to a
          function, they should be grouped together, e.g.::

              @fileOrImage('a', 'b')
Paul McCarthy's avatar
Paul McCarthy committed
63
              @fileOrArray('c', 'd')
64
65
66
67
68
              @fslwrapper
              def func(**kwargs):
                  ...


69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
Command outputs can also be loaded back into memory by using the special
:data:`LOAD` value when calling a wrapper function. For example::


    @fileOrImage('src', 'ref', 'out')
    @fslwrapper
    def flirt(src, ref, **kwargs):
        cmd  = ['flirt', '-in', src, '-ref', ref]
        return cmd + applyArgStyle('-=', **kwargs)


If we set the ``out`` argument to ``LOAD``, the output image will be loaded
and returned::

    src     = nib.load('src.nii')
    ref     = nib.load('ref.nii')
    aligned = flirt(src, ref, out=LOAD)['out']
Paul McCarthy's avatar
Paul McCarthy committed
86
"""
87
88


89
90
91
92
93
94
95
96
97
import itertools       as it
import os.path         as op
import collections.abc as abc
import                    os
import                    re
import                    sys
import                    glob
import                    random
import                    string
98
import                    pathlib
99
100
101
102
103
104
105
import                    fnmatch
import                    inspect
import                    logging
import                    tempfile
import                    warnings
import                    functools

106
import nibabel as nib
Paul McCarthy's avatar
Paul McCarthy committed
107
import numpy   as np
108

109
110
111
112
113
import fsl.utils.run        as run
import fsl.utils.assertions as asrt
import fsl.utils.path       as fslpath
import fsl.utils.tempdir    as tempdir
import fsl.data.image       as fslimage
114
115


116
117
118
log = logging.getLogger(__name__)


Paul McCarthy's avatar
Paul McCarthy committed
119
120
121
122
123
def _update_wrapper(wrapper, wrapped, *args, **kwargs):
    """Replacement for the built-in ``functools.update_wrapper``. This
    implementation ensures that the wrapper function has an attribute
    called ``__wrapped__``, which refers to the ``wrapped`` function.

124
    This custom function is only needed in Python versions < 3.4.
Paul McCarthy's avatar
Paul McCarthy committed
125
126
127
128
129
    """

    wrapper = functools.update_wrapper(wrapper, wrapped, *args, **kwargs)

    # Python >= 3.4 does things right
Paul McCarthy's avatar
Paul McCarthy committed
130
    if (sys.version_info[0] * 10 + sys.version_info[1]) < 34:
Paul McCarthy's avatar
Paul McCarthy committed
131
132
        wrapper.__wrapped__ = wrapped
    return wrapper
133
134


Paul McCarthy's avatar
Paul McCarthy committed
135
136
137
138
139
140
def _unwrap(func):
    """Replacement for the built-in ``inspect.unwrap`` function, which
    is not present in Python versions prior to 3.4.
    """

    # Python >= 3.4 has an inspect.unwrap function
Paul McCarthy's avatar
Paul McCarthy committed
141
    if (sys.version_info[0] * 10 + sys.version_info[1]) >= 34:
Paul McCarthy's avatar
Paul McCarthy committed
142
143
144
145
146
147
148
149
150
        return inspect.unwrap(func)

    # Otherwise we follow the __wrapped__ chain ourselves
    if hasattr(func, '__wrapped__'):
        return _unwrap(func.__wrapped__)

    return func


151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
def genxwrapper(func, runner):
    """This function is used by :func:`cmdwrapper` and :func:`fslwrapper`.
    It is not intended to be used in any other circumstances.

    This function generates a wrapper function which calls ``func`` to
    generate a command-line call, and then uses ``runner`` to invoke that
    command.

    ``func`` is assumed to be a wrapper function which generates a command-
    line. ``runner`` is assumed to be Either :func:`.run.run` or
    :func:`.run.runfsl`.

    The generated wrapper function will pass all of its arguments to ``func``,
    and will then pass the generated command-line to ``runner``, returning
    whatever is returned.

    The following keyword arguments will be intercepted by the wrapper
    function, and will *not* be passed to ``func``:
Paul McCarthy's avatar
Paul McCarthy committed
169

170
171
172
173
174
      - ``stdout``:   Passed to ``runner``. Defaults to ``True``.
      - ``stderr``:   Passed to ``runner``. Defaults to ``True``.
      - ``exitcode``: Passed to ``runner``. Defaults to ``False``.
      - ``submit``:   Passed to ``runner``. Defaults to ``None``.
      - ``log``:      Passed to ``runner``. Defaults to ``{'tee':True}``.
175
      - ``cmdonly``:  Passed to ``runner``. Defaults to ``False``.
176
177
178

    :arg func:   A function which generates a command line.
    :arg runner: Either :func:`.run.run` or :func:`.run.runfsl`.
179
    """
180

181
    def wrapper(*args, **kwargs):
182
183
        stdout   = kwargs.pop('stdout',   True)
        stderr   = kwargs.pop('stderr',   True)
184
        exitcode = kwargs.pop('exitcode', False)
185
        submit   = kwargs.pop('submit',   None)
186
        cmdonly  = kwargs.pop('cmdonly',  False)
187
        log      = kwargs.pop('log',      {'tee' : True})
188
189
190
191
192
193
194
195

        # many wrapper functions use fsl.utils.assertions
        # statements to check that input arguments are
        # valid. Disable these if the cmdonly argument is
        # being used to generate a command without running
        # it.
        with asrt.disabled(cmdonly):
            cmd = func(*args, **kwargs)
196

197
198
199
200
201
202
203
204
        return runner(cmd,
                      stderr=stderr,
                      log=log,
                      submit=submit,
                      cmdonly=cmdonly,
                      stdout=stdout,
                      exitcode=exitcode)

205
206
207
    return _update_wrapper(wrapper, func)


208
209
210
211
212
213
214
215
216
217
def cmdwrapper(func):
    """This decorator can be used on functions which generate a command line.
    It will pass the return value of the function to the
    :func:`fsl.utils.run.run` function in a standardised manner.

    See the :func:`genxwrapper` function for details.
    """
    return genxwrapper(func, run.run)


218
219
220
221
def fslwrapper(func):
    """This decorator can be used on functions which generate a FSL command
    line. It will pass the return value of the function to the
    :func:`fsl.utils.run.runfsl` function in a standardised manner.
222
223

    See the :func:`genxwrapper` function for details.
224
    """
225
    return genxwrapper(func, run.runfsl)
226
227


Paul McCarthy's avatar
Paul McCarthy committed
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
SHOW_IF_TRUE = object()
"""Constant to be used in the ``valmap`` passed to the :func:`applyArgStyle`
function.

When a ``SHOW_IF_TRUE`` argument is ``True``, it is added to the generated
command line arguments.
"""


HIDE_IF_TRUE = object()
"""Constant to be used in the ``valmap`` passed to the :func:`applyArgStyle`
function.

When a ``HIDE_IF_TRUE`` argument is ``True``, it is suppressed from the
generated command line arguments.
"""
244
245


246
247
248
249
250
251
def applyArgStyle(style,
                  valsep=None,
                  argmap=None,
                  valmap=None,
                  singlechar_args=False,
                  **kwargs):
Paul McCarthy's avatar
Paul McCarthy committed
252
253
254
255
    """Turns the given ``kwargs`` into command line options. This function
    is intended to be used to automatically generate command line options
    from arguments passed into a Python function.

256
257
258
259
260
261
262
    The ``style`` and ``valsep`` arguments control how key-value pairs
    are converted into command-line options:


    =========  ==========  ===========================
    ``style``  ``valsep``  Result
    =========  ==========  ===========================
Paul McCarthy's avatar
Paul McCarthy committed
263
264
265
266
267
268
269
270
271
272
273
274
    ``'-'``    ``' '``     ``-name val1 val2 val3``
    ``'-'``    ``'"'``     ``-name "val1 val2 val3"``
    ``'-'``    ``','``     ``-name val1,val2,val3``
    ``'--'``   ``' '``     ``--name val1 val2 val3``
    ``'--'``   ``'"'``     ``--name "val1 val2 val3"``
    ``'--'``   ``','``     ``--name val1,val2,val3``
    ``'-='``   ``' '``     Not supported
    ``'-='``   ``'"'``     ``-name="val1 val2 val3"``
    ``'-='``   ``','``     ``-name=val1,val2,val3``
    ``'--='``  ``' '``     Not supported
    ``'--='``  ``'"'``     ``--name="val1 val2 val3"``
    ``'--='``  ``','``     ``--name=val1,val2,val3``
275
276
277
    =========  ==========  ===========================


Paul McCarthy's avatar
Paul McCarthy committed
278
    :arg style:  Controls how the ``kwargs`` are converted into command-line
279
280
281
282
283
                 options - must be one of ``'-'``, ``'--'``, ``'-='``, or
                 ``'--='``.

    :arg valsep: Controls how the values passed to command-line options
                 which expect multiple arguments are delimited - must be
284
285
                 one of ``' '``, ``','`` or ``'"'``. Defaults to ``' '``
                 if ``'=' not in style``, ``','`` otherwise.
Paul McCarthy's avatar
Paul McCarthy committed
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308

    :arg argmap: Dictionary of ``{kwarg-name : cli-name}`` mappings. This can
                 be used if you want to use different argument names in your
                 Python function for the command-line options.

    :arg valmap: Dictionary of ``{cli-name : value}`` mappings. This can be
                 used to define specific semantics for some command-line
                 options. Acceptable values for ``value`` are as follows

                  - :data:`SHOW_IF_TRUE` - if the argument is present, and
                    ``True`` in ``kwargs``, the command line option
                    will be added (without any arguments).

                  - :data:`HIDE_IF_TRUE` - if the argument is present, and
                    ``False`` in ``kwargs``, the command line option
                    will be added (without any arguments).

                  - Any other constant value. If the argument is present
                    in ``kwargs``, its command-line option will be
                    added, with the constant value as its argument.

                 The argument for any options not specified in the ``valmap``
                 will be converted into strings.
309

310
311
312
    :arg singlechar_args: If True, single character arguments always take a
                          single hyphen prefix (e.g. -h) regardless of the
                          style.
313

Paul McCarthy's avatar
Paul McCarthy committed
314
315
316
317
318
    :arg kwargs: Arguments to be converted into command-line options.

    :returns:    A list containing the generated command-line options.
    """

319
320
321
322
    if valsep is None:
        if '=' in style: valsep = ','
        else:            valsep = ' '

Paul McCarthy's avatar
Paul McCarthy committed
323
324
    if style not in ('-', '--', '-=', '--='):
        raise ValueError('Invalid style: {}'.format(style))
325
326
327
328
329
330
331
332
333
    if valsep not in (' ', ',', '"'):
        raise ValueError('Invalid valsep: {}'.format(valsep))

    # we don't handle the case where '=' in
    # style, and valsep == ' ', because no
    # sane CLI app would do this. Right?
    if '=' in style and valsep == ' ':
        raise ValueError('Incompatible style and valsep: s={} v={}'.format(
            style, valsep))
Paul McCarthy's avatar
Paul McCarthy committed
334
335
336
337
338

    if argmap is None: argmap = {}
    if valmap is None: valmap = {}

    def fmtarg(arg):
339
340
341
342
        if   style in ('-',  '-=') or (singlechar_args and len(arg) == 1):
            arg =  '-{}'.format(arg)
        elif style in ('--', '--='):
            arg = '--{}'.format(arg)
343
344
        return arg

345
    # always returns a sequence
Paul McCarthy's avatar
Paul McCarthy committed
346
    def fmtval(val):
Paul McCarthy's avatar
Paul McCarthy committed
347
        if isinstance(val, abc.Sequence) and (not isinstance(val, str)):
348
349
350
351
352

            val = [str(v) for v in val]
            if   valsep == ' ': return val
            elif valsep == '"': return [' '   .join(val)]
            else:               return [valsep.join(val)]
353
        else:
354
355
356
357
358
359
360
361
            return [str(val)]

    # val is assumed to be a sequence
    def fmtargval(arg, val):
        # if '=' in style, val will
        # always be a single string
        if '=' in style: return ['{}={}'.format(arg, val[0])]
        else:            return [arg] + val
362
363
364
365
366

    args = []

    for k, v in kwargs.items():

367
368
        if v is None: continue

369
        k    = argmap.get(k, k)
Paul McCarthy's avatar
Paul McCarthy committed
370
371
372
        mapv = valmap.get(k, fmtval(v))
        k    = fmtarg(k)

373
374
375
376
        if mapv in (SHOW_IF_TRUE, HIDE_IF_TRUE):
            if (mapv is SHOW_IF_TRUE and     v) or \
               (mapv is HIDE_IF_TRUE and not v):
                args.append(k)
377
        else:
378
            args.extend(fmtargval(k, mapv))
379
380
381
382

    return args


383
def namedPositionals(func, args):
384
    """Given a function, and a sequence of positional arguments destined
385
    for that function, identifies the name for each positional argument.
386
    Variable positional arguments are given an automatic name.
387

388
389
    :arg func: Function which will accept ``args`` as positionals.
    :arg args: Tuple of positional arguments to be passed to ``func``.
390
    """
Paul McCarthy's avatar
Paul McCarthy committed
391

392
393
394
395
396
397
398
399
400
401
    # Current implementation will
    # result in naming collisions
    # for something like this:
    #
    # def func(args0, *args):
    #     ...
    # because of automatic vararg
    # naming. But who would write
    # a function like that anyway?

402
403
    # Remove any decorators
    # from the function
Paul McCarthy's avatar
Paul McCarthy committed
404
405
    func = _unwrap(func)

406
407
408
    # getargspec is the only way to
    # get the names of positional
    # arguments in Python 2.x.
Paul McCarthy's avatar
Paul McCarthy committed
409
    if sys.version_info[0] < 3:
410
411
412
        spec     = inspect.getargspec(func)
        argnames = spec.args
        varargs  = spec.varargs
Paul McCarthy's avatar
Paul McCarthy committed
413

414
415
    # But getargspec is deprecated
    # in python 3.x
Paul McCarthy's avatar
Paul McCarthy committed
416
417
418
419
420
421
    else:

        # getfullargspec is deprecated in
        # python 3.5, but not in python 3.6.
        with warnings.catch_warnings():
            warnings.filterwarnings('ignore', category=DeprecationWarning)
422
423
424
            spec     = inspect.getfullargspec(func)
            argnames = spec.args
            varargs  = spec.varargs
425

426
427
428
    # we only care about the arguments
    # that are being passed in
    argnames = argnames[:len(args)]
429

430
431
432
433
    # make up names for varargs
    nvarargs = len(args) - len(argnames)
    if varargs is not None and nvarargs > 0:
        argnames += ['{}{}'.format(varargs, i) for i in range(nvarargs)]
434

435
    return argnames
436
437


Paul McCarthy's avatar
Paul McCarthy committed
438
LOAD = object()
439
"""Constant used by the :class:`FileOrThing` class to indicate that an output
Paul McCarthy's avatar
Paul McCarthy committed
440
file should be loaded into memory and returned as a Python object.
441
442
443
"""


Paul McCarthy's avatar
Paul McCarthy committed
444
class FileOrThing:
Paul McCarthy's avatar
Paul McCarthy committed
445
446
447
448
449
    """Decorator which ensures that certain arguments which are passed into the
    decorated function are always passed as file names. Both positional and
    keyword arguments can be specified.


450
    The ``FileOrThing`` class is not intended to be used directly - see the
Paul McCarthy's avatar
Paul McCarthy committed
451
452
453
    :func:`fileOrImage` and :func:`fileOrArray` decorator functions for more
    details.

454

Paul McCarthy's avatar
Paul McCarthy committed
455
456
457
458
459
460
461
462
463
464
465
466
    These decorators are intended for functions which wrap a command-line tool,
    i.e. where some inputs/outputs need to be specified as file names.


    **Inputs**


    Any arguments which are not of type ``Thing`` are passed through to the
    decorated function unmodified.  Arguments which are of type ``Thing`` are
    saved to a temporary file, and the name of that file is passed to the
    function.

467

Paul McCarthy's avatar
Paul McCarthy committed
468
    **Outputs**
469
470


Paul McCarthy's avatar
Paul McCarthy committed
471
    If an argument is given the special :data:`LOAD` value, it is assumed
Paul McCarthy's avatar
Paul McCarthy committed
472
473
474
    to be an output argument. In this case, it is replaced with a temporary
    file name then, after the function has completed, that file is loaded
    into memory, and the value returned (along with the function's output,
Paul McCarthy's avatar
Paul McCarthy committed
475
    and any other arguments with a value of ``LOAD``).
Paul McCarthy's avatar
Paul McCarthy committed
476
477
478
479
480


    **Return value**


481
    Functions decorated with a ``FileOrThing`` decorator will always return a
482
    ``dict``-like object, where the function's actual return value is
483
    accessible via an attribute called ``stdout``. All output arguments with a
484
    value of ``LOAD`` will be present as dictionary entries, with the keyword
485
486
487
488
    argument names used as keys; these values will also be accessible as
    attributes of the results dict, when possible. Any ``LOAD`` output
    arguments which were not generated by the function will not be present in
    the dictionary.
Paul McCarthy's avatar
Paul McCarthy committed
489
490


491
    **Exceptions**
492
493


494
495
496
497
498
    The above description holds in all situations, except when arguments called
    ``submit`` and/or ``cmdonly`` are passed, and are set to values which
    evaluate to ``True``. In this case, the ``FileOrThing`` decorator will pass
    all arguments straight through to the decorated function, and will return
    its return value unchanged.
499
500
501

    This is because most functions that are decorated with the
    :func:`fileOrImage` or :func:`fileOrArray` decorators will invoke a call
502
    to :func:`.run.run` or :func:`.runfsl`, where:
503

504
505
506
507
      - a value of ``submit=True`` will cause the command to be executed
        asynchronously on a cluster platform.
      - a value of ``cmdonly=True`` will cause the command to *not* be executed,
        but instead the command that would have been executed is returned.
508
509

    A :exc:`ValueError` will be raised if the decorated function is called
510
511
    with ``submit=True`` and/or ``cmdonly=True``, and with any in-memory
    objects or ``LOAD`` symbols.
512
513


Paul McCarthy's avatar
Paul McCarthy committed
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
    **Example**


    As an example of using the ``fileOrArray`` decorator on a function
    which concatenates two files containing affine transformations, and
    saves the output to a file::

        # if atob, btoc, or output are passed
        # in as arrays, they are converted to
        # file names.
        @fileOrArray('atob', 'btoc', 'output')
        def concat(atob, btoc, output=None):

            # inputs are guaranteed to be files
            atob = np.loadtxt(atob)
            btoc = np.loadtxt(atoc)

            atoc = np.dot(btoc, atob)

            if output is not None:
                np.savetxt(output, atoc)

536
537
            return 'Done'

Paul McCarthy's avatar
Paul McCarthy committed
538
539
540
541

    Because we have decorated the ``concat`` function with :func:`fileToArray`,
    it can be called with either file names, or Numpy arrays::

542

Paul McCarthy's avatar
Paul McCarthy committed
543
544
        # All arguments are passed through
        # unmodified - the output will be
545
        # saved to a file called atoc.mat.
Paul McCarthy's avatar
Paul McCarthy committed
546
547
        concat('atob.txt', 'btoc.txt', 'atoc.mat')

548
549
        # The function's return value
        # is accessed via an attribute called
550
551
        # "stdout" on the dict
        assert concat('atob.txt', 'btoc.txt', 'atoc.mat').stdout == 'Done'
552
553
554

        # Outputs to be loaded into memory
        # are returned in a dictionary,
555
556
557
        # with argument names as keys. Values
        # can be accessed as dict items, or
        # as attributes.
558
        atoc = concat('atob.txt', 'btoc.txt', LOAD)['atoc']
559
        atoc = concat('atob.txt', 'btoc.txt', LOAD).atoc
Paul McCarthy's avatar
Paul McCarthy committed
560

561
562
563
564
565
        # In-memory inputs are saved to
        # temporary files, and those file
        # names are passed to the concat
        # function.
        atoc = concat(np.diag([2, 2, 2, 0]),
566
                      np.diag([3, 3, 3, 3]), LOAD).atoc
567
568
569


    **Using with other decorators**
Paul McCarthy's avatar
Paul McCarthy committed
570
571


572
573
    ``FileOrThing`` decorators can be chained with other ``FileOrThing``
    decorators, and other decorators.  When multiple ``FileOrThing``
Paul McCarthy's avatar
Paul McCarthy committed
574
575
576
577
    decorators are used on a single function, the outputs from each decorator
    are merged together into a single dict-like object.


578
    ``FileOrThing`` decorators can be used with any other decorators
579
    **as long as** they do not manipulate the return value, and as long as
580
    the ``FileOrThing`` decorators are adjacent to each other.
581
582
583
    """


584
    class Results(dict):
585
        """A custom ``dict`` type used to return outputs from a function
586
        decorated with ``FileOrThing``. All outputs are stored as dictionary
587
588
589
        items, with the argument name as key, and the output object (the
        "thing") as value.

590
591
        Where possible (i.e. for outputs named with a valid Python
        identifier), the outputs are also made accessible as attributes of
592
        the ``Results`` object.
593

594
        The decorated function's actual return value is accessible via the
595
        :meth:`stdout` property.
596
        """
597
598


599
        def __init__(self, stdout):
600
            """Create a ``Results`` dict.
601

Paul McCarthy's avatar
Paul McCarthy committed
602
            :arg stdout: Return value of the decorated function (typically a
603
604
                         tuple containing the standard output and error of the
                         underlying command).
605
            """
606
            super().__init__()
607
608
609
610
            self.__stdout = stdout


        def __setitem__(self, key, val):
611
            """Add an item to the dict. The item is also added as an attribute.
612
613
            """
            super().__setitem__(key, val)
614
            setattr(self, key, val)
615

616

617
        @property
618
        def stdout(self):
619
            """Access the return value of the decorated function. """
620
            return self.__stdout
621
622


623
624
625
626
627
628
    def __init__(self,
                 func,
                 prepIn,
                 prepOut,
                 load,
                 removeExt,
Paul McCarthy's avatar
Paul McCarthy committed
629
630
                 *args,
                 **kwargs):
631
        """Initialise a ``FileOrThing`` decorator.
Paul McCarthy's avatar
Paul McCarthy committed
632

633
634
635
636
        :arg func:      The function to be decorated.

        :arg prepIn:    Function which returns a file name to be used in
                        place of an input argument.
637

638
639
        :arg prepOut:   Function which generates a file name to use for
                        arguments that were set to :data:`LOAD`.
Paul McCarthy's avatar
Paul McCarthy committed
640

641
        :arg load:      Function which is called to load items for arguments
642
643
                        that were set to :data:`LOAD`. Must accept the
                        following arguments:
Paul McCarthy's avatar
Paul McCarthy committed
644

645
646
                         - the name of the argument
                         - path to the file to be loaded
647

648
649
650
        :arg removeExt: Function which can remove a file extension from a file
                        path.

Paul McCarthy's avatar
Paul McCarthy committed
651
652
653
654
655
656
657
658
        :arg outprefix: Must be passed as a keyword argument. The name of a
                        positional or keyword argument to the function, which
                        specifies an output file name prefix.  All other
                        arguments with names that begin with this prefix may
                        be interpreted as things to ``LOAD``.

        All other positional arguments are interpreted as the names of the
        arguments to the function which will be handled by this
659
        ``FileOrThing`` decorator. If not provided, *all* arguments passed to
Paul McCarthy's avatar
Paul McCarthy committed
660
        the function will be handled.
661

662
663
664
665
666
667

        The ``prepIn`` and ``prepOut`` functions must accept the following
        positional arguments:

          - A directory in which all temporary input/output files should be
            stored
Paul McCarthy's avatar
Paul McCarthy committed
668

669
670
671
672
          - The name of the keyword argument to be processed

          - The argument value that was passed in
        """
673
674
675
676
        self.__func      = func
        self.__prepIn    = prepIn
        self.__prepOut   = prepOut
        self.__load      = load
677
        self.__removeExt = removeExt
Paul McCarthy's avatar
Paul McCarthy committed
678
679
        self.__things    = args
        self.__outprefix = kwargs.get('outprefix', None)
Paul McCarthy's avatar
Paul McCarthy committed
680
681


682
    def __call__(self, *args, **kwargs):
683
        """Function which calls ``func``, ensuring that any arguments of
Paul McCarthy's avatar
Paul McCarthy committed
684
        type ``Thing`` are saved to temporary files, and any arguments
Paul McCarthy's avatar
Paul McCarthy committed
685
        with the value :data:`LOAD` are loaded and returned.
686

687
        All other arguments are passed through to ``func``.
688
        """
Paul McCarthy's avatar
Paul McCarthy committed
689

690
691
        func     = self.__func
        argnames = namedPositionals(func, args)
692

693
694
        # Special case - if fsl.utils.run[fsl] is
        # being decorated (e.g. via cmdwrapper/
695
696
697
698
699
        # fslwrapper), and submit=True or
        # cmdonly=True, this call will ultimately
        # submit the job to the cluster, or will
        # return the command that would have been
        # executed, and will return immediately.
700
701
702
703
704
705
706
        #
        # We error if we are given any in-memory
        # things, or LOAD symbols.
        #
        # n.b. testing values to be strings could
        # interfere with the fileOrText decorator.
        # Possible solution is to use pathlib?
707
708
        if kwargs.get('submit',  False) or \
           kwargs.get('cmdonly', False):
709
710
            allargs = {**dict(zip(argnames, args)), **kwargs}
            for name, val in allargs.items():
Paul McCarthy's avatar
Paul McCarthy committed
711
                if (name in self.__things) and (not isinstance(val, str)):
712
713
714
715
                    raise ValueError('Cannot use in-memory objects '
                                     'or LOAD with submit=True!')
            return func(*args, **kwargs)

716
717
        # If this FileOrThing is being called
        # by another FileOrThing don't create
718
719
720
721
722
723
724
725
        # another working directory. We do this
        # sneakily, by setting an attribute on
        # the wrapped function which stores the
        # current working directory.
        wrapped     = _unwrap(func)
        fot_workdir = getattr(wrapped, '_fot_workdir', None)
        parent      = fot_workdir is None

726
        # Create a tempdir to store any temporary
Paul McCarthy's avatar
Paul McCarthy committed
727
        # input/output things, but don't change
728
729
        # into it, as file paths passed to the
        # function may be relative.
730
        with tempdir.tempdir(changeto=False, override=fot_workdir) as td:
731

732
733
            log.debug('Redirecting LOADed outputs to %s', td)

734
735
            # Replace any things with file names.
            # Also get a list of LOAD outputs
736
            args = self.__prepareArgs(parent, td, argnames, args, kwargs)
737
            args, kwargs, outprefix, outfiles, prefixes = args
738

739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
            # The prefix/patterns may be
            # overridden by a parent FoT
            outprefix = getattr(wrapped, '_fot_outprefix', outprefix)
            prefixes  = getattr(wrapped, '_fot_prefixes',  prefixes)

            # if there are any other FileOrThings
            # in the decorator chain, get them to
            # use our working directory, and
            # prefixes, instead of creating their
            # own.
            if parent:
                setattr(wrapped, '_fot_workdir',   td)
                setattr(wrapped, '_fot_outprefix', outprefix)
                setattr(wrapped, '_fot_prefixes',  prefixes)

754
            # Call the function
755
756
757
758
759
760
761
762
763
764
765
            try:
                result = func(*args, **kwargs)

            finally:
                # if we're the top-level FileOrThing
                # decorator, remove the attributes we
                # added above.
                if parent:
                    delattr(wrapped, '_fot_workdir')
                    delattr(wrapped, '_fot_outprefix')
                    delattr(wrapped, '_fot_prefixes')
766

767
768
            return self.__generateResult(
                td, result, outprefix, outfiles, prefixes)
769
770


771
    def __prepareArgs(self, parent, workdir, argnames, args, kwargs):
772
773
774
775
776
        """Prepares all input and output arguments to be passed to the
        decorated function. Any arguments with a value of :data:`LOAD` are
        passed to the ``prepOut`` function specified at :meth:`__init__`.
        All other arguments are passed through the ``prepIn`` function.

777
778
        :arg parent:  ``True`` if this ``FileOrThing`` is the first in a
                      chain of ``FileOrThing`` decorators.
779

780
781
        :arg workdir: Directory in which all temporary files should be stored.

782
783
784
        :arg args:    Positional arguments to be passed to the decorated
                      function.

785
786
787
788
        :arg kwargs:  Keyword arguments to be passed to the decorated function.

        :returns:     A tuple containing:

789
790
791
                        - An updated copy of ``args``.

                        - An updated copy of ``kwargs``.
792

793
794
795
796
797
798
799
                        - The output file prefix that was actually passed in
                          (it is subsequently modified so that prefixed outputs
                          are redirected to a temporary location). All prefixed
                          outputs that are not ``LOAD``ed should be moved into
                          this directory. ``None`` if there is no output
                          prefix.

800
801
                        - A dictionary of ``{ name : filename }`` mappings,
                          for all arguments with a value of ``LOAD``.
802

803
                        - A dictionary of ``{ filepat : replstr }`` paths, for
804
                          all output-prefix arguments with a value of ``LOAD``.
805
806
        """

807
808
809
810
        # These containers keep track
        # of output files which are to
        # be loaded into memory
        outfiles      = dict()
811
        prefixedFiles = dict()
812

813
814
815
        allargs  = {k : v for k, v in zip(argnames, args)}
        allargs.update(kwargs)

816
817
818
819
        # Has an output prefix been specified?
        prefix     = allargs.get(self.__outprefix, None)
        realPrefix = None

820
821
        # Prefixed outputs are only
        # managed by the parent
822
        # FileOrthing in a chain of
823
824
825
826
        # FoT decorators.
        if not parent:
            prefix = None

827
828
829
830
831
832
833
834
835
836
        # If so, replace it with a new output
        # prefix which will redirect all output
        # to the temp dir.
        #
        # Importantly, here we assume that the
        # underlying function (and hence the
        # underlying command-line tool) will
        # accept an output prefix which contains
        # a directory path.
        if prefix is not None:
837
838
839
840
841
842
843
844
845
846

            # If prefix is set to LOAD,
            # all generated output files
            # should be loaded - we use a
            # randomly generated prefix,
            # and add it to prefixedFiles,
            # so that every file which
            # starts with it will be
            # loaded.
            if prefix is LOAD:
847
848
849
                prefix                = random.sample(string.ascii_letters, 10)
                prefix                = ''.join(prefix)
                prefixedFiles[prefix] = self.__outprefix
850

851
            realPrefix                = prefix
852
853
854
            fakePrefix                = op.join(workdir, prefix)
            allargs[self.__outprefix] = fakePrefix

855
856
            log.debug('Replacing output prefix: %s -> %s',
                      realPrefix, fakePrefix)
857

858
859
860
861
            # If the prefix specifies a
            # directory, make sure it
            # exists (remember that we're
            # in a temporary directory)
862
863
864
            pdir = op.dirname(fakePrefix)
            if pdir != '' and not op.exists(pdir):
                os.makedirs(pdir)
865

866
867
        if len(self.__things) > 0: things = self.__things
        else:                      things = allargs.keys()
868

869
        for name, val in list(allargs.items()):
870

871
872
873
874
875
            # don't process the
            # outprefix argument
            if name == self.__outprefix:
                continue

876
            # is this argument referring
877
            # to a prefixed output?
878
879
            isprefixed = (prefix is not None and
                          name.startswith(prefix))
880

881
            if not (isprefixed or name in things):
882
883
                continue

884
885
886
887
            # Prefixed output files may only
            # be given a value of LOAD
            if isprefixed and val is not LOAD:
                raise ValueError('Cannot specify name of prefixed file - the '
888
889
                                 'name is defined by the output prefix: '
                                 '{}'.format(name))
890

891
            if val is LOAD:
892

893
894
895
896
897
898
899
                # this argument refers to an output
                # that is generated from the output
                # prefix argument, and doesn't map
                # directly to an argument of the
                # function. So we don't pass it
                # through.
                if isprefixed:
900
                    prefixedFiles[name] = name
901
902
903
904
905
                    allargs.pop(name)

                # regular output-file argument
                else:
                    outfile = self.__prepOut(workdir, name, val)
906
                    outfiles[name] = outfile
907
                    allargs[ name] = outfile
Paul McCarthy's avatar
Paul McCarthy committed
908

909
            # Assumed to be an input file
910
            else:
911
912
                # sequences may be
                # accepted for inputs
913
914
915
916
917
918
919
920
921
                if isinstance(val, (list, tuple)):
                    infile = list(val)
                    for i, v in enumerate(val):
                        v = self.__prepIn(workdir, name, v)
                        if v is not None:
                            infile[i] = v

                else:
                    infile = self.__prepIn(workdir, name, val)
Paul McCarthy's avatar
Paul McCarthy committed
922

923
                if infile is not None:
924
925
                    allargs[name] = infile

926
927
928
        if realPrefix is not None and len(prefixedFiles) == 0:
            allargs[self.__outprefix] = realPrefix

929
930
        args   = [allargs.pop(k) for k in argnames]
        kwargs = allargs
931

932
        return args, kwargs, realPrefix, outfiles, prefixedFiles
933
934


935
936
    def __generateResult(
            self, workdir, result, outprefix, outfiles, prefixes):
937
        """Loads function outputs and returns a :class:`Results` object.
938
939
940

        Called by :meth:`__call__` after the decorated function has been
        called. Figures out what files should be loaded, and loads them into
941
        a ``Results`` object.
942
943
944
945
946
947
948
949
950
951

        :arg workdir:   Directory which contains the function outputs.
        :arg result:    Function return value.
        :arg outprefix: Original output prefix that was passed into the
                        function (or ``None`` if one wasn't passed)
        :arg outfiles:  Dictionary containing output files to be loaded (see
                        :meth:`__prepareArgs`).
        :arg prefixes:  Dictionary containing output-prefix patterns to be
                        loaded (see :meth:`__prepareArgs`).

952
        :returns:       A ``Results`` object containing all loaded outputs.
953
954
        """

955
        # make a Results object to store
956
        # the output. If we are decorating
957
        # another FileOrThing, the
958
        # results will get merged together
959
960
961
        # into a single Results dict.
        if not isinstance(result, FileOrThing.Results):
            result = FileOrThing.Results(result)
962
963
964
965
966
967

        # Load the LOADed outputs
        for oname, ofile in outfiles.items():

            log.debug('Loading output %s: %s', oname, ofile)

968
            if op.exists(ofile): oval = self.__load(oname, ofile)
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
            else:                oval = None

            result[oname] = oval

        # No output prefix - we're done
        if outprefix is None or len(prefixes) == 0:
            return result

        # Load or move output-prefixed files.
        # Find all files with a name that
        # matches the prefix that was passed
        # in (recursing into matching sub-
        # directories too).
        allPrefixed = glob.glob(op.join(workdir, '{}*'.format(outprefix)))
        allPrefixed = [fslpath.allFiles(f) if op.isdir(f) else [f]
                       for f in allPrefixed]

        for prefixed in it.chain(*allPrefixed):
            fullpath = prefixed
            prefixed = op.relpath(prefixed, workdir)
            for prefPat, prefName in prefixes.items():
                if not fnmatch.fnmatch(prefixed, '{}*'.format(prefPat)):
                    continue

                log.debug('Loading prefixed output %s [%s]: %s',
                          prefPat, prefName, prefixed)

996
997
998
999
1000
                noext   = self.__removeExt(prefixed)
                prefPat = prefPat.replace('\\', '\\\\')
                noext   = re.sub('^' + prefPat, prefName, noext)
                withext = re.sub('^' + prefPat, prefName, prefixed)

1001
1002
1003
                # if the load function returns
                # None, this file is probably
                # not of the correct type.
1004
                fval = self.__load(noext, fullpath)
1005
                if fval is not None:
1006

1007
1008
1009
                    # If there is already an item in result with the
                    # name (stripped of prefix), then instead store
                    # the result with the full prefixed name
1010
1011
1012
1013
                    if noext not in result:
                        result[noext] = fval
                    else:
                        result[withext] = fval
1014
1015
1016
1017
1018
                    break

        return result


1019
def fileOrImage(*args, **kwargs):
Paul McCarthy's avatar
Paul McCarthy committed
1020
1021
    """Decorator which can be used to ensure that any NIfTI images are saved
    to file, and output images can be loaded and returned as ``nibabel``
1022
    image objects or :class:`.Image` objects.
Paul McCarthy's avatar
Paul McCarthy committed
1023
1024
    """

1025
1026
1027
1028
1029
1030
    # keep track of the input argument
    # types on each call, so we know
    # whether to return a fsl.Image or
    # a nibabel image
    intypes = []

1031
    def prepIn(workdir, name, val):
Paul McCarthy's avatar
Paul McCarthy committed
1032

1033
        infile = None
Paul McCarthy's avatar
Paul McCarthy committed
1034

1035
1036
1037
1038
1039
        if isinstance(val, fslimage.Image):
            intypes.append(fslimage.Image)

        elif isinstance(val, nib.nifti1.Nifti1Image):
            intypes.append(nib.nifti1.Nifti1Image)
1040
1041
1042
1043

        if isinstance(val, fslimage.Image):
            val = val.nibImage

Paul McCarthy's avatar
Paul McCarthy committed
1044
        if isinstance(val, nib.nifti1.Nifti1Image):
1045
            infile = val.get_filename()
Paul McCarthy's avatar
Paul McCarthy committed
1046
1047
1048

            # in-memory image - we have
            # to save it out to a file
1049
1050
            if infile is None:
                hd, infile = tempfile.mkstemp(fslimage.defaultExt())
Paul McCarthy's avatar
Paul McCarthy committed
1051
                os.close(hd)
1052
                val.to_filename(infile)
Paul McCarthy's avatar
Paul McCarthy committed
1053

1054
        return infile
Paul McCarthy's avatar
Paul McCarthy committed
1055

1056
1057
    def prepOut(workdir, name, val):
        return op.join(workdir, '{}.nii.gz'.format(name))
Paul McCarthy's avatar
Paul McCarthy committed
1058

1059
    def load(name, path):
1060
1061
1062
1063

        if not fslimage.looksLikeImage(path):
            return None

Paul McCarthy's avatar
Paul McCarthy committed
1064
1065
        # create an independent in-memory
        # copy of the image file
Paul McCarthy's avatar
Paul McCarthy committed
1066
1067
        img  = nib.load(path, mmap=False)
        data = np.asanyarray(img.dataobj)
1068
1069
1070
1071

        # if any arguments were fsl images,
        # that takes precedence.
        if fslimage.Image in intypes:
1072
1073
            return fslimage.Image(data, header=img.header, name=name)

1074
1075
1076
        # but if all inputs were file names,
        # nibabel takes precedence
        elif nib.nifti1.Nifti1Image in intypes or len(intypes) == 0:
Paul McCarthy's avatar
Paul McCarthy committed
1077
            return nib.nifti1.Nifti1Image(data, None, img.header)
1078
1079
1080
1081
1082

        # this function should not be called
        # under any other circumstances
        else:
            raise RuntimeError('Cannot handle type: {}'.format(intypes))
Paul McCarthy's avatar
Paul McCarthy committed
1083

1084
    def decorator(func):
1085
1086
1087
1088
1089
1090
1091
        fot = FileOrThing(func,
                          prepIn,
                          prepOut,
                          load,
                          fslimage.removeExt,
                          *args,
                          **kwargs)
1092
1093

        def wrapper(*args, **kwargs):
1094
1095
1096
            result = fot(*args, **kwargs)
            intypes[:] = []
            return result
1097
1098
1099
1100

        return _update_wrapper(wrapper, func)

    return decorator
Paul McCarthy's avatar
Paul McCarthy committed
1101
1102


1103
def fileOrArray(*args, **kwargs):
Paul McCarthy's avatar
Paul McCarthy committed
1104
1105
1106
1107
    """Decorator which can be used to ensure that any Numpy arrays are saved
    to text files, and output files can be loaded and returned as Numpy arrays.
    """

1108
    def prepIn(workdir, name, val):
Paul McCarthy's avatar
Paul McCarthy committed
1109

1110
        infile = None
Paul McCarthy's avatar
Paul McCarthy committed
1111
1112

        if isinstance(val, np.ndarray):
1113
            hd, infile = tempfile.mkstemp('.txt')
Paul McCarthy's avatar
Paul McCarthy committed
1114
            os.close(hd)
1115
            np.savetxt(infile, val, fmt='%0.18f')
Paul McCarthy's avatar
Paul McCarthy committed
1116

1117
        return infile
Paul McCarthy's avatar
Paul McCarthy committed
1118

1119
1120
    def prepOut(workdir, name, val):
        return op.join(workdir, '{}.txt'.format(name))
Paul McCarthy's avatar
Paul McCarthy committed
1121

1122
    def load(_, path):
1123
1124
        try:              return np.loadtxt(path)
        except Exception: return None
Paul McCarthy's avatar
Paul McCarthy committed
1125

1126
    def decorator(func):
1127
1128
1129
1130
1131
1132
1133
        fot = FileOrThing(func,
                          prepIn,
                          prepOut,
                          load,
                          fslpath.removeExt,
                          *args,
                          **kwargs)
1134
1135
1136
1137
1138
1139
1140

        def wrapper(*args, **kwargs):
            return fot(*args, **kwargs)

        return _update_wrapper(wrapper, func)

    return decorator
1141

1142

1143
def fileOrText(*args, **kwargs):
1144
1145
1146
    """Decorator which can be used to ensure that any text output (e.g. log
    file) are saved to text files, and output files can be loaded and returned
    as strings.
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169

    To be able to distinguish between input values and input file paths, the
    ``fileOrText`` decorator requires that input and output file paths are
    passed in as ``pathlib.Path`` objects. For example, given a function
    like this::

        @fileOrText()
        def myfunc(infile, outfile):
            ...

    if we want to pass file paths for both ``infile`` and ``outfile``, we would
    do this::

        from pathlib import Path
        myfunc(Path('input.txt'), Path('output.txt'))

    Input values may be passed in as normal strings, e.g.::

        myfunc('input data', Path('output.txt'))

    Output values can be loaded as normal via the :attr:`LOAD` symbol, e.g.::

        myfunc(Path('input.txt'), LOAD)
1170
1171
1172
1173
1174
1175
    """

    def prepIn(workdir, name, val):

        infile = None

1176
1177
1178
1179
        if not isinstance(val, pathlib.Path):
            with tempfile.NamedTemporaryFile(mode='w',
                                             suffix='.txt',
                                             delete=False) as f:
1180
1181
1182
1183
1184
1185
1186
                f.write(val)
                infile = f.name
        return infile

    def prepOut(workdir, name, val):
        return op.join(workdir, '{}.txt'.format(name))

1187
    def load(_, path):
1188
1189
1190
1191
1192
1193
        try:
            with open(path, "r") as f:
                return f.read()
        except Exception: return None

    def decorator(func):
1194
1195
1196
1197
1198
1199
1200
        fot = FileOrThing(func,
                          prepIn,
                          prepOut,
                          load,
                          fslpath.removeExt,
                          *args,
                          **kwargs)
1201
1202
1203
1204
1205
1206
1207

        def wrapper(*args, **kwargs):
            return fot(*args, **kwargs)

        return _update_wrapper(wrapper, func)

    return decorator