wrapperutils.py 39.5 KB
Newer Older
1
2
#!/usr/bin/env python
#
Paul McCarthy's avatar
Paul McCarthy committed
3
4
# wrapperutils.py - Functions and decorators used by the FSL wrapper
# functions.
5
6
#
# Author: Paul McCarthy <pauldmccarthy@gmail.com>
7
# Author: Martin Craig <martin.craig@eng.ox.ac.uk>
8
#
Paul McCarthy's avatar
Paul McCarthy committed
9
10
11
12
"""This module contains functions and decorators used by the FSL wrapper
functions.


Paul McCarthy's avatar
Paul McCarthy committed
13
The :func:`cmdwrapper` and :func:`fslwrapper` functions are convenience
14
15
16
17
18
19
20
21
22
23
24
25
26
27
decorators which allow you to write your wrapper function such that it simply
generates the command-line needed to respectively run a standard shell
command or a FSL command. For example::


    @fslwrapper
    def fslreorient2std(input, output):
        return ['fslreorient2std', input, output]


When this ``fslreorient2std`` function is called, the ``fslwrapper`` decorator
will take care of invoking the command in a standardised way.


Paul McCarthy's avatar
Paul McCarthy committed
28
The :func:`applyArgStyle` function can be used to automatically convert
29
30
31
32
33
34
35
36
37
38
39
40
41
keyword arguments into command-line arguments, based on a set of standard
patterns. For example::


    @fslwrapper
    def flirt(src, ref, **kwargs):
        cmd  = ['flirt', '-in', src, '-ref', ref]
        return cmd + applyArgStyle('-=', **kwargs)


The :func:`fileOrImage` and :func:`fileOrArray` functions can be used to
decorate a wrapper function such that in-memory ``nibabel`` images or Numpy
arrays can be passed in as arguments - they will be automatically saved out to
Paul McCarthy's avatar
Paul McCarthy committed
42
files, and then the file names passed into the wrapper function. For example::
43
44
45
46
47
48
49
50
51
52
53
54
55


    @fileOrImage('src', 'ref')
    @fslwrapper
    def flirt(src, ref, **kwargs):
        cmd  = ['flirt', '-in', src, '-ref', ref]
        return cmd + applyArgStyle('-=', **kwargs)


Now this ``flirt`` function can be called either with file names, or
``nibabel`` images.


56
57
58
59
60
61
62
.. note:: Because the :func:`fileOrImage` and :func:`fileOrArray` decorators
          manipulate the return value of the decorated function, they should
          be applied *after* any other decorators. Furthermore, if you need to
          apply both a ``fileOrImage`` and ``fileOrArray`` decorator to a
          function, they should be grouped together, e.g.::

              @fileOrImage('a', 'b')
Paul McCarthy's avatar
Paul McCarthy committed
63
              @fileOrArray('c', 'd')
64
65
66
67
68
              @fslwrapper
              def func(**kwargs):
                  ...


69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
Command outputs can also be loaded back into memory by using the special
:data:`LOAD` value when calling a wrapper function. For example::


    @fileOrImage('src', 'ref', 'out')
    @fslwrapper
    def flirt(src, ref, **kwargs):
        cmd  = ['flirt', '-in', src, '-ref', ref]
        return cmd + applyArgStyle('-=', **kwargs)


If we set the ``out`` argument to ``LOAD``, the output image will be loaded
and returned::

    src     = nib.load('src.nii')
    ref     = nib.load('ref.nii')
    aligned = flirt(src, ref, out=LOAD)['out']
Paul McCarthy's avatar
Paul McCarthy committed
86
"""
87
88


89
90
91
92
93
94
95
96
97
import itertools       as it
import os.path         as op
import collections.abc as abc
import                    os
import                    re
import                    sys
import                    glob
import                    random
import                    string
98
import                    pathlib
99
100
101
102
103
104
105
import                    fnmatch
import                    inspect
import                    logging
import                    tempfile
import                    warnings
import                    functools

106
107
108

import            six
import nibabel as nib
Paul McCarthy's avatar
Paul McCarthy committed
109
import numpy   as np
110

111
import fsl.utils.run     as run
112
113
import fsl.utils.path    as fslpath
import fsl.utils.tempdir as tempdir
114
115
116
import fsl.data.image    as fslimage


117
118
119
log = logging.getLogger(__name__)


Paul McCarthy's avatar
Paul McCarthy committed
120
121
122
123
124
def _update_wrapper(wrapper, wrapped, *args, **kwargs):
    """Replacement for the built-in ``functools.update_wrapper``. This
    implementation ensures that the wrapper function has an attribute
    called ``__wrapped__``, which refers to the ``wrapped`` function.

125
    This custom function is only needed in Python versions < 3.4.
Paul McCarthy's avatar
Paul McCarthy committed
126
127
128
129
130
    """

    wrapper = functools.update_wrapper(wrapper, wrapped, *args, **kwargs)

    # Python >= 3.4 does things right
Paul McCarthy's avatar
Paul McCarthy committed
131
    if (sys.version_info[0] * 10 + sys.version_info[1]) < 34:
Paul McCarthy's avatar
Paul McCarthy committed
132
133
        wrapper.__wrapped__ = wrapped
    return wrapper
134
135


Paul McCarthy's avatar
Paul McCarthy committed
136
137
138
139
140
141
def _unwrap(func):
    """Replacement for the built-in ``inspect.unwrap`` function, which
    is not present in Python versions prior to 3.4.
    """

    # Python >= 3.4 has an inspect.unwrap function
Paul McCarthy's avatar
Paul McCarthy committed
142
    if (sys.version_info[0] * 10 + sys.version_info[1]) >= 34:
Paul McCarthy's avatar
Paul McCarthy committed
143
144
145
146
147
148
149
150
151
        return inspect.unwrap(func)

    # Otherwise we follow the __wrapped__ chain ourselves
    if hasattr(func, '__wrapped__'):
        return _unwrap(func.__wrapped__)

    return func


152
153
154
155
156
157
def cmdwrapper(func):
    """This decorator can be used on functions which generate a command line.
    It will pass the return value of the function to the
    :func:`fsl.utils.run.run` function in a standardised manner.
    """
    def wrapper(*args, **kwargs):
158
159
        stdout   = kwargs.pop('stdout',   True)
        stderr   = kwargs.pop('stderr',   True)
160
        exitcode = kwargs.pop('exitcode', False)
161
162
163
        submit   = kwargs.pop('submit',   None)
        log      = kwargs.pop('log',      {'tee' : True})
        cmd      = func(*args, **kwargs)
164
165
166
167
168
169
        return run.run(cmd,
                       stderr=stderr,
                       log=log,
                       submit=submit,
                       stdout=stdout,
                       exitcode=exitcode)
170
171
172
173
174
175
176
177
178
    return _update_wrapper(wrapper, func)


def fslwrapper(func):
    """This decorator can be used on functions which generate a FSL command
    line. It will pass the return value of the function to the
    :func:`fsl.utils.run.runfsl` function in a standardised manner.
    """
    def wrapper(*args, **kwargs):
179
180
        stdout   = kwargs.pop('stdout',   True)
        stderr   = kwargs.pop('stderr',   True)
181
        exitcode = kwargs.pop('exitcode', False)
182
183
184
        submit   = kwargs.pop('submit',   None)
        log      = kwargs.pop('log',      {'tee' : True})
        cmd      = func(*args, **kwargs)
185
186
187
188
189
190
        return run.runfsl(cmd,
                          stderr=stderr,
                          log=log,
                          submit=submit,
                          stdout=stdout,
                          exitcode=exitcode)
191
192
193
    return _update_wrapper(wrapper, func)


Paul McCarthy's avatar
Paul McCarthy committed
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
SHOW_IF_TRUE = object()
"""Constant to be used in the ``valmap`` passed to the :func:`applyArgStyle`
function.

When a ``SHOW_IF_TRUE`` argument is ``True``, it is added to the generated
command line arguments.
"""


HIDE_IF_TRUE = object()
"""Constant to be used in the ``valmap`` passed to the :func:`applyArgStyle`
function.

When a ``HIDE_IF_TRUE`` argument is ``True``, it is suppressed from the
generated command line arguments.
"""
210
211


212
213
214
215
216
217
def applyArgStyle(style,
                  valsep=None,
                  argmap=None,
                  valmap=None,
                  singlechar_args=False,
                  **kwargs):
Paul McCarthy's avatar
Paul McCarthy committed
218
219
220
221
    """Turns the given ``kwargs`` into command line options. This function
    is intended to be used to automatically generate command line options
    from arguments passed into a Python function.

222
223
224
225
226
227
228
    The ``style`` and ``valsep`` arguments control how key-value pairs
    are converted into command-line options:


    =========  ==========  ===========================
    ``style``  ``valsep``  Result
    =========  ==========  ===========================
Paul McCarthy's avatar
Paul McCarthy committed
229
230
231
232
233
234
235
236
237
238
239
240
    ``'-'``    ``' '``     ``-name val1 val2 val3``
    ``'-'``    ``'"'``     ``-name "val1 val2 val3"``
    ``'-'``    ``','``     ``-name val1,val2,val3``
    ``'--'``   ``' '``     ``--name val1 val2 val3``
    ``'--'``   ``'"'``     ``--name "val1 val2 val3"``
    ``'--'``   ``','``     ``--name val1,val2,val3``
    ``'-='``   ``' '``     Not supported
    ``'-='``   ``'"'``     ``-name="val1 val2 val3"``
    ``'-='``   ``','``     ``-name=val1,val2,val3``
    ``'--='``  ``' '``     Not supported
    ``'--='``  ``'"'``     ``--name="val1 val2 val3"``
    ``'--='``  ``','``     ``--name=val1,val2,val3``
241
242
243
    =========  ==========  ===========================


Paul McCarthy's avatar
Paul McCarthy committed
244
    :arg style:  Controls how the ``kwargs`` are converted into command-line
245
246
247
248
249
                 options - must be one of ``'-'``, ``'--'``, ``'-='``, or
                 ``'--='``.

    :arg valsep: Controls how the values passed to command-line options
                 which expect multiple arguments are delimited - must be
250
251
                 one of ``' '``, ``','`` or ``'"'``. Defaults to ``' '``
                 if ``'=' not in style``, ``','`` otherwise.
Paul McCarthy's avatar
Paul McCarthy committed
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274

    :arg argmap: Dictionary of ``{kwarg-name : cli-name}`` mappings. This can
                 be used if you want to use different argument names in your
                 Python function for the command-line options.

    :arg valmap: Dictionary of ``{cli-name : value}`` mappings. This can be
                 used to define specific semantics for some command-line
                 options. Acceptable values for ``value`` are as follows

                  - :data:`SHOW_IF_TRUE` - if the argument is present, and
                    ``True`` in ``kwargs``, the command line option
                    will be added (without any arguments).

                  - :data:`HIDE_IF_TRUE` - if the argument is present, and
                    ``False`` in ``kwargs``, the command line option
                    will be added (without any arguments).

                  - Any other constant value. If the argument is present
                    in ``kwargs``, its command-line option will be
                    added, with the constant value as its argument.

                 The argument for any options not specified in the ``valmap``
                 will be converted into strings.
275

276
277
278
    :arg singlechar_args: If True, single character arguments always take a
                          single hyphen prefix (e.g. -h) regardless of the
                          style.
279

Paul McCarthy's avatar
Paul McCarthy committed
280
281
282
283
284
    :arg kwargs: Arguments to be converted into command-line options.

    :returns:    A list containing the generated command-line options.
    """

285
286
287
288
    if valsep is None:
        if '=' in style: valsep = ','
        else:            valsep = ' '

Paul McCarthy's avatar
Paul McCarthy committed
289
290
    if style not in ('-', '--', '-=', '--='):
        raise ValueError('Invalid style: {}'.format(style))
291
292
293
294
295
296
297
298
299
    if valsep not in (' ', ',', '"'):
        raise ValueError('Invalid valsep: {}'.format(valsep))

    # we don't handle the case where '=' in
    # style, and valsep == ' ', because no
    # sane CLI app would do this. Right?
    if '=' in style and valsep == ' ':
        raise ValueError('Incompatible style and valsep: s={} v={}'.format(
            style, valsep))
Paul McCarthy's avatar
Paul McCarthy committed
300
301
302
303
304

    if argmap is None: argmap = {}
    if valmap is None: valmap = {}

    def fmtarg(arg):
305
306
307
308
        if   style in ('-',  '-=') or (singlechar_args and len(arg) == 1):
            arg =  '-{}'.format(arg)
        elif style in ('--', '--='):
            arg = '--{}'.format(arg)
309
310
        return arg

311
    # always returns a sequence
Paul McCarthy's avatar
Paul McCarthy committed
312
    def fmtval(val):
313
        if     isinstance(val, abc.Sequence) and \
314
           not isinstance(val, six.string_types):
315
316
317
318
319

            val = [str(v) for v in val]
            if   valsep == ' ': return val
            elif valsep == '"': return [' '   .join(val)]
            else:               return [valsep.join(val)]
320
        else:
321
322
323
324
325
326
327
328
            return [str(val)]

    # val is assumed to be a sequence
    def fmtargval(arg, val):
        # if '=' in style, val will
        # always be a single string
        if '=' in style: return ['{}={}'.format(arg, val[0])]
        else:            return [arg] + val
329
330
331
332
333

    args = []

    for k, v in kwargs.items():

334
335
        if v is None: continue

336
        k    = argmap.get(k, k)
Paul McCarthy's avatar
Paul McCarthy committed
337
338
339
        mapv = valmap.get(k, fmtval(v))
        k    = fmtarg(k)

340
341
342
343
        if mapv in (SHOW_IF_TRUE, HIDE_IF_TRUE):
            if (mapv is SHOW_IF_TRUE and     v) or \
               (mapv is HIDE_IF_TRUE and not v):
                args.append(k)
344
        else:
345
            args.extend(fmtargval(k, mapv))
346
347
348
349

    return args


350
def namedPositionals(func, args):
351
    """Given a function, and a sequence of positional arguments destined
352
    for that function, identifies the name for each positional argument.
353
    Variable positional arguments are given an automatic name.
354

355
356
    :arg func: Function which will accept ``args`` as positionals.
    :arg args: Tuple of positional arguments to be passed to ``func``.
357
    """
Paul McCarthy's avatar
Paul McCarthy committed
358

359
360
361
362
363
364
365
366
367
368
    # Current implementation will
    # result in naming collisions
    # for something like this:
    #
    # def func(args0, *args):
    #     ...
    # because of automatic vararg
    # naming. But who would write
    # a function like that anyway?

369
370
    # Remove any decorators
    # from the function
Paul McCarthy's avatar
Paul McCarthy committed
371
372
    func = _unwrap(func)

373
374
375
    # getargspec is the only way to
    # get the names of positional
    # arguments in Python 2.x.
Paul McCarthy's avatar
Paul McCarthy committed
376
    if sys.version_info[0] < 3:
377
378
379
        spec     = inspect.getargspec(func)
        argnames = spec.args
        varargs  = spec.varargs
Paul McCarthy's avatar
Paul McCarthy committed
380

381
382
    # But getargspec is deprecated
    # in python 3.x
Paul McCarthy's avatar
Paul McCarthy committed
383
384
385
386
387
388
    else:

        # getfullargspec is deprecated in
        # python 3.5, but not in python 3.6.
        with warnings.catch_warnings():
            warnings.filterwarnings('ignore', category=DeprecationWarning)
389
390
391
            spec     = inspect.getfullargspec(func)
            argnames = spec.args
            varargs  = spec.varargs
392

393
394
395
    # we only care about the arguments
    # that are being passed in
    argnames = argnames[:len(args)]
396

397
398
399
400
    # make up names for varargs
    nvarargs = len(args) - len(argnames)
    if varargs is not None and nvarargs > 0:
        argnames += ['{}{}'.format(varargs, i) for i in range(nvarargs)]
401

402
    return argnames
403
404


Paul McCarthy's avatar
Paul McCarthy committed
405
LOAD = object()
406
"""Constant used by the :class:`FileOrThing` class to indicate that an output
Paul McCarthy's avatar
Paul McCarthy committed
407
file should be loaded into memory and returned as a Python object.
408
409
410
"""


411
class FileOrThing(object):
Paul McCarthy's avatar
Paul McCarthy committed
412
413
414
415
416
    """Decorator which ensures that certain arguments which are passed into the
    decorated function are always passed as file names. Both positional and
    keyword arguments can be specified.


417
    The ``FileOrThing`` class is not intended to be used directly - see the
Paul McCarthy's avatar
Paul McCarthy committed
418
419
420
    :func:`fileOrImage` and :func:`fileOrArray` decorator functions for more
    details.

421

Paul McCarthy's avatar
Paul McCarthy committed
422
423
424
425
426
427
428
429
430
431
432
433
    These decorators are intended for functions which wrap a command-line tool,
    i.e. where some inputs/outputs need to be specified as file names.


    **Inputs**


    Any arguments which are not of type ``Thing`` are passed through to the
    decorated function unmodified.  Arguments which are of type ``Thing`` are
    saved to a temporary file, and the name of that file is passed to the
    function.

434

Paul McCarthy's avatar
Paul McCarthy committed
435
    **Outputs**
436
437


Paul McCarthy's avatar
Paul McCarthy committed
438
    If an argument is given the special :data:`LOAD` value, it is assumed
Paul McCarthy's avatar
Paul McCarthy committed
439
440
441
    to be an output argument. In this case, it is replaced with a temporary
    file name then, after the function has completed, that file is loaded
    into memory, and the value returned (along with the function's output,
Paul McCarthy's avatar
Paul McCarthy committed
442
    and any other arguments with a value of ``LOAD``).
Paul McCarthy's avatar
Paul McCarthy committed
443
444
445
446
447


    **Return value**


448
    Functions decorated with a ``FileOrThing`` decorator will always return a
449
    ``dict``-like object, where the function's actual return value is
450
    accessible via an attribute called ``stdout``. All output arguments with a
451
    value of ``LOAD`` will be present as dictionary entries, with the keyword
452
453
454
455
    argument names used as keys; these values will also be accessible as
    attributes of the results dict, when possible. Any ``LOAD`` output
    arguments which were not generated by the function will not be present in
    the dictionary.
Paul McCarthy's avatar
Paul McCarthy committed
456
457


458
459
460
461
    **Cluster submission**


    The above description holds in all situations, except when an argument
462
    called ``submit`` is passed, and is set to a value which evaluates to
463
    ``True``. In this case, the ``FileOrThing`` decorator will pass all
464
465
    arguments straight through to the decorated function, and will return its
    return value unchanged.
466
467
468
469


    This is because most functions that are decorated with the
    :func:`fileOrImage` or :func:`fileOrArray` decorators will invoke a call
Paul McCarthy's avatar
Paul McCarthy committed
470
471
472
    to :func:`.run.run` or :func:`.runfsl`, where a value of ``submit=True``
    will cause the command to be executed asynchronously on a cluster
    platform.
473
474
475
476
477
478


    A :exc:`ValueError` will be raised if the decorated function is called
    with ``submit=True``, and with any in-memory objects or ``LOAD`` symbols.


Paul McCarthy's avatar
Paul McCarthy committed
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
    **Example**


    As an example of using the ``fileOrArray`` decorator on a function
    which concatenates two files containing affine transformations, and
    saves the output to a file::

        # if atob, btoc, or output are passed
        # in as arrays, they are converted to
        # file names.
        @fileOrArray('atob', 'btoc', 'output')
        def concat(atob, btoc, output=None):

            # inputs are guaranteed to be files
            atob = np.loadtxt(atob)
            btoc = np.loadtxt(atoc)

            atoc = np.dot(btoc, atob)

            if output is not None:
                np.savetxt(output, atoc)

501
502
            return 'Done'

Paul McCarthy's avatar
Paul McCarthy committed
503
504
505
506

    Because we have decorated the ``concat`` function with :func:`fileToArray`,
    it can be called with either file names, or Numpy arrays::

507

Paul McCarthy's avatar
Paul McCarthy committed
508
509
        # All arguments are passed through
        # unmodified - the output will be
510
        # saved to a file called atoc.mat.
Paul McCarthy's avatar
Paul McCarthy committed
511
512
        concat('atob.txt', 'btoc.txt', 'atoc.mat')

513
514
        # The function's return value
        # is accessed via an attribute called
515
516
        # "stdout" on the dict
        assert concat('atob.txt', 'btoc.txt', 'atoc.mat').stdout == 'Done'
517
518
519

        # Outputs to be loaded into memory
        # are returned in a dictionary,
520
521
522
        # with argument names as keys. Values
        # can be accessed as dict items, or
        # as attributes.
523
        atoc = concat('atob.txt', 'btoc.txt', LOAD)['atoc']
524
        atoc = concat('atob.txt', 'btoc.txt', LOAD).atoc
Paul McCarthy's avatar
Paul McCarthy committed
525

526
527
528
529
530
        # In-memory inputs are saved to
        # temporary files, and those file
        # names are passed to the concat
        # function.
        atoc = concat(np.diag([2, 2, 2, 0]),
531
                      np.diag([3, 3, 3, 3]), LOAD).atoc
532
533
534


    **Using with other decorators**
Paul McCarthy's avatar
Paul McCarthy committed
535
536


537
538
    ``FileOrThing`` decorators can be chained with other ``FileOrThing``
    decorators, and other decorators.  When multiple ``FileOrThing``
Paul McCarthy's avatar
Paul McCarthy committed
539
540
541
542
    decorators are used on a single function, the outputs from each decorator
    are merged together into a single dict-like object.


543
    ``FileOrThing`` decorators can be used with any other decorators
544
    **as long as** they do not manipulate the return value, and as long as
545
    the ``FileOrThing`` decorators are adjacent to each other.
546
547
548
    """


549
    class Results(dict):
550
        """A custom ``dict`` type used to return outputs from a function
551
        decorated with ``FileOrThing``. All outputs are stored as dictionary
552
553
554
        items, with the argument name as key, and the output object (the
        "thing") as value.

555
556
        Where possible (i.e. for outputs named with a valid Python
        identifier), the outputs are also made accessible as attributes of
557
        the ``Results`` object.
558

559
        The decorated function's actual return value is accessible via the
560
        :meth:`stdout` property.
561
        """
562
563


564
        def __init__(self, stdout):
565
            """Create a ``Results`` dict.
566

Paul McCarthy's avatar
Paul McCarthy committed
567
            :arg stdout: Return value of the decorated function (typically a
568
569
                         tuple containing the standard output and error of the
                         underlying command).
570
            """
571
            super().__init__()
572
573
574
575
            self.__stdout = stdout


        def __setitem__(self, key, val):
576
            """Add an item to the dict. The item is also added as an attribute.
577
578
            """
            super().__setitem__(key, val)
579
            setattr(self, key, val)
580

581

582
        @property
583
        def stdout(self):
584
            """Access the return value of the decorated function. """
585
            return self.__stdout
586
587


588
589
590
591
592
593
    def __init__(self,
                 func,
                 prepIn,
                 prepOut,
                 load,
                 removeExt,
Paul McCarthy's avatar
Paul McCarthy committed
594
595
                 *args,
                 **kwargs):
596
        """Initialise a ``FileOrThing`` decorator.
Paul McCarthy's avatar
Paul McCarthy committed
597

598
599
600
601
        :arg func:      The function to be decorated.

        :arg prepIn:    Function which returns a file name to be used in
                        place of an input argument.
602

603
604
        :arg prepOut:   Function which generates a file name to use for
                        arguments that were set to :data:`LOAD`.
Paul McCarthy's avatar
Paul McCarthy committed
605

606
607
608
        :arg load:      Function which is called to load items for arguments
                        that were set to :data:`LOAD`. Must accept a file path
                        as its sole argument.
609

610
611
612
        :arg removeExt: Function which can remove a file extension from a file
                        path.

Paul McCarthy's avatar
Paul McCarthy committed
613
614
615
616
617
618
619
620
        :arg outprefix: Must be passed as a keyword argument. The name of a
                        positional or keyword argument to the function, which
                        specifies an output file name prefix.  All other
                        arguments with names that begin with this prefix may
                        be interpreted as things to ``LOAD``.

        All other positional arguments are interpreted as the names of the
        arguments to the function which will be handled by this
621
        ``FileOrThing`` decorator. If not provided, *all* arguments passed to
Paul McCarthy's avatar
Paul McCarthy committed
622
        the function will be handled.
623

624
625
626
627
628
629

        The ``prepIn`` and ``prepOut`` functions must accept the following
        positional arguments:

          - A directory in which all temporary input/output files should be
            stored
Paul McCarthy's avatar
Paul McCarthy committed
630

631
632
633
634
          - The name of the keyword argument to be processed

          - The argument value that was passed in
        """
635
636
637
638
        self.__func      = func
        self.__prepIn    = prepIn
        self.__prepOut   = prepOut
        self.__load      = load
639
        self.__removeExt = removeExt
Paul McCarthy's avatar
Paul McCarthy committed
640
641
        self.__things    = args
        self.__outprefix = kwargs.get('outprefix', None)
Paul McCarthy's avatar
Paul McCarthy committed
642
643


644
    def __call__(self, *args, **kwargs):
645
        """Function which calls ``func``, ensuring that any arguments of
Paul McCarthy's avatar
Paul McCarthy committed
646
        type ``Thing`` are saved to temporary files, and any arguments
Paul McCarthy's avatar
Paul McCarthy committed
647
        with the value :data:`LOAD` are loaded and returned.
648

649
        All other arguments are passed through to ``func``.
650
        """
Paul McCarthy's avatar
Paul McCarthy committed
651

652
653
        func     = self.__func
        argnames = namedPositionals(func, args)
654

655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
        # Special case - if fsl.utils.run[fsl] is
        # being decorated (e.g. via cmdwrapper/
        # fslwrapper), and submit=True, this call
        # will ultimately submit the job to the
        # cluster, and will return immediately.
        #
        # We error if we are given any in-memory
        # things, or LOAD symbols.
        #
        # n.b. testing values to be strings could
        # interfere with the fileOrText decorator.
        # Possible solution is to use pathlib?
        if kwargs.get('submit', False):
            allargs = {**dict(zip(argnames, args)), **kwargs}
            for name, val in allargs.items():
                if (name in self.__things) and \
                   (not isinstance(val, six.string_types)):
                    raise ValueError('Cannot use in-memory objects '
                                     'or LOAD with submit=True!')
            return func(*args, **kwargs)

676
677
        # If this FileOrThing is being called
        # by another FileOrThing don't create
678
679
680
681
682
683
684
685
        # another working directory. We do this
        # sneakily, by setting an attribute on
        # the wrapped function which stores the
        # current working directory.
        wrapped     = _unwrap(func)
        fot_workdir = getattr(wrapped, '_fot_workdir', None)
        parent      = fot_workdir is None

686
        # Create a tempdir to store any temporary
Paul McCarthy's avatar
Paul McCarthy committed
687
        # input/output things, but don't change
688
689
        # into it, as file paths passed to the
        # function may be relative.
690
        with tempdir.tempdir(changeto=False, override=fot_workdir) as td:
691

692
693
            log.debug('Redirecting LOADed outputs to %s', td)

694
695
            # Replace any things with file names.
            # Also get a list of LOAD outputs
696
            args = self.__prepareArgs(parent, td, argnames, args, kwargs)
697
            args, kwargs, outprefix, outfiles, prefixes = args
698

699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
            # The prefix/patterns may be
            # overridden by a parent FoT
            outprefix = getattr(wrapped, '_fot_outprefix', outprefix)
            prefixes  = getattr(wrapped, '_fot_prefixes',  prefixes)

            # if there are any other FileOrThings
            # in the decorator chain, get them to
            # use our working directory, and
            # prefixes, instead of creating their
            # own.
            if parent:
                setattr(wrapped, '_fot_workdir',   td)
                setattr(wrapped, '_fot_outprefix', outprefix)
                setattr(wrapped, '_fot_prefixes',  prefixes)

714
            # Call the function
715
716
717
718
719
720
721
722
723
724
725
            try:
                result = func(*args, **kwargs)

            finally:
                # if we're the top-level FileOrThing
                # decorator, remove the attributes we
                # added above.
                if parent:
                    delattr(wrapped, '_fot_workdir')
                    delattr(wrapped, '_fot_outprefix')
                    delattr(wrapped, '_fot_prefixes')
726

727
728
            return self.__generateResult(
                td, result, outprefix, outfiles, prefixes)
729
730


731
    def __prepareArgs(self, parent, workdir, argnames, args, kwargs):
732
733
734
735
736
        """Prepares all input and output arguments to be passed to the
        decorated function. Any arguments with a value of :data:`LOAD` are
        passed to the ``prepOut`` function specified at :meth:`__init__`.
        All other arguments are passed through the ``prepIn`` function.

737
738
        :arg parent:  ``True`` if this ``FileOrThing`` is the first in a
                      chain of ``FileOrThing`` decorators.
739

740
741
        :arg workdir: Directory in which all temporary files should be stored.

742
743
744
        :arg args:    Positional arguments to be passed to the decorated
                      function.

745
746
747
748
        :arg kwargs:  Keyword arguments to be passed to the decorated function.

        :returns:     A tuple containing:

749
750
751
                        - An updated copy of ``args``.

                        - An updated copy of ``kwargs``.
752

753
754
755
756
757
758
759
                        - The output file prefix that was actually passed in
                          (it is subsequently modified so that prefixed outputs
                          are redirected to a temporary location). All prefixed
                          outputs that are not ``LOAD``ed should be moved into
                          this directory. ``None`` if there is no output
                          prefix.

760
761
                        - A dictionary of ``{ name : filename }`` mappings,
                          for all arguments with a value of ``LOAD``.
762

763
764
                        - A dictionary   ``{ filepat : replstr }`` paths, for
                          all output-prefix arguments with a value of ``LOAD``.
765
766
        """

767
768
769
770
        # These containers keep track
        # of output files which are to
        # be loaded into memory
        outfiles      = dict()
771
        prefixedFiles = dict()
772

773
774
775
        allargs  = {k : v for k, v in zip(argnames, args)}
        allargs.update(kwargs)

776
777
778
779
        # Has an output prefix been specified?
        prefix     = allargs.get(self.__outprefix, None)
        realPrefix = None

780
781
        # Prefixed outputs are only
        # managed by the parent
782
        # FileOrthing in a chain of
783
784
785
786
        # FoT decorators.
        if not parent:
            prefix = None

787
788
789
790
791
792
793
794
795
796
        # If so, replace it with a new output
        # prefix which will redirect all output
        # to the temp dir.
        #
        # Importantly, here we assume that the
        # underlying function (and hence the
        # underlying command-line tool) will
        # accept an output prefix which contains
        # a directory path.
        if prefix is not None:
797
798
799
800
801
802
803
804
805
806

            # If prefix is set to LOAD,
            # all generated output files
            # should be loaded - we use a
            # randomly generated prefix,
            # and add it to prefixedFiles,
            # so that every file which
            # starts with it will be
            # loaded.
            if prefix is LOAD:
807
808
809
                prefix                = random.sample(string.ascii_letters, 10)
                prefix                = ''.join(prefix)
                prefixedFiles[prefix] = self.__outprefix
810

811
            realPrefix                = prefix
812
813
814
            fakePrefix                = op.join(workdir, prefix)
            allargs[self.__outprefix] = fakePrefix

815
816
            log.debug('Replacing output prefix: %s -> %s',
                      realPrefix, fakePrefix)
817

818
819
820
821
            # If the prefix specifies a
            # directory, make sure it
            # exists (remember that we're
            # in a temporary directory)
822
823
824
            pdir = op.dirname(fakePrefix)
            if pdir != '' and not op.exists(pdir):
                os.makedirs(pdir)
825

826
827
        if len(self.__things) > 0: things = self.__things
        else:                      things = allargs.keys()
828

829
        for name, val in list(allargs.items()):
830

831
832
833
834
835
            # don't process the
            # outprefix argument
            if name == self.__outprefix:
                continue

836
            # is this argument referring
837
            # to a prefixed output?
838
839
            isprefixed = (prefix is not None and
                          name.startswith(prefix))
840

841
            if not (isprefixed or name in things):
842
843
                continue

844
845
846
847
            # Prefixed output files may only
            # be given a value of LOAD
            if isprefixed and val is not LOAD:
                raise ValueError('Cannot specify name of prefixed file - the '
848
849
                                 'name is defined by the output prefix: '
                                 '{}'.format(name))
850

851
            if val is LOAD:
852

853
854
855
856
857
858
859
                # this argument refers to an output
                # that is generated from the output
                # prefix argument, and doesn't map
                # directly to an argument of the
                # function. So we don't pass it
                # through.
                if isprefixed:
860
                    prefixedFiles[name] = name
861
862
863
864
865
                    allargs.pop(name)

                # regular output-file argument
                else:
                    outfile = self.__prepOut(workdir, name, val)
866
                    outfiles[name] = outfile
867
                    allargs[ name] = outfile
Paul McCarthy's avatar
Paul McCarthy committed
868

869
            # Assumed to be an input file
870
            else:
871
872
                # sequences may be
                # accepted for inputs
873
874
875
876
877
878
879
880
881
                if isinstance(val, (list, tuple)):
                    infile = list(val)
                    for i, v in enumerate(val):
                        v = self.__prepIn(workdir, name, v)
                        if v is not None:
                            infile[i] = v

                else:
                    infile = self.__prepIn(workdir, name, val)
Paul McCarthy's avatar
Paul McCarthy committed
882

883
                if infile is not None:
884
885
                    allargs[name] = infile

886
887
888
        if realPrefix is not None and len(prefixedFiles) == 0:
            allargs[self.__outprefix] = realPrefix

889
890
        args   = [allargs.pop(k) for k in argnames]
        kwargs = allargs
891

892
        return args, kwargs, realPrefix, outfiles, prefixedFiles
893
894


895
896
    def __generateResult(
            self, workdir, result, outprefix, outfiles, prefixes):
897
        """Loads function outputs and returns a :class:`Results` object.
898
899
900

        Called by :meth:`__call__` after the decorated function has been
        called. Figures out what files should be loaded, and loads them into
901
        a ``Results`` object.
902
903
904
905
906
907
908
909
910
911

        :arg workdir:   Directory which contains the function outputs.
        :arg result:    Function return value.
        :arg outprefix: Original output prefix that was passed into the
                        function (or ``None`` if one wasn't passed)
        :arg outfiles:  Dictionary containing output files to be loaded (see
                        :meth:`__prepareArgs`).
        :arg prefixes:  Dictionary containing output-prefix patterns to be
                        loaded (see :meth:`__prepareArgs`).

912
        :returns:       A ``Results`` object containing all loaded outputs.
913
914
        """

915
        # make a Results object to store
916
        # the output. If we are decorating
917
        # another FileOrThing, the
918
        # results will get merged together
919
920
921
        # into a single Results dict.
        if not isinstance(result, FileOrThing.Results):
            result = FileOrThing.Results(result)
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960

        # Load the LOADed outputs
        for oname, ofile in outfiles.items():

            log.debug('Loading output %s: %s', oname, ofile)

            if op.exists(ofile): oval = self.__load(ofile)
            else:                oval = None

            result[oname] = oval

        # No output prefix - we're done
        if outprefix is None or len(prefixes) == 0:
            return result

        # Load or move output-prefixed files.
        # Find all files with a name that
        # matches the prefix that was passed
        # in (recursing into matching sub-
        # directories too).
        allPrefixed = glob.glob(op.join(workdir, '{}*'.format(outprefix)))
        allPrefixed = [fslpath.allFiles(f) if op.isdir(f) else [f]
                       for f in allPrefixed]

        for prefixed in it.chain(*allPrefixed):
            fullpath = prefixed
            prefixed = op.relpath(prefixed, workdir)
            for prefPat, prefName in prefixes.items():
                if not fnmatch.fnmatch(prefixed, '{}*'.format(prefPat)):
                    continue

                log.debug('Loading prefixed output %s [%s]: %s',
                          prefPat, prefName, prefixed)

                # if the load function returns
                # None, this file is probably
                # not of the correct type.
                fval = self.__load(fullpath)
                if fval is not None:
961
                    noext = self.__removeExt(prefixed)
Paul McCarthy's avatar
Paul McCarthy committed
962
                    prefPat  = prefPat.replace('\\', '\\\\')
963
                    noext = re.sub('^' + prefPat, prefName, noext)
964
965
966
                    # If there is already an item in result with the
                    # name (stripped of prefix), then instead store
                    # the result with the full prefixed name
967
968
969
970
971
                    if noext not in result:
                        result[noext] = fval
                    else:
                        withext = re.sub('^' + prefPat, prefName, prefixed)
                        result[withext] = fval
972
973
974
975
976
                    break

        return result


977
def fileOrImage(*args, **kwargs):
Paul McCarthy's avatar
Paul McCarthy committed
978
979
    """Decorator which can be used to ensure that any NIfTI images are saved
    to file, and output images can be loaded and returned as ``nibabel``
980
    image objects or :class:`.Image` objects.
Paul McCarthy's avatar
Paul McCarthy committed
981
982
    """

983
984
985
986
987
988
    # keep track of the input argument
    # types on each call, so we know
    # whether to return a fsl.Image or
    # a nibabel image
    intypes = []

989
    def prepIn(workdir, name, val):
Paul McCarthy's avatar
Paul McCarthy committed
990

991
        infile = None
Paul McCarthy's avatar
Paul McCarthy committed
992

993
994
995
996
997
        if isinstance(val, fslimage.Image):
            intypes.append(fslimage.Image)

        elif isinstance(val, nib.nifti1.Nifti1Image):
            intypes.append(nib.nifti1.Nifti1Image)
998
999
1000
1001

        if isinstance(val, fslimage.Image):
            val = val.nibImage

Paul McCarthy's avatar
Paul McCarthy committed
1002
        if isinstance(val, nib.nifti1.Nifti1Image):
1003
            infile = val.get_filename()
Paul McCarthy's avatar
Paul McCarthy committed
1004
1005
1006

            # in-memory image - we have
            # to save it out to a file
1007
1008
            if infile is None:
                hd, infile = tempfile.mkstemp(fslimage.defaultExt())
Paul McCarthy's avatar
Paul McCarthy committed
1009
                os.close(hd)
1010
                val.to_filename(infile)
Paul McCarthy's avatar
Paul McCarthy committed
1011

1012
        return infile
Paul McCarthy's avatar
Paul McCarthy committed
1013

1014
1015
    def prepOut(workdir, name, val):
        return op.join(workdir, '{}.nii.gz'.format(name))
Paul McCarthy's avatar
Paul McCarthy committed
1016

1017
    def load(path):
1018
1019
1020
1021

        if not fslimage.looksLikeImage(path):
            return None

Paul McCarthy's avatar
Paul McCarthy committed
1022
1023
        # create an independent in-memory
        # copy of the image file
Paul McCarthy's avatar
Paul McCarthy committed
1024
1025
        img  = nib.load(path, mmap=False)
        data = np.asanyarray(img.dataobj)
1026
1027
1028
1029

        # if any arguments were fsl images,
        # that takes precedence.
        if fslimage.Image in intypes:
Paul McCarthy's avatar
Paul McCarthy committed
1030
            return fslimage.Image(data, header=img.header)
1031
1032
1033
        # but if all inputs were file names,
        # nibabel takes precedence
        elif nib.nifti1.Nifti1Image in intypes or len(intypes) == 0:
Paul McCarthy's avatar
Paul McCarthy committed
1034
            return nib.nifti1.Nifti1Image(data, None, img.header)
1035
1036
1037
1038
1039

        # this function should not be called
        # under any other circumstances
        else:
            raise RuntimeError('Cannot handle type: {}'.format(intypes))
Paul McCarthy's avatar
Paul McCarthy committed
1040

1041
    def decorator(func):
1042
1043
1044
1045
1046
1047
1048
        fot = FileOrThing(func,
                          prepIn,
                          prepOut,
                          load,
                          fslimage.removeExt,
                          *args,
                          **kwargs)
1049
1050

        def wrapper(*args, **kwargs):
1051
1052
1053
            result = fot(*args, **kwargs)
            intypes[:] = []
            return result
1054
1055
1056
1057

        return _update_wrapper(wrapper, func)

    return decorator
Paul McCarthy's avatar
Paul McCarthy committed
1058
1059


1060
def fileOrArray(*args, **kwargs):
Paul McCarthy's avatar
Paul McCarthy committed
1061
1062
1063
1064
    """Decorator which can be used to ensure that any Numpy arrays are saved
    to text files, and output files can be loaded and returned as Numpy arrays.
    """

1065
    def prepIn(workdir, name, val):
Paul McCarthy's avatar
Paul McCarthy committed
1066

1067
        infile = None
Paul McCarthy's avatar
Paul McCarthy committed
1068
1069

        if isinstance(val, np.ndarray):
1070
            hd, infile = tempfile.mkstemp('.txt')
Paul McCarthy's avatar
Paul McCarthy committed
1071
            os.close(hd)
1072
            np.savetxt(infile, val, fmt='%0.18f')
Paul McCarthy's avatar
Paul McCarthy committed
1073

1074
        return infile
Paul McCarthy's avatar
Paul McCarthy committed
1075

1076
1077
    def prepOut(workdir, name, val):
        return op.join(workdir, '{}.txt'.format(name))
Paul McCarthy's avatar
Paul McCarthy committed
1078

1079
1080
1081
    def load(path):
        try:              return np.loadtxt(path)
        except Exception: return None
Paul McCarthy's avatar
Paul McCarthy committed
1082

1083
    def decorator(func):
1084
1085
1086
1087
1088
1089
1090
        fot = FileOrThing(func,
                          prepIn,
                          prepOut,
                          load,
                          fslpath.removeExt,
                          *args,
                          **kwargs)
1091
1092
1093
1094
1095
1096
1097

        def wrapper(*args, **kwargs):
            return fot(*args, **kwargs)

        return _update_wrapper(wrapper, func)

    return decorator
1098

1099

1100
def fileOrText(*args, **kwargs):
1101
1102
1103
    """Decorator which can be used to ensure that any text output (e.g. log
    file) are saved to text files, and output files can be loaded and returned
    as strings.
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126

    To be able to distinguish between input values and input file paths, the
    ``fileOrText`` decorator requires that input and output file paths are
    passed in as ``pathlib.Path`` objects. For example, given a function
    like this::

        @fileOrText()
        def myfunc(infile, outfile):
            ...

    if we want to pass file paths for both ``infile`` and ``outfile``, we would
    do this::

        from pathlib import Path
        myfunc(Path('input.txt'), Path('output.txt'))

    Input values may be passed in as normal strings, e.g.::

        myfunc('input data', Path('output.txt'))

    Output values can be loaded as normal via the :attr:`LOAD` symbol, e.g.::

        myfunc(Path('input.txt'), LOAD)
1127
1128
1129
1130
1131
1132
    """

    def prepIn(workdir, name, val):

        infile = None

1133
1134
1135
1136
        if not isinstance(val, pathlib.Path):
            with tempfile.NamedTemporaryFile(mode='w',
                                             suffix='.txt',
                                             delete=False) as f:
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
                f.write(val)
                infile = f.name
        return infile

    def prepOut(workdir, name, val):
        return op.join(workdir, '{}.txt'.format(name))

    def load(path):
        try:
            with open(path, "r") as f:
                return f.read()
        except Exception: return None

    def decorator(func):
1151
1152
1153
1154
1155
1156
1157
        fot = FileOrThing(func,
                          prepIn,
                          prepOut,
                          load,
                          fslpath.removeExt,
                          *args,
                          **kwargs)
1158
1159
1160
1161
1162
1163
1164

        def wrapper(*args, **kwargs):
            return fot(*args, **kwargs)

        return _update_wrapper(wrapper, func)

    return decorator