wrapperutils.py 40.8 KB
Newer Older
1
2
#!/usr/bin/env python
#
Paul McCarthy's avatar
Paul McCarthy committed
3
4
# wrapperutils.py - Functions and decorators used by the FSL wrapper
# functions.
5
6
#
# Author: Paul McCarthy <pauldmccarthy@gmail.com>
7
# Author: Martin Craig <martin.craig@eng.ox.ac.uk>
8
#
Paul McCarthy's avatar
Paul McCarthy committed
9
10
11
12
"""This module contains functions and decorators used by the FSL wrapper
functions.


Paul McCarthy's avatar
Paul McCarthy committed
13
The :func:`cmdwrapper` and :func:`fslwrapper` functions are convenience
14
15
16
17
18
19
20
21
22
23
24
25
26
27
decorators which allow you to write your wrapper function such that it simply
generates the command-line needed to respectively run a standard shell
command or a FSL command. For example::


    @fslwrapper
    def fslreorient2std(input, output):
        return ['fslreorient2std', input, output]


When this ``fslreorient2std`` function is called, the ``fslwrapper`` decorator
will take care of invoking the command in a standardised way.


Paul McCarthy's avatar
Paul McCarthy committed
28
The :func:`applyArgStyle` function can be used to automatically convert
29
30
31
32
33
34
35
36
37
38
39
40
41
keyword arguments into command-line arguments, based on a set of standard
patterns. For example::


    @fslwrapper
    def flirt(src, ref, **kwargs):
        cmd  = ['flirt', '-in', src, '-ref', ref]
        return cmd + applyArgStyle('-=', **kwargs)


The :func:`fileOrImage` and :func:`fileOrArray` functions can be used to
decorate a wrapper function such that in-memory ``nibabel`` images or Numpy
arrays can be passed in as arguments - they will be automatically saved out to
Paul McCarthy's avatar
Paul McCarthy committed
42
files, and then the file names passed into the wrapper function. For example::
43
44
45
46
47
48
49
50
51
52
53
54
55


    @fileOrImage('src', 'ref')
    @fslwrapper
    def flirt(src, ref, **kwargs):
        cmd  = ['flirt', '-in', src, '-ref', ref]
        return cmd + applyArgStyle('-=', **kwargs)


Now this ``flirt`` function can be called either with file names, or
``nibabel`` images.


56
57
58
59
60
61
62
.. note:: Because the :func:`fileOrImage` and :func:`fileOrArray` decorators
          manipulate the return value of the decorated function, they should
          be applied *after* any other decorators. Furthermore, if you need to
          apply both a ``fileOrImage`` and ``fileOrArray`` decorator to a
          function, they should be grouped together, e.g.::

              @fileOrImage('a', 'b')
Paul McCarthy's avatar
Paul McCarthy committed
63
              @fileOrArray('c', 'd')
64
65
66
67
68
              @fslwrapper
              def func(**kwargs):
                  ...


69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
Command outputs can also be loaded back into memory by using the special
:data:`LOAD` value when calling a wrapper function. For example::


    @fileOrImage('src', 'ref', 'out')
    @fslwrapper
    def flirt(src, ref, **kwargs):
        cmd  = ['flirt', '-in', src, '-ref', ref]
        return cmd + applyArgStyle('-=', **kwargs)


If we set the ``out`` argument to ``LOAD``, the output image will be loaded
and returned::

    src     = nib.load('src.nii')
    ref     = nib.load('ref.nii')
    aligned = flirt(src, ref, out=LOAD)['out']
Paul McCarthy's avatar
Paul McCarthy committed
86
"""
87
88


89
90
91
92
93
94
95
96
97
import itertools       as it
import os.path         as op
import collections.abc as abc
import                    os
import                    re
import                    sys
import                    glob
import                    random
import                    string
98
import                    pathlib
99
100
101
102
103
104
105
import                    fnmatch
import                    inspect
import                    logging
import                    tempfile
import                    warnings
import                    functools

106
107
108

import            six
import nibabel as nib
Paul McCarthy's avatar
Paul McCarthy committed
109
import numpy   as np
110

111
import fsl.utils.run     as run
112
113
import fsl.utils.path    as fslpath
import fsl.utils.tempdir as tempdir
114
115
116
import fsl.data.image    as fslimage


117
118
119
log = logging.getLogger(__name__)


Paul McCarthy's avatar
Paul McCarthy committed
120
121
122
123
124
def _update_wrapper(wrapper, wrapped, *args, **kwargs):
    """Replacement for the built-in ``functools.update_wrapper``. This
    implementation ensures that the wrapper function has an attribute
    called ``__wrapped__``, which refers to the ``wrapped`` function.

125
    This custom function is only needed in Python versions < 3.4.
Paul McCarthy's avatar
Paul McCarthy committed
126
127
128
129
130
    """

    wrapper = functools.update_wrapper(wrapper, wrapped, *args, **kwargs)

    # Python >= 3.4 does things right
Paul McCarthy's avatar
Paul McCarthy committed
131
    if (sys.version_info[0] * 10 + sys.version_info[1]) < 34:
Paul McCarthy's avatar
Paul McCarthy committed
132
133
        wrapper.__wrapped__ = wrapped
    return wrapper
134
135


Paul McCarthy's avatar
Paul McCarthy committed
136
137
138
139
140
141
def _unwrap(func):
    """Replacement for the built-in ``inspect.unwrap`` function, which
    is not present in Python versions prior to 3.4.
    """

    # Python >= 3.4 has an inspect.unwrap function
Paul McCarthy's avatar
Paul McCarthy committed
142
    if (sys.version_info[0] * 10 + sys.version_info[1]) >= 34:
Paul McCarthy's avatar
Paul McCarthy committed
143
144
145
146
147
148
149
150
151
        return inspect.unwrap(func)

    # Otherwise we follow the __wrapped__ chain ourselves
    if hasattr(func, '__wrapped__'):
        return _unwrap(func.__wrapped__)

    return func


152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
def genxwrapper(func, runner):
    """This function is used by :func:`cmdwrapper` and :func:`fslwrapper`.
    It is not intended to be used in any other circumstances.

    This function generates a wrapper function which calls ``func`` to
    generate a command-line call, and then uses ``runner`` to invoke that
    command.

    ``func`` is assumed to be a wrapper function which generates a command-
    line. ``runner`` is assumed to be Either :func:`.run.run` or
    :func:`.run.runfsl`.

    The generated wrapper function will pass all of its arguments to ``func``,
    and will then pass the generated command-line to ``runner``, returning
    whatever is returned.

    The following keyword arguments will be intercepted by the wrapper
    function, and will *not* be passed to ``func``:
      - ``stdout``:   Passed to ``runner``. Defaults to ``True``.
      - ``stderr``:   Passed to ``runner``. Defaults to ``True``.
      - ``exitcode``: Passed to ``runner``. Defaults to ``False``.
      - ``submit``:   Passed to ``runner``. Defaults to ``None``.
      - ``log``:      Passed to ``runner``. Defaults to ``{'tee':True}``.
      - ``cmdonly``:  Defaults to ``False``. If ``True``, the return
        value of ``func`` is returned directly, instead of it being passed
        to ``runner``,

    :arg func:   A function which generates a command line.
    :arg runner: Either :func:`.run.run` or :func:`.run.runfsl`.
181
    """
182

183
    def wrapper(*args, **kwargs):
184
185
        stdout   = kwargs.pop('stdout',   True)
        stderr   = kwargs.pop('stderr',   True)
186
        exitcode = kwargs.pop('exitcode', False)
187
        cmdonly  = kwargs.pop('cmdonly',  False)
188
189
190
        submit   = kwargs.pop('submit',   None)
        log      = kwargs.pop('log',      {'tee' : True})
        cmd      = func(*args, **kwargs)
191
192
193
194
195
196
197
198
199
200

        if cmdonly:
            return cmd
        else:
            return runner(cmd,
                          stderr=stderr,
                          log=log,
                          submit=submit,
                          stdout=stdout,
                          exitcode=exitcode)
201
202
203
    return _update_wrapper(wrapper, func)


204
205
206
207
208
209
210
211
212
213
def cmdwrapper(func):
    """This decorator can be used on functions which generate a command line.
    It will pass the return value of the function to the
    :func:`fsl.utils.run.run` function in a standardised manner.

    See the :func:`genxwrapper` function for details.
    """
    return genxwrapper(func, run.run)


214
215
216
217
def fslwrapper(func):
    """This decorator can be used on functions which generate a FSL command
    line. It will pass the return value of the function to the
    :func:`fsl.utils.run.runfsl` function in a standardised manner.
218
219

    See the :func:`genxwrapper` function for details.
220
    """
221
    return genxwrapper(func, run.runfsl)
222
223


Paul McCarthy's avatar
Paul McCarthy committed
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
SHOW_IF_TRUE = object()
"""Constant to be used in the ``valmap`` passed to the :func:`applyArgStyle`
function.

When a ``SHOW_IF_TRUE`` argument is ``True``, it is added to the generated
command line arguments.
"""


HIDE_IF_TRUE = object()
"""Constant to be used in the ``valmap`` passed to the :func:`applyArgStyle`
function.

When a ``HIDE_IF_TRUE`` argument is ``True``, it is suppressed from the
generated command line arguments.
"""
240
241


242
243
244
245
246
247
def applyArgStyle(style,
                  valsep=None,
                  argmap=None,
                  valmap=None,
                  singlechar_args=False,
                  **kwargs):
Paul McCarthy's avatar
Paul McCarthy committed
248
249
250
251
    """Turns the given ``kwargs`` into command line options. This function
    is intended to be used to automatically generate command line options
    from arguments passed into a Python function.

252
253
254
255
256
257
258
    The ``style`` and ``valsep`` arguments control how key-value pairs
    are converted into command-line options:


    =========  ==========  ===========================
    ``style``  ``valsep``  Result
    =========  ==========  ===========================
Paul McCarthy's avatar
Paul McCarthy committed
259
260
261
262
263
264
265
266
267
268
269
270
    ``'-'``    ``' '``     ``-name val1 val2 val3``
    ``'-'``    ``'"'``     ``-name "val1 val2 val3"``
    ``'-'``    ``','``     ``-name val1,val2,val3``
    ``'--'``   ``' '``     ``--name val1 val2 val3``
    ``'--'``   ``'"'``     ``--name "val1 val2 val3"``
    ``'--'``   ``','``     ``--name val1,val2,val3``
    ``'-='``   ``' '``     Not supported
    ``'-='``   ``'"'``     ``-name="val1 val2 val3"``
    ``'-='``   ``','``     ``-name=val1,val2,val3``
    ``'--='``  ``' '``     Not supported
    ``'--='``  ``'"'``     ``--name="val1 val2 val3"``
    ``'--='``  ``','``     ``--name=val1,val2,val3``
271
272
273
    =========  ==========  ===========================


Paul McCarthy's avatar
Paul McCarthy committed
274
    :arg style:  Controls how the ``kwargs`` are converted into command-line
275
276
277
278
279
                 options - must be one of ``'-'``, ``'--'``, ``'-='``, or
                 ``'--='``.

    :arg valsep: Controls how the values passed to command-line options
                 which expect multiple arguments are delimited - must be
280
281
                 one of ``' '``, ``','`` or ``'"'``. Defaults to ``' '``
                 if ``'=' not in style``, ``','`` otherwise.
Paul McCarthy's avatar
Paul McCarthy committed
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304

    :arg argmap: Dictionary of ``{kwarg-name : cli-name}`` mappings. This can
                 be used if you want to use different argument names in your
                 Python function for the command-line options.

    :arg valmap: Dictionary of ``{cli-name : value}`` mappings. This can be
                 used to define specific semantics for some command-line
                 options. Acceptable values for ``value`` are as follows

                  - :data:`SHOW_IF_TRUE` - if the argument is present, and
                    ``True`` in ``kwargs``, the command line option
                    will be added (without any arguments).

                  - :data:`HIDE_IF_TRUE` - if the argument is present, and
                    ``False`` in ``kwargs``, the command line option
                    will be added (without any arguments).

                  - Any other constant value. If the argument is present
                    in ``kwargs``, its command-line option will be
                    added, with the constant value as its argument.

                 The argument for any options not specified in the ``valmap``
                 will be converted into strings.
305

306
307
308
    :arg singlechar_args: If True, single character arguments always take a
                          single hyphen prefix (e.g. -h) regardless of the
                          style.
309

Paul McCarthy's avatar
Paul McCarthy committed
310
311
312
313
314
    :arg kwargs: Arguments to be converted into command-line options.

    :returns:    A list containing the generated command-line options.
    """

315
316
317
318
    if valsep is None:
        if '=' in style: valsep = ','
        else:            valsep = ' '

Paul McCarthy's avatar
Paul McCarthy committed
319
320
    if style not in ('-', '--', '-=', '--='):
        raise ValueError('Invalid style: {}'.format(style))
321
322
323
324
325
326
327
328
329
    if valsep not in (' ', ',', '"'):
        raise ValueError('Invalid valsep: {}'.format(valsep))

    # we don't handle the case where '=' in
    # style, and valsep == ' ', because no
    # sane CLI app would do this. Right?
    if '=' in style and valsep == ' ':
        raise ValueError('Incompatible style and valsep: s={} v={}'.format(
            style, valsep))
Paul McCarthy's avatar
Paul McCarthy committed
330
331
332
333
334

    if argmap is None: argmap = {}
    if valmap is None: valmap = {}

    def fmtarg(arg):
335
336
337
338
        if   style in ('-',  '-=') or (singlechar_args and len(arg) == 1):
            arg =  '-{}'.format(arg)
        elif style in ('--', '--='):
            arg = '--{}'.format(arg)
339
340
        return arg

341
    # always returns a sequence
Paul McCarthy's avatar
Paul McCarthy committed
342
    def fmtval(val):
343
        if     isinstance(val, abc.Sequence) and \
344
           not isinstance(val, six.string_types):
345
346
347
348
349

            val = [str(v) for v in val]
            if   valsep == ' ': return val
            elif valsep == '"': return [' '   .join(val)]
            else:               return [valsep.join(val)]
350
        else:
351
352
353
354
355
356
357
358
            return [str(val)]

    # val is assumed to be a sequence
    def fmtargval(arg, val):
        # if '=' in style, val will
        # always be a single string
        if '=' in style: return ['{}={}'.format(arg, val[0])]
        else:            return [arg] + val
359
360
361
362
363

    args = []

    for k, v in kwargs.items():

364
365
        if v is None: continue

366
        k    = argmap.get(k, k)
Paul McCarthy's avatar
Paul McCarthy committed
367
368
369
        mapv = valmap.get(k, fmtval(v))
        k    = fmtarg(k)

370
371
372
373
        if mapv in (SHOW_IF_TRUE, HIDE_IF_TRUE):
            if (mapv is SHOW_IF_TRUE and     v) or \
               (mapv is HIDE_IF_TRUE and not v):
                args.append(k)
374
        else:
375
            args.extend(fmtargval(k, mapv))
376
377
378
379

    return args


380
def namedPositionals(func, args):
381
    """Given a function, and a sequence of positional arguments destined
382
    for that function, identifies the name for each positional argument.
383
    Variable positional arguments are given an automatic name.
384

385
386
    :arg func: Function which will accept ``args`` as positionals.
    :arg args: Tuple of positional arguments to be passed to ``func``.
387
    """
Paul McCarthy's avatar
Paul McCarthy committed
388

389
390
391
392
393
394
395
396
397
398
    # Current implementation will
    # result in naming collisions
    # for something like this:
    #
    # def func(args0, *args):
    #     ...
    # because of automatic vararg
    # naming. But who would write
    # a function like that anyway?

399
400
    # Remove any decorators
    # from the function
Paul McCarthy's avatar
Paul McCarthy committed
401
402
    func = _unwrap(func)

403
404
405
    # getargspec is the only way to
    # get the names of positional
    # arguments in Python 2.x.
Paul McCarthy's avatar
Paul McCarthy committed
406
    if sys.version_info[0] < 3:
407
408
409
        spec     = inspect.getargspec(func)
        argnames = spec.args
        varargs  = spec.varargs
Paul McCarthy's avatar
Paul McCarthy committed
410

411
412
    # But getargspec is deprecated
    # in python 3.x
Paul McCarthy's avatar
Paul McCarthy committed
413
414
415
416
417
418
    else:

        # getfullargspec is deprecated in
        # python 3.5, but not in python 3.6.
        with warnings.catch_warnings():
            warnings.filterwarnings('ignore', category=DeprecationWarning)
419
420
421
            spec     = inspect.getfullargspec(func)
            argnames = spec.args
            varargs  = spec.varargs
422

423
424
425
    # we only care about the arguments
    # that are being passed in
    argnames = argnames[:len(args)]
426

427
428
429
430
    # make up names for varargs
    nvarargs = len(args) - len(argnames)
    if varargs is not None and nvarargs > 0:
        argnames += ['{}{}'.format(varargs, i) for i in range(nvarargs)]
431

432
    return argnames
433
434


Paul McCarthy's avatar
Paul McCarthy committed
435
LOAD = object()
436
"""Constant used by the :class:`FileOrThing` class to indicate that an output
Paul McCarthy's avatar
Paul McCarthy committed
437
file should be loaded into memory and returned as a Python object.
438
439
440
"""


441
class FileOrThing(object):
Paul McCarthy's avatar
Paul McCarthy committed
442
443
444
445
446
    """Decorator which ensures that certain arguments which are passed into the
    decorated function are always passed as file names. Both positional and
    keyword arguments can be specified.


447
    The ``FileOrThing`` class is not intended to be used directly - see the
Paul McCarthy's avatar
Paul McCarthy committed
448
449
450
    :func:`fileOrImage` and :func:`fileOrArray` decorator functions for more
    details.

451

Paul McCarthy's avatar
Paul McCarthy committed
452
453
454
455
456
457
458
459
460
461
462
463
    These decorators are intended for functions which wrap a command-line tool,
    i.e. where some inputs/outputs need to be specified as file names.


    **Inputs**


    Any arguments which are not of type ``Thing`` are passed through to the
    decorated function unmodified.  Arguments which are of type ``Thing`` are
    saved to a temporary file, and the name of that file is passed to the
    function.

464

Paul McCarthy's avatar
Paul McCarthy committed
465
    **Outputs**
466
467


Paul McCarthy's avatar
Paul McCarthy committed
468
    If an argument is given the special :data:`LOAD` value, it is assumed
Paul McCarthy's avatar
Paul McCarthy committed
469
470
471
    to be an output argument. In this case, it is replaced with a temporary
    file name then, after the function has completed, that file is loaded
    into memory, and the value returned (along with the function's output,
Paul McCarthy's avatar
Paul McCarthy committed
472
    and any other arguments with a value of ``LOAD``).
Paul McCarthy's avatar
Paul McCarthy committed
473
474
475
476
477


    **Return value**


478
    Functions decorated with a ``FileOrThing`` decorator will always return a
479
    ``dict``-like object, where the function's actual return value is
480
    accessible via an attribute called ``stdout``. All output arguments with a
481
    value of ``LOAD`` will be present as dictionary entries, with the keyword
482
483
484
485
    argument names used as keys; these values will also be accessible as
    attributes of the results dict, when possible. Any ``LOAD`` output
    arguments which were not generated by the function will not be present in
    the dictionary.
Paul McCarthy's avatar
Paul McCarthy committed
486
487


488
489
490
491
    **Cluster submission**


    The above description holds in all situations, except when an argument
492
    called ``submit`` is passed, and is set to a value which evaluates to
493
    ``True``. In this case, the ``FileOrThing`` decorator will pass all
494
495
    arguments straight through to the decorated function, and will return its
    return value unchanged.
496
497
498
499


    This is because most functions that are decorated with the
    :func:`fileOrImage` or :func:`fileOrArray` decorators will invoke a call
Paul McCarthy's avatar
Paul McCarthy committed
500
501
502
    to :func:`.run.run` or :func:`.runfsl`, where a value of ``submit=True``
    will cause the command to be executed asynchronously on a cluster
    platform.
503
504
505
506
507
508


    A :exc:`ValueError` will be raised if the decorated function is called
    with ``submit=True``, and with any in-memory objects or ``LOAD`` symbols.


Paul McCarthy's avatar
Paul McCarthy committed
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
    **Example**


    As an example of using the ``fileOrArray`` decorator on a function
    which concatenates two files containing affine transformations, and
    saves the output to a file::

        # if atob, btoc, or output are passed
        # in as arrays, they are converted to
        # file names.
        @fileOrArray('atob', 'btoc', 'output')
        def concat(atob, btoc, output=None):

            # inputs are guaranteed to be files
            atob = np.loadtxt(atob)
            btoc = np.loadtxt(atoc)

            atoc = np.dot(btoc, atob)

            if output is not None:
                np.savetxt(output, atoc)

531
532
            return 'Done'

Paul McCarthy's avatar
Paul McCarthy committed
533
534
535
536

    Because we have decorated the ``concat`` function with :func:`fileToArray`,
    it can be called with either file names, or Numpy arrays::

537

Paul McCarthy's avatar
Paul McCarthy committed
538
539
        # All arguments are passed through
        # unmodified - the output will be
540
        # saved to a file called atoc.mat.
Paul McCarthy's avatar
Paul McCarthy committed
541
542
        concat('atob.txt', 'btoc.txt', 'atoc.mat')

543
544
        # The function's return value
        # is accessed via an attribute called
545
546
        # "stdout" on the dict
        assert concat('atob.txt', 'btoc.txt', 'atoc.mat').stdout == 'Done'
547
548
549

        # Outputs to be loaded into memory
        # are returned in a dictionary,
550
551
552
        # with argument names as keys. Values
        # can be accessed as dict items, or
        # as attributes.
553
        atoc = concat('atob.txt', 'btoc.txt', LOAD)['atoc']
554
        atoc = concat('atob.txt', 'btoc.txt', LOAD).atoc
Paul McCarthy's avatar
Paul McCarthy committed
555

556
557
558
559
560
        # In-memory inputs are saved to
        # temporary files, and those file
        # names are passed to the concat
        # function.
        atoc = concat(np.diag([2, 2, 2, 0]),
561
                      np.diag([3, 3, 3, 3]), LOAD).atoc
562
563
564


    **Using with other decorators**
Paul McCarthy's avatar
Paul McCarthy committed
565
566


567
568
    ``FileOrThing`` decorators can be chained with other ``FileOrThing``
    decorators, and other decorators.  When multiple ``FileOrThing``
Paul McCarthy's avatar
Paul McCarthy committed
569
570
571
572
    decorators are used on a single function, the outputs from each decorator
    are merged together into a single dict-like object.


573
    ``FileOrThing`` decorators can be used with any other decorators
574
    **as long as** they do not manipulate the return value, and as long as
575
    the ``FileOrThing`` decorators are adjacent to each other.
576
577
578
    """


579
    class Results(dict):
580
        """A custom ``dict`` type used to return outputs from a function
581
        decorated with ``FileOrThing``. All outputs are stored as dictionary
582
583
584
        items, with the argument name as key, and the output object (the
        "thing") as value.

585
586
        Where possible (i.e. for outputs named with a valid Python
        identifier), the outputs are also made accessible as attributes of
587
        the ``Results`` object.
588

589
        The decorated function's actual return value is accessible via the
590
        :meth:`stdout` property.
591
        """
592
593


594
        def __init__(self, stdout):
595
            """Create a ``Results`` dict.
596

Paul McCarthy's avatar
Paul McCarthy committed
597
            :arg stdout: Return value of the decorated function (typically a
598
599
                         tuple containing the standard output and error of the
                         underlying command).
600
            """
601
            super().__init__()
602
603
604
605
            self.__stdout = stdout


        def __setitem__(self, key, val):
606
            """Add an item to the dict. The item is also added as an attribute.
607
608
            """
            super().__setitem__(key, val)
609
            setattr(self, key, val)
610

611

612
        @property
613
        def stdout(self):
614
            """Access the return value of the decorated function. """
615
            return self.__stdout
616
617


618
619
620
621
622
623
    def __init__(self,
                 func,
                 prepIn,
                 prepOut,
                 load,
                 removeExt,
Paul McCarthy's avatar
Paul McCarthy committed
624
625
                 *args,
                 **kwargs):
626
        """Initialise a ``FileOrThing`` decorator.
Paul McCarthy's avatar
Paul McCarthy committed
627

628
629
630
631
        :arg func:      The function to be decorated.

        :arg prepIn:    Function which returns a file name to be used in
                        place of an input argument.
632

633
634
        :arg prepOut:   Function which generates a file name to use for
                        arguments that were set to :data:`LOAD`.
Paul McCarthy's avatar
Paul McCarthy committed
635

636
        :arg load:      Function which is called to load items for arguments
637
638
                        that were set to :data:`LOAD`. Must accept the
                        following arguments:
Paul McCarthy's avatar
Paul McCarthy committed
639

640
641
                         - the name of the argument
                         - path to the file to be loaded
642

643
644
645
        :arg removeExt: Function which can remove a file extension from a file
                        path.

Paul McCarthy's avatar
Paul McCarthy committed
646
647
648
649
650
651
652
653
        :arg outprefix: Must be passed as a keyword argument. The name of a
                        positional or keyword argument to the function, which
                        specifies an output file name prefix.  All other
                        arguments with names that begin with this prefix may
                        be interpreted as things to ``LOAD``.

        All other positional arguments are interpreted as the names of the
        arguments to the function which will be handled by this
654
        ``FileOrThing`` decorator. If not provided, *all* arguments passed to
Paul McCarthy's avatar
Paul McCarthy committed
655
        the function will be handled.
656

657
658
659
660
661
662

        The ``prepIn`` and ``prepOut`` functions must accept the following
        positional arguments:

          - A directory in which all temporary input/output files should be
            stored
Paul McCarthy's avatar
Paul McCarthy committed
663

664
665
666
667
          - The name of the keyword argument to be processed

          - The argument value that was passed in
        """
668
669
670
671
        self.__func      = func
        self.__prepIn    = prepIn
        self.__prepOut   = prepOut
        self.__load      = load
672
        self.__removeExt = removeExt
Paul McCarthy's avatar
Paul McCarthy committed
673
674
        self.__things    = args
        self.__outprefix = kwargs.get('outprefix', None)
Paul McCarthy's avatar
Paul McCarthy committed
675
676


677
    def __call__(self, *args, **kwargs):
678
        """Function which calls ``func``, ensuring that any arguments of
Paul McCarthy's avatar
Paul McCarthy committed
679
        type ``Thing`` are saved to temporary files, and any arguments
Paul McCarthy's avatar
Paul McCarthy committed
680
        with the value :data:`LOAD` are loaded and returned.
681

682
        All other arguments are passed through to ``func``.
683
        """
Paul McCarthy's avatar
Paul McCarthy committed
684

685
686
        func     = self.__func
        argnames = namedPositionals(func, args)
687

688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
        # Special case - if fsl.utils.run[fsl] is
        # being decorated (e.g. via cmdwrapper/
        # fslwrapper), and submit=True, this call
        # will ultimately submit the job to the
        # cluster, and will return immediately.
        #
        # We error if we are given any in-memory
        # things, or LOAD symbols.
        #
        # n.b. testing values to be strings could
        # interfere with the fileOrText decorator.
        # Possible solution is to use pathlib?
        if kwargs.get('submit', False):
            allargs = {**dict(zip(argnames, args)), **kwargs}
            for name, val in allargs.items():
                if (name in self.__things) and \
                   (not isinstance(val, six.string_types)):
                    raise ValueError('Cannot use in-memory objects '
                                     'or LOAD with submit=True!')
            return func(*args, **kwargs)

709
710
        # If this FileOrThing is being called
        # by another FileOrThing don't create
711
712
713
714
715
716
717
718
        # another working directory. We do this
        # sneakily, by setting an attribute on
        # the wrapped function which stores the
        # current working directory.
        wrapped     = _unwrap(func)
        fot_workdir = getattr(wrapped, '_fot_workdir', None)
        parent      = fot_workdir is None

719
        # Create a tempdir to store any temporary
Paul McCarthy's avatar
Paul McCarthy committed
720
        # input/output things, but don't change
721
722
        # into it, as file paths passed to the
        # function may be relative.
723
        with tempdir.tempdir(changeto=False, override=fot_workdir) as td:
724

725
726
            log.debug('Redirecting LOADed outputs to %s', td)

727
728
            # Replace any things with file names.
            # Also get a list of LOAD outputs
729
            args = self.__prepareArgs(parent, td, argnames, args, kwargs)
730
            args, kwargs, outprefix, outfiles, prefixes = args
731

732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
            # The prefix/patterns may be
            # overridden by a parent FoT
            outprefix = getattr(wrapped, '_fot_outprefix', outprefix)
            prefixes  = getattr(wrapped, '_fot_prefixes',  prefixes)

            # if there are any other FileOrThings
            # in the decorator chain, get them to
            # use our working directory, and
            # prefixes, instead of creating their
            # own.
            if parent:
                setattr(wrapped, '_fot_workdir',   td)
                setattr(wrapped, '_fot_outprefix', outprefix)
                setattr(wrapped, '_fot_prefixes',  prefixes)

747
            # Call the function
748
749
750
751
752
753
754
755
756
757
758
            try:
                result = func(*args, **kwargs)

            finally:
                # if we're the top-level FileOrThing
                # decorator, remove the attributes we
                # added above.
                if parent:
                    delattr(wrapped, '_fot_workdir')
                    delattr(wrapped, '_fot_outprefix')
                    delattr(wrapped, '_fot_prefixes')
759

760
761
            return self.__generateResult(
                td, result, outprefix, outfiles, prefixes)
762
763


764
    def __prepareArgs(self, parent, workdir, argnames, args, kwargs):
765
766
767
768
769
        """Prepares all input and output arguments to be passed to the
        decorated function. Any arguments with a value of :data:`LOAD` are
        passed to the ``prepOut`` function specified at :meth:`__init__`.
        All other arguments are passed through the ``prepIn`` function.

770
771
        :arg parent:  ``True`` if this ``FileOrThing`` is the first in a
                      chain of ``FileOrThing`` decorators.
772

773
774
        :arg workdir: Directory in which all temporary files should be stored.

775
776
777
        :arg args:    Positional arguments to be passed to the decorated
                      function.

778
779
780
781
        :arg kwargs:  Keyword arguments to be passed to the decorated function.

        :returns:     A tuple containing:

782
783
784
                        - An updated copy of ``args``.

                        - An updated copy of ``kwargs``.
785

786
787
788
789
790
791
792
                        - The output file prefix that was actually passed in
                          (it is subsequently modified so that prefixed outputs
                          are redirected to a temporary location). All prefixed
                          outputs that are not ``LOAD``ed should be moved into
                          this directory. ``None`` if there is no output
                          prefix.

793
794
                        - A dictionary of ``{ name : filename }`` mappings,
                          for all arguments with a value of ``LOAD``.
795

796
                        - A dictionary of ``{ filepat : replstr }`` paths, for
797
                          all output-prefix arguments with a value of ``LOAD``.
798
799
        """

800
801
802
803
        # These containers keep track
        # of output files which are to
        # be loaded into memory
        outfiles      = dict()
804
        prefixedFiles = dict()
805

806
807
808
        allargs  = {k : v for k, v in zip(argnames, args)}
        allargs.update(kwargs)

809
810
811
812
        # Has an output prefix been specified?
        prefix     = allargs.get(self.__outprefix, None)
        realPrefix = None

813
814
        # Prefixed outputs are only
        # managed by the parent
815
        # FileOrthing in a chain of
816
817
818
819
        # FoT decorators.
        if not parent:
            prefix = None

820
821
822
823
824
825
826
827
828
829
        # If so, replace it with a new output
        # prefix which will redirect all output
        # to the temp dir.
        #
        # Importantly, here we assume that the
        # underlying function (and hence the
        # underlying command-line tool) will
        # accept an output prefix which contains
        # a directory path.
        if prefix is not None:
830
831
832
833
834
835
836
837
838
839

            # If prefix is set to LOAD,
            # all generated output files
            # should be loaded - we use a
            # randomly generated prefix,
            # and add it to prefixedFiles,
            # so that every file which
            # starts with it will be
            # loaded.
            if prefix is LOAD:
840
841
842
                prefix                = random.sample(string.ascii_letters, 10)
                prefix                = ''.join(prefix)
                prefixedFiles[prefix] = self.__outprefix
843

844
            realPrefix                = prefix
845
846
847
            fakePrefix                = op.join(workdir, prefix)
            allargs[self.__outprefix] = fakePrefix

848
849
            log.debug('Replacing output prefix: %s -> %s',
                      realPrefix, fakePrefix)
850

851
852
853
854
            # If the prefix specifies a
            # directory, make sure it
            # exists (remember that we're
            # in a temporary directory)
855
856
857
            pdir = op.dirname(fakePrefix)
            if pdir != '' and not op.exists(pdir):
                os.makedirs(pdir)
858

859
860
        if len(self.__things) > 0: things = self.__things
        else:                      things = allargs.keys()
861

862
        for name, val in list(allargs.items()):
863

864
865
866
867
868
            # don't process the
            # outprefix argument
            if name == self.__outprefix:
                continue

869
            # is this argument referring
870
            # to a prefixed output?
871
872
            isprefixed = (prefix is not None and
                          name.startswith(prefix))
873

874
            if not (isprefixed or name in things):
875
876
                continue

877
878
879
880
            # Prefixed output files may only
            # be given a value of LOAD
            if isprefixed and val is not LOAD:
                raise ValueError('Cannot specify name of prefixed file - the '
881
882
                                 'name is defined by the output prefix: '
                                 '{}'.format(name))
883

884
            if val is LOAD:
885

886
887
888
889
890
891
892
                # this argument refers to an output
                # that is generated from the output
                # prefix argument, and doesn't map
                # directly to an argument of the
                # function. So we don't pass it
                # through.
                if isprefixed:
893
                    prefixedFiles[name] = name
894
895
896
897
898
                    allargs.pop(name)

                # regular output-file argument
                else:
                    outfile = self.__prepOut(workdir, name, val)
899
                    outfiles[name] = outfile
900
                    allargs[ name] = outfile
Paul McCarthy's avatar
Paul McCarthy committed
901

902
            # Assumed to be an input file
903
            else:
904
905
                # sequences may be
                # accepted for inputs
906
907
908
909
910
911
912
913
914
                if isinstance(val, (list, tuple)):
                    infile = list(val)
                    for i, v in enumerate(val):
                        v = self.__prepIn(workdir, name, v)
                        if v is not None:
                            infile[i] = v

                else:
                    infile = self.__prepIn(workdir, name, val)
Paul McCarthy's avatar
Paul McCarthy committed
915

916
                if infile is not None:
917
918
                    allargs[name] = infile

919
920
921
        if realPrefix is not None and len(prefixedFiles) == 0:
            allargs[self.__outprefix] = realPrefix

922
923
        args   = [allargs.pop(k) for k in argnames]
        kwargs = allargs
924

925
        return args, kwargs, realPrefix, outfiles, prefixedFiles
926
927


928
929
    def __generateResult(
            self, workdir, result, outprefix, outfiles, prefixes):
930
        """Loads function outputs and returns a :class:`Results` object.
931
932
933

        Called by :meth:`__call__` after the decorated function has been
        called. Figures out what files should be loaded, and loads them into
934
        a ``Results`` object.
935
936
937
938
939
940
941
942
943
944

        :arg workdir:   Directory which contains the function outputs.
        :arg result:    Function return value.
        :arg outprefix: Original output prefix that was passed into the
                        function (or ``None`` if one wasn't passed)
        :arg outfiles:  Dictionary containing output files to be loaded (see
                        :meth:`__prepareArgs`).
        :arg prefixes:  Dictionary containing output-prefix patterns to be
                        loaded (see :meth:`__prepareArgs`).

945
        :returns:       A ``Results`` object containing all loaded outputs.
946
947
        """

948
        # make a Results object to store
949
        # the output. If we are decorating
950
        # another FileOrThing, the
951
        # results will get merged together
952
953
954
        # into a single Results dict.
        if not isinstance(result, FileOrThing.Results):
            result = FileOrThing.Results(result)
955
956
957
958
959
960

        # Load the LOADed outputs
        for oname, ofile in outfiles.items():

            log.debug('Loading output %s: %s', oname, ofile)

961
            if op.exists(ofile): oval = self.__load(oname, ofile)
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
            else:                oval = None

            result[oname] = oval

        # No output prefix - we're done
        if outprefix is None or len(prefixes) == 0:
            return result

        # Load or move output-prefixed files.
        # Find all files with a name that
        # matches the prefix that was passed
        # in (recursing into matching sub-
        # directories too).
        allPrefixed = glob.glob(op.join(workdir, '{}*'.format(outprefix)))
        allPrefixed = [fslpath.allFiles(f) if op.isdir(f) else [f]
                       for f in allPrefixed]

        for prefixed in it.chain(*allPrefixed):
            fullpath = prefixed
            prefixed = op.relpath(prefixed, workdir)
            for prefPat, prefName in prefixes.items():
                if not fnmatch.fnmatch(prefixed, '{}*'.format(prefPat)):
                    continue

                log.debug('Loading prefixed output %s [%s]: %s',
                          prefPat, prefName, prefixed)

989
990
991
992
993
                noext   = self.__removeExt(prefixed)
                prefPat = prefPat.replace('\\', '\\\\')
                noext   = re.sub('^' + prefPat, prefName, noext)
                withext = re.sub('^' + prefPat, prefName, prefixed)

994
995
996
                # if the load function returns
                # None, this file is probably
                # not of the correct type.
997
                fval = self.__load(noext, fullpath)
998
                if fval is not None:
999

1000
                    # If there is already an item in result with the
For faster browsing, not all history is shown. View entire blame