wrapperutils.py 19.2 KB
Newer Older
1
2
#!/usr/bin/env python
#
Paul McCarthy's avatar
Paul McCarthy committed
3
4
# wrapperutils.py - Functions and decorators used by the FSL wrapper
# functions.
5
6
7
#
# Author: Paul McCarthy <pauldmccarthy@gmail.com>
#
Paul McCarthy's avatar
Paul McCarthy committed
8
9
10
11
12
13
14
15
16
17
18
"""This module contains functions and decorators used by the FSL wrapper
functions.

.. autosummary::
   :nosignatures:

   applyArgStyle
   required
   fileOrImage
   fileOrArray
"""
19
20
21
22


import os.path as op
import            os
Paul McCarthy's avatar
Paul McCarthy committed
23
import            sys
24
25
26
27
28
29
30
31
import            inspect
import            tempfile
import            warnings
import            functools
import            collections

import            six
import nibabel as nib
Paul McCarthy's avatar
Paul McCarthy committed
32
import numpy   as np
33
34
35
36
37

import fsl.utils.tempdir as tempdir
import fsl.data.image    as fslimage


Paul McCarthy's avatar
Paul McCarthy committed
38
39
40
41
42
43
44
45
46
47
48
49
50
51
def _update_wrapper(wrapper, wrapped, *args, **kwargs):
    """Replacement for the built-in ``functools.update_wrapper``. This
    implementation ensures that the wrapper function has an attribute
    called ``__wrapped__``, which refers to the ``wrapped`` function.

    This behaviour is only required in Python versions < 3.4.
    """

    wrapper = functools.update_wrapper(wrapper, wrapped, *args, **kwargs)

    # Python >= 3.4 does things right
    if sys.version_info[0] * 10 + sys.version_info[1] < 3.4:
        wrapper.__wrapped__ = wrapped
    return wrapper
52
53


Paul McCarthy's avatar
Paul McCarthy committed
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
def _unwrap(func):
    """Replacement for the built-in ``inspect.unwrap`` function, which
    is not present in Python versions prior to 3.4.
    """

    # Python >= 3.4 has an inspect.unwrap function
    if sys.version_info[0] * 10 + sys.version_info[1] < 3.4:
        return inspect.unwrap(func)

    # Otherwise we follow the __wrapped__ chain ourselves
    if hasattr(func, '__wrapped__'):
        return _unwrap(func.__wrapped__)

    return func


SHOW_IF_TRUE = object()
"""Constant to be used in the ``valmap`` passed to the :func:`applyArgStyle`
function.

When a ``SHOW_IF_TRUE`` argument is ``True``, it is added to the generated
command line arguments.
"""


HIDE_IF_TRUE = object()
"""Constant to be used in the ``valmap`` passed to the :func:`applyArgStyle`
function.

When a ``HIDE_IF_TRUE`` argument is ``True``, it is suppressed from the
generated command line arguments.
"""
86
87


88
def applyArgStyle(style, valsep=None, argmap=None, valmap=None, **kwargs):
Paul McCarthy's avatar
Paul McCarthy committed
89
90
91
92
    """Turns the given ``kwargs`` into command line options. This function
    is intended to be used to automatically generate command line options
    from arguments passed into a Python function.

93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
    The ``style`` and ``valsep`` arguments control how key-value pairs
    are converted into command-line options:


    =========  ==========  ===========================
    ``style``  ``valsep``  Result
    =========  ==========  ===========================
    ``'-'``    ' '         ``-name val1 val2 val3``
    ``'-'``    '"'         ``-name "val1 val2 val3"``
    ``'-'``    ','         ``-name val1,val2,val3``
    ``'--'``   ' '         ``--name val1 val2 val3``
    ``'--'``   '"'         ``--name "val1 val2 val3"``
    ``'--'``   ','         ``--name val1,val2,val3``
    ``'-='``   ' '         Not supported
    ``'-='``   '"'         ``-name="val1 val2 val3"``
    ``'-='``   ','         ``-name=val1,val2,val3``
    ``'--='``  ' '         Not supported
    ``'--='``  '"'         ``--name="val1 val2 val3"``
    ``'--='``  ','         ``--name=val1,val2,val3``
    =========  ==========  ===========================


Paul McCarthy's avatar
Paul McCarthy committed
115
    :arg style:  Controls how the ``kwargs`` are converted into command-line
116
117
118
119
120
                 options - must be one of ``'-'``, ``'--'``, ``'-='``, or
                 ``'--='``.

    :arg valsep: Controls how the values passed to command-line options
                 which expect multiple arguments are delimited - must be
121
122
                 one of ``' '``, ``','`` or ``'"'``. Defaults to ``' '``
                 if ``'=' not in style``, ``','`` otherwise.
Paul McCarthy's avatar
Paul McCarthy committed
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145

    :arg argmap: Dictionary of ``{kwarg-name : cli-name}`` mappings. This can
                 be used if you want to use different argument names in your
                 Python function for the command-line options.

    :arg valmap: Dictionary of ``{cli-name : value}`` mappings. This can be
                 used to define specific semantics for some command-line
                 options. Acceptable values for ``value`` are as follows

                  - :data:`SHOW_IF_TRUE` - if the argument is present, and
                    ``True`` in ``kwargs``, the command line option
                    will be added (without any arguments).

                  - :data:`HIDE_IF_TRUE` - if the argument is present, and
                    ``False`` in ``kwargs``, the command line option
                    will be added (without any arguments).

                  - Any other constant value. If the argument is present
                    in ``kwargs``, its command-line option will be
                    added, with the constant value as its argument.

                 The argument for any options not specified in the ``valmap``
                 will be converted into strings.
146

Paul McCarthy's avatar
Paul McCarthy committed
147
148
149
150
151
    :arg kwargs: Arguments to be converted into command-line options.

    :returns:    A list containing the generated command-line options.
    """

152
153
154
155
    if valsep is None:
        if '=' in style: valsep = ','
        else:            valsep = ' '

Paul McCarthy's avatar
Paul McCarthy committed
156
157
    if style not in ('-', '--', '-=', '--='):
        raise ValueError('Invalid style: {}'.format(style))
158
159
160
161
162
163
164
165
166
    if valsep not in (' ', ',', '"'):
        raise ValueError('Invalid valsep: {}'.format(valsep))

    # we don't handle the case where '=' in
    # style, and valsep == ' ', because no
    # sane CLI app would do this. Right?
    if '=' in style and valsep == ' ':
        raise ValueError('Incompatible style and valsep: s={} v={}'.format(
            style, valsep))
Paul McCarthy's avatar
Paul McCarthy committed
167
168
169
170
171

    if argmap is None: argmap = {}
    if valmap is None: valmap = {}

    def fmtarg(arg):
172
173
174
175
        if   style in ('-',  '-='):  arg =  '-{}'.format(arg)
        elif style in ('--', '--='): arg = '--{}'.format(arg)
        return arg

176
    # always returns a sequence
Paul McCarthy's avatar
Paul McCarthy committed
177
    def fmtval(val):
178
179
        if     isinstance(val, collections.Sequence) and \
           not isinstance(val, six.string_types):
180
181
182
183
184

            val = [str(v) for v in val]
            if   valsep == ' ': return val
            elif valsep == '"': return [' '   .join(val)]
            else:               return [valsep.join(val)]
185
        else:
186
187
188
189
190
191
192
193
            return [str(val)]

    # val is assumed to be a sequence
    def fmtargval(arg, val):
        # if '=' in style, val will
        # always be a single string
        if '=' in style: return ['{}={}'.format(arg, val[0])]
        else:            return [arg] + val
194
195
196
197
198
199

    args = []

    for k, v in kwargs.items():

        k    = argmap.get(k, k)
Paul McCarthy's avatar
Paul McCarthy committed
200
201
202
        mapv = valmap.get(k, fmtval(v))
        k    = fmtarg(k)

203

204
205
206
207
        if mapv in (SHOW_IF_TRUE, HIDE_IF_TRUE):
            if (mapv is SHOW_IF_TRUE and     v) or \
               (mapv is HIDE_IF_TRUE and not v):
                args.append(k)
208
        else:
209
            args.extend(fmtargval(k, mapv))
210
211
212
213
214

    return args


def required(*reqargs):
215
216
217
218
219
220
221
    """Decorator which makes sure that all specified arguments are present
    before calling the decorated function. Arguments which are not present
    will result in an :exc:`AssertionError`. Use as follows::

        @required('foo')
        def funcWhichRequires_foo(**kwargs):
            foo = kwargs['foo']
222
    """
Paul McCarthy's avatar
Paul McCarthy committed
223

224
    def decorator(func):
Paul McCarthy's avatar
Paul McCarthy committed
225
        def wrapper(*args, **kwargs):
226
            kwargs = argsToKwargs(func, args, kwargs)
227
228
229
            for reqarg in reqargs:
                assert reqarg in kwargs
            return func(**kwargs)
230
        return _update_wrapper(wrapper, func)
231
232
233
    return decorator


234
def argsToKwargs(func, args, kwargs=None):
235
236
    """Given a function, and a sequence of positional arguments destined
    for that function, converts the positional arguments into a dict
Paul McCarthy's avatar
Paul McCarthy committed
237
    of keyword arguments. Used by the :class:`_FileOrThing` class.
238
239
240
241
242
243
244
245
246

    :arg func:   Function which will accept ``args`` as positionals.

    :arg args:   Tuple of positional arguments to be passed to ``func``.

    :arg kwargs: Optional. If provided, assumed to be keyword arguments
                 to be passed to ``func``. The ``args`` are merged into
                 ``kwargs``. A :exc:`ValueError` is raised if one of
                 ``args`` is already present in ``kwargs``.
247
    """
Paul McCarthy's avatar
Paul McCarthy committed
248

249
250
    # Remove any decorators
    # from the function
Paul McCarthy's avatar
Paul McCarthy committed
251
252
    func = _unwrap(func)

253
254
255
    # getargspec is the only way to
    # get the names of positional
    # arguments in Python 2.x.
Paul McCarthy's avatar
Paul McCarthy committed
256
257
258
    if sys.version_info[0] < 3:
        argnames = inspect.getargspec(func).args

259
260
    # But getargspec is deprecated
    # in python 3.x
Paul McCarthy's avatar
Paul McCarthy committed
261
262
263
264
265
266
267
    else:

        # getfullargspec is deprecated in
        # python 3.5, but not in python 3.6.
        with warnings.catch_warnings():
            warnings.filterwarnings('ignore', category=DeprecationWarning)
            argnames = inspect.getfullargspec(func).args
268

269
270
271
    if kwargs is None: kwargs = dict()
    else:              kwargs = dict(kwargs)

Paul McCarthy's avatar
Paul McCarthy committed
272
    for name, val in zip(argnames, args):
273
274
        if name in kwargs:
            raise ValueError('Argument {} repeated'.format(name))
275
276
277
278
279
        kwargs[name] = val

    return kwargs


Paul McCarthy's avatar
Paul McCarthy committed
280
LOAD = object()
Paul McCarthy's avatar
Paul McCarthy committed
281
282
"""Constant used by the :class:`_FileOrThing` class to indicate that an output
file should be loaded into memory and returned as a Python object.
283
284
285
"""


Paul McCarthy's avatar
Paul McCarthy committed
286
287
288
289
290
291
292
293
294
295
class _FileOrThing(object):
    """Decorator which ensures that certain arguments which are passed into the
    decorated function are always passed as file names. Both positional and
    keyword arguments can be specified.


    The ``_FileOrThing`` class is not intended to be used directly - see the
    :func:`fileOrImage` and :func:`fileOrArray` decorator functions for more
    details.

296

Paul McCarthy's avatar
Paul McCarthy committed
297
298
299
300
301
302
303
304
305
306
307
308
    These decorators are intended for functions which wrap a command-line tool,
    i.e. where some inputs/outputs need to be specified as file names.


    **Inputs**


    Any arguments which are not of type ``Thing`` are passed through to the
    decorated function unmodified.  Arguments which are of type ``Thing`` are
    saved to a temporary file, and the name of that file is passed to the
    function.

309

Paul McCarthy's avatar
Paul McCarthy committed
310
    **Outputs**
311
312


Paul McCarthy's avatar
Paul McCarthy committed
313
    If an argument is given the special :data:`LOAD` value, it is assumed
Paul McCarthy's avatar
Paul McCarthy committed
314
315
316
    to be an output argument. In this case, it is replaced with a temporary
    file name then, after the function has completed, that file is loaded
    into memory, and the value returned (along with the function's output,
Paul McCarthy's avatar
Paul McCarthy committed
317
    and any other arguments with a value of ``LOAD``).
Paul McCarthy's avatar
Paul McCarthy committed
318
319
320
321
322
323


    **Return value**


    Functions decorated with a ``_FileOrThing`` decorator will always return a
324
325
326
327
328
    ``dict``-like object, where the function's actual return value is
    accessible via an attribute called `output`. All output arguments with a
    value of ``LOAD`` will be present as dictionary entries, with the keyword
    argument names used as keys. Any ``LOAD``ed output arguments which were not
    generated by the function will not be present in the dictionary.
Paul McCarthy's avatar
Paul McCarthy committed
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352


    **Example**


    As an example of using the ``fileOrArray`` decorator on a function
    which concatenates two files containing affine transformations, and
    saves the output to a file::

        # if atob, btoc, or output are passed
        # in as arrays, they are converted to
        # file names.
        @fileOrArray('atob', 'btoc', 'output')
        def concat(atob, btoc, output=None):

            # inputs are guaranteed to be files
            atob = np.loadtxt(atob)
            btoc = np.loadtxt(atoc)

            atoc = np.dot(btoc, atob)

            if output is not None:
                np.savetxt(output, atoc)

353
354
            return 'Done'

Paul McCarthy's avatar
Paul McCarthy committed
355
356
357
358

    Because we have decorated the ``concat`` function with :func:`fileToArray`,
    it can be called with either file names, or Numpy arrays::

359

Paul McCarthy's avatar
Paul McCarthy committed
360
361
        # All arguments are passed through
        # unmodified - the output will be
362
        # saved to a file called atoc.mat.
Paul McCarthy's avatar
Paul McCarthy committed
363
364
        concat('atob.txt', 'btoc.txt', 'atoc.mat')

365
366
367
368
369
370
371
372
373
        # The function's return value
        # is accessed via an attribute called
        # "output" on the dict
        assert concat('atob.txt', 'btoc.txt', 'atoc.mat').output == 'Done'

        # Outputs to be loaded into memory
        # are returned in a dictionary,
        # with argument names as keys.
        atoc = concat('atob.txt', 'btoc.txt', LOAD)['atoc']
Paul McCarthy's avatar
Paul McCarthy committed
374

375
376
377
378
379
380
381
382
383
        # In-memory inputs are saved to
        # temporary files, and those file
        # names are passed to the concat
        # function.
        atoc = concat(np.diag([2, 2, 2, 0]),
                      np.diag([3, 3, 3, 3]), LOAD)['atoc']


    **Using with other decorators**
Paul McCarthy's avatar
Paul McCarthy committed
384
385
386
387
388
389
390
391
392
393


    ``_FileOrThing`` decorators can be chained with other ``_FileOrThing``
    decorators, and other decorators.  When multiple ``_FileOrThing``
    decorators are used on a single function, the outputs from each decorator
    are merged together into a single dict-like object.


    ``_FileOrThing`` decorators can be used with any other decorators
    __as long as__ they do not manipulate the return value.
394
395
396
    """


397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
    class _Results(dict):
        """A custom ``dict`` type used to return outputs from a function
        decorated with ``_FileOrThing``. All outputs are stored as dictionary
        items, with the argument name as key, and the output object (the
        "thing") as value.

        The decorated function's actual return value is accessible via the
        :meth:`output` property.
        """
        def __init__(self, output):
            self.__output = output

        @property
        def output(self):
            """Access the return value of the decorated function. """
            return self.__output


    def __init__(self, prepIn, prepOut, load, *things):
Paul McCarthy's avatar
Paul McCarthy committed
416
417
        """Initialise a ``_FileOrThing`` decorator.

418
419
        :arg prepIn:  Function which returns a file name to be used in
                      place of an input argument.
Paul McCarthy's avatar
Paul McCarthy committed
420

421
422
        :arg prepOut: Function which generates a file name to use for
                      arguments that were set to :data:`LOAD`.
423

424
425
426
        :arg load:    Function which is called to load items for arguments
                      that were set to :data:`LOAD`. Must accept a file path
                      as its sole argument.
427

428
429
430
431
432
433
434
435
        :arg things:  Names of all arguments which will be handled by
                      this ``_FileOrThing`` decorator.

        The ``prepIn`` and ``prepOut`` functions must accept the following
        positional arguments:

          - A directory in which all temporary input/output files should be
            stored
Paul McCarthy's avatar
Paul McCarthy committed
436

437
438
439
440
441
442
443
444
          - The name of the keyword argument to be processed

          - The argument value that was passed in
        """
        self.__prepIn  = prepIn
        self.__prepOut = prepOut
        self.__load    = load
        self.__things  = things
Paul McCarthy's avatar
Paul McCarthy committed
445
446


447
448
449
450
    def __call__(self, func):
        """Creates and returns the decorated function. """
        wrapper = functools.partial(self.__wrapper, func)
        return _update_wrapper(wrapper, func)
Paul McCarthy's avatar
Paul McCarthy committed
451

452

453
454
    def __wrapper(self, func, *args, **kwargs):
        """Function which calls ``func``, ensuring that any arguments of
Paul McCarthy's avatar
Paul McCarthy committed
455
        type ``Thing`` are saved to temporary files, and any arguments
Paul McCarthy's avatar
Paul McCarthy committed
456
        with the value :data:`LOAD` are loaded and returned.
457

458
        :arg func: The function being wrapped.
459

460
        All other arguments are passed through to ``func``.
461
        """
Paul McCarthy's avatar
Paul McCarthy committed
462

463
464
        # Turn all positionals into keywords
        kwargs = argsToKwargs(func, args, kwargs)
465
466

        # Create a tempdir to store any temporary
Paul McCarthy's avatar
Paul McCarthy committed
467
        # input/output things, but don't change
468
469
470
471
        # into it, as file paths passed to the
        # function may be relative.
        with tempdir.tempdir(changeto=False) as td:

472
473
474
            # Replace any things with file names.
            # Also get a list of LOAD outputs
            kwargs, outfiles = self.__prepareArgs(td, kwargs)
475
476

            # Call the function
Paul McCarthy's avatar
Paul McCarthy committed
477
            result = func(**kwargs)
478

479
480
481
482
483
484
485
            # make a _Reults object to store
            # the output. If we are decorating
            # another _FileOrThing, the
            # results will get merged together
            # into a single _Results dict.
            if not isinstance(result, _FileOrThing._Results):
                result = _FileOrThing._Results(result)
486

487
488
            # Load the LOADed outputs
            for oname, ofile in outfiles.items():
489

490
491
                if not op.exists(ofile): oval = None
                else:                    oval = self.__load(ofile)
492

493
                result[oname] = oval
494

495
            return result
496
497


498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
    def __prepareArgs(self, workdir, kwargs):
        """Prepares all input and output arguments to be passed to the
        decorated function. Any arguments with a value of :data:`LOAD` are
        passed to the ``prepOut`` function specified at :meth:`__init__`.
        All other arguments are passed through the ``prepIn`` function.

        :arg workdir: Directory in which all temporary files should be stored.

        :arg kwargs:  Keyword arguments to be passed to the decorated function.

        :returns:     A tuple containing:

                        - An updated copy of ``kwargs``, ready to be passed
                          into the function

                        - A dictionary of ``{ name : filename }`` mappings,
                          for all arguments with a value of ``LOAD``.
515
516
517
        """

        kwargs   = dict(kwargs)
518
        outfiles = dict()
519

520
        for name in self.__things:
521

522
            val = kwargs.get(name, None)
523

524
            if val is None:
525
526
                continue

527
            if val is LOAD:
528
529
530
531
532
533
534

                outfile = self.__prepOut(workdir, name, val)

                if outfile is not None:
                    kwargs[  name] = outfile
                    outfiles[name] = outfile
            else:
Paul McCarthy's avatar
Paul McCarthy committed
535

536
                infile = self.__prepIn(workdir, name, val)
Paul McCarthy's avatar
Paul McCarthy committed
537

538
539
                if infile is not None:
                    kwargs[name] = infile
540

541
        return kwargs, outfiles
542
543


Paul McCarthy's avatar
Paul McCarthy committed
544
545
546
547
548
549
def fileOrImage(*imgargs):
    """Decorator which can be used to ensure that any NIfTI images are saved
    to file, and output images can be loaded and returned as ``nibabel``
    image objects.
    """

550
    def prepIn(workdir, name, val):
Paul McCarthy's avatar
Paul McCarthy committed
551

552
        infile = None
Paul McCarthy's avatar
Paul McCarthy committed
553
554

        if isinstance(val, nib.nifti1.Nifti1Image):
555
            infile = val.get_filename()
Paul McCarthy's avatar
Paul McCarthy committed
556
557
558

            # in-memory image - we have
            # to save it out to a file
559
560
            if infile is None:
                hd, infile = tempfile.mkstemp(fslimage.defaultExt())
Paul McCarthy's avatar
Paul McCarthy committed
561
                os.close(hd)
562
                val.to_filename(infile)
Paul McCarthy's avatar
Paul McCarthy committed
563

564
        return infile
Paul McCarthy's avatar
Paul McCarthy committed
565

566
567
    def prepOut(workdir, name, val):
        return op.join(workdir, '{}.nii.gz'.format(name))
Paul McCarthy's avatar
Paul McCarthy committed
568

569
    def load(path):
Paul McCarthy's avatar
Paul McCarthy committed
570
571
572
573
574
        # create an independent in-memory
        # copy of the image file
        img = nib.load(path)
        return nib.nifti1.Nifti1Image(img.get_data(), None, img.header)

575
    return _FileOrThing(prepIn, prepOut, load, *imgargs)
Paul McCarthy's avatar
Paul McCarthy committed
576
577
578
579
580
581
582


def fileOrArray(*arrargs):
    """Decorator which can be used to ensure that any Numpy arrays are saved
    to text files, and output files can be loaded and returned as Numpy arrays.
    """

583
    def prepIn(workdir, name, val):
Paul McCarthy's avatar
Paul McCarthy committed
584

585
        infile = None
Paul McCarthy's avatar
Paul McCarthy committed
586
587

        if isinstance(val, np.ndarray):
588
            hd, infile = tempfile.mkstemp('.txt')
Paul McCarthy's avatar
Paul McCarthy committed
589
            os.close(hd)
590
            np.savetxt(infile, val, fmt='%0.18f')
Paul McCarthy's avatar
Paul McCarthy committed
591

592
        return infile
Paul McCarthy's avatar
Paul McCarthy committed
593

594
595
    def prepOut(workdir, name, val):
        return op.join(workdir, '{}.txt'.format(name))
Paul McCarthy's avatar
Paul McCarthy committed
596

597
    load = np.loadtxt
Paul McCarthy's avatar
Paul McCarthy committed
598

599
    return _FileOrThing(prepIn, prepOut, load, *arrargs)