Commit a34bfb2d authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

RF: Do all config/table/plugin file lookup using

add_argument(type=findFile). New FUNPACK_CONFIG_DIR variable to make things
more flexible
parent c1e50148
......@@ -5,9 +5,10 @@
# Author: Paul McCarthy <>
"""This module contains functions for parsing ``funpack`` command line
arguments and configuration files. A ``funpack`` configuration file
is simply a plain-text file which contains command-line options,
arguments and configuration files.
A ``funpack`` configuration file is simply a plain-text file which contains
command-line options, sans-hyphens.
......@@ -15,12 +16,13 @@ import os.path as op
import functools as ft
import itertools as it
import multiprocessing as mp
import os
import sys
import site
import shlex
import logging
import argparse
import collections
import numpy as np
import funpack
import funpack.util as util
......@@ -56,14 +58,17 @@ CLI_ARGUMENTS = collections.OrderedDict((
'default' : DEFAULT_MERGE_AXIS}),
(('ms', 'merge_strategy'), {'choices' : AVAILABLE_MERGE_STRATEGIES,
(('rm', 'remove_duplicates'), {'action' : 'store_true'}),
(('rd', 'rename_duplicates'), {'action' : 'store_true'}),
(('cfg', 'config_file'), {'action' : 'append'}),
(('vf', 'variable_file'), {'action' : 'append'}),
(('df', 'datacoding_file'), {'action' : 'append'}),
(('tf', 'type_file'), {}),
(('pf', 'processing_file'), {}),
(('cf', 'category_file'), {})]),
(('rm', 'remove_duplicates'), {'action' : 'store_true'}),
(('rd', 'rename_duplicates'), {'action' : 'store_true'}),
(('cfg', 'config_file'), {'action' : 'append',
'type' : util.findConfigFile}),
(('vf', 'variable_file'), {'action' : 'append',
'type' : util.findTableFile}),
(('df', 'datacoding_file'), {'action' : 'append',
'type' : util.findTableFile}),
(('tf', 'type_file'), {'type' : util.findTableFile}),
(('pf', 'processing_file'), {'type' : util.findTableFile}),
(('cf', 'category_file'), {'type' : util.findTableFile})]),
('Import options', [
(('s', 'subject'), {'action' : 'append'}),
......@@ -158,7 +163,8 @@ CLI_ARGUMENTS = collections.OrderedDict((
(('nj', 'num_jobs'), {'type' : int,
'default' : 1}),
(('p', 'plugin_file'), {'action' : 'append',
'metavar' : 'FILE'}),
'metavar' : 'FILE',
'type' : util.findPluginFile}),
(('ow', 'overwrite'), {'action' : 'store_true'}),
(('n', 'noisy'), {'action' : 'count'}),
(('q', 'quiet'), {'action' : 'store_true'})])))
......@@ -568,15 +574,6 @@ def loadConfigFile(cfgfile, namespace=None):
argv = []
# If the specified config file does
# not exist, assume it is a reference
# to a built-in config file
if not op.exists(cfgfile):
if not cfgfile.endswith('.cfg'):
cfgfile = cfgfile + '.cfg'
moddir = op.dirname(op.abspath(__file__))
cfgfile = op.join(moddir, 'configs', cfgfile)
log.debug('Loading arguments from configuration file %s', cfgfile)
with open(cfgfile, 'rt') as f:
......@@ -668,7 +665,6 @@ def parseArgs(argv=None, namespace=None):
if args.quiet: args.noisy = 0
_prepareInputAndOutputFiles( args)
_prepareTableFiles( args)
_prepareAuxFileNames( args)
_prepareCategorySelectors( args)
......@@ -744,48 +740,6 @@ def _prepareInputAndOutputFiles(args):
args.var_format = var_format
def _prepareTableFiles(args):
"""Sub-function of :func:`parseArgs`. Prepares arguments related to
the input table files.
:arg args: ``argparse.Namespace`` object.
# The variable_file, datacoding_file, type_file,
# processing_file, and category_file options
# can be specified relative to funpackdir
configdir = op.join(op.dirname(__file__), 'configs')
def fixPath(f):
if f is None:
return f
options = [
op.join(configdir, f),
op.join(configdir, f.replace('.', op.sep) + '.tsv')
for o in options:
if op.exists(o):
return o
# if the fixed version does not
# exist, allow processing to
# continue - it will fail later on.
return f
if args.variable_file is not None:
args.variable_file = [fixPath(f) for f in args.variable_file]
if args.datacoding_file is not None:
args.datacoding_file = [fixPath(f) for f in args.datacoding_file]
args.type_file = fixPath(args.type_file)
args.processing_file = fixPath(args.processing_file)
args.category_file = fixPath(args.category_file)
def _prepareAuxFileNames(args):
"""Sub-function of :func:`parseArgs`. Prepares arguments related to
the auxillary output files.
......@@ -135,15 +135,6 @@ class PluginRegistry:
plugin functions.
# If file does not exist, assume
# it is a direct reference to a
# file in the plugins directory.
if not op.exists(filename):
if not filename.endswith('.py'):
filename = filename + '.py'
moddir = op.dirname(op.abspath(__file__))
filename = op.join(moddir, 'plugins', filename)
filename = op.abspath(filename)
name = re.sub('[^a-zA-Z]', '_', filename)
......@@ -9,15 +9,19 @@ classes, and constants.
import os
import re
import sys
import enum
import site
import time
import shutil
import logging
import warnings
import functools
import contextlib
import itertools as it
import os.path as op
import subprocess as sp
import multiprocessing as mp
......@@ -177,6 +181,84 @@ def generateColumnName(variable, visit, instance):
return '{}-{}.{}'.format(variable, visit, instance)
def findTableFile(filename):
"""Searches for a FUNPACK table tile - see :func:`findConfigFile`. """
return findConfigFile(filename, '.tsv')
def findPluginFile(filename):
"""Searches for a FUNPACK plugin tile - see :func:`findConfigFile`. """
return findConfigFile(filename, '.py', dirname='plugins')
def findConfigFile(filename, suffix='.cfg', dirname='configs'):
"""Searches for a FUNPACK configuration file in a number of locations.
:arg filename: Name of file to search for
:arg suffix: Suffix to append, if the filename is specfied without one
(must include the leading period).
:arg dirname: Name of internal/built-in directory to search - assumed to
be within the ``funpack`` package directory, e.g.
:returns: Absolute path to the found file, or ``filename`` unmodified
if a match was not found.
# Make things easier for users of this function
if filename is None:
return filename
# Suffix is just appended straight onto the
# file name, so empty string is a no-op
if suffix is None:
suffix = ''
# config files may be absolute / relative
# paths to an arbitrary location
if op.exists(filename):
return op.abspath(filename)
# The user can also refer to "built-in"
# config files just by giving a file path
# with/without suffix, relative to one of
# the following locations (in order of
# precedence):
# - within the funpack package directory, (<thisdir>/configs/) or,
# - if we are running from a git checkout, installed in the running
# python env (<pyenv>/lib/pythonX.Y/site-packages/funpack/configs/)
cfgdirs = []
if 'FUNPACK_CONFIG_DIR' in os.environ:
cfgdirs.append(op.join(op.dirname(__file__), dirname))
cfgdirs.extend(op.join(sitedir, 'funpack', dirname)
for sitedir in site.getsitepackages())
# Built-in config files can be specified
# with (in order of precedence):
# - file path with suffix (e.g. "fmrib/categories.tsv")
# - file path without suffix (e.g. "fmrib/categories")
# - file path without suffix, with dots instead of slashes
# (e.g. "fmrib.categories")
candidates = [filename,
f'{filename.replace(".", op.sep)}{suffix}']
for cfgdir, cand in it.product(cfgdirs, candidates):
cand = op.abspath(op.join(cfgdir, cand))
if op.exists(cand):
return cand
# Can't find the file - return the
# path unmodified, which will result
# in an error at some other point.
return filename
def parseMatlabRange(r):
"""Parses a string containing a MATLAB-style ``start:stop`` or
``start:step:stop`` range, where the ``stop`` is inclusive).
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment