ENH,RF: - Renamed "overrides" to "selectors".

        - Flatten dictionaries in main config file, e.g. "[struct_T1] k = v"
          is flattened to struct_T1_k = v
        - New gettuple method
        - Added ability to override values e.g. from command-line.
        - Basic type validation applied to overrides - error if their
          parsed type does not match the stored type

bip/utils/config.py

 #!/usr/bin/env python
+"""This module provides the BIP Config class. The Config class is used
+throughout BIP for accessing settings stored in BIP configuration files.
 
 
+# Configuration file format
+
+TODO
+
+# Selectors - alternate setting values for specific scenarios
+
+TODO
+
+# Type validation
+
+TODO
+"""
+
+
+import              copy
 import              glob
 import functools as ft
 import itertools as it
@@ -24,6 +41,19 @@ import bip
 log = logging.getLogger(__name__)
 
 
+MAIN_CONFIG_FILE_NAME = 'config.toml'
+"""Name of the primary BIP configuration file. This file contains global
+settings, and may contain pipeline-specific settings which take precedence
+over equivalent settings in secondary configuration files.
+"""
+
+
+DEFAULT_CONFIG_DIRECTORY = op.join(op.dirname(bip.__file__), 'data', 'config')
+"""Default configuration directory. Used when a Config object is created with
+specifying a directory.
+"""
+
+
 def nested_lookup(d, key):
     """Look up a value in a nested dictionary based on the given key.
     For example, imagine we have a dictionary d:
@@ -46,7 +76,74 @@ def nested_lookup(d, key):
     return nested_lookup(d, key[1:])
 
 
+def flatten_dictionary(d, nlevels=1):
+    """Flattens a nested dictionary.
+
+    Given a dictionary such as:
+
+        {
+            'k1' : 'v1',
+            'k2' : {
+                'sk1' : 'sv1',
+                'sk2' : 'sv2'
+            }
+        }
+
+    this function will adjust the dictionary to have structure:
+        {
+            'k1'     : 'v1',
+            'k2_sk1' : 'sv1'
+            'k2_sk2' : 'sv2'
+        }
+
+    If nlevels is 1 (the default), the function will only flatten
+    sub-dictionaries that are one-level deep.
+    """
+
+    d = copy.deepcopy(d)
+
+    for key, val in list(d.items()):
+        if not isinstance(val, dict):
+            continue
+
+        if nlevels > 1:
+            val = flatten_dictionary(val, nlevels - 1)
+
+        for subkey, subval in val.items():
+            subkey    = f'{key}_{subkey}'
+            d[subkey] = subval
+
+        d.pop(key)
+
+    return d
+
+
+def parse_override_value(name, origval, val):
+    """Use tomllib to coerce a string to a suitable type."""
+
+    # If the configuration file value is a string,
+    # return the override value unchanged
+    if isinstance(origval, str):
+        return val
+
+    # Otherwise use tomllib to convert the value.
+    # Leave as a string on failures, but emit
+    # a warning.
+    try:
+        return tomllib.loads(f'val = {val}')['val']
+    except Exception:
+        log.warning('Cannot parse override value for %s (%s) - '
+                    'leaving value as a string.', name, val)
+        return val
+
+
 class Config:
+    """The Config class is a dictionary containing BIP settings. A Config
+    can be created by specifying the directory which contains all BIP
+    configuration files, e.g.:
+
+        cfg = Config('path/to/config/dir/')
+    """
 
 
     @staticmethod
@@ -67,7 +164,10 @@ class Config:
         BIP settings may be grouped according to the file identifier - the file
         prefix is used solely for enforcing a particular ordering.
         """
-        base = op.basename(fname).removesuffix('.toml')
+        base = op.basename(fname)
+        if base == MAIN_CONFIG_FILE_NAME:
+            return 'config'
+        base = base.removesuffix('.toml')
         # Drop file prefix if present
         return base.split('.')[-1]
 
@@ -80,17 +180,17 @@ class Config:
 
         A BIP configuration is a directory containing one or more .toml files.
 
-        BIP configuration files are intended to be loaded in reverse-
-        alphabetical order, with settings in later files overwriting settings
-        in earlier files.
+        BIP configuration files are intended to be loaded in alphabetical
+        order, with settings in later files overwriting settings in earlier
+        files.
 
         The "config.toml" file must be loaded last, so that settings
         contained within it take precedence over all other files.
         """
 
         cfgfiles = glob.glob(op.join(cfgdir, '*.toml'))
-        cfgfiles = sorted(cfgfiles, key=str.lower, reverse=True)
-        maincfg  = op.join(cfgdir, 'config.toml')
+        cfgfiles = sorted(cfgfiles, key=str.lower)
+        maincfg  = op.join(cfgdir, MAIN_CONFIG_FILE_NAME)
 
         # make sure main config is loaded last
         if maincfg in cfgfiles:
@@ -101,28 +201,28 @@ class Config:
 
 
     @staticmethod
-    def resolve_overrides(settings, **overrides):
-        """Resolves and applies any "override" configuration settings.
+    def resolve_selectors(settings, selectors):
+        """Resolves and applies any "selector" configuration settings.
 
         A BIP configuration file may contain one default value for a
         setting, but may also contain alternate values for that setting
         which should be used in certain scenarios. Each alternate value
-        is associated with a set of "override" parameters, which are
+        is associated with a set of "selector" parameters, which are
         just key-value pairs.
 
         For example, a file may contain a default value for a setting called
         "param1", and an alternate value for param1 to be used when the
-        "subject" override parameter is set to "12345":
+        "subject" selector parameter is set to "12345":
 
             param1               = 0.5
             subject.12345.param1 = 0.4
 
-        If resolve_overrides is given these settings along with subject=12345,
-        the default value for param1 will be replaced with the override value.
+        If resolve_selectors is given these settings along with subject=12345,
+        the default value for param1 will be replaced with the selector value.
 
-        Multiple override parameters may be specified - in the configuration
-        file, the override parameters must be ordered alphabetically. For
-        example, if we have override parameters "subject" and "visit", the
+        Multiple selector parameters may be specified - in the configuration
+        file, the selector parameters must be ordered alphabetically. For
+        example, if we have selector parameters "subject" and "visit", the
         alternate values for subject=123 and visit=2 must be specified as:
 
             param1                     = 0.5
@@ -134,20 +234,21 @@ class Config:
             visit.2.subject.123.param1 = 0.3
         """
 
-        # Take the override parameters, and generate all possible
+        settings = copy.deepcopy(settings)
+
+        # Take the selector parameters, and generate all possible
         # candidate keys - e.g. {'subject' : '123', 'visit' : '1'}
         # will result in:
         #
         #   ['subject', '123']
         #   ['visit',   '1']
         #   ['subject', '123', 'visit', '1']
-        kvps     = [[str(k), str(v)] for k, v in overrides.items()]
+        kvps     = [[str(k), str(v)] for k, v in selectors.items()]
         patterns = [it.combinations(kvps, i) for i in range(1, len(kvps) + 1)]
         patterns = it.chain(*patterns)
         patterns = [ft.reduce(operator.add, p) for p in patterns]
 
         for pat in patterns:
-
             try:
                 val = nested_lookup(settings, pat)
             except KeyError:
@@ -155,7 +256,7 @@ class Config:
             if not isinstance(val, dict):
                 continue
 
-            log.debug('Updating settings from override: %s', pat)
+            log.debug('Updating settings from selector: %s', pat)
             for k, v in val.items():
 
                 # If we have a configuration like:
@@ -163,14 +264,54 @@ class Config:
                 #   subject.123.visit.1.param = 0.5
                 #   visit.2.param             = 0.5
                 #
-                # and we are processing ['subject', '123'],
-                # we do not want to override 'visit'.
-                if k not in overrides:
+                # and we are processing ['subject', '123'], we
+                # do not want to clobber the original 'visit'.
+                if k not in selectors:
                     settings[k] = v
 
+        return settings
+
 
     @staticmethod
-    def load_config_file(cfgfile, **overrides):
+    def apply_overrides(settings, overrides=None):
+        """Override some settings with override values.
+
+        Any value read from the configuration files can be overridden.
+
+        If an override value has an incompatible type with the value from the
+        configuration file, a ValueError is raised.
+        """
+
+        def types_match(old, new):
+            """Return True if old and new are of compatible types."""
+            if isinstance(old, (float, int)):
+                return isinstance(new, (float, int))
+            else:
+                return isinstance(new, type(old))
+
+        settings = copy.deepcopy(settings)
+
+        for key, val in overrides.items():
+            origval = settings.get(key, None)
+
+            # Coerce strings to a toml type
+            if isinstance(val, str):
+                val = parse_override_value(key, origval, val)
+
+            # Reject the new value if it has a
+            # different type to the original value
+            if origval is not None and not types_match(origval, val):
+                raise ValueError(f'{key}: override value has wrong type (got '
+                                 f'{type(val)}, expected {type(origval)}')
+
+            log.debug('Overriding %s (%s -> %s)', key, origval, val)
+            settings[key] = val
+
+        return settings
+
+
+    @staticmethod
+    def load_config_file(cfgfile, selectors=None):
         """Load a BIP configuration file. The file is assumed to be a TOML
         file, named as described in the config_file_identifier documentation.
 
@@ -190,36 +331,53 @@ class Config:
         file.
         """
 
+        if selectors is None:
+            selectors = {}
+
         with open(cfgfile, 'rb') as f:
             settings = tomllib.load(f)
 
         ident     = Config.config_file_identifier(cfgfile)
-        overrides = {k : overrides[k] for k in sorted(overrides.keys())}
+        selectors = {k : selectors[k] for k in sorted(selectors.keys())}
+        settings  = Config.resolve_selectors(settings, selectors)
 
-        Config.resolve_overrides(settings, **overrides)
-
-        # TODO if main file, unpack dictionaries - e.g. we want
-        # to be able to have things like this in config.yaml:
+        # Any tables in the main config file are relabelled
+        # to "<tablename>_<setting>.  For example, if the
+        # main config.toml contains:
         #
-        # some_global_param = 75
+        #     some_global_param = 75
         #
-        # [T1_struct]
-        # bet_f = 0.2  # should be accessible via "T1_struct_bet_f"
-
-        if ident == 'config' : ident = ''
-        else:                  ident = f'{ident}_'
+        #     [T1_struct]
+        #     bet_f = 0.2
+        #
+        # bet_f is relabelled to T1_struct_bet_f
+        if ident == 'config':
+            settings = flatten_dictionary(settings, 1)
 
-        settings = {f'{ident}{k}' : v for k, v in settings.items()}
+        # All settings in secondary configuration files
+        # are relabelled to "<ident>_<setting>". For example,
+        # if T1_struct.toml contains:
+        #
+        #     bet_f = 0.5
+        #
+        # bet_f is relabelled to T1_struct_bet_f
+        else:
+            ident    = f'{ident}_'
+            settings = {f'{ident}{k}' : v for k, v in settings.items()}
 
         return settings
 
 
-    def __init__(self, cfgdir=None, **overrides):
-        """
+    def __init__(self, cfgdir=None, selectors=None, overrides=None):
+        """Create a Config object. Read configuration files from cfgdir.
+
+        Selectors are applied using Config.resolve_selectors.
+        Override values are applied using Config.apply_overrides.
         """
 
-        if cfgdir is None:
-            cfgdir = op.join(op.dirname(bip.__file__), 'data', 'config')
+        if selectors is None: selectors = {}
+        if overrides is None: overrides = {}
+        if cfgdir    is None: cfgdir    = DEFAULT_CONFIG_DIRECTORY
 
         cfgfiles = Config.list_config_files(cfgdir)
 
@@ -230,7 +388,9 @@ class Config:
 
         for fname in cfgfiles:
             log.debug('Loading settings from %s', fname)
-            settings.update(Config.load_config_file(fname, **overrides))
+            settings.update(Config.load_config_file(fname, selectors))
+
+        settings = Config.apply_overrides(settings, overrides)
 
         self.__settings = settings
         self.__cfgdir   = cfgdir
@@ -238,43 +398,64 @@ class Config:
 
     @property
     def cfgdir(self):
-        """
-        """
+        """Return the directory that the config files were loaded from. """
         return self.__cfgdir
 
 
-    def __contains__(self, name):
-        return name in self.__settings
+    def __contains__(self, key):
+        """Return True if a configuration item with key exists. """
+        return key in self.__settings
 
 
-    def __getitem__(self, name):
-        """Return the configuration item with the specified name. """
-        return self.__settings[name]
+    def __getitem__(self, key):
+        """Return the configuration item with the specified key. """
+        return self.__settings[key]
 
 
-    def __getattr__(self, name):
-        """Return the configuration item with the specified name. """
-        return self[name]
+    def __getattr__(self, key):
+        """Return the configuration item with the specified key. """
+        return self[kwey]
 
 
-    def get(self, name, default=None):
-        if name not in self:
+    def get(self, key, default=None):
+        """Return value associated with key.
+        Return default if there is no value associated with key.
+        """
+        if key not in self:
             return default
-        return self[name]
+        return self[key]
+
 
+    def getall(self, prefix=None, **kwargs):
+        """Return all requested values with given key prefix.
+        The specified default value is used for any keys which are not present.
+        The values are returned as a dictionary, with the keys un-prefixed.
+        """
+        if prefix is None: prefix = ''
+        else:              prefix = f'{prefix}_'
 
-    def getall(self, prefix, **kwargs):
         values = {}
         for k, v in kwargs.items():
-            values[k] = self.get(k, v)
+            values[k] = self.get(f'{prefix}{k}', v)
+
         return values
 
 
+    def gettuple(self, prefix=None, **kwargs):
+        """Same as getall, but values are returned as a tuple. """
+        return tuple(self.getall(prefix, **kwargs))
+
+
     def keys(self):
+        """Return all keys present in the config. """
         return self.__settings.keys()
 
+
     def values(self):
+        """Return all values present in the config. """
         return self.__settings.values()
 
+
     def items(self):
+        """Return all key-value pairs present in the config. """
         return self.__settings.items()