ENH: Initial work on new conda-based installer script and

FSLrelease/environment management system

.gitlab-ci.yml

+# CI / CD for the fsl/conda/installer project.
+#
+# Two pipelines may be executed:
+#
+#   - Commits pushed to any branch cause the test stage to run, which runs a
+#     set of unit tests.
+#
+#   - New tags cause the release-* stages to run:
+#
+#      1.A set of FSL conda environment specifications (conda YAML environment
+#         files) are generated
+#      2. The environment specifications are sanity-checked for validity
+#      3. The environment specifications and the fslinstaller.py script are
+#         deployed to a server to be made available for download (this job must
+#         be manually triggered).
+
+
+stages:
+  - test
+  - release-build
+  - release-test
+  - release-deploy
+
+
+variables:
+
+  # Path to the FSL installer directory where the installer
+  # script and environment files are deployed to - assumed
+  # to be locally accessible to docker jobs that are
+  # executed by the fslconda-channel-host gitlab runner.
+  FSL_INSTALLER_DIRECTORY: /fslinstaller
+
+
+# Unit tests run on every push to any branch
+unit-tests:
+  stage: test
+  image: continuumio/miniconda3
+  tags:
+    - fsl-ci
+    - docker
+  rules:
+    - if '$CI_COMMIT_TAG == null'
+  script:
+    - true
+
+
+# Generate FSL conda environment specifications on new tags
+generate-environment-specs:
+  stage: release-build
+  image: python:3.9
+  tags:
+    - fsl-ci
+    - docker
+  rules:
+    - if '$CI_COMMIT_TAG != null'
+
+  variables:
+    - PREFIX: fsl-$CI_COMMIT_TAG
+
+  script:
+    - python utils/generate_environment_spec.py ${PREFIX}-linux-64.yml           linux-64
+    - python utils/generate_environment_spec.py ${PREFIX}-macos-64.yml           macos-64
+    - python utils/generate_environment_spec.py ${PREFIX}-linux-64-cuda-9.2.yml  linux-64  9.2
+    - python utils/generate_environment_spec.py ${PREFIX}-linux-64-cuda-10.2.yml linux-64 10.2
+    - python utils/generate_environment_spec.py ${PREFIX}-linux-64-cuda-11.0.yml linux-64 11.0
+
+  artifacts:
+    paths:
+      - ${PREFIX}*.yml
+
+
+# Validate newly generated FSL conda environment specifications
+test-release:
+  stage: release-test
+  image: continuumio/miniconda3
+  tags:
+    - fsl-ci
+    - docker
+  dependencies:
+    - generate-environment-specs
+  script:
+    - true
+
+
+# Deploy FSL conda environment specifications and installer script
+upload-installer-and-environment-specs:
+  stage: release-deploy
+  image: python:3.9
+  when:  manual
+  tags:
+    - fsl-ci
+    - fslconda-channel-host
+  dependencies:
+    - generate-environment-specs
+
+  script:
+    - cp fsl-*.yml fslinstaller.py ${FSL_INSTALLER_DIRECTORY}

README.md

+# FSL installer
+
+
+This repository contains:
+
+  - `fslinstaller.py`:  The FSL installer script for conda-based FSL releases
+  - `fsl-packages.txt`: The list of packages that are included in official FSL
+    releases
+
+
+The `fslinstaller.py` script in this repository is the successor to the
+`fslinstaller.py` script from the fsl/installer> repository.
+
+
+The `fsl-packages.txt` file in this repository is the successor to the
+`FSL_SourceManifest.yml` file from the `fsl/FslBuildManifests` repository.

fsl-packages.txt

+boost-cpp=1.74.*
+fsl-avwutils
+fsl-eddy
+numpy=1.20.*
+openblas=0.3.*
+python=3.9.*
+tk=8.6.*
+zlib=1.2.*

fslinstaller.py

+#!/usr/bin/env python
+
+
+import sys
+import argparse
+
+
+def parseArgs(argv=None):
+
+    helptext = {
+        'dest'         : 'Install FSL into this folder (default: /usr/local/fsl)',
+        'listversions' : 'List available versions of FSL',
+        'version'      : 'Download this specific version of FSL',
+    }
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-d', '--dest', help=helptext['dest'],
+                        metavar='DESTDIR', default='/usr/local/fsl/')
+    parser.add_argument('-l', '--listversions', help=helptext['listversions'],
+                        action='store_true')
+    parser.add_argument('-V', '--version', help=helptext['version'])
+
+    return parser.parse_args(argv)
+
+
+def main(argv=None):
+    args = parseArgs(argv)
+
+
+if __name__ == '__main__':
+    sys.exit(main())

generate_environment_spec.py

+#!/usr/bin/env python
+"""Generate conda YAML environment specifications for specific platforms.
+
+This script reads the fsl-environment-yml.template template file, and generates
+a new file from it for a specific platform and, optionally, CUDA version.
+"""
+
+
+import os.path as op
+import            re
+import            sys
+
+
+USAGE = """Usage: generate_environment_spec.py outfile platform [cudaver]
+
+Generates a conda YAML environment specification file for a specific platform
+(and, optionally, CUDA version).
+
+Arguments:
+
+  - outfile:  File to write generated specification to
+
+  - platform: Platform to generate specification for (linux, macOS)
+
+  - cudaver:  (Optional) CUDA version to generate specification for
+              (9.2 10.2 11.1 11.2).
+"""
+
+
+CHANNELS = [
+    'http://18.133.213.73/production/',
+    'conda-forge',
+    'defaults'
+]
+
+
+def parse_selector(selector):
+    """Parses a selector specification. A selector specification is a
+    sequence of key-value pairs which are appended to lines in the
+    fsl-environment.yml template file. For example:
+
+       platform:linux, cudaver:9.2
+
+    Returns a dictionary containing the key-value pairs.
+    """
+    selector = selector.lower()
+    kvps     = selector.split(',')
+    kvps     = selector.split(',')
+    kvps     = [kvp.strip().split(':')  for kvp in kvps]
+
+    return dict((k.strip(), v.strip()) for k, v in kvps)
+
+
+def filter_packages(packages, selectors):
+    """Filters the given package list according to the given selectors. """
+
+    pat     = r'^(.+) *# *\[(.+)\]$'
+    include = []
+
+    for pkg in packages:
+
+        match = re.fullmatch(pat, pkg)
+
+        if match is None:
+            include.append(pkg)
+            continue
+
+        pkg          = match.group(1)
+        pkg_selector = match.group(2)
+
+        pkg_selector = parse_selector(pkg_selector)
+        selected      = all(selectors.get(k, None) == v
+                            for k, v in pkg_selector.items())
+
+        if selected:
+            include.append(pkg)
+
+    return include
+
+
+def load_packages(pkgfile):
+    """Load the fsl-packages.txt file, returning a list of package entries. """
+
+    with open(pkgfile) as f:
+        packages = f.readlines()
+
+    packages = [p.strip() for p in packages]
+    packages = [p for p in packages if p != '' and not p.startswith('#')]
+
+    return packages
+
+
+def generate_environment_spec(outfile, channels, packages):
+    """Writes a conda YAML environment speciifcation to the given file,
+    containing the given channels and packages.
+    """
+
+    with open(outfile, 'wt') as f:
+
+        f.write('channels:\n')
+        for channel in channels:
+            f.write(f'  - {channel}\n')
+
+        f.write('dependencies:\n')
+        for pkg in packages:
+            f.write(f'  - {pkg}\n')
+
+
+def main():
+    """Generates a conda YAML environment specification file. """
+    if len(sys.argv) not in (3, 4):
+        print(USAGE)
+        return 1
+
+    thisdir = op.abspath(op.dirname(__file__))
+    pkgfile = op.join(thisdir, 'fsl-packages.txt')
+    outfile = sys.argv[1]
+
+    selectors             = {}
+    selectors['platform'] = sys.argv[2].lower()
+
+    if len(sys.argv) == 4:
+        selectors['cudaver'] = sys.argv[3].lower()
+
+    packages = load_packages(pkgfile)
+    packages = filter_packages(packages, selectors)
+
+    generate_environment_spec(outfile, CHANNELS, packages)
+
+
+if __name__ == '__main__':
+    sys.exit(main())