Commit 1f58c92a authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

ENH: Initial work on new conda-based installer script and

FSLrelease/environment management system
parents
# CI / CD for the fsl/conda/installer project.
#
# Two pipelines may be executed:
#
# - Commits pushed to any branch cause the test stage to run, which runs a
# set of unit tests.
#
# - New tags cause the release-* stages to run:
#
# 1.A set of FSL conda environment specifications (conda YAML environment
# files) are generated
# 2. The environment specifications are sanity-checked for validity
# 3. The environment specifications and the fslinstaller.py script are
# deployed to a server to be made available for download (this job must
# be manually triggered).
stages:
- test
- release-build
- release-test
- release-deploy
variables:
# Path to the FSL installer directory where the installer
# script and environment files are deployed to - assumed
# to be locally accessible to docker jobs that are
# executed by the fslconda-channel-host gitlab runner.
FSL_INSTALLER_DIRECTORY: /fslinstaller
# Unit tests run on every push to any branch
unit-tests:
stage: test
image: continuumio/miniconda3
tags:
- fsl-ci
- docker
rules:
- if '$CI_COMMIT_TAG == null'
script:
- true
# Generate FSL conda environment specifications on new tags
generate-environment-specs:
stage: release-build
image: python:3.9
tags:
- fsl-ci
- docker
rules:
- if '$CI_COMMIT_TAG != null'
variables:
- PREFIX: fsl-$CI_COMMIT_TAG
script:
- python utils/generate_environment_spec.py ${PREFIX}-linux-64.yml linux-64
- python utils/generate_environment_spec.py ${PREFIX}-macos-64.yml macos-64
- python utils/generate_environment_spec.py ${PREFIX}-linux-64-cuda-9.2.yml linux-64 9.2
- python utils/generate_environment_spec.py ${PREFIX}-linux-64-cuda-10.2.yml linux-64 10.2
- python utils/generate_environment_spec.py ${PREFIX}-linux-64-cuda-11.0.yml linux-64 11.0
artifacts:
paths:
- ${PREFIX}*.yml
# Validate newly generated FSL conda environment specifications
test-release:
stage: release-test
image: continuumio/miniconda3
tags:
- fsl-ci
- docker
dependencies:
- generate-environment-specs
script:
- true
# Deploy FSL conda environment specifications and installer script
upload-installer-and-environment-specs:
stage: release-deploy
image: python:3.9
when: manual
tags:
- fsl-ci
- fslconda-channel-host
dependencies:
- generate-environment-specs
script:
- cp fsl-*.yml fslinstaller.py ${FSL_INSTALLER_DIRECTORY}
# FSL installer
This repository contains:
- `fslinstaller.py`: The FSL installer script for conda-based FSL releases
- `fsl-packages.txt`: The list of packages that are included in official FSL
releases
The `fslinstaller.py` script in this repository is the successor to the
`fslinstaller.py` script from the fsl/installer> repository.
The `fsl-packages.txt` file in this repository is the successor to the
`FSL_SourceManifest.yml` file from the `fsl/FslBuildManifests` repository.
boost-cpp=1.74.*
fsl-avwutils
fsl-eddy
numpy=1.20.*
openblas=0.3.*
python=3.9.*
tk=8.6.*
zlib=1.2.*
#!/usr/bin/env python
import sys
import argparse
def parseArgs(argv=None):
helptext = {
'dest' : 'Install FSL into this folder (default: /usr/local/fsl)',
'listversions' : 'List available versions of FSL',
'version' : 'Download this specific version of FSL',
}
parser = argparse.ArgumentParser()
parser.add_argument('-d', '--dest', help=helptext['dest'],
metavar='DESTDIR', default='/usr/local/fsl/')
parser.add_argument('-l', '--listversions', help=helptext['listversions'],
action='store_true')
parser.add_argument('-V', '--version', help=helptext['version'])
return parser.parse_args(argv)
def main(argv=None):
args = parseArgs(argv)
if __name__ == '__main__':
sys.exit(main())
#!/usr/bin/env python
"""Generate conda YAML environment specifications for specific platforms.
This script reads the fsl-environment-yml.template template file, and generates
a new file from it for a specific platform and, optionally, CUDA version.
"""
import os.path as op
import re
import sys
USAGE = """Usage: generate_environment_spec.py outfile platform [cudaver]
Generates a conda YAML environment specification file for a specific platform
(and, optionally, CUDA version).
Arguments:
- outfile: File to write generated specification to
- platform: Platform to generate specification for (linux, macOS)
- cudaver: (Optional) CUDA version to generate specification for
(9.2 10.2 11.1 11.2).
"""
CHANNELS = [
'http://18.133.213.73/production/',
'conda-forge',
'defaults'
]
def parse_selector(selector):
"""Parses a selector specification. A selector specification is a
sequence of key-value pairs which are appended to lines in the
fsl-environment.yml template file. For example:
platform:linux, cudaver:9.2
Returns a dictionary containing the key-value pairs.
"""
selector = selector.lower()
kvps = selector.split(',')
kvps = selector.split(',')
kvps = [kvp.strip().split(':') for kvp in kvps]
return dict((k.strip(), v.strip()) for k, v in kvps)
def filter_packages(packages, selectors):
"""Filters the given package list according to the given selectors. """
pat = r'^(.+) *# *\[(.+)\]$'
include = []
for pkg in packages:
match = re.fullmatch(pat, pkg)
if match is None:
include.append(pkg)
continue
pkg = match.group(1)
pkg_selector = match.group(2)
pkg_selector = parse_selector(pkg_selector)
selected = all(selectors.get(k, None) == v
for k, v in pkg_selector.items())
if selected:
include.append(pkg)
return include
def load_packages(pkgfile):
"""Load the fsl-packages.txt file, returning a list of package entries. """
with open(pkgfile) as f:
packages = f.readlines()
packages = [p.strip() for p in packages]
packages = [p for p in packages if p != '' and not p.startswith('#')]
return packages
def generate_environment_spec(outfile, channels, packages):
"""Writes a conda YAML environment speciifcation to the given file,
containing the given channels and packages.
"""
with open(outfile, 'wt') as f:
f.write('channels:\n')
for channel in channels:
f.write(f' - {channel}\n')
f.write('dependencies:\n')
for pkg in packages:
f.write(f' - {pkg}\n')
def main():
"""Generates a conda YAML environment specification file. """
if len(sys.argv) not in (3, 4):
print(USAGE)
return 1
thisdir = op.abspath(op.dirname(__file__))
pkgfile = op.join(thisdir, 'fsl-packages.txt')
outfile = sys.argv[1]
selectors = {}
selectors['platform'] = sys.argv[2].lower()
if len(sys.argv) == 4:
selectors['cudaver'] = sys.argv[3].lower()
packages = load_packages(pkgfile)
packages = filter_packages(packages, selectors)
generate_environment_spec(outfile, CHANNELS, packages)
if __name__ == '__main__':
sys.exit(main())
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment