Commit 1454a7eb authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

MNT: initial commit

parents
# CI / CD for the fsl/conda/manifest project
#
# The fsl/conda/manifest project is used to manage
# internal and public FSL releases. The FSL release
# manifest and environment files are generated
# whenever changes are made to this project.
#
# The scripts used to generate and test files files
# are contained in the fsl/conda/manifest-rules
# project. All jobs are executed in an environment
# in which conda is available.
default:
before_script:
- conda create -c conda-forge -p ./env python=3.9
- source activate ./env
- python -m pip install git+https://git.fmrib.ox.ac.uk/fsl/conda/manifest-rules.git
# Don't run pipeline on merge requests
workflow:
rules:
- if: '$CI_COMMIT_TAG != null || $CI_COMMIT_BRANCH != null'
# Manifest and environment files can only be published
# from tags, or from these branches. Attempts to run a
# publish job from a different branch will fail.
# Additionally, the test stages will only perform a
# full test on these branches - a fast test is
# performed on other branches. See these files for more
# details:
#
# - fsl/conda/manifest-rules:manifest_rules/test_environment.py
# - fsl/conda/manifest-rules:manifest_rules/deploy_files.py
variables:
PUBLISH_FROM_BRANCHES: "master internal"
stages:
# Generate FSL conda environment specification
# files for all platforms and CUDA versions
- generate-environment-files
# Do a test installation for all environment
# files, and calculate the number of expected
# output lines so that the fslinstaller
# script can report progress to the user.
- test-environments
# Generate release or development manifest.json
# files
- generate-manifest-files
# Run the fslinstaller.py script against
# the generated manifest files
- test-manifests
# Publish the manifest and/or environment files.
# All jobs in this stage must be manually triggered.
- publish
# Generate FSL conda environment.yml files,
# for all supported platforms and CUDA
# versions, from fsl-release.yml. Refer to
# the scripts/generate_environment_files.py
# script for details.
generate-environment-files:
stage: generate-environment-files
image: continuumio/miniconda3
tags:
- fsl-ci
- docker
script:
- generate_environment_files ./environments
artifacts:
expire_in: 2 hrs
paths:
- environments
# Do a test install of all environment files on
# all platforms. Information about the
# installation of each environment file is saved
# to test_install/<environment_file>.install.txt.
# The macos-64 test job is run on a macOS shell
# runner - it is assumed that that conda is
# installed in the environment.
# This is a template. Concrete test jobs (below)
# specify a SUFFIX variable, denoting the
# environment file to test.
.test-environments:
stage: test-environments
dependencies:
- generate-environment-files
# tags and image is overridden by the macOS job
tags:
- fsl-ci
- docker
script:
- mkdir ./test_environments
- test_environment ./environments/*${SUFFIX}.yml 2>&1 | tee test_environments/${SUFFIX}.install.txt
artifacts:
expire_in: 2 hrs
when: always
paths:
- test_environments
test-environment:macos-64:
extends: .test-environments
tags:
- fsl-ci
- macOS
variables:
SUFFIX: "macos-64"
test-environment:linux-64:
extends: .test-environments
image: continuumio/miniconda3
variables:
SUFFIX: "linux-64"
test-environment:linux-64_cudaX.Y:
extends: .test-environments
image: fsldevelopment/fsl-linux-64
parallel:
matrix:
- CUDA: ["9.2", "10.2", "11.0"]
variables:
SUFFIX: "linux-64_cuda${CUDA}"
# Generate release manifest files. Both release
# and development manifest files are generated
# on every commit, but as the deploy jobs must
# be manually invoked, the user can choose which
# one they want to publish.
.generate-manifest-file:
stage: generate-manifest-files
image: continuumio/miniconda3
tags:
- fsl-ci
- docker
dependencies:
- generate-environment-files
- test-environment:linux-64
- test-environment:macos-64
- test-environment:linux-64_cudaX.Y
script:
- mkdir -p ./manifest
- generate_manifest_file "$OFFICIAL" ./manifest ./environments ./test_environments
artifacts:
expire_in: 2 hrs
paths:
- manifest
generate-release-manifest-file:
extends: .generate-manifest-file
variables:
OFFICIAL: "true"
generate-development-manifest-file:
extends: .generate-manifest-file
variables:
OFFICIAL: "false"
# Run the latest fslinstaller.py script
# on the generated manifest and environment
# files.
#
# This is a template. Each concrete job
# specifies a variable SUFFIX which denotes
# the environment file to be installed.
.test-manifest:
stage: test-manifests
script:
# there should only be one manifest file, so
# manifest/*.json will resolve to a single
# file. Concrete jobs should define SUFFIX
# so that environments/*${SUFFIX}.yml should
# also resolve to one file.
- test_manifest manifest/*.json environments/*${SUFFIX}.yml
test-release-manifest:linux-64:
stage: test-manifests
extends: .test-manifest
image: fsldevelopment/fsl-linux-64
variables:
SUFFIX: "linux-64"
tags:
- fsl-ci
- docker
dependencies:
- generate-environment-files
- generate-release-manifest-file
test-development-manifest:linux-64:
stage: test-manifests
extends: .test-manifest
image: fsldevelopment/fsl-linux-64
variables:
SUFFIX: "linux-64"
tags:
- fsl-ci
- docker
dependencies:
- generate-environment-files
- generate-development-manifest-file
test-release-manifest:linux-64_cudaX.Y:
stage: test-manifests
extends: .test-manifest
image: fsldevelopment/fsl-linux-64
parallel:
matrix:
- CUDA: ["9.2", "10.2", "11.0"]
variables:
SUFFIX: "linux-64_cuda$CUDA"
tags:
- fsl-ci
- docker
dependencies:
- generate-environment-files
- generate-release-manifest-file
test-development-manifest:linux-64_cudaX.Y:
stage: test-manifests
extends: .test-manifest
image: fsldevelopment/fsl-linux-64
parallel:
matrix:
- CUDA: ["9.2", "10.2", "11.0"]
variables:
SUFFIX: "linux-64_cuda${CUDA}"
tags:
- fsl-ci
- docker
dependencies:
- generate-environment-files
- generate-development-manifest-file
test-release-manifest:macos-64:
stage: test-manifests
extends: .test-manifest
variables:
SUFFIX: "macos-64"
tags:
- fsl-ci
- macOS
dependencies:
- generate-environment-files
- generate-release-manifest-file
test-development-manifest:macos-64:
stage: test-manifests
extends: .test-manifest
variables:
SUFFIX: "macos-64"
tags:
- fsl-ci
- macOS
dependencies:
- generate-environment-files
- generate-development-manifest-file
# Deploy/publish environment and manifest files.
# All deploy jobs must be manually invoked.
# Deployment destination is assumed to be
# available as a mount point at /releases/
.deploy-job-template:
stage: publish
image: continuumio/miniconda3
tags:
- fsl-ci
- fslconda-channel-host
when: manual
deploy-environment-files:
extends: .deploy-job-template
dependencies:
- generate-environment-files
script:
- deploy_files /releases/ ./environments/*.yml
deploy-development-manifest:
extends: .deploy-job-template
dependencies:
- generate-development-manifest-file
script:
- deploy_files /releases/ ./manifest/*.json
deploy-release-manifest:
extends: .deploy-job-template
dependencies:
- generate-release-manifest-file
script:
- deploy_files /releases/ ./manifest/*.json
# FSL release manifest
This repository is used to manage FSL releases. The `fsl-release.yml` file
defines the packages that are installed as part of a FSL release.
## The `fslinstaller.py` script
FSL is installed using a Python 2/3 script called
[fslinstaller.py](https://git.fmrib.ox.ac.uk/fsl/conda/installer).
The `fslinstaller` script starts by downloading a _manifest_ file (referred
to as `manifest.json`), a JSON file which contains information about:
- available FSL releases (past and present)
- miniconda installer releases, upon which FSL installations are based
- The latest available version of the fslinstaller script itself (used
for self-updating)
Once the appropriate FSL release has been determined, the `fslinstaller`
downloads a FSL release file - A FSL release is defined by a conda
environment specification file (referred to as `environment.yml`), which
is a YAML file that contains a list of the individual packages and their
respective versions that comprise a specific FSL release. Separate
`environment.yml` files exist for each supported platform and CUDA version.
## Generating the `manifest.json` and `environment.yml` files
The `manifest.json` and `environment.yml` files are automatically generated
from information stored in this repository:
- The `fsl-release.yml` file contains information about the `fslinstaller`,
miniconda installers, and packages for the latest FSL release.
- Information about past FSL releases is obtained from the most recently
generated `manifest.json` file.
The `fsl-release.yml` file is a YAML file with the following structure:
```yaml
# URL to FSL release information. The fslinstaller.py
# script, and previously generated manifest.json
# file is assumed to be downloadable from this URL.
# URLs for newly generated environment files are
# constructed using this URL.
release_url: http://18.133.213.73/releases/
# Latest fslinstaller version
installer: 1.2.3
# URLs to miniconda installers to
# use for all supported platforms
miniconda:
linux-64: https://github.com/conda-forge/miniforge/releases/download/4.10.3-1/Miniforge3-4.10.3-1-Linux-x86_64.sh
macos-64: https://github.com/conda-forge/miniforge/releases/download/4.10.3-1/Miniforge3-4.10.3-1-MacOSX-x86_64.sh
# Versions of CUDA for which FSL packages are
# built (currently only available on Linux).
cuda:
- 9.2
- 10.2
- 11.0
# Conda channels to source packages from
channels:
- http://18.133.213.73/public/
- conda-forge
- defaults
# Private internal channel for development and
# testing releases. This is prepended to the
# channels list for development releases.
internal_channel: http://${FSLCONDA_USERNAME}:${FSLCONDA_PASSWORD}@18.133.213.73/internal/
# List of FSL conda packages and their
# dependencies which comprise the FSL
# release.
packages:
- fsl-avwutils=2007.0
- fsl-eddy-cpu=2106.0
- python=3.9.6
...
```
When changes are made to the `fsl-release.yml` file, new versions of the
`manifest.json` and `environment.yml` files are generated and made available
for download.
Note that there are two types of manifest file:
- The official release manifest, containing information about past and
present public FSL releases - this is always called `manifest.json`, and
may be re-generated on new public releases, and on updates to e.g. the
fslinstaller script.
- Development manifests, containing information about one
internal/development FSL release. Development manifests are generated for
every internal release. Development manifests are named according to a
standard convention described in the **Internal/development FSL releases**
section, below.
Manifest and environment files can only be published when they are generated
from a new tag, or when they are generated from specific branches (currently
`master` or `internal`).
## New FSL releases
New public FSL releases are denoted by adding a tag to this repository. The
tag is used as the FSL version identifier, which must be of the form
`X.Y.Z[.W]`, where X, Y, Z, and W are integers.
When a new tag is added to this repository, the `manifest.json` and
`environment.yml` files are generated from the contents of `fsl-release.yml`,
and the contents of the previous `manifest.json` (used for obtaining
information about past FSL releases). These files are named according to these
conventions:
- `manifest.json` - the release manifest
- `fsl-<tag>_<platform>.yml` - FSL version `<tag>` for platform `<platform>`
(e.g. `fsl-6.1.0_macos-64.yml`).
- `fsl-<tag>_<platform>_cuda<cudaver>.yml` - FSL version `<tag>` for
platform `<platform>` and CUDA version `<cudaver>`
(e.g. `fsl-6.1.0_linux-64_cuda11.0.yml`).
After they have been generated, the manifest and environment files are
published to the FSL release web server. This publishing step must be manually
triggered through the GitLab web interface, via the `deploy-environment-files`
and `deploy-release-manifest` jobs.
## Internal/development FSL releases
When commits are added to _any_ branch of this repository, separate
`manifest.json` and `environment.yml` files are generated from the contents of
`fsl-release.yml` and made available for download. These files are assigned a
version identifier of the form `<tag>.<date>.<commit>.<branch>`, where:
where:
- `<tag>` is the _most recent_ public FSL release version identifier (e.g
`6.1.0`).
- `<date>` is the date of the commit that the files are generated from,
in the form `YYYYMMDD`.
- `<commit>` is the short hash of the git commit that the files are
generated from.
- `<branchname>` is the name of the branch on this repository.
The released files are then named as follows, where:
- `manifest-<tag>.<date>.<commit>.<branch>.json` - the development release
manifest file
- `fsl-<tag>.<date>.<commit>.<branch>_<platform>.yml` - FSL environment file
for platform `<platform>`.
- `fsl-<tag>.<date>.<commit>.<branch>_<platform>_cuda<cudaver>.yml` - FSL
environment file for platform `<platform>` (e.g. `linux-64`) and CUDA
version `<cudaver>` (e.g. `11.0`).
After they have been generated, the manifest and environment files may be
published. As with public releases, the process of publishing the manifest and
environment files must be manually triggered through the GitLab web interface,
via the `deploy-environment-files` and `deploy-development-manifest` jobs.
## Updates to `manifest.json`
Sometimes the contents of the official release `manifest.json` file may need
to be changed independently of a public FSL release - for example, a new
version of the `fslinstaller.py` script may be released, or the `miniconda`
installer may need to be changed.
Normally, `manifest.json` files that are not associated with a tag are named
`manifest-<tag>.<date>.<commit>.<branch>.json` as outlined above. However, a
new release `manifest.json` file is always re-generated, and can be deployed
through the Gitlab web interface, via the `deploy-release-manifest` job (with
the caveat that only files generated from the `master` or `internal` branches
may be published).
Release `manifest.json` files which are generated from branches will only
contain information about past official FSL releases. Release `manifest.json`
files which are generated from tags will contain information about past
official FSL release, and also about the new FSL release.
# URL to FSL release information. The fslinstaller.py
# script, and previously generated manifest.json
# file is assumed to be downloadable from this URL.
# URLs for newly generated environment files are
# constructed using this URL.
#release_url: http://18.133.213.73/releases/
release_url: http://localhost:8888/
# Latest fslinstaller version
installer: 1.0.8
# URLs to miniconda installers to
# use for all supported platforms
miniconda:
linux-64: https://github.com/conda-forge/miniforge/releases/download/4.10.3-1/Miniforge3-4.10.3-1-Linux-x86_64.sh
macos-64: https://github.com/conda-forge/miniforge/releases/download/4.10.3-1/Miniforge3-4.10.3-1-MacOSX-x86_64.sh
# Versions of CUDA for which FSL packages are
# built (currently only available on Linux).
cuda:
- 9.2
- 10.2
- 11.0
# Conda channels to source packages from
channels:
- http://18.133.213.73/public/
- conda-forge
- defaults
# Private internal channel for development and
# testing releases. This is prepended to the
# channels list for development releases.
internal_channel: http://${FSLCONDA_USERNAME}:${FSLCONDA_PASSWORD}@18.133.213.73/internal/
# List of FSL conda packages and their
# dependencies which comprise the FSL
# release.
packages:
# Packages sourced from anaconda.org / conda-forge
- boost-cpp 1.74
- fsleyes 1.0.14
- fslpy 3.6.1
- openblas 0.3.*
- python 3.9.*
- numpy 1.21.*
- tk 8.6.*
- zlib 1.2.*
# Packages sourced from the FSL conda channels
- ciftilib 1.5.3
- fsl-add_module 0.3.1
- fsl-armawrap 0.5.1
- fsl-asl_mfree v1.0.2
- fsl-avwutils 2101.0
- fsl-base 2107.3
- fsl-basil v4.0.4
- fsl-basisfield 1910.0
- fsl-baycest 2012.0
- fsl-bet2 2103.0
- fsl-bianca 2106.0
- fsl-bint 2012.0
- fsl-cluster 2004.0
- fsl-copain 2106.0
- fsl-cprob 2006.0
- fsl-cudabasisfield-cuda-10.2 2101.0
- fsl-cudabasisfield-cuda-11.0 2101.0
- fsl-cudabasisfield-cuda-9.2 2101.0
- fsl-cudimot 2006.0
- fsl-data_atlases 2103.0
- fsl-data_atlases_xtract 1.1.0
- fsl-data_first_models_317_bin 2006.0
- fsl-data_first_models_336_bin 2006.0
- fsl-data_linearmni 1.0.0
- fsl-data_possum 2106.0
- fsl-data_standard 2006.0
- fsl-deface 1.2.0
- fsl-discreteopt 2008.0
- fsl-dpm 1910.0
- fsl-dwssfp 2010.0
- fsl-eddy 2106.1
- fsl-eddy-cuda-10.2 2106.1
- fsl-eddy-cuda-11.0 2106.1
- fsl-eddy-cuda-9.2 2106.1
- fsl-eddy_qc v1.0.3
- fsl-fabber_core v4.0.10
- fsl-fabber_models_asl v2.0.7
- fsl-fabber_models_cest v2.1.6
- fsl-fabber_models_dce v2.1.5
- fsl-fabber_models_dsc v2.0.5
- fsl-fabber_models_dualecho v2.0.4
- fsl-fabber_models_dwi v2.0.3
- fsl-fabber_models_pet v1.0.1
- fsl-fabber_models_qbold v1.0.2
- fsl-fabber_models_t1 v2.0.2
- fsl-fast4 2012.0
- fsl-fastpdlib 2012.0
- fsl-fdt 2103.0
- fsl-fdt-cuda-10.2 2103.0
- fsl-fdt-cuda-11.0 2103.0
- fsl-fdt-cuda-9.2 2103.0
- fsl-feat5 2104.0
- fsl-film 2007.0
- fsl-filmbabe 2012.0
- fsl-first 2006.0
- fsl-first_lib 2012.0
- fsl-flameo 2012.0
- fsl-flirt 2010.0
- fsl-fnirt 2012.0
- fsl-fugue 2010.0
- fsl-get_standard 0.1.0
- fsl-giftiio 2012.0
- fsl-gps 2012.0
- fsl-lesions 1909.1
- fsl-libgdc 2006.0
- fsl-libmeshutils 2012.0
- fsl-libvis 1909.0
- fsl-load_dicom 2012.0
- fsl-load_varian 1904.0
- fsl-mcflirt 2012.0
- fsl-melodic 2012.0
- fsl-meshclass 2102.0
- fsl-misc_c 2106.0
- fsl-misc_scripts 2004.0
- fsl-misc_tcl 2010.0
- fsl-miscmaths 2010.0
- fsl-miscvis 2007.0
- fsl-mist 2010.0
- fsl-mm 1909.0
- fsl-mrs 1.1.2
- fsl-msm 2008.0
- fsl-msmreglib 2008.0
- fsl-mvdisc 2012.0
- fsl-newimage 2104.1
- fsl-newmesh 1904.0
- fsl-newnifti 3.0.6
- fsl-newran 2012.0
- fsl-oxford_asl v4.0.24
- fsl-possum 2012.0
- fsl-ptx2 2101.1
- fsl-ptx2-cuda-10.2 2101.1
- fsl-ptx2-cuda-11.0 2101.1
- fsl-ptx2-cuda-9.2 2101.1
- fsl-pyfeeds 0.9.7
- fsl-pyfeeds-tests 2106.0
- fsl-qboot 2010.0