Merge remote-tracking branch 'origin/master' into mnt/devrelease

README.md

@@ -9,10 +9,15 @@ generated from information stored in this repository, using CI rules and
 scripts in the fsl/conda/manifest-rules> repository.
 
 
-This repository contains two branches:
- - Public FSL releases are created from tags on the `external` branch.
- - Development releases are created from commits on the `external` branch.
- - Internal FSL releases are created from the `master` branch
+This repository contains two branches, `master` and `external`. These two
+branches are used to create three different types of FSL release:
+
+ - Public FSL releases (e.g. FSL 6.0.6) are created from **tags** on the
+   `external` branch.
+ - Development releases (development versions of public releases) are created
+   from **commits** on the `external` branch.
+ - Internal FSL releases (for internal use only) are created from **commits**
+   on the `master` branch
 
 
 ## The `fslinstaller.py` script
@@ -59,7 +64,7 @@ The `fsl-release.yml` file is a YAML file with the following structure:
 # file is assumed to be downloadable from this URL.
 # URLs for newly generated environment files are
 # constructed using this URL.
-release_url: http://18.133.213.73/releases/
+release-url: https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/releases/
 
 # Latest fslinstaller version
 installer: 1.2.3
@@ -79,23 +84,51 @@ cuda:
 
 # Conda channels to source packages from
 channels:
- - http://18.133.213.73/public/
+ - https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/public/
  - conda-forge
  - defaults
 
 # Private channel containing packages that are
 # only released internally. This is prepended
 # to the channels list for internal releases.
-internal_channel: http://${FSLCONDA_USERNAME}:${FSLCONDA_PASSWORD}@18.133.213.73/internal/
+internal-channel: http://${FSLCONDA_USERNAME}:${FSLCONDA_PASSWORD}@https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/internal/
+
+# The FSL installation is created in two stages.
+# First of all, the base_packages are installed.
+# Versions of these packages are specified in
+# the packages list below
+base-packages:
+ - fsl-base
+ - fslpy
+ - fsl-sub
+ - libopenblas
+ ...
 
-# List of FSL conda packages and their
-# dependencies which comprise the FSL
-# release.
+# List of all packages that comprise a FSL
+# installation. All packages not installed
+# in the first stage (see base_packages
+# above) are installed in the second stage.
 packages:
- - fsl-avwutils=2007.0
- - fsl-eddy-cpu=2106.0
- - python=3.9.6
+ - ciftilib 1.5.3
+ - fsleyes 1.3.0
+ - fslpy 3.7.0
+ - fsl-avwutils 2007.0
+ - fsl-base 2109.1
+ - fsl-eddy-cpu 2106.0
+ - fsl-sub 2.5.8
+ - libopenblas 0.3.18 *openmp*
  ...
+
+# List of additional packages that
+# are added to linux-64 installations.
+linux-64-packages: []
+
+# List of additional packages that
+# are added to macos-64 installations.
+macos-64-packages:
+ # Newer libffi causes FSLeyes to segfault:
+ # https://github.com/conda-forge/pyobjc-core-feedstock/issues/37
+ - libffi 3.3
 ```
 
 
@@ -235,3 +268,44 @@ Release `manifest.json` files which are generated from branches will only
 contain information about past official FSL releases. Release `manifest.json`
 files which are generated from new tags will contain information about past
 official FSL release, and also about the new FSL release.
+
+
+## Development workflow
+
+
+The `fsl-release.yml` file on the `master` branch should be updated as new
+versions of FSL projects are released. This will result in new manifest and
+environment files for internal installations to be created.
+
+
+In order to create manifest and environment files for a new public FSL
+release, the `fsl-release.yml` file on the `external` branch must be updated.
+This can be accomplished using the following process:
+
+1. Clone the `fsl/conda/manifest` repository locally:
+   ```
+   git clone git@git.fmrib.ox.ac.uk:fsl/conda/manifest.git
+   cd manifest
+   ```
+2. Create a new branch off the `external` branch:
+   ```
+   git checkout -b rel/6.0.7 origin/external
+   ```
+3. Merge all changes from the `master` branch onto this branch:
+   ```
+   git merge origin/master
+   ```
+4. Resolve any conflicts that arise, making sure that internal-only FSL
+   packages are not included in the package lists. A good idea is to compare
+   your branch with the `master` branch - usually the only difference should
+   be that your branch does not contain any internal-only packages:
+   ```
+   git diff origin/master HEAD
+   ```
+5. Merge your branch into the `external` branch on the `fsl/conda/manifest`
+   GitLab repository via a merge request. This will result in new manifest
+   and environment files for a new development release to be created.
+
+6. If this is a public FSL release, create a new tag on the `external` branch.
+   This will result in a new `manifest.json`, and environment files for the
+   new FSL release to be created.

fsl-release.yml

@@ -3,16 +3,16 @@
 # file is assumed to be downloadable from this URL.
 # URLs for newly generated environment files are
 # constructed using this URL.
-release_url: http://18.133.213.73/releases/
+release-url: https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/releases/
 
 # Latest fslinstaller version
-installer: 1.0.15
+installer: 1.4.3
 
 # URLs to miniconda installers to
 # use for all supported platforms
 miniconda:
- linux-64: https://github.com/conda-forge/miniforge/releases/download/4.10.3-1/Miniforge3-4.10.3-1-Linux-x86_64.sh
- macos-64: https://github.com/conda-forge/miniforge/releases/download/4.10.3-1/Miniforge3-4.10.3-1-MacOSX-x86_64.sh
+ linux-64: https://github.com/conda-forge/miniforge/releases/download/4.10.3-7/Miniforge3-4.10.3-7-Linux-x86_64.sh
+ macos-64: https://github.com/conda-forge/miniforge/releases/download/4.10.3-7/Miniforge3-4.10.3-7-MacOSX-x86_64.sh
 
 # Versions of CUDA for which FSL packages are
 # built (currently only available on Linux).
@@ -20,40 +20,46 @@ cuda:
  - 9.2
  - 10.2
  - 11.0
+ - 11.1
+ - 11.3
 
 # Conda channels to source packages from
 channels:
- - http://18.133.213.73/public/
+ - https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/public/
  - conda-forge
  - defaults
 
 # Private internal channel for development and
 # testing releases. This is prepended to the
 # channels list for development releases.
-internal_channel: http://${FSLCONDA_USERNAME}:${FSLCONDA_PASSWORD}@18.133.213.73/internal/
+internal-channel: http://${FSLCONDA_USERNAME}:${FSLCONDA_PASSWORD}@fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/internal/
 
-# List of FSL conda packages and their
-# dependencies which comprise the FSL
-# release.
-packages:
-
- # Packages sourced from anaconda.org / conda-forge
- - boost-cpp 1.74
- - fsleyes 1.0.14
- - fslpy 3.6.1
- - openblas 0.3.*
- - python 3.9.*
- - numpy 1.21.*
- - tk 8.6.*
- - zlib 1.2.*
+# The FSL installation is created in two stages.
+# First of all, the base_packages are installed.
+# Versions of these packages are specified in
+# the packages list below
+base-packages:
+ - fslpy
+ - fsl-base
+ - fsl-sub
+ - fsl-sub_plugin_sge
+ - fsl-sub_plugin_slurm
+ - libopenblas
 
- # Packages sourced from the FSL conda channels
+# List of all packages that comprise a FSL
+# installation. All packages not installed
+# in the first stage (see base_packages
+# above) are installed in the second stage.
+packages:
  - ciftilib 1.5.3
- - fsl-add_module 0.3.1
- - fsl-armawrap 0.5.1
+ - fmrib-unpack 2.8.0
+ - fsleyes 1.3.0
+ - fslpy 3.7.0
+ - fsl-add_module 0.3.2
+ - fsl-armawrap 0.6.0
  - fsl-asl_mfree v1.0.2
  - fsl-avwutils 2101.0
- - fsl-base 2107.3
+ - fsl-base 2109.1
  - fsl-basil v4.0.4
  - fsl-basisfield 1910.0
  - fsl-baycest 2012.0
@@ -62,9 +68,11 @@ packages:
  - fsl-bint 2012.0
  - fsl-cluster 2004.0
  - fsl-cprob 2006.0
+ - fsl-cudabasisfield-cuda-9.2 2101.0
  - fsl-cudabasisfield-cuda-10.2 2101.0
  - fsl-cudabasisfield-cuda-11.0 2101.0
- - fsl-cudabasisfield-cuda-9.2 2101.0
+ - fsl-cudabasisfield-cuda-11.1 2101.0
+ - fsl-cudabasisfield-cuda-11.3 2101.0
  - fsl-cudimot 2006.0
  - fsl-data_atlases 2103.0
  - fsl-data_atlases_xtract 1.1.0
@@ -76,9 +84,11 @@ packages:
  - fsl-deface 1.2.0
  - fsl-discreteopt 2008.0
  - fsl-eddy 2106.1
+ - fsl-eddy-cuda-9.2 2106.1
  - fsl-eddy-cuda-10.2 2106.1
  - fsl-eddy-cuda-11.0 2106.1
- - fsl-eddy-cuda-9.2 2106.1
+ - fsl-eddy-cuda-11.1 2106.1
+ - fsl-eddy-cuda-11.3 2106.1
  - fsl-eddy_qc v1.0.3
  - fsl-fabber_core v4.0.10
  - fsl-fabber_models_asl v2.0.7
@@ -93,16 +103,18 @@ packages:
  - fsl-fast4 2012.0
  - fsl-fastpdlib 2012.0
  - fsl-fdt 2103.0
+ - fsl-fdt-cuda-9.2 2103.0
  - fsl-fdt-cuda-10.2 2103.0
  - fsl-fdt-cuda-11.0 2103.0
- - fsl-fdt-cuda-9.2 2103.0
+ - fsl-fdt-cuda-11.1 2103.0
+ - fsl-fdt-cuda-11.3 2103.0
  - fsl-feat5 2104.0
  - fsl-film 2007.0
  - fsl-filmbabe 2012.0
  - fsl-first 2006.0
  - fsl-first_lib 2012.0
  - fsl-flameo 2012.0
- - fsl-flirt 2010.0
+ - fsl-flirt 2108.0
  - fsl-fnirt 2012.0
  - fsl-fugue 2010.0
  - fsl-get_standard 0.1.0
@@ -117,12 +129,12 @@ packages:
  - fsl-meshclass 2102.0
  - fsl-misc_c 2106.0
  - fsl-misc_scripts 2004.0
- - fsl-misc_tcl 2010.0
+ - fsl-misc_tcl 2109.0
  - fsl-miscmaths 2010.0
  - fsl-miscvis 2007.0
  - fsl-mist 2010.0
  - fsl-mm 1909.0
- - fsl-mrs 1.1.2
+ - fsl-mrs 1.1.8
  - fsl-msm 2008.0
  - fsl-msmreglib 2008.0
  - fsl-mvdisc 2012.0
@@ -133,17 +145,21 @@ packages:
  - fsl-oxford_asl v4.0.24
  - fsl-possum 2012.0
  - fsl-ptx2 2101.1
+ - fsl-ptx2-cuda-9.2 2101.1
  - fsl-ptx2-cuda-10.2 2101.1
  - fsl-ptx2-cuda-11.0 2101.1
- - fsl-ptx2-cuda-9.2 2101.1
- - fsl-pyfeeds 0.9.8
- - fsl-pyfeeds-tests 2106.0
+ - fsl-ptx2-cuda-11.1 2101.1
+ - fsl-ptx2-cuda-11.3 2101.1
+ - fsl-pyfeeds 0.9.9
+ - fsl-pyfeeds-tests 2110.0
  - fsl-qboot 2010.0
  - fsl-randomise 2101.0
  - fsl-shapemodel 2012.0
  - fsl-siena 2006.0
  - fsl-slicetimer 2012.0
- - fsl-sub 2.5.6
+ - fsl-sub 2.5.8
+ - fsl-sub_plugin_sge 1.4.7
+ - fsl-sub_plugin_slurm 1.3.8
  - fsl-surface 2101.0
  - fsl-susan 2012.0
  - fsl-swe v1.0.3
@@ -160,6 +176,18 @@ packages:
  - fsl-znzlib 2012.0
  - ghalton 0.6.1
  - hlsvdpropy 2.0.1
+ - libopenblas 0.3.18 *openmp*
  - nidmresults 2.1.0
  - nidmresults-fsl 2.2.0
  - openslide-python v1.1.2
+
+# List of additional packages that
+# are added to linux-64 installations.
+linux-64-packages: []
+
+# List of additional packages that
+# are added to macos-64 installations.
+macos-64-packages:
+ # Newer libffi causes FSLeyes to segfault:
+ # https://github.com/conda-forge/pyobjc-core-feedstock/issues/37
+ - libffi 3.3

manifest.example.json

@@ -13,13 +13,13 @@
     "installer" : {
 
         // Latest version of fslinstaller script
-        "version" : "1.2.3",
+        "version" : "1.3.4",
 
         // URL to download fslinstaller script
-        "url" : "http://localhost:8888/fslinstaller.py",
+        "url" : "https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/releases/fslinstaller.py",
 
         // SHA256 checksum of installer script
-        "sha256"  : "f6b234c11ccabd6454b0c5b2bdf1a2a69e521c596187d887d69938a3c344af3e"
+        "sha256"  : "2626342288e232d56bd16898ab6b46a348e21bb7f06bb8371fbf7479d7b5e4e8"
     },
 
     // URLs to download miniconda/miniforge
@@ -60,6 +60,8 @@
         //                     builds.
         //   - "environment"   Conda environment.yml file
         //   - "sha256"        SHA256 checksum of environment.yml file
+        //   - "base_packages" List of "base" packages, which are
+        //                     installed before all other packages.
         //   - "output"        Dictionary containing number of lines of
         //                     output to expect when installing the FSL
         //                     environment ("install" : "100"), or when
@@ -69,8 +71,9 @@
         "6.1.0"  : [
             {
                 "platform"      : "linux-64",
-                "environment" : "http://localhost:8888/fsl-6.1.0-linux-64.yml",
+                "environment"   : "https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/releases/fsl-6.1.0-linux-64.yml",
                 "sha256"        : "952a0101fc19948e785572f8c06436c59f19c4ef92df9425f2e2033d4b986918",
+                "base_packages" : ["fsl-base", "libopenblas", "fslpy"],
                 "output"        : {
                     "install"   : "92",
                     "6.0.6"     : "11"
@@ -78,8 +81,9 @@
             },
             {
                 "platform"      : "macos-64",
-                "environment" : "http://localhost:8888/fsl-6.1.0-macos-64.yml",
+                "environment"   : "https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/releases/fsl-6.1.0-macos-64.yml",
                 "sha256"        : "952a0101fc19948e785572f8c06436c59f19c4ef92df9425f2e2033d4b986918",
+                "base_packages" : ["fsl-base", "libopenblas", "fslpy"],
                 "output"        : {
                     "install"   : "92",
                     "6.0.6"     : "11"
@@ -88,8 +92,9 @@
             {
                 "platform"      : "linux-64",
                 "cuda"          : "9.2",
-                "environment" : "http://localhost:8888/fsl-6.1.0-linux-64-cuda9.2.yml",
+                "environment"   : "https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/releases/fsl-6.1.0-linux-64-cuda9.2.yml",
                 "sha256"        : "952a0101fc19948e785572f8c06436c59f19c4ef92df9425f2e2033d4b986918",
+                "base_packages" : ["fsl-base", "libopenblas", "fslpy"],
                 "output"        : {
                     "install"   : "92",
                     "6.0.6"     : "11"
@@ -98,8 +103,9 @@
             {
                 "platform"      : "linux-64",
                 "cuda"          : "10.2",
-                "environment" : "http://localhost:8888/fsl-6.1.0-linux-64-cuda10.2.yml",
+                "environment"   : "https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/releases/fsl-6.1.0-linux-64-cuda10.2.yml",
                 "sha256"        : "85ef650586c1d7f3475fe7d16108bb94b737decbfc7506ad4deff2917365d05f",
+                "base_packages" : ["fsl-base", "libopenblas", "fslpy"],
                 "output"        : {
                     "install"   : "92",
                     "6.0.6"     : "11"
@@ -109,8 +115,9 @@
         "6.2.0" : [
             {
                 "platform"      : "linux-64",
-                "environment" : "http://localhost:8888/fsl-6.2.0-linux-64.yml",
+                "environment"   : "https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/releases/fsl-6.2.0-linux-64.yml",
                 "sha256"        : "952a0101fc19948e785572f8c06436c59f19c4ef92df9425f2e2033d4b986918",
+                "base_packages" : ["fsl-base", "libopenblas", "fslpy"],
                 "output"        : {
                     "install"   : "92",
                     "6.0.6"     : "11",
@@ -119,8 +126,9 @@
             },
             {
                 "platform"      : "macos-64",
-                "environment" : "http://localhost:8888/fsl-6.2.0-macos-64.yml",
+                "environment"   : "https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/releases/fsl-6.2.0-macos-64.yml",
                 "sha256"        : "952a0101fc19948e785572f8c06436c59f19c4ef92df9425f2e2033d4b986918",
+                "base_packages" : ["fsl-base", "libopenblas", "fslpy"],
                 "output"        : {
                     "install"   : "92",
                     "6.0.6"     : "11",
@@ -130,8 +138,9 @@
             {
                 "platform"      : "linux-64",
                 "cuda"          : "9.2",
-                "environment" : "http://localhost:8888/fsl-6.2.0-linux-64-cuda9.2.yml",
+                "environment"   : "https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/releases/fsl-6.2.0-linux-64-cuda9.2.yml",
                 "sha256"        : "952a0101fc19948e785572f8c06436c59f19c4ef92df9425f2e2033d4b986918",
+                "base_packages" : ["fsl-base", "libopenblas", "fslpy"],
                 "output"        : {
                     "install"   : "92",
                     "6.0.6"     : "11",
@@ -141,8 +150,9 @@
             {
                 "platform"      : "linux-64",
                 "cuda"          : "10.2",
-                "environment" : "http://localhost:8888/fsl-6.2.0-linux-64-cuda10.2.yml",
+                "environment"   : "https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/releases/fsl-6.2.0-linux-64-cuda10.2.yml",
                 "sha256"        : "6499385a2f5d2da48f4a44c95e090a5156319b0acbbd47029d4270abc691a290",
+                "base_packages" : ["fsl-base", "libopenblas", "fslpy"],
                 "output"        : {
                     "install"   : "92",
                     "6.0.6"     : "11",