Merge branch 'enh/m1' into 'master'

Enh/m1

See merge request fsl/conda/fsl-ci-rules!100

README.md

@@ -241,11 +241,12 @@ trigger_build -t "$TOKEN" ...
 
 FSL conda packages are built for the following platforms/architectures:
 
- - `noarch`:           Platform-independent, e.g. Python, TCL, Bash projects
- - `linux-64`:         C/C++ projects compiled for Linux (x86-64)
- - `macos-64`:         C/C++ projects compiled for macOS (x86-64)
+ - `noarch`:            Platform-independent, e.g. Python, TCL, Bash projects
+ - `linux-64`:          C/C++ projects compiled for Linux (x86-64)
+ - `macos-64`:          C/C++ projects compiled for macOS (x86-64)
+ - `macos-M1`:          C/C++ projects compiled for macOS (M1)
  - `linux-64-cuda-X.Y`: CUDA projects compiled for Linux (x86-64), with CUDA
-                      Toolkit version X.Y.
+                        Toolkit version X.Y.
 
 
 > *Note*: The above platform labels are used as identifiers - they must be used
@@ -267,7 +268,7 @@ recipe repository is a recipe for the
 [`fsl/add_module`](https://git.fmrib.ox.ac.uk/fsl/add_module) project, which
 is a pure Python project that is built as a `noarch` package. When a build is
 triggered on the `fsl/conda/fsl-add_module` repository, jobs for `noarch`,
-`linux-64`, `macos-64`, and supported `cuda` variants will all be
+`linux-64`, `macos-64`, `macos-M1`, and supported `cuda` variants will all be
 started. However, only the `noarch` job will proceed - all of the other jobs
 will detect that they are unnecessary, and will immediately abort. The
 `build_conda_package` script determines the type of a project by inspecting
@@ -277,7 +278,8 @@ the `meta.yaml` file in the conda recipe.
 To clarify, the CI conda build jobs are configured so that:
 
 1. Platform independent recipes will be built as `noarch` packages.
-2. C/C++ recipes will be built as `linux-64` and `macos-64` packages.
+2. C/C++ recipes will be built as `linux-64`, `macos-64`, and `macos-M1`
+   packages.
 3. CUDA recipes will be built as `linux-64-cuda-X.Y` packages.
 
 
@@ -303,8 +305,8 @@ separate recipes:
 
 
  - [`fsl/conda/fsl-fdt`](https://git.fmrib.ox.ac.uk/fsl/conda/fsl-fdt), which
-   builds the CPU-only executables - these recipes are built as `linux-64` and
-   `macos-64` packages.
+   builds the CPU-only executables - these recipes are built as `linux-64`,
+   `macos-64`, and `macos-M1` packages.
  - [`fsl/conda/fsl-fdt-cuda`](https://git.fmrib.ox.ac.uk/fsl/conda/fsl-fdt-cuda),
    which builds the CUDA/GPU-enabled executables - these recipes are built as
    `linux-64-cuda-X.Y` packages. A separate package is built for each supported
@@ -324,6 +326,7 @@ platform names are:
 
  - `linux-64`
  - `macos-64`
+ - `macos-M1`
  - `noarch`
  - `linux-64-cuda` (skips all CUDA builds)
  - `linux-64-cuda-9.2`
@@ -358,13 +361,18 @@ Gitlab CI runner:
 
  - One or more runners with the `fsl-ci` and `docker` tags. These runners will
    be used to execute any CI jobs that do not involve deploying conda
-   packages, including building `noarch` packages, and binary packages for
-   Linux.
-
- - One or more runners with the `fsl-ci` and `macOS` tags. These runners will
-   be used to build binary packages for macOS. An assumption is made that
-   `conda` is installed and available in the environment provided by these
-   runners.
+   packages, including building `noarch` packages, and binary x86-64 packages
+   for Linux.
+
+ - One or more runners with the `fsl-ci` and `macOS-64` tags. These runners
+   will be used to build binary x86-64 packages for macOS. An assumption is
+   made that `conda` is installed and available in the environment provided by
+   these runners.
+
+ - One or more runners with the `fsl-ci` and `macOS-M1` tags. These runners
+   will be used to build binary M1 packages for macOS. An assumption is made
+   that `conda` is installed and available in the environment provided by
+   these runners.
 
 Next, on **both project and recipe repositories**, for a FSL project which does
 not require any custom CI rules of its own (most FSL projects fall into this

fsl_ci/__init__.py

@@ -20,7 +20,7 @@ import subprocess   as sp
 import yaml
 
 
-__version__ = '0.18.4'
+__version__ = '0.19.0'
 """Current version of the fsl-ci-rules."""
 
 

fsl_ci/platform.py

@@ -14,7 +14,7 @@ from typing import List
 
 # Note: These lists must be updated whenever new
 # platforms / CUDA versions become supported.
-COMPILATION_TARGETS = ['linux-64', 'macos-64']
+COMPILATION_TARGETS = ['linux-64', 'macos-64', 'macos-M1']
 CUDA_VERSIONS       = ['9.2', '10.2', '11.0', '11.1', '11.3']
 
 
@@ -56,6 +56,7 @@ def get_platform_shortcut_if_not_applicable(
       - noarch
       - linux-64
       - macos-64
+      - macos-M1
       - linux-64-cuda-9.2
       - etc
     """
@@ -66,6 +67,7 @@ def get_platform_shortcut_if_not_applicable(
     # e.g.:
     #  - build-linux-64-conda-package
     #  - build-macos-64-conda-package
+    #  - build-macos-M1-conda-package
     #  - build-noarch-conda-package
     #  - build-noarch-conda-package-direct-deploy
     #  - build-linux-cuda-9.2-conda-package

fsl_ci/scripts/deploy_conda_package.py

@@ -35,12 +35,14 @@ def copy_packages(builddir, channeldir):
 
     # The build_conda_package.py script stores its
     # outputs in a platform-specific subdirectory -
-    # one of _linux-64_, _macos-64_, _noarch_,
-    # _linux-64-cuda-9.2_, etc. Within each one of
-    # those is a conventional conda channel
-    # structure.
+    # one of _linux-64_, _macos-64_, _macos-M1_,
+    # _noarch_, _linux-64-cuda-9.2_, etc. Within
+    # each one of those is a conventional conda
+    # channel structure.
     fslplats   = glob.glob(op.join(builddir, '_*_'))
-    condaplats = ('noarch', 'linux-64', 'osx-64', 'win-32', 'win64')
+    condaplats = ('noarch', 'linux-64',
+                  'osx-64', 'osx-arm64',
+                  'win-32', 'win64')
 
     for fslplat, condaplat in it.product(fslplats, condaplats):
         # copy any package files that we find in

fsl_ci/tests/test_platform.py

@@ -42,6 +42,7 @@ def test_get_platform_shorcut_if_not_applicable():
         # skip platform
         (({}, 'pkg', 'build-noarch-conda-package',   ['noarch']),   False),
         (({}, 'pkg', 'build-macos-64-conda-package', ['macos-64']), False),
+        (({}, 'pkg', 'build-macos-M1-conda-package', ['macos-M1']), False),
         (({}, 'pkg', 'build-linux-64-conda-package', ['linux-64']), False),
         (({}, 'pkg', 'build-linux-64-cuda-9.2-conda-package',
           ['linux-64-cuda-9.2']), False),
@@ -68,6 +69,8 @@ def test_get_platform_shorcut_if_not_applicable():
          'linux-64'),
         ((binary_meta, 'pkg', 'build-macos-64-conda-package', []),
          'macos-64'),
+        ((binary_meta, 'pkg', 'build-macos-M1-conda-package', []),
+         'macos-M1'),
         ((cuda_meta, 'pkg', 'build-linux-64-cuda-9.2-conda-package', []),
          'linux-64-cuda-9.2'),
 
@@ -100,12 +103,14 @@ def test_get_skip_platform():
     tests = [
         (noarch_meta, ('linux-64 '
                        'macos-64 '
+                       'macos-M1 '
                        'linux-64-cuda')),
         (binary_meta, ('noarch '
                        'linux-64-cuda')),
         (cuda_meta,   ('noarch '
                        'linux-64 '
-                       'macos-64'))
+                       'macos-64 '
+                       'macos-M1'))
     ]
 
     for input, expected in tests:
@@ -121,7 +126,7 @@ def test_get_platform_ids():
     # (input, expected)
     tests = [
         (noarch_meta, ['noarch']),
-        (binary_meta, ['linux-64', 'macos-64']),
+        (binary_meta, ['linux-64', 'macos-64', 'macos-M1']),
         (cuda_meta,   ['linux-64-cuda-9.2',
                        'linux-64-cuda-10.2',
                        'linux-64-cuda-11.0',

fsl_ci/utils/configure_repositories.py

@@ -43,7 +43,8 @@ command-line.
 
 RUNNER_TAGSETS = ['fslconda-channel-host',
                   'docker,fsl-ci',
-                  'macOS,fsl-ci']
+                  'macOS-64,fsl-ci',
+                  'macOS-M1,fsl-ci']
 """Runners which match any of these tag sets are enabled on the specified
 project/recipe repositories.
 """

fsl_ci/utils/package_status.py

@@ -50,7 +50,7 @@ PUBLIC_CHANNEL_URL = "https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/public/"
 """Default public channel URL, if not specified on the command.line."""
 
 
-PLATFORMS = ['noarch', 'linux-64', 'osx-64']
+PLATFORMS = ['noarch', 'linux-64', 'osx-64', 'osx-arm64']
 """Default platforms to check package availability for. """
 
 
@@ -63,9 +63,10 @@ MISSING = '-'
 
 
 PLATFORM_CODES = {
-    'noarch'   : 'N',
-    'linux-64' : 'L',
-    'osx-64'   : 'M'
+    'noarch'    : 'N',
+    'linux-64'  : 'L',
+    'osx-64'    : 'O',
+    'osx-arm64' : 'M'
 }
 """Identifiers for all platforms, used in printed table."""
 

rules/fsl-ci-build-rules.yml

@@ -145,15 +145,19 @@ update-conda-recipe:
 #
 #
 # We define the common logic in a template, which is re-used below for
-# concrete definitions of platform-specific jobs. Linux/noarch packages are
-# built using Gitlab runners with a "docker" tag, and macOS packages are
-# built using Gitlab runners with a "macOS" and "shell" tag.
+# concrete definitions of platform-specific jobs.
+#   - Linux/noarch packages are built using Gitlab runners with tags "fsl-ci"
+#     and "docker".
+#   - macOS/x86_64 packages are built using Gitlab runners with tags "fsl-ci"
+#     and "macOS-x64".
+#   - macOS/x86_64 packages are built using Gitlab runners with tags "fsl-ci"
+#     and "macOS-M1".
 #
 #
 # All concrete jobs are started for all packages, but the
 # build_conda_package.py script will short-cut when it detects that it has
 # been asked to build a package on the wrong platform (e.g. the
-# build-linux-conda-package and build-macos-conda-package jobs will not
+# build-linux-64-conda-package and build-macos-64-conda-package jobs will not
 # attempt to build a noarch package, but rather will exit early with an exit
 # code of 0). Jobs can also be disabled with the FSLCONDA_SKIP_PLATFORM
 # variable.
@@ -182,6 +186,8 @@ update-conda-recipe:
             $FSLCONDA_SKIP_PLATFORM =~ /(^| )linux-64($| )/)                    ||
            ($CI_JOB_NAME            == "build-macos-64-conda-package"           &&
             $FSLCONDA_SKIP_PLATFORM =~ /(^| )macos-64($| )/)                    ||
+           ($CI_JOB_NAME            == "build-macos-M1-conda-package"           &&
+            $FSLCONDA_SKIP_PLATFORM =~ /(^| )macos-M1($| )/)                    ||
            ($CI_JOB_NAME            =~ /^build-linux-64-cuda-.*-conda-package$/ &&
             $FSLCONDA_SKIP_PLATFORM =~ /(^| )linux-64-cuda($| )/)               ||
            ($CI_JOB_NAME            == "build-linux-64-cuda-9.2-conda-package"  &&
@@ -221,8 +227,9 @@ update-conda-recipe:
   extends: .fsl-ci-conda-build-job-base
 
   # Platform-specific jobs will direct their outputs
-  # into "noarch", "linux" and "macos" sub-directories
-  # within conda_build - see build_conda_package.py.
+  # into "noarch", "linux-644", "macos-64", and "macos-M1"
+  # sub-directories within conda_build - see
+  # build_conda_package.py.
   artifacts:
     expire_in: 2 hrs
     paths:
@@ -259,7 +266,14 @@ build-macos-64-conda-package:
   extends: .fsl-ci-conda-build-job
   tags:
     - fsl-ci
-    - macOS
+    - macOS-64
+
+
+build-macos-M1-conda-package:
+  extends: .fsl-ci-conda-build-job
+  tags:
+    - fsl-ci
+    - macOS-M1
 
 
 build-linux-64-cuda-9.2-conda-package:

rules/fsl-ci-deploy-rules.yml

@@ -36,6 +36,8 @@
             $FSLCONDA_SKIP_PLATFORM =~ /(^| )linux-64($| )/)                     ||
            ($CI_JOB_NAME            == "deploy-macos-64-conda-package"           &&
             $FSLCONDA_SKIP_PLATFORM =~ /(^| )macos-64($| )/)                     ||
+           ($CI_JOB_NAME            == "deploy-macos-M1-conda-package"           &&
+            $FSLCONDA_SKIP_PLATFORM =~ /(^| )macos-M1($| )/)                     ||
            ($CI_JOB_NAME            =~ /^deploy-linux-64-cuda-.*-conda-package$/ &&
             $FSLCONDA_SKIP_PLATFORM =~ /(^| )linux-64-cuda($| )/)                ||
            ($CI_JOB_NAME            == "deploy-linux-64-cuda-9.2-conda-package"  &&
@@ -80,6 +82,10 @@ deploy-macos-64-conda-package:
   dependencies:
     - build-macos-64-conda-package
 
+deploy-macos-M1-conda-package:
+  extends: .deploy-conda-package
+  dependencies:
+    - build-macos-M1-conda-package
 
 deploy-linux-64-cuda-9.2-conda-package:
   extends: .deploy-conda-package

rules/fsl-ci-test-rules.yml

@@ -11,9 +11,10 @@
 
 
 # In the same manner as build jobs, a test job is started for every possible
-# conda package platform type (linux-64, macos-64, noarch, linux-64-cuda-9.2,
-# etc). Jobs which are not relevant for a particular platform (e.g. the
-# linux-64 job for a noarch package) are aborted immediately.
+# conda package platform type (linux-64, macos-64, macos-M1, noarch,
+# linux-64-cuda-9.2, etc). Jobs which are not relevant for a particular
+# platform (e.g. the linux-64 job for a noarch package) are aborted
+# immediately.
 
 .fsl-ci-test-job:
   stage:   fsl-ci-test
@@ -29,7 +30,9 @@
            ($CI_JOB_NAME            == "test-linux-64-conda-package" &&
             $FSLCONDA_SKIP_PLATFORM =~ /(^| )linux-64($| )/)         ||
            ($CI_JOB_NAME            == "test-macos-64-conda-package" &&
-            $FSLCONDA_SKIP_PLATFORM =~ /(^| )macos-64($| )/)'
+            $FSLCONDA_SKIP_PLATFORM =~ /(^| )macos-64($| )/)         ||
+           ($CI_JOB_NAME            == "test-macos-M1-conda-package" &&
+            $FSLCONDA_SKIP_PLATFORM =~ /(^| )macos-M1($| )/)'
       when: never
 
     # This job is only run on branches
@@ -82,10 +85,19 @@ test-macos-64-conda-package:
   extends: .fsl-ci-test-job
   tags:
     - fsl-ci
-    - macOS
+    - macOS-64
   dependencies:
     - build-macos-64-conda-package
 
 
+test-macos-M1-conda-package:
+  extends: .fsl-ci-test-job
+  tags:
+    - fsl-ci
+    - macOS-M1
+  dependencies:
+    - build-macos-M1-conda-package
+
+
 # Setting up a test environment for
 # CUDA packages is on the TODO list
\ No newline at end of file