Merge branch 'rf/skip-platforms' into 'master'

BF: Handle case where conda is not in runner environment $PATH (or conda is a

See merge request fsl/conda/manifest-rules!39

manifest-gitlab-ci.yml

@@ -16,7 +16,8 @@ default:
       mkdir ./conda-dir;
       export CONDA_ENVS_DIRS=$(pwd)/conda-dir/;
       export CONDA_PKGS_DIRS=$(pwd)/conda-dir/.pkgs;
-      CONDABINDIR=$(dirname $(which conda))/../bin/;
+      CONDABIN=$(which conda || echo "${HOME}/miniconda3/bin/conda");
+      CONDABINDIR=$(dirname ${CONDABIN})/../bin/;
       conda create -c conda-forge -p ./env python=3.9;
       if [ -f "$CONDABINDIR"/activate ]; then source $CONDABINDIR/activate ./env;
       else                                    conda               activate ./env; fi;

manifest_rules/__init__.py

 #!/usr/bin/env python
 
 
-__version__ = '0.7.1'
+__version__ = '0.7.2'

manifest_rules/generate_environment_files.py

@@ -18,6 +18,7 @@ import               sys
 
 from fsl_ci.conda         import  get_channel_packages
 from manifest_rules.utils import (load_release_info,
+                                  get_platform_identifiers,
                                   generate_environment_file_name,
                                   generate_development_version_identifier)
 
@@ -87,14 +88,9 @@ def generate_variants(release_info):
     """Generate a list of (platform, cudaver) pairs for which an environment
     file should be generated. For non-CUDA environments, cudaver will be None.
     """
-    cudas      = [None] + list(release_info['cuda'])
-    platforms  = os.environ.get('FSLCONDA_PLATFORMS', None)
-    if platforms is None:
-        platforms = list(release_info['miniconda'].keys())
-    else:
-        platforms = platforms = platforms.split(',')
-
-    variants   = list(it.product(platforms, cudas))
+    cudas     = [None] + list(release_info['cuda'])
+    platforms = get_platform_identifiers(release_info)
+    variants  = list(it.product(platforms, cudas))
 
     # CUDA builds are only supported
     # on linux-64 at this time

manifest_rules/generate_manifest_file.py

@@ -16,6 +16,7 @@ from manifest_rules.utils import (load_release_info,
                                   download_file,
                                   sha256,
                                   parse_environment_file_name,
+                                  get_platform_identifiers,
                                   generate_development_version_identifier)
 
 
@@ -29,9 +30,15 @@ def load_test_install_info(install_dir):
     for fname in inst_files:
         with open(fname, 'rt') as f:
             lines = f.readlines()
+        try:
             info = lines[-1].strip()
-
-        envfile, inst_lines, env_lines = info.split()
+            envfile, inst_lines, env_lines = info.split()
+        except Exception:
+            print(f'Could not parse test install info from: {fname}')
+            print('Perhaps it corresponds to a platform that is not in '
+                  'FSLCONDA_PLATFORMS?',
+                  os.environ.get('FSLCONDA_PLATFORMS', 'n/a'))
+            continue
         platform        = parse_environment_file_name(envfile)[1]
         inst_lines      = int(inst_lines)
         env_lines       = int(env_lines)
@@ -115,11 +122,13 @@ def generate_installer_section(release_info):
 
 def generate_miniconda_section(release_info, install_info):
     """Generates the "miniconda" manifest section, returning a dict. """
-    section = {}
-
-    outputs = install_info.get('miniconda', {})
+    section   = {}
+    outputs   = install_info.get('miniconda', {})
+    platforms = get_platform_identifiers(release_info)
 
     for platform, url in release_info['miniconda'].items():
+        if platform not in platforms:
+            continue
         output = outputs.get(platform, None)
         with tempdir():
             download_file(url, 'miniconda.sh')
@@ -155,8 +164,9 @@ def generate_version_section(version,
     if not include_current_release:
         return section
 
-    versions = []
-    section  = {version : versions}
+    versions  = []
+    section   = {version : versions}
+    platforms = get_platform_identifiers(release_info)
 
     for envfile in glob.glob(op.join(envdir, '*.yml')):
 
@@ -166,6 +176,9 @@ def generate_version_section(version,
         url      = urlparse.urljoin(release_info['release-url'], envfile)
         version, platform, cuda = parse_environment_file_name(envfile)
 
+        if platform not in platforms:
+            continue
+
         build = {
             'platform'      : platform,
             'environment'   : url,

manifest_rules/test_environment.py

@@ -17,6 +17,7 @@ from manifest_rules.utils import (sprun,
                                   tempdir,
                                   download_file,
                                   load_release_info,
+                                  get_platform_identifiers,
                                   parse_environment_file_name)
 
 
@@ -114,9 +115,18 @@ def full_test(envfile, release_info):
 
 
 def main():
+    job_name      = os.environ['CI_JOB_NAME']
     release_file  = op.abspath(sys.argv[1])
     envfile       = op.abspath(sys.argv[2])
     release_info  = load_release_info(release_file)
+    platforms     = get_platform_identifiers(release_info)
+
+    for platform in platforms:
+        if platform in job_name:
+            break
+    else:
+        print('This platform is disabled')
+        sys.exit(0)
 
     publish_from_branches = os.environ.get('PUBLISH_FROM_BRANCHES', '')
     branch_name           = os.environ.get('CI_COMMIT_BRANCH', None)

manifest_rules/test_manifest.py

@@ -7,12 +7,27 @@ import os
 import sys
 import json
 
-from manifest_rules.utils import sprun, tempdir, download_file
+from manifest_rules.utils import (sprun,
+                                  tempdir,
+                                  download_file,
+                                  load_release_info,
+                                  get_platform_identifiers)
 
 
 def main():
-    manifest_file   = op.abspath(sys.argv[1])
-    environment_dir = op.abspath(sys.argv[2])
+    job_name        = os.environ['CI_JOB_NAME']
+    release_file    = op.abspath(sys.argv[1])
+    manifest_file   = op.abspath(sys.argv[2])
+    environment_dir = op.abspath(sys.argv[3])
+    release_info    = load_release_info(release_file)
+    platforms       = get_platform_identifiers(release_info)
+
+    for platform in platforms:
+        if platform in job_name:
+            break
+    else:
+        print('This platform is disabled')
+        sys.exit(0)
 
     # credentials for logging into
     # internal FSL conda channel

manifest_rules/utils.py

@@ -174,6 +174,18 @@ def load_release_info(fsl_release_file):
         return yaml.load(f.read(), Loader=yaml.Loader)
 
 
+def get_platform_identifiers(release_info):
+    """Returns a list of all platform identifiers for which FSL environment
+    files are being generated, e.g. ['linux-64', 'macos-64', 'macos-M1'].
+    """
+    platforms  = os.environ.get('FSLCONDA_PLATFORMS', None)
+    if platforms is None:
+        platforms = list(release_info['miniconda'].keys())
+    else:
+        platforms = platforms.split(',')
+    return platforms
+
+
 @contextlib.contextmanager
 def tempdir():
     """Returns a context manager which creates, changes into, and returns a