Merge branch 'rf/skip-test-dev-manifest' into 'master'

Rf/skip test dev manifest

See merge request fsl/conda/manifest-rules!11

.ci/check_manifest_gitlab_ci_yml.py

+#!/usr/bin/env python
+import yaml
+with open('manifest-gitlab-ci.yml', 'rt') as f:
+    print(yaml.load(f.read(), Loader=yaml.Loader))

.gitlab-ci.yml

@@ -33,6 +33,17 @@ test:
     - python -c 'import manifest_rules; print(manifest_rules.__version__)'
 
 
+check-manifest-gitab-ci-yml:
+  stage: test
+  image: python:3.9
+  tags:
+    - fsl-ci
+    - docker
+  script:
+    - pip install git+https://git.fmrib.ox.ac.uk/fsl/fsl-ci-rules.git
+    - python ./.ci/check_manifest_gitlab_ci_yml.py
+
+
 check-version:
   stage: check-version
   image: python:3.9

manifest-gitlab-ci.yml

@@ -174,29 +174,33 @@ generate-release-manifest-file:
 
 generate-development-manifest-file:
   extends: .generate-manifest-file
+  rules:
+    - if: '$CI_COMMIT_TAG == null'
   variables:
     OFFICIAL: "false"
 
 
 # Run the latest fslinstaller.py script
 # on the generated manifest and environment
-# files.
+# files. Only the release manifest is tested.
 #
 # This is a template. Each concrete job
 # specifies a variable SUFFIX which denotes
 # the environment file to be installed.
 .test-manifest:
   stage: test-manifests
+  rules:
+    - if: '$CI_COMMIT_TAG != null'
   script:
     # there should only be one manifest file, so
     # manifest/*.json will resolve to a single
     # file. Concrete jobs should define SUFFIX
     # so that environments/*${SUFFIX}.yml should
     # also resolve to one file.
-    - test_manifest manifest/*.json environments/*${SUFFIX}.yml
+    - test_manifest manifest/*.json environments/
 
 
-test-release-manifest:linux-64:
+test-manifest:linux-64:
   stage:   test-manifests
   extends: .test-manifest
   image:   fsldevelopment/fsl-linux-64
@@ -209,21 +213,7 @@ test-release-manifest:linux-64:
     - generate-environment-files
     - generate-release-manifest-file
 
-
-test-development-manifest:linux-64:
-  stage:   test-manifests
-  extends: .test-manifest
-  image:   fsldevelopment/fsl-linux-64
-  variables:
-    SUFFIX: "linux-64"
-  tags:
-    - fsl-ci
-    - docker
-  dependencies:
-    - generate-environment-files
-    - generate-development-manifest-file
-
-test-release-manifest:linux-64_cudaX.Y:
+test-manifest:linux-64_cudaX.Y:
   stage:   test-manifests
   extends: .test-manifest
   image:   fsldevelopment/fsl-linux-64
@@ -239,25 +229,7 @@ test-release-manifest:linux-64_cudaX.Y:
     - generate-environment-files
     - generate-release-manifest-file
 
-
-test-development-manifest:linux-64_cudaX.Y:
-  stage:   test-manifests
-  extends: .test-manifest
-  image:   fsldevelopment/fsl-linux-64
-  parallel:
-    matrix:
-      - CUDA: ["9.2", "10.2", "11.0"]
-  variables:
-    SUFFIX: "linux-64_cuda${CUDA}"
-  tags:
-    - fsl-ci
-    - docker
-  dependencies:
-    - generate-environment-files
-    - generate-development-manifest-file
-
-
-test-release-manifest:macos-64:
+test-manifest:macos-64:
   stage:   test-manifests
   extends: .test-manifest
   variables:
@@ -270,19 +242,6 @@ test-release-manifest:macos-64:
     - generate-release-manifest-file
 
 
-test-development-manifest:macos-64:
-  stage:   test-manifests
-  extends: .test-manifest
-  variables:
-    SUFFIX: "macos-64"
-  tags:
-    - fsl-ci
-    - macOS
-  dependencies:
-    - generate-environment-files
-    - generate-development-manifest-file
-
-
 # Deploy/publish environment and manifest files.
 # All deploy jobs must be manually invoked.
 # Deployment destination is assumed to be

manifest_rules/__init__.py

 #!/usr/bin/env python
 
 
-__version__ = '0.3.7'
+__version__ = '0.3.8'

manifest_rules/test_manifest.py

@@ -11,39 +11,30 @@ from manifest_rules.utils import sprun, tempdir, download_file
 
 
 def main():
-    manifest_file    = op.abspath(sys.argv[1])
-    environment_file = op.abspath(sys.argv[2])
+    manifest_file   = op.abspath(sys.argv[1])
+    environment_dir = op.abspath(sys.argv[2])
 
     # credentials for logging into
     # internal FSL conda channel
     username = os.environ['FSLCONDA_USERNAME']
     password = os.environ['FSLCONDA_PASSWORD']
 
-    publish_from_branches = os.environ.get('PUBLISH_FROM_BRANCHES', '')
-    branch                = os.environ.get('CI_COMMIT_BRANCH', None)
-    tag                   = os.environ.get('CI_COMMIT_TAG',    None)
-
-    run_test = ((tag is not None) or
-                ((branch is not None) and
-                 (branch in publish_from_branches)))
-
-    if not run_test:
-        print('This test is only run on tags and these '
-              f'branches: {publish_from_branches}')
-        sys.exit(0)
+    version  = os.environ['CI_COMMIT_TAG']
+    cuda     = os.environ.get('CUDA', None)
 
     with open(manifest_file, 'rt') as f:
         manifest = json.loads(f.read())
 
     installer_url = manifest['installer']['url']
-    latest        = manifest['versions'].get('latest', None)
 
-    # This skip can be removed when we
-    # actually have a conda-based FSL release
-    if latest is None:
-        print('Manifest file does not specify a "latest" FSL '
-              'release - cannot run fslinstaller.py')
-        sys.exit(0)
+    # patch the manifest to refer to local environment files
+    for build in manifest['versions'][version]:
+        envfname = build['environment'].split('//')[-1]
+        build['environment'] = op.join(environment_dir, envfname)
+
+    with open(manifest_file, 'wt') as f:
+        manifest = json.dumps(manifest, indent=4, sort_keys=True)
+        f.write(manifest)
 
     with tempdir():
         download_file(installer_url, 'fslinstaller.py')
@@ -55,12 +46,15 @@ def main():
                     f'--manifest {manifest_file} '
                     '--listversions', text=True))
 
+        if cuda is not None: cuda = f'--cuda {cuda}'
+        else:                cuda = ''
+
         try:
             print(sprun(f'{sys.executable} ./fslinstaller.py '
-                  f'--manifest {manifest_file} '
-                  f'--environment {environment_file} '
-                  f'--username {username} --password {password} '
-                 '--dest fsl --workdir workdir --homedir homedir', text=True))
+                        f'--manifest {manifest_file} {cuda} '
+                        f'--username {username} --password {password} '
+                        '--dest fsl --workdir workdir --homedir homedir',
+                        text=True))
         finally:
             logfile = op.join('workdir', 'fslinstaller.log')
             if op.exists(logfile):