MNT: initial commit

.gitlab-ci.yml

+stages:
+  - check-version
+
+
+# Only run on merge requests
+workflow:
+  rules:
+    - if: '$CI_MERGE_REQUEST_ID != null'
+
+
+check-version:
+  stage: check-version
+  image: python:3.9
+  tags:
+    - fsl-ci
+    - docker
+  script:
+    - thisver=$(  cat manifest_rules/__init__.py             | grep __version__ | cut -d " " -f 3 | tr -d "'")
+    - masterver=$(git show master:manifest_rules/__init__.py | grep __version__ | cut -d " " -f 3 | tr -d "'")
+    - |
+      if [ "$thisver" = "$masterver" ]; then
+        echo "Version has not been updated!"
+        echo "Version on master branch: $masterver"
+        echo "Version in this MR:       $thisver"
+        echo "The version number must be updated before this MR can be merged."
+        exit 1
+      else
+        echo "Version on master branch: $masterver"
+        echo "Version in this MR:       $thisver"
+      fi

README.md

+# FSL release manifest rules
+
+This repository contains scripts which are used in the fsl/conda/manifest>
+repository to generate, test, and deploy FSL release manifest and environment
+files. Refer to the fsl/conda/manifest> repository for more details.
+
+These scripts are hosted separately from the `fsl/conda/manifest` project so
+that they can be updated independently of FSL releases.
+
+Whenever changes to these scripts are made, the version number in
+`manifest_rules/__init__.py` must be incremented.

manifest_rules/__init__.py

+#!/usr/bin/env python
+
+
+__version__ = '0.1.0'

manifest_rules/deploy_files.py

+#!/usr/bin/env python
+
+
+import os
+import sys
+import shutil
+
+
+def main():
+
+    dest                  = sys.argv[1]
+    files                 = sys.argv[2:]
+    publish_from_branches = os.environ.get('PUBLISH_FROM_BRANCHES', '')
+    branch                = os.environ.get('CI_COMMIT_BRANCH', None)
+    tag                   = os.environ.get('CI_COMMIT_TAG',    None)
+
+    allow_publish = ((tag is not None) or
+                     ((branch is not None) and
+                      (branch in publish_from_branches)))
+
+    if not allow_publish:
+        print('Publishing is only allowed from tags or '
+              f'these branches: {publish_from_branches}')
+        sys.exit(1)
+
+    if tag is not None:
+        print(f'Publishing files from tag {tag}')
+    else:
+        print(f'Publishing files from branch {branch}')
+
+    for src in files:
+        print(f'Copying {src} to {dest}')
+        shutil.copy(src, dest)
+
+
+if __name__ == '__main__':
+    main()

manifest_rules/generate_environment_files.py

+#!/usr/bin/env python
+#
+# Generate FSL conda environment files from fsl-release.yml
+
+
+import itertools  as it
+import os.path    as op
+import               os
+import               re
+import               sys
+
+from manifest_rules.utils import (sprun,
+                                  load_release_info,
+                                  generate_environment_file_name,
+                                  generate_development_version_identifier)
+
+
+def filter_packages(release_info, platform, cudaver):
+    """Return a list of packages that should be included in the environment
+    file for the platform and CUDA version.
+    """
+    packages  = release_info['packages']
+    cudapat   = r'cuda-[\d]+\.[\d]+'
+    cudapkgs  = [p for p in packages if re.search(cudapat, p) is not None]
+    otherpkgs = [p for p in packages if p not in cudapkgs]
+
+    if cudaver is None:
+        return otherpkgs
+
+    else:
+        include  = f'cuda-{cudaver}'
+        cudapkgs = [p for p in cudapkgs if include in p]
+
+        return otherpkgs + cudapkgs
+
+
+def generate_environment(release_info, version, platform, cudaver, outfile):
+    """Genereate an environment file for the platform and CUDA version.
+    """
+    channels = list(release_info['channels'])
+    packages = filter_packages(release_info, platform, cudaver)
+
+    # Dev release
+    if version is None:
+        channels = [release_info['internal_channel']] + channels
+
+    with open(outfile, 'wt') as f:
+        f.write('name: FSL\n')
+        f.write('channels:\n')
+        for channel in channels:
+            f.write(f' - {channel}\n')
+        f.write('dependencies:\n')
+        for package in packages:
+            f.write(f' - {package}\n')
+
+
+def generate_variants(release_info):
+    """Generate a list of (platform, cudaver) pairs for which an environment
+    file should be generated. For non-CUDA environments, cudaver will be None.
+    """
+    cudas     = [None] + list(release_info['cuda'])
+    platforms = list(release_info['miniconda'].keys())
+    variants  = list(it.product(platforms, cudas))
+
+    # CUDA builds are only supported
+    # on linux-64 at this time
+    variants = [(plat, cuda) for plat, cuda in variants
+                if not (('linux-64' not in plat) and cuda is not None)]
+
+    return variants
+
+
+def main():
+    # Save generated environment files here
+    release_file = op.abspath(sys.argv[1])
+    outdir       = op.abspath(sys.argv[2])
+
+    # Tags on fsl/conda/manifest denote a
+    # public FSL release.  All other
+    # commits denote an internal release.
+    version      = os.environ.get('CI_COMMIT_TAG', None)
+    release_info = load_release_info(release_file)
+
+    # Generate environment files
+    # for all platforms+CUDA versions
+    variants = generate_variants(release_info)
+
+    if not op.exists(outdir):
+        os.mkdir(outdir)
+
+    generated = []
+
+    for platform, cuda in variants:
+        filename = generate_environment_file_name(version, platform, cuda)
+        filename = op.join(outdir, filename)
+        generate_environment(release_info, version, platform, cuda, filename)
+        generated.append((platform, cuda, op.basename(filename)))
+
+    for platform, cuda, filename in generated:
+        print(f'Platform {platform} [CUDA {cuda}]: {filename}')
+
+
+if __name__ == '__main__':
+    sys.exit(main())

manifest_rules/generate_manifest_file.py

+#!/usr/bin/env python
+#
+# Generate FSL release manifest file from fsl-release.yml
+
+
+import os.path      as     op
+import                     os
+import                     sys
+import                     glob
+import                     json
+import urllib.parse as     urlparse
+from   collections  import defaultdict
+
+from manifest_rules.utils import (load_release_info,
+                                  tempdir,
+                                  download_file,
+                                  sha256,
+                                  parse_environment_file_name,
+                                  generate_development_version_identifier)
+
+
+def load_test_install_info(install_dir):
+    """Returns information (number of output lines produced) about
+    the test installations. Returns them in a dict.
+    """
+    inst_info  = defaultdict(dict)
+    inst_files = glob.glob(op.join(install_dir, '*.install.txt'))
+
+    for fname in inst_files:
+        with open(fname, 'rt') as f:
+            lines = f.readlines()
+            info = lines[-1].strip()
+
+        envfile, inst_lines, env_lines = info.split()
+        platform        = parse_environment_file_name(envfile)[1]
+        inst_lines      = int(inst_lines)
+        env_lines       = int(env_lines)
+        prev_inst_lines = inst_info['miniconda'].get('platform', 0)
+        inst_lines      = max((inst_lines, prev_inst_lines))
+
+        inst_info['miniconda'][platform] = inst_lines
+        inst_info[envfile]               = env_lines
+
+    return inst_info
+
+
+def load_previous_manifest(release_info):
+    """Downloads the previous official FSL manifest, returning it
+    as a dict. Returns None if the manifest cannot be downloaded.
+    """
+    url = urlparse.urljoin(release_info['release_url'], 'manifest.json')
+    with tempdir():
+
+        try:
+            download_file(url, 'manifest.json')
+        except Exception:
+            return None
+
+        with open('manifest.json', 'rt') as f:
+            lines = f.readlines()
+
+    # Drop comments
+    lines = [l for l in lines if not l.lstrip().startswith('//')]
+
+    manifest = json.loads('\n'.join(lines))
+
+    # Filter out any releases/builds for which
+    # the environment file is not available
+    versions = dict(manifest.get('versions', {}))
+    for version, builds in versions.items():
+        for i, build in enumerate(list(builds)):
+            try:
+                with tempdir():
+                    download_file(build['url'], 'env.yml')
+            except Exception:
+                print(f'WARNING: Build environment file '
+                      f'is not available: {build}')
+                builds.pop(i)
+
+        if len(builds) == 0:
+            versions.pop(version)
+        else:
+            versions[version] = builds
+
+    latest = versions.get('latest', None)
+    if (latest is not None) and (latest not in versions):
+        print(f'WARNING: Build for latest FSL '
+              f'version {latest} is not available')
+        versions.pop('latest')
+
+    manifest['versions'] = versions
+
+    return manifest
+
+
+def generate_installer_section(release_info):
+    """Generates the "installer" manifest section, returning a dict. """
+    version  = release_info['installer']
+    url      = urlparse.urljoin(release_info['release_url'], 'fslinstaller.py')
+
+    try:
+        with tempdir():
+            download_file(url, 'fslinstaller.py')
+            checksum = sha256('fslinstaller.py')
+    except Exception:
+        print(f'WARNING: fslinstaller.py script not available at {url}!')
+        checksum = ''
+
+    return {'version' : version,
+            'url'     : url,
+            'sha256'  : checksum}
+
+
+def generate_miniconda_section(release_info, install_info):
+    """Generates the "miniconda" manifest section, returning a dict. """
+    section = {}
+
+    outputs = install_info.get('miniconda', {})
+
+    for platform, url in release_info['miniconda'].items():
+        output = outputs.get(platform, None)
+        with tempdir():
+            download_file(url, 'miniconda.sh')
+            checksum = sha256('miniconda.sh')
+
+            section[platform] = {
+                'url'    : url,
+                'sha256' : checksum,
+            }
+            if output is not None:
+                section[platform]['output'] = str(output)
+
+    return section
+
+
+def generate_version_section(version,
+                             include_current_release,
+                             include_past_releases,
+                             envdir,
+                             release_info,
+                             install_info):
+    """Generates the "versions" manifest section, returning a dict.
+    Does not add the "latest" entry.
+    """
+
+    section = {}
+
+    if include_past_releases:
+        prev_manifest = load_previous_manifest(release_info)
+        if prev_manifest is not None:
+            section = prev_manifest['versions']
+
+    if not include_current_release:
+        return section
+
+    versions = []
+    section  = {version : versions}
+
+    for envfile in glob.glob(op.join(envdir, '*.yml')):
+
+        checksum = sha256(envfile)
+        envfile  = op.basename(envfile)
+        output   = install_info.get(envfile, None)
+        url      = urlparse.urljoin(release_info['release_url'], envfile)
+        version, platform, cuda = parse_environment_file_name(envfile)
+
+        build = {
+            'platform'    : platform,
+            'environment' : url,
+            'sha256'      : checksum
+        }
+        if cuda is not None:
+            build['cuda'] = cuda
+
+        versions.append(build)
+        if output is not None:
+            versions[-1]['output'] = {'install' : str(output)}
+
+    return section
+
+
+def main():
+    # This script is run in the following scenarios:
+    #   - New public FSL release - generate a new official
+    #     manifest, describing the new release, and all past
+    #     public releases
+    #   - New internal/development release - generate a
+    #     development manifest describing just the development
+    #     release
+    #   - Update to official manifest - re-generate the
+    #     official manifest, describing all past public
+    #     releases
+    # We use the "official" and "add_current_release"
+    # variables to figure out the scenario we are in
+
+    release_file = op.abspath(sys.argv[1])
+
+    # See .gitlab-ci.yml - we are passed "true" or "false"
+    # to determine whether we are to generate an official
+    # or development manifest
+    official = sys.argv[2] == 'true'
+
+    # Directory to save the generated manifest file
+    outdir = op.abspath(sys.argv[3])
+
+    # Directory containing the environment files
+    # to be added to/described by the manifest
+    envdir = op.abspath(sys.argv[4])
+
+    # Directory containing outputs of the test
+    # installation
+    installdir = op.abspath(sys.argv[5])
+
+    # FSL version identifier
+    version = os.environ.get('CI_COMMIT_TAG', None)
+
+    # Add information about the current release
+    # (tag or commit) to the generated manifest?
+    # We do this for new official releases, or
+    # for development releases
+    add_current_release = (version is not None) or (not official)
+
+    # Generate dev version identnfier
+    if version is None:
+        version = generate_development_version_identifier()
+
+    # Load fsl-release.yml, and load
+    # outputs of test install stage
+    release_info = load_release_info(release_file)
+    install_info = load_test_install_info(installdir)
+
+    # Generate the manifest
+    manifest = {
+        'installer' : generate_installer_section(release_info),
+        'miniconda' : generate_miniconda_section(release_info, install_info),
+        'versions'  : generate_version_section(version,
+                                               add_current_release,
+                                               official,
+                                               envdir,
+                                               release_info,
+                                               install_info),
+    }
+
+    # Set the "latest" field for for
+    # new public or developmenmt releases
+    if add_current_release:
+        manifest['versions']['latest'] = version
+
+    if official:  fname = 'manifest.json'
+    else:         fname = f'manifest-{version}.json'
+
+    manifest = json.dumps(manifest, indent=4, sort_keys=True)
+
+    with open(op.join(outdir, fname), 'wt') as f:
+        f.write(manifest)
+
+    print(f'Generated manifest file {fname}:')
+    print(manifest)
+
+
+if __name__ == '__main__':
+    sys.exit(main())

manifest_rules/test_environment.py

+#!/usr/bin/env python
+#
+# Performs a test installation for a FSL conda environment file. Counts
+# the number of output lines produced when installing miniconda, and
+# when installing the FSL environment.
+#
+
+
+import os.path  as op
+import             os
+import             sys
+import             shutil
+
+import yaml
+
+from manifest_rules.utils import (sprun,
+                                  tempdir,
+                                  download_file,
+                                  load_release_info,
+                                  parse_environment_file_name)
+
+
+def preprocess_environment(envfile, condadir, regen_envfile=False):
+    with open(envfile, 'rt') as f:
+        env = f.read()
+    env = yaml.load(env, Loader=yaml.Loader)
+
+    packages = env['dependencies']
+    channels = env['channels']
+
+    for i, pkg in enumerate(packages):
+        pkg, ver    = pkg.split()
+        packages[i] = f'{pkg}={ver}'
+
+    with open(op.join(condadir, 'condarc'), 'wt') as f:
+        f.write('channels:\n')
+        for c in channels:
+            f.write(f' - {c}\n')
+
+    if regen_envfile:
+        with open(envfile, 'wt') as f:
+            f.write('channels:\n')
+            for c in channels:
+                f.write(f' - {c}\n')
+            f.write('dependencies:\n')
+            for p in packages:
+                f.write(f' - {p}\n')
+
+    return packages
+
+
+def fast_test(envfile, release_info):
+
+    install_out = sprun('conda create -y -p fsl')
+
+    packages = preprocess_environment(envfile, 'fsl')
+
+    env = os.environ.copy()
+    env['CONDARC'] = op.abspath(op.join('fsl', 'condarc'))
+
+    cmd = 'conda install -p fsl -y --dry-run ' + ' '.join(packages)
+    env_out = sprun(cmd, env=env)
+
+    return install_out, env_out
+
+
+def full_test(envfile, release_info):
+
+    platform      = parse_environment_file_name(envfile)[1]
+    miniconda_url = release_info['miniconda'][platform]
+
+    download_file(miniconda_url, 'miniconda.sh')
+
+    install_out = sprun('sh ./miniconda.sh -b -p ./fsl')
+
+    preprocess_environment(envfile, 'fsl', True)
+
+    env = os.environ.copy()
+    env['CONDARC'] = op.abspath(op.join('fsl', 'condarc'))
+
+    env_out =  sprun(f'./fsl/bin/conda env update -n base -f {envfile}',
+                     env=env)
+
+    return install_out, env_out
+
+
+def main():
+    release_file  = op.abspath(sys.argv[1])
+    envfile       = op.abspath(sys.argv[2])
+    release_info  = load_release_info(release_file)
+
+    publish_from_branches = os.environ.get('PUBLISH_FROM_BRANCHES', '')
+    branch_name           = os.environ.get('CI_COMMIT_BRANCH', None)
+    is_tag                = 'CI_COMMIT_TAG' in os.environ
+
+    run_full_test = is_tag or ((branch_name is not None) and
+                               (branch_name in publish_from_branches))
+
+    username = os.environ.get('FSLCONDA_USERNAME', 'username')
+    password = os.environ.get('FSLCONDA_PASSWORD', 'password')
+
+    with tempdir():
+
+        shutil.copy(envfile, '.environment.yml')
+
+        with open('.environment.yml', 'rt') as inf, \
+             open(envfile,            'wt') as outf:
+            contents = inf.read()
+            contents = contents.replace('${FSLCONDA_USERNAME}', username)
+            contents = contents.replace('${FSLCONDA_PASSWORD}', password)
+            outf.write(contents)
+
+        if run_full_test:
+            install_out, env_out = full_test(envfile, release_info)
+        else:
+            install_out, env_out = fast_test(envfile, release_info)
+
+        print(install_out)
+        print(env_out)
+
+        # TODO carriage returns in output are being
+        #      interpreted as newlines. Maybe need to
+        #      change sprun to capture output as binary
+
+        install_lines = len(install_out.split('\n'))
+        env_lines     = len(env_out    .split('\n'))
+        envfile       = op.basename(envfile)
+
+        print(f'{envfile} {install_lines} {env_lines}')
+
+
+if __name__ == '__main__':
+    sys.exit(main())

manifest_rules/test_manifest.py

+#!/usr/bin/env python
+#
+# test_manifest.py - Run the fslinstaller script on a generated manifest file.
+
+import os.path as op
+import os
+import sys
+import json
+
+from manifest_rules.utils import sprun, tempdir, download_file
+
+
+def main():
+    manifest_file    = op.abspath(sys.argv[1])
+    environment_file = op.abspath(sys.argv[2])
+
+    # credentials for logging into
+    # internal FSL conda channel
+    username = os.environ['FSLCONDA_USERNAME']
+    password = os.environ['FSLCONDA_PASSWORD']