Commit 44353549 authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

MNT: initial commit

parents
stages:
- check-version
# Only run on merge requests
workflow:
rules:
- if: '$CI_MERGE_REQUEST_ID != null'
check-version:
stage: check-version
image: python:3.9
tags:
- fsl-ci
- docker
script:
- thisver=$( cat manifest_rules/__init__.py | grep __version__ | cut -d " " -f 3 | tr -d "'")
- masterver=$(git show master:manifest_rules/__init__.py | grep __version__ | cut -d " " -f 3 | tr -d "'")
- |
if [ "$thisver" = "$masterver" ]; then
echo "Version has not been updated!"
echo "Version on master branch: $masterver"
echo "Version in this MR: $thisver"
echo "The version number must be updated before this MR can be merged."
exit 1
else
echo "Version on master branch: $masterver"
echo "Version in this MR: $thisver"
fi
# FSL release manifest rules
This repository contains scripts which are used in the fsl/conda/manifest>
repository to generate, test, and deploy FSL release manifest and environment
files. Refer to the fsl/conda/manifest> repository for more details.
These scripts are hosted separately from the `fsl/conda/manifest` project so
that they can be updated independently of FSL releases.
Whenever changes to these scripts are made, the version number in
`manifest_rules/__init__.py` must be incremented.
#!/usr/bin/env python
__version__ = '0.1.0'
#!/usr/bin/env python
import os
import sys
import shutil
def main():
dest = sys.argv[1]
files = sys.argv[2:]
publish_from_branches = os.environ.get('PUBLISH_FROM_BRANCHES', '')
branch = os.environ.get('CI_COMMIT_BRANCH', None)
tag = os.environ.get('CI_COMMIT_TAG', None)
allow_publish = ((tag is not None) or
((branch is not None) and
(branch in publish_from_branches)))
if not allow_publish:
print('Publishing is only allowed from tags or '
f'these branches: {publish_from_branches}')
sys.exit(1)
if tag is not None:
print(f'Publishing files from tag {tag}')
else:
print(f'Publishing files from branch {branch}')
for src in files:
print(f'Copying {src} to {dest}')
shutil.copy(src, dest)
if __name__ == '__main__':
main()
#!/usr/bin/env python
#
# Generate FSL conda environment files from fsl-release.yml
import itertools as it
import os.path as op
import os
import re
import sys
from manifest_rules.utils import (sprun,
load_release_info,
generate_environment_file_name,
generate_development_version_identifier)
def filter_packages(release_info, platform, cudaver):
"""Return a list of packages that should be included in the environment
file for the platform and CUDA version.
"""
packages = release_info['packages']
cudapat = r'cuda-[\d]+\.[\d]+'
cudapkgs = [p for p in packages if re.search(cudapat, p) is not None]
otherpkgs = [p for p in packages if p not in cudapkgs]
if cudaver is None:
return otherpkgs
else:
include = f'cuda-{cudaver}'
cudapkgs = [p for p in cudapkgs if include in p]
return otherpkgs + cudapkgs
def generate_environment(release_info, version, platform, cudaver, outfile):
"""Genereate an environment file for the platform and CUDA version.
"""
channels = list(release_info['channels'])
packages = filter_packages(release_info, platform, cudaver)
# Dev release
if version is None:
channels = [release_info['internal_channel']] + channels
with open(outfile, 'wt') as f:
f.write('name: FSL\n')
f.write('channels:\n')
for channel in channels:
f.write(f' - {channel}\n')
f.write('dependencies:\n')
for package in packages:
f.write(f' - {package}\n')
def generate_variants(release_info):
"""Generate a list of (platform, cudaver) pairs for which an environment
file should be generated. For non-CUDA environments, cudaver will be None.
"""
cudas = [None] + list(release_info['cuda'])
platforms = list(release_info['miniconda'].keys())
variants = list(it.product(platforms, cudas))
# CUDA builds are only supported
# on linux-64 at this time
variants = [(plat, cuda) for plat, cuda in variants
if not (('linux-64' not in plat) and cuda is not None)]
return variants
def main():
# Save generated environment files here
release_file = op.abspath(sys.argv[1])
outdir = op.abspath(sys.argv[2])
# Tags on fsl/conda/manifest denote a
# public FSL release. All other
# commits denote an internal release.
version = os.environ.get('CI_COMMIT_TAG', None)
release_info = load_release_info(release_file)
# Generate environment files
# for all platforms+CUDA versions
variants = generate_variants(release_info)
if not op.exists(outdir):
os.mkdir(outdir)
generated = []
for platform, cuda in variants:
filename = generate_environment_file_name(version, platform, cuda)
filename = op.join(outdir, filename)
generate_environment(release_info, version, platform, cuda, filename)
generated.append((platform, cuda, op.basename(filename)))
for platform, cuda, filename in generated:
print(f'Platform {platform} [CUDA {cuda}]: {filename}')
if __name__ == '__main__':
sys.exit(main())
#!/usr/bin/env python
#
# Generate FSL release manifest file from fsl-release.yml
import os.path as op
import os
import sys
import glob
import json
import urllib.parse as urlparse
from collections import defaultdict
from manifest_rules.utils import (load_release_info,
tempdir,
download_file,
sha256,
parse_environment_file_name,
generate_development_version_identifier)
def load_test_install_info(install_dir):
"""Returns information (number of output lines produced) about
the test installations. Returns them in a dict.
"""
inst_info = defaultdict(dict)
inst_files = glob.glob(op.join(install_dir, '*.install.txt'))
for fname in inst_files:
with open(fname, 'rt') as f:
lines = f.readlines()
info = lines[-1].strip()
envfile, inst_lines, env_lines = info.split()
platform = parse_environment_file_name(envfile)[1]
inst_lines = int(inst_lines)
env_lines = int(env_lines)
prev_inst_lines = inst_info['miniconda'].get('platform', 0)
inst_lines = max((inst_lines, prev_inst_lines))
inst_info['miniconda'][platform] = inst_lines
inst_info[envfile] = env_lines
return inst_info
def load_previous_manifest(release_info):
"""Downloads the previous official FSL manifest, returning it
as a dict. Returns None if the manifest cannot be downloaded.
"""
url = urlparse.urljoin(release_info['release_url'], 'manifest.json')
with tempdir():
try:
download_file(url, 'manifest.json')
except Exception:
return None
with open('manifest.json', 'rt') as f:
lines = f.readlines()
# Drop comments
lines = [l for l in lines if not l.lstrip().startswith('//')]
manifest = json.loads('\n'.join(lines))
# Filter out any releases/builds for which
# the environment file is not available
versions = dict(manifest.get('versions', {}))
for version, builds in versions.items():
for i, build in enumerate(list(builds)):
try:
with tempdir():
download_file(build['url'], 'env.yml')
except Exception:
print(f'WARNING: Build environment file '
f'is not available: {build}')
builds.pop(i)
if len(builds) == 0:
versions.pop(version)
else:
versions[version] = builds
latest = versions.get('latest', None)
if (latest is not None) and (latest not in versions):
print(f'WARNING: Build for latest FSL '
f'version {latest} is not available')
versions.pop('latest')
manifest['versions'] = versions
return manifest
def generate_installer_section(release_info):
"""Generates the "installer" manifest section, returning a dict. """
version = release_info['installer']
url = urlparse.urljoin(release_info['release_url'], 'fslinstaller.py')
try:
with tempdir():
download_file(url, 'fslinstaller.py')
checksum = sha256('fslinstaller.py')
except Exception:
print(f'WARNING: fslinstaller.py script not available at {url}!')
checksum = ''
return {'version' : version,
'url' : url,
'sha256' : checksum}
def generate_miniconda_section(release_info, install_info):
"""Generates the "miniconda" manifest section, returning a dict. """
section = {}
outputs = install_info.get('miniconda', {})
for platform, url in release_info['miniconda'].items():
output = outputs.get(platform, None)
with tempdir():
download_file(url, 'miniconda.sh')
checksum = sha256('miniconda.sh')
section[platform] = {
'url' : url,
'sha256' : checksum,
}
if output is not None:
section[platform]['output'] = str(output)
return section
def generate_version_section(version,
include_current_release,
include_past_releases,
envdir,
release_info,
install_info):
"""Generates the "versions" manifest section, returning a dict.
Does not add the "latest" entry.
"""
section = {}
if include_past_releases:
prev_manifest = load_previous_manifest(release_info)
if prev_manifest is not None:
section = prev_manifest['versions']
if not include_current_release:
return section
versions = []
section = {version : versions}
for envfile in glob.glob(op.join(envdir, '*.yml')):
checksum = sha256(envfile)
envfile = op.basename(envfile)
output = install_info.get(envfile, None)
url = urlparse.urljoin(release_info['release_url'], envfile)
version, platform, cuda = parse_environment_file_name(envfile)
build = {
'platform' : platform,
'environment' : url,
'sha256' : checksum
}
if cuda is not None:
build['cuda'] = cuda
versions.append(build)
if output is not None:
versions[-1]['output'] = {'install' : str(output)}
return section
def main():
# This script is run in the following scenarios:
# - New public FSL release - generate a new official
# manifest, describing the new release, and all past
# public releases
# - New internal/development release - generate a
# development manifest describing just the development
# release
# - Update to official manifest - re-generate the
# official manifest, describing all past public
# releases
# We use the "official" and "add_current_release"
# variables to figure out the scenario we are in
release_file = op.abspath(sys.argv[1])
# See .gitlab-ci.yml - we are passed "true" or "false"
# to determine whether we are to generate an official
# or development manifest
official = sys.argv[2] == 'true'
# Directory to save the generated manifest file
outdir = op.abspath(sys.argv[3])
# Directory containing the environment files
# to be added to/described by the manifest
envdir = op.abspath(sys.argv[4])
# Directory containing outputs of the test
# installation
installdir = op.abspath(sys.argv[5])
# FSL version identifier
version = os.environ.get('CI_COMMIT_TAG', None)
# Add information about the current release
# (tag or commit) to the generated manifest?
# We do this for new official releases, or
# for development releases
add_current_release = (version is not None) or (not official)
# Generate dev version identnfier
if version is None:
version = generate_development_version_identifier()
# Load fsl-release.yml, and load
# outputs of test install stage
release_info = load_release_info(release_file)
install_info = load_test_install_info(installdir)
# Generate the manifest
manifest = {
'installer' : generate_installer_section(release_info),
'miniconda' : generate_miniconda_section(release_info, install_info),
'versions' : generate_version_section(version,
add_current_release,
official,
envdir,
release_info,
install_info),
}
# Set the "latest" field for for
# new public or developmenmt releases
if add_current_release:
manifest['versions']['latest'] = version
if official: fname = 'manifest.json'
else: fname = f'manifest-{version}.json'
manifest = json.dumps(manifest, indent=4, sort_keys=True)
with open(op.join(outdir, fname), 'wt') as f:
f.write(manifest)
print(f'Generated manifest file {fname}:')
print(manifest)
if __name__ == '__main__':
sys.exit(main())
#!/usr/bin/env python
#
# Performs a test installation for a FSL conda environment file. Counts
# the number of output lines produced when installing miniconda, and
# when installing the FSL environment.
#
import os.path as op
import os
import sys
import shutil
import yaml
from manifest_rules.utils import (sprun,
tempdir,
download_file,
load_release_info,
parse_environment_file_name)
def preprocess_environment(envfile, condadir, regen_envfile=False):
with open(envfile, 'rt') as f:
env = f.read()
env = yaml.load(env, Loader=yaml.Loader)
packages = env['dependencies']
channels = env['channels']
for i, pkg in enumerate(packages):
pkg, ver = pkg.split()
packages[i] = f'{pkg}={ver}'
with open(op.join(condadir, 'condarc'), 'wt') as f:
f.write('channels:\n')
for c in channels:
f.write(f' - {c}\n')
if regen_envfile:
with open(envfile, 'wt') as f:
f.write('channels:\n')
for c in channels:
f.write(f' - {c}\n')
f.write('dependencies:\n')
for p in packages:
f.write(f' - {p}\n')
return packages
def fast_test(envfile, release_info):
install_out = sprun('conda create -y -p fsl')
packages = preprocess_environment(envfile, 'fsl')
env = os.environ.copy()
env['CONDARC'] = op.abspath(op.join('fsl', 'condarc'))
cmd = 'conda install -p fsl -y --dry-run ' + ' '.join(packages)
env_out = sprun(cmd, env=env)
return install_out, env_out
def full_test(envfile, release_info):
platform = parse_environment_file_name(envfile)[1]
miniconda_url = release_info['miniconda'][platform]
download_file(miniconda_url, 'miniconda.sh')
install_out = sprun('sh ./miniconda.sh -b -p ./fsl')
preprocess_environment(envfile, 'fsl', True)
env = os.environ.copy()
env['CONDARC'] = op.abspath(op.join('fsl', 'condarc'))
env_out = sprun(f'./fsl/bin/conda env update -n base -f {envfile}',
env=env)
return install_out, env_out
def main():
release_file = op.abspath(sys.argv[1])
envfile = op.abspath(sys.argv[2])
release_info = load_release_info(release_file)
publish_from_branches = os.environ.get('PUBLISH_FROM_BRANCHES', '')
branch_name = os.environ.get('CI_COMMIT_BRANCH', None)
is_tag = 'CI_COMMIT_TAG' in os.environ
run_full_test = is_tag or ((branch_name is not None) and
(branch_name in publish_from_branches))
username = os.environ.get('FSLCONDA_USERNAME', 'username')
password = os.environ.get('FSLCONDA_PASSWORD', 'password')
with tempdir():
shutil.copy(envfile, '.environment.yml')
with open('.environment.yml', 'rt') as inf, \
open(envfile, 'wt') as outf:
contents = inf.read()
contents = contents.replace('${FSLCONDA_USERNAME}', username)
contents = contents.replace('${FSLCONDA_PASSWORD}', password)
outf.write(contents)
if run_full_test:
install_out, env_out = full_test(envfile, release_info)
else:
install_out, env_out = fast_test(envfile, release_info)
print(install_out)
print(env_out)
# TODO carriage returns in output are being
# interpreted as newlines. Maybe need to
# change sprun to capture output as binary
install_lines = len(install_out.split('\n'))
env_lines = len(env_out .split('\n'))
envfile = op.basename(envfile)
print(f'{envfile} {install_lines} {env_lines}')
if __name__ == '__main__':
sys.exit(main())
#!/usr/bin/env python
#
# test_manifest.py - Run the fslinstaller script on a generated manifest file.
import os.path as op
import os
import sys
import json
from manifest_rules.utils import sprun, tempdir, download_file
def main():
manifest_file = op.abspath(sys.argv[1])
environment_file = op.abspath(sys.argv[2])
# credentials for logging into
# internal FSL conda channel
username = os.environ['FSLCONDA_USERNAME']
password = os.environ['FSLCONDA_PASSWORD']
with open(manifest_file, 'rt') as f:
manifest = json.loads(f.read())
installer_url = manifest['installer']['url']
latest = manifest['versions'].get('latest', None)
# This skip can be removed when we