Commit b379727f authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

Merge branch 'rf/skip-platforms' into 'master'

BF: Handle case where conda is not in runner environment $PATH (or conda is a

See merge request fsl/conda/manifest-rules!39
parents aba63973 95c6c9bc
...@@ -16,7 +16,8 @@ default: ...@@ -16,7 +16,8 @@ default:
mkdir ./conda-dir; mkdir ./conda-dir;
export CONDA_ENVS_DIRS=$(pwd)/conda-dir/; export CONDA_ENVS_DIRS=$(pwd)/conda-dir/;
export CONDA_PKGS_DIRS=$(pwd)/conda-dir/.pkgs; export CONDA_PKGS_DIRS=$(pwd)/conda-dir/.pkgs;
CONDABINDIR=$(dirname $(which conda))/../bin/; CONDABIN=$(which conda || echo "${HOME}/miniconda3/bin/conda");
CONDABINDIR=$(dirname ${CONDABIN})/../bin/;
conda create -c conda-forge -p ./env python=3.9; conda create -c conda-forge -p ./env python=3.9;
if [ -f "$CONDABINDIR"/activate ]; then source $CONDABINDIR/activate ./env; if [ -f "$CONDABINDIR"/activate ]; then source $CONDABINDIR/activate ./env;
else conda activate ./env; fi; else conda activate ./env; fi;
......
#!/usr/bin/env python #!/usr/bin/env python
__version__ = '0.7.1' __version__ = '0.7.2'
...@@ -18,6 +18,7 @@ import sys ...@@ -18,6 +18,7 @@ import sys
from fsl_ci.conda import get_channel_packages from fsl_ci.conda import get_channel_packages
from manifest_rules.utils import (load_release_info, from manifest_rules.utils import (load_release_info,
get_platform_identifiers,
generate_environment_file_name, generate_environment_file_name,
generate_development_version_identifier) generate_development_version_identifier)
...@@ -87,14 +88,9 @@ def generate_variants(release_info): ...@@ -87,14 +88,9 @@ def generate_variants(release_info):
"""Generate a list of (platform, cudaver) pairs for which an environment """Generate a list of (platform, cudaver) pairs for which an environment
file should be generated. For non-CUDA environments, cudaver will be None. file should be generated. For non-CUDA environments, cudaver will be None.
""" """
cudas = [None] + list(release_info['cuda']) cudas = [None] + list(release_info['cuda'])
platforms = os.environ.get('FSLCONDA_PLATFORMS', None) platforms = get_platform_identifiers(release_info)
if platforms is None: variants = list(it.product(platforms, cudas))
platforms = list(release_info['miniconda'].keys())
else:
platforms = platforms = platforms.split(',')
variants = list(it.product(platforms, cudas))
# CUDA builds are only supported # CUDA builds are only supported
# on linux-64 at this time # on linux-64 at this time
......
...@@ -16,6 +16,7 @@ from manifest_rules.utils import (load_release_info, ...@@ -16,6 +16,7 @@ from manifest_rules.utils import (load_release_info,
download_file, download_file,
sha256, sha256,
parse_environment_file_name, parse_environment_file_name,
get_platform_identifiers,
generate_development_version_identifier) generate_development_version_identifier)
...@@ -29,9 +30,15 @@ def load_test_install_info(install_dir): ...@@ -29,9 +30,15 @@ def load_test_install_info(install_dir):
for fname in inst_files: for fname in inst_files:
with open(fname, 'rt') as f: with open(fname, 'rt') as f:
lines = f.readlines() lines = f.readlines()
try:
info = lines[-1].strip() info = lines[-1].strip()
envfile, inst_lines, env_lines = info.split()
envfile, inst_lines, env_lines = info.split() except Exception:
print(f'Could not parse test install info from: {fname}')
print('Perhaps it corresponds to a platform that is not in '
'FSLCONDA_PLATFORMS?',
os.environ.get('FSLCONDA_PLATFORMS', 'n/a'))
continue
platform = parse_environment_file_name(envfile)[1] platform = parse_environment_file_name(envfile)[1]
inst_lines = int(inst_lines) inst_lines = int(inst_lines)
env_lines = int(env_lines) env_lines = int(env_lines)
...@@ -115,11 +122,13 @@ def generate_installer_section(release_info): ...@@ -115,11 +122,13 @@ def generate_installer_section(release_info):
def generate_miniconda_section(release_info, install_info): def generate_miniconda_section(release_info, install_info):
"""Generates the "miniconda" manifest section, returning a dict. """ """Generates the "miniconda" manifest section, returning a dict. """
section = {} section = {}
outputs = install_info.get('miniconda', {})
outputs = install_info.get('miniconda', {}) platforms = get_platform_identifiers(release_info)
for platform, url in release_info['miniconda'].items(): for platform, url in release_info['miniconda'].items():
if platform not in platforms:
continue
output = outputs.get(platform, None) output = outputs.get(platform, None)
with tempdir(): with tempdir():
download_file(url, 'miniconda.sh') download_file(url, 'miniconda.sh')
...@@ -155,8 +164,9 @@ def generate_version_section(version, ...@@ -155,8 +164,9 @@ def generate_version_section(version,
if not include_current_release: if not include_current_release:
return section return section
versions = [] versions = []
section = {version : versions} section = {version : versions}
platforms = get_platform_identifiers(release_info)
for envfile in glob.glob(op.join(envdir, '*.yml')): for envfile in glob.glob(op.join(envdir, '*.yml')):
...@@ -166,6 +176,9 @@ def generate_version_section(version, ...@@ -166,6 +176,9 @@ def generate_version_section(version,
url = urlparse.urljoin(release_info['release-url'], envfile) url = urlparse.urljoin(release_info['release-url'], envfile)
version, platform, cuda = parse_environment_file_name(envfile) version, platform, cuda = parse_environment_file_name(envfile)
if platform not in platforms:
continue
build = { build = {
'platform' : platform, 'platform' : platform,
'environment' : url, 'environment' : url,
......
...@@ -17,6 +17,7 @@ from manifest_rules.utils import (sprun, ...@@ -17,6 +17,7 @@ from manifest_rules.utils import (sprun,
tempdir, tempdir,
download_file, download_file,
load_release_info, load_release_info,
get_platform_identifiers,
parse_environment_file_name) parse_environment_file_name)
...@@ -114,9 +115,18 @@ def full_test(envfile, release_info): ...@@ -114,9 +115,18 @@ def full_test(envfile, release_info):
def main(): def main():
job_name = os.environ['CI_JOB_NAME']
release_file = op.abspath(sys.argv[1]) release_file = op.abspath(sys.argv[1])
envfile = op.abspath(sys.argv[2]) envfile = op.abspath(sys.argv[2])
release_info = load_release_info(release_file) release_info = load_release_info(release_file)
platforms = get_platform_identifiers(release_info)
for platform in platforms:
if platform in job_name:
break
else:
print('This platform is disabled')
sys.exit(0)
publish_from_branches = os.environ.get('PUBLISH_FROM_BRANCHES', '') publish_from_branches = os.environ.get('PUBLISH_FROM_BRANCHES', '')
branch_name = os.environ.get('CI_COMMIT_BRANCH', None) branch_name = os.environ.get('CI_COMMIT_BRANCH', None)
......
...@@ -7,12 +7,27 @@ import os ...@@ -7,12 +7,27 @@ import os
import sys import sys
import json import json
from manifest_rules.utils import sprun, tempdir, download_file from manifest_rules.utils import (sprun,
tempdir,
download_file,
load_release_info,
get_platform_identifiers)
def main(): def main():
manifest_file = op.abspath(sys.argv[1]) job_name = os.environ['CI_JOB_NAME']
environment_dir = op.abspath(sys.argv[2]) release_file = op.abspath(sys.argv[1])
manifest_file = op.abspath(sys.argv[2])
environment_dir = op.abspath(sys.argv[3])
release_info = load_release_info(release_file)
platforms = get_platform_identifiers(release_info)
for platform in platforms:
if platform in job_name:
break
else:
print('This platform is disabled')
sys.exit(0)
# credentials for logging into # credentials for logging into
# internal FSL conda channel # internal FSL conda channel
......
...@@ -174,6 +174,18 @@ def load_release_info(fsl_release_file): ...@@ -174,6 +174,18 @@ def load_release_info(fsl_release_file):
return yaml.load(f.read(), Loader=yaml.Loader) return yaml.load(f.read(), Loader=yaml.Loader)
def get_platform_identifiers(release_info):
"""Returns a list of all platform identifiers for which FSL environment
files are being generated, e.g. ['linux-64', 'macos-64', 'macos-M1'].
"""
platforms = os.environ.get('FSLCONDA_PLATFORMS', None)
if platforms is None:
platforms = list(release_info['miniconda'].keys())
else:
platforms = platforms.split(',')
return platforms
@contextlib.contextmanager @contextlib.contextmanager
def tempdir(): def tempdir():
"""Returns a context manager which creates, changes into, and returns a """Returns a context manager which creates, changes into, and returns a
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment