Commit 72a01f51 authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

Merge branch 'mnt/cudaversions' into 'master'

Mnt/cudaversions

See merge request fsl/fsl-ci-rules!93
parents 4b92d5a3 4b03452c
......@@ -19,7 +19,7 @@ import subprocess as sp
import yaml
__version__ = '0.16.8'
__version__ = '0.16.9'
"""Current version of the fsl-ci-rules."""
......
......@@ -3,7 +3,10 @@
# The git repository for this conda recipe is: {{ template_recipe_repository }}
{% if 'cuda' not in sub_template %}
{{ '{%' }} set name = '{{ template_name }}' {{ '%}' }}
{{ '{%' }} set name = '{{ template_name }}' {{ '%}' }}
{{ '{%' }} set version = '{{ template_version }}' {{ '%}' }}
{{ '{%' }} set repository = '{{ template_repository }}' {{ '%}' }}
{{ '{%' }} set build = '{{ template_build }}' {{ '%}' }}
{% else %}
# Note that multiple conda packages are built from
# this recipe - one package is built for each
......@@ -20,11 +23,10 @@
{{ '{%' }} set cuda_version = os.environ["CUDA_VER"] {{ '%}' }}
{{ '{%' }} set cuda_label = '-cuda-' + cuda_version {{ '%}' }}
{{ '{%' }} set name = '{{ template_name }}' + cuda_label {{ '%}' }}
{% endif %}
{{ '{%' }} set version = '{{ template_version }}' {{ '%}' }}
{{ '{%' }} set repository = '{{ template_repository }}' {{ '%}' }}
{{ '{%' }} set build = '{{ template_build }}' {{ '%}' }}
{% endif %}
package:
name: {{ '{{ name }}' }}
......
......@@ -17,15 +17,21 @@ requirements:
{% endif %}
build:
# Different versions of nvcc need
# different versions of gcc
# different versions of gcc. This is
# outlined in the "System requirements"
# section of the CUDA installation
# guide for each CUDA version, e.g.:
# https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html
{{ '{% if cuda_version in ("9.2", "10.0") %}' }}
- {{ '{{ compiler("cxx") }} 7.*' }} # [linux]
{{ '{% elif cuda_version in ("10.1", "10.2") %}' }}
- {{ '{{ compiler("cxx") }} 8.*' }} # [linux]
{{ '{% elif cuda_version in ("11.0") %}' }}
{{ '{% elif cuda_version in ("11.0", "11.1", "11.2", "11.3") %}' }}
- {{ '{{ compiler("cxx") }} 9.*' }} # [linux]
{{ '{% elif cuda_version in ("11.4") %}' }}
- {{ '{{ compiler("cxx") }} 11.*' }} # [linux]
{{ '{% else %}' }}
- {{ '{{ compiler("cxx") }}' }} # [linux]
{{ '{% endif %}' }}
- make
{% endif %}
\ No newline at end of file
{% endif %}
......@@ -2,20 +2,27 @@
# The project git repository is: https://git.fmrib.ox.ac.uk/fsl/ptx2.git
# The git repository for this conda recipe is: https://git.fmrib.ox.ac.uk/fsl/conda/fsl-ptx2.git
{% set name = 'fsl-ptx2' %}
{% set version = '@@ project_version @@' %}
{% set repository = 'https://git.fmrib.ox.ac.uk/fsl/ptx2.git' %}
{% set build = '0' %}
# Note that multiple conda packages are built from
# this recipe - one package is built for each
# supported CUDA version, and named
# "<name>-cuda-X.Y", where X.Y is the CUDA version
# that the package was compiled against.
# The CUDA_VER environment variable must be set when
# a CUDA project is built. CUDA_VER should contain
# the major.minor CUDA toolkit version. CUDA_VER is
# pre-set in the conda-forge and FSL development
# CUDA build docker images.
# CUDA build docker images, but needs to be explicitly
# set when building locally.
{% set cuda_version = os.environ["CUDA_VER"] %}
{% set cuda_label = '-cuda-' + cuda_version %}
{% set name = 'fsl-ptx2' + cuda_label %}
{% set version = '@@ project_version @@' %}
{% set repository = 'https://git.fmrib.ox.ac.uk/fsl/ptx2.git' %}
{% set build = '0' %}
package:
name: {{ name + cuda_label }}
name: {{ name }}
version: {{ version }}
source:
......@@ -36,12 +43,20 @@ requirements:
- fsl-base >=@@ fsl_base_version @@
build:
# Different versions of nvcc need
# different versions of gcc
# different versions of gcc. This is
# outlined in the "System requirements"
# section of the CUDA installation
# guide for each CUDA version, e.g.:
# https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html
{% if cuda_version in ("9.2", "10.0") %}
- {{ compiler("cxx") }} 7.* # [linux]
{% elif cuda_version in ("10.1", "10.2") %}
- {{ compiler("cxx") }} 8.* # [linux]
{% elif cuda_version in ("11.0", "11.1") %}
{% elif cuda_version in ("11.0", "11.1", "11.2", "11.3") %}
- {{ compiler("cxx") }} 9.* # [linux]
{% elif cuda_version in ("11.4") %}
- {{ compiler("cxx") }} 11.* # [linux]
{% else %}
- {{ compiler("cxx") }} # [linux]
{% endif %}
- make
......@@ -2,10 +2,10 @@
# The project git repository is: https://git.fmrib.ox.ac.uk/fsl/feat5.git
# The git repository for this conda recipe is: https://git.fmrib.ox.ac.uk/fsl/conda/fsl-feat5.git
{% set name = 'fsl-feat5' %}
{% set version = '@@ project_version @@' %}
{% set repository = 'https://git.fmrib.ox.ac.uk/fsl/feat5.git' %}
{% set build = '0' %}
{% set name = 'fsl-feat5' %}
{% set version = '@@ project_version @@' %}
{% set repository = 'https://git.fmrib.ox.ac.uk/fsl/feat5.git' %}
{% set build = '0' %}
package:
name: {{ name }}
......@@ -26,7 +26,7 @@ build:
requirements:
host:
- fsl-base >=2108.1
- fsl-base >=@@ fsl_base_version @@
build:
- {{ compiler("cxx") }}
- make
......@@ -10,6 +10,7 @@ import os.path as op
import itertools as it
import contextlib as ctxlib
import shlex
import difflib
import shutil
from unittest import mock
import jinja2 as j2
......@@ -48,35 +49,37 @@ def environment():
def compare_directories(project, benchmarkdir, directory):
tkn = token().strip('"').strip()
basever = get_project_version('fsl/base', GITLAB_URL, tkn)
projver = get_project_version(project, GITLAB_URL, tkn)
tvars = {'fsl_base_version' : basever, 'project_version' : projver}
tkn = token().strip('"').strip()
basever = get_project_version('fsl/base', GITLAB_URL, tkn)
projver = get_project_version(project, GITLAB_URL, tkn)
tvars = {'fsl_base_version' : basever, 'project_version' : projver}
directory = op.abspath(directory)
with tempdir():
shutil.copytree(benchmarkdir, 'benchmark')
with indir('benchmark'):
benchmarkdir = op.abspath('benchmark')
with indir(benchmarkdir):
filenames = []
for dirpath, _, dirfiles in os.walk(''):
for dirpath, _, dirfiles in os.walk('.'):
if '.git' in dirpath:
continue
for f in dirfiles:
for fname in dirfiles:
loader = j2.FileSystemLoader('.')
env = j2.Environment(loader=loader,
block_start_string='@@',
block_end_string='@@',
variable_start_string='@=',
variable_end_string='=@')
with open(f, 'rt') as f:
template = env.from_string(f.read())
with open(f, 'wt') as f:
block_start_string='@=',
block_end_string='=@',
variable_start_string='@@',
variable_end_string='@@')
template = env.get_template(fname)
with open(fname, 'wt') as f:
f.write(template.render(**tvars))
filenames.extend([op.join(dirpath, f) for f in dirfiles])
for filename in filenames:
with open(op.join(benchmarkdir, filename), 'rt') as f: exp = f.read()
with open(op.join(directory, filename), 'rt') as f: got = f.read()
assert exp.strip() == got.strip()
for filename in filenames:
with open(op.join(benchmarkdir, filename), 'rt') as f: exp = f.read()
with open(op.join(directory, filename), 'rt') as f: got = f.read()
assert exp.strip() == got.strip()
def test_create_cpp_conda_recipe():
......
......@@ -148,7 +148,9 @@ def parseArgs(argv=None):
'server' : f'Gitlab server (default: {SERVER_URL})',
'project' : 'Project(s) to build',
'production' : 'Build production/stable version '
'(default: label built package as staging/development)'
'(default: label built package as staging/development)',
'sequential' : 'Build projects sequentially (default: build in '
'parallel',
}
parser = argparse.ArgumentParser(usage=usage, description=desc)
......@@ -160,6 +162,8 @@ def parseArgs(argv=None):
help=helps['token'])
parser.add_argument('-p', '--production', action='store_true',
help=helps['production'])
parser.add_argument('-q', '--sequential', action='store_true',
help=helps['sequential'])
return parser.parse_args(argv)
......@@ -171,28 +175,26 @@ def main(argv=None):
args = parseArgs(argv)
projects = args.project
pool = mp.Pool(len(projects))
build = ft.partial(trigger_build,
server=args.server,
token=args.token,
production=args.production)
deploy = ft.partial(trigger_deploy,
server=args.server,
token=args.token,
production=args.production)
# Build packages in parallel
pids = pool.map(build, projects)
pool.close()
pool.join()
# Deploy sequentially, to avoid conflicts
# arising from simultaneous access to the
# channel directory
for project, pid in zip(projects, pids):
if pid is not None:
build = ft.partial(trigger_build,
server=args.server,
token=args.token,
production=args.production)
deploy = ft.partial(trigger_deploy,
server=args.server,
token=args.token,
production=args.production)
if args.sequential:
for project in projects:
pid = build(project)
deploy(project, pid)
else:
pool = mp.Pool(len(projects))
pids = pool.map(build, projects)
pool.starmap(deploy, zip(projects, pids))
pool.close()
pool.join()
if __name__ == '__main__':
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment