Commit a685dab1 authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

Merge branch 'mnt/auth' into 'master'

Mnt/auth

See merge request fsl/fsl-ci-rules!94
parents 72a01f51 c2ac5316
...@@ -6,20 +6,21 @@ ...@@ -6,20 +6,21 @@
# #
import os.path as op import os.path as op
import os import os
import sys import sys
import errno import errno
import shlex import shlex
import time import time
import tempfile import tempfile
import contextlib as ctxlib import urllib.parse as urlparse
import subprocess as sp import contextlib as ctxlib
import subprocess as sp
import yaml import yaml
__version__ = '0.16.9' __version__ = '0.17.0'
"""Current version of the fsl-ci-rules.""" """Current version of the fsl-ci-rules."""
...@@ -133,6 +134,14 @@ def dumpyaml(o): ...@@ -133,6 +134,14 @@ def dumpyaml(o):
return yaml.dump(o, sort_keys=False) return yaml.dump(o, sort_keys=False)
def add_credentials(url, username, password):
"""Returns a URL with the username/password added."""
username = urlparse.quote_plus(username)
password = urlparse.quote_plus(password)
scheme, path = url.split('://')
return f'{scheme}://{username}:{password}@{path}'
class CaptureStdout: class CaptureStdout:
"""Context manager which captures stdout and stderr. """ """Context manager which captures stdout and stderr. """
......
...@@ -13,7 +13,8 @@ import urllib.parse as urlparse ...@@ -13,7 +13,8 @@ import urllib.parse as urlparse
from fsl_ci import (sprun, from fsl_ci import (sprun,
indir, indir,
fprint) fprint,
add_credentials)
from fsl_ci.versioning import generate_staging_version from fsl_ci.versioning import generate_staging_version
from fsl_ci.platform import get_platform_shortcut_if_not_applicable from fsl_ci.platform import get_platform_shortcut_if_not_applicable
from fsl_ci.conda import load_meta_yaml from fsl_ci.conda import load_meta_yaml
...@@ -119,6 +120,8 @@ def main(): ...@@ -119,6 +120,8 @@ def main():
token = os.environ['FSL_CI_API_TOKEN'] token = os.environ['FSL_CI_API_TOKEN']
staging = os.environ['STAGING'].lower() == 'true' staging = os.environ['STAGING'].lower() == 'true'
internal = 'FSLCONDA_INTERNAL' in os.environ internal = 'FSLCONDA_INTERNAL' in os.environ
username = os.environ.get('FSLCONDA_INTERNAL_CHANNEL_USERNAME', None)
password = os.environ.get('FSLCONDA_INTERNAL_CHANNEL_PASSWORD', None)
skip_platforms = os.environ.get('FSLCONDA_SKIP_PLATFORM', '') skip_platforms = os.environ.get('FSLCONDA_SKIP_PLATFORM', '')
skip_platforms = skip_platforms.split() skip_platforms = skip_platforms.split()
...@@ -145,11 +148,12 @@ def main(): ...@@ -145,11 +148,12 @@ def main():
# we output to _platform_, rather thah platform. # we output to _platform_, rather thah platform.
output_dir = op.join(os.getcwd(), 'conda_build', f'_{platform}_') output_dir = op.join(os.getcwd(), 'conda_build', f'_{platform}_')
if internal: pubchan = os.environ['FSLCONDA_PUBLIC_CHANNEL_URL']
channel_urls = [os.environ['FSLCONDA_INTERNAL_CHANNEL_URL'], intchan = os.environ['FSLCONDA_INTERNAL_CHANNEL_URL']
os.environ['FSLCONDA_PUBLIC_CHANNEL_URL']] if username is not None:
else: intchan = add_credentials(intchan, username, password)
channel_urls = [os.environ['FSLCONDA_PUBLIC_CHANNEL_URL']] if internal: channel_urls = [intchan, pubchan]
else: channel_urls = [pubchan]
os.makedirs(output_dir) os.makedirs(output_dir)
......
...@@ -12,7 +12,7 @@ import os ...@@ -12,7 +12,7 @@ import os
import sys import sys
import glob import glob
from fsl_ci import sprun from fsl_ci import sprun, add_credentials
from fsl_ci.platform import get_platform_shortcut_if_not_applicable from fsl_ci.platform import get_platform_shortcut_if_not_applicable
from fsl_ci.conda import (load_meta_yaml, from fsl_ci.conda import (load_meta_yaml,
get_project_repository_and_revision) get_project_repository_and_revision)
...@@ -55,6 +55,11 @@ def create_test_env(env_dir, build_dir): ...@@ -55,6 +55,11 @@ def create_test_env(env_dir, build_dir):
internal = 'FSLCONDA_INTERNAL' in os.environ internal = 'FSLCONDA_INTERNAL' in os.environ
intchannel = os.environ['FSLCONDA_INTERNAL_CHANNEL_URL'] intchannel = os.environ['FSLCONDA_INTERNAL_CHANNEL_URL']
pubchannel = os.environ['FSLCONDA_PUBLIC_CHANNEL_URL'] pubchannel = os.environ['FSLCONDA_PUBLIC_CHANNEL_URL']
username = os.environ.get('FSLCONDA_INTERNAL_CHANNEL_USERNAME', None)
password = os.environ.get('FSLCONDA_INTERNAL_CHANNEL_PASSWORD', None)
if username is not None:
intchannel = add_credentials(intchannel, username, password)
if internal: pkgchannels = [intchannel, pubchannel] if internal: pkgchannels = [intchannel, pubchannel]
else: pkgchannels = [pubchannel] else: pkgchannels = [pubchannel]
......
...@@ -7,10 +7,12 @@ ...@@ -7,10 +7,12 @@
# #
import sys
import os.path as op import os.path as op
import functools as ft import functools as ft
import textwrap as tw import textwrap as tw
import argparse import argparse
import datetime
import multiprocessing.dummy as mp import multiprocessing.dummy as mp
from fsl_ci.recipe import get_recipe_variable from fsl_ci.recipe import get_recipe_variable
...@@ -32,6 +34,11 @@ SERVER_URL = 'https://git.fmrib.ox.ac.uk' ...@@ -32,6 +34,11 @@ SERVER_URL = 'https://git.fmrib.ox.ac.uk'
"""Default gitlab instance URL, if not specified on the command.line.""" """Default gitlab instance URL, if not specified on the command.line."""
def now():
"""Returns the current time as a string. """
return datetime.datetime.now().strftime('%H:%M:%S')
def get_revision(recipe_path, server, token): def get_revision(recipe_path, server, token):
"""Return the value of the FSLCONDA_REVISION variable on the given """Return the value of the FSLCONDA_REVISION variable on the given
conda recipe repository, or None if it is not set. conda recipe repository, or None if it is not set.
...@@ -85,7 +92,7 @@ def trigger_build(project, server, token, production): ...@@ -85,7 +92,7 @@ def trigger_build(project, server, token, production):
pid = pipeline['id'] pid = pipeline['id']
print(f'Pipeline triggered on {project} ({channel} build) ' print(f'{now()} Pipeline triggered on {project} ({channel} build) '
f'- see {pipeline["web_url"]}') f'- see {pipeline["web_url"]}')
try: try:
...@@ -93,7 +100,7 @@ def trigger_build(project, server, token, production): ...@@ -93,7 +100,7 @@ def trigger_build(project, server, token, production):
except Exception: except Exception:
return None return None
print(f'Build pipeline for {project} has finished: {status}') print(f'{now()} Build pipeline for {project} has finished: {status}')
if status != 'manual': if status != 'manual':
return None return None
...@@ -106,6 +113,11 @@ def trigger_deploy(project, pid, server, token, production): ...@@ -106,6 +113,11 @@ def trigger_deploy(project, pid, server, token, production):
and waits for it to complete. and waits for it to complete.
""" """
# trigger_build returns None
# if the build failed
if pid is None:
return False
# deployment to staging/production gets set at # deployment to staging/production gets set at
# build time, so we don't need to pass the STAGING # build time, so we don't need to pass the STAGING
# variable here, like we do in trigger_build # variable here, like we do in trigger_build
...@@ -116,7 +128,7 @@ def trigger_deploy(project, pid, server, token, production): ...@@ -116,7 +128,7 @@ def trigger_deploy(project, pid, server, token, production):
meta = load_meta_yaml(meta) meta = load_meta_yaml(meta)
platforms = get_platform_ids(meta) platforms = get_platform_ids(meta)
print(f'Triggering deploy-{platforms}-conda-package jobs ' print(f'{now()} Triggering deploy-{platforms}-conda-package jobs '
f'on {project} (deploying to {channel} channel)') f'on {project} (deploying to {channel} channel)')
try: try:
...@@ -128,9 +140,11 @@ def trigger_deploy(project, pid, server, token, production): ...@@ -128,9 +140,11 @@ def trigger_deploy(project, pid, server, token, production):
trigger_job(project, j['id'], server, token) trigger_job(project, j['id'], server, token)
status = wait_on_pipeline(project, pid, server, token) status = wait_on_pipeline(project, pid, server, token)
print(f'Deploy job {project} has finished: {status}') print(f'{now()} Deploy job {project} has finished: {status}')
return True
except Exception as e: except Exception as e:
print(f'Error triggering deploy job on {project}: {e}') print(f'Error triggering deploy job on {project}: {e}')
return False
def parseArgs(argv=None): def parseArgs(argv=None):
...@@ -183,19 +197,22 @@ def main(argv=None): ...@@ -183,19 +197,22 @@ def main(argv=None):
server=args.server, server=args.server,
token=args.token, token=args.token,
production=args.production) production=args.production)
result = True
if args.sequential: if args.sequential:
for project in projects: for project in projects:
pid = build(project) pid = build(project)
deploy(project, pid) if not deploy(project, pid):
result = False
else: else:
pool = mp.Pool(len(projects)) pool = mp.Pool(len(projects))
pids = pool.map(build, projects) pids = pool.map(build, projects)
pool.starmap(deploy, zip(projects, pids)) result = all(pool.starmap(deploy, zip(projects, pids)))
pool.close() pool.close()
pool.join() pool.join()
return 0 if result else 1
if __name__ == '__main__': if __name__ == '__main__':
main() sys.exit(main())
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment