Commit a685dab1 authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

Merge branch 'mnt/auth' into 'master'

Mnt/auth

See merge request fsl/fsl-ci-rules!94
parents 72a01f51 c2ac5316
......@@ -13,13 +13,14 @@ import errno
import shlex
import time
import tempfile
import urllib.parse as urlparse
import contextlib as ctxlib
import subprocess as sp
import yaml
__version__ = '0.16.9'
__version__ = '0.17.0'
"""Current version of the fsl-ci-rules."""
......@@ -133,6 +134,14 @@ def dumpyaml(o):
return yaml.dump(o, sort_keys=False)
def add_credentials(url, username, password):
"""Returns a URL with the username/password added."""
username = urlparse.quote_plus(username)
password = urlparse.quote_plus(password)
scheme, path = url.split('://')
return f'{scheme}://{username}:{password}@{path}'
class CaptureStdout:
"""Context manager which captures stdout and stderr. """
......
......@@ -13,7 +13,8 @@ import urllib.parse as urlparse
from fsl_ci import (sprun,
indir,
fprint)
fprint,
add_credentials)
from fsl_ci.versioning import generate_staging_version
from fsl_ci.platform import get_platform_shortcut_if_not_applicable
from fsl_ci.conda import load_meta_yaml
......@@ -119,6 +120,8 @@ def main():
token = os.environ['FSL_CI_API_TOKEN']
staging = os.environ['STAGING'].lower() == 'true'
internal = 'FSLCONDA_INTERNAL' in os.environ
username = os.environ.get('FSLCONDA_INTERNAL_CHANNEL_USERNAME', None)
password = os.environ.get('FSLCONDA_INTERNAL_CHANNEL_PASSWORD', None)
skip_platforms = os.environ.get('FSLCONDA_SKIP_PLATFORM', '')
skip_platforms = skip_platforms.split()
......@@ -145,11 +148,12 @@ def main():
# we output to _platform_, rather thah platform.
output_dir = op.join(os.getcwd(), 'conda_build', f'_{platform}_')
if internal:
channel_urls = [os.environ['FSLCONDA_INTERNAL_CHANNEL_URL'],
os.environ['FSLCONDA_PUBLIC_CHANNEL_URL']]
else:
channel_urls = [os.environ['FSLCONDA_PUBLIC_CHANNEL_URL']]
pubchan = os.environ['FSLCONDA_PUBLIC_CHANNEL_URL']
intchan = os.environ['FSLCONDA_INTERNAL_CHANNEL_URL']
if username is not None:
intchan = add_credentials(intchan, username, password)
if internal: channel_urls = [intchan, pubchan]
else: channel_urls = [pubchan]
os.makedirs(output_dir)
......
......@@ -12,7 +12,7 @@ import os
import sys
import glob
from fsl_ci import sprun
from fsl_ci import sprun, add_credentials
from fsl_ci.platform import get_platform_shortcut_if_not_applicable
from fsl_ci.conda import (load_meta_yaml,
get_project_repository_and_revision)
......@@ -55,6 +55,11 @@ def create_test_env(env_dir, build_dir):
internal = 'FSLCONDA_INTERNAL' in os.environ
intchannel = os.environ['FSLCONDA_INTERNAL_CHANNEL_URL']
pubchannel = os.environ['FSLCONDA_PUBLIC_CHANNEL_URL']
username = os.environ.get('FSLCONDA_INTERNAL_CHANNEL_USERNAME', None)
password = os.environ.get('FSLCONDA_INTERNAL_CHANNEL_PASSWORD', None)
if username is not None:
intchannel = add_credentials(intchannel, username, password)
if internal: pkgchannels = [intchannel, pubchannel]
else: pkgchannels = [pubchannel]
......
......@@ -7,10 +7,12 @@
#
import sys
import os.path as op
import functools as ft
import textwrap as tw
import argparse
import datetime
import multiprocessing.dummy as mp
from fsl_ci.recipe import get_recipe_variable
......@@ -32,6 +34,11 @@ SERVER_URL = 'https://git.fmrib.ox.ac.uk'
"""Default gitlab instance URL, if not specified on the command.line."""
def now():
"""Returns the current time as a string. """
return datetime.datetime.now().strftime('%H:%M:%S')
def get_revision(recipe_path, server, token):
"""Return the value of the FSLCONDA_REVISION variable on the given
conda recipe repository, or None if it is not set.
......@@ -85,7 +92,7 @@ def trigger_build(project, server, token, production):
pid = pipeline['id']
print(f'Pipeline triggered on {project} ({channel} build) '
print(f'{now()} Pipeline triggered on {project} ({channel} build) '
f'- see {pipeline["web_url"]}')
try:
......@@ -93,7 +100,7 @@ def trigger_build(project, server, token, production):
except Exception:
return None
print(f'Build pipeline for {project} has finished: {status}')
print(f'{now()} Build pipeline for {project} has finished: {status}')
if status != 'manual':
return None
......@@ -106,6 +113,11 @@ def trigger_deploy(project, pid, server, token, production):
and waits for it to complete.
"""
# trigger_build returns None
# if the build failed
if pid is None:
return False
# deployment to staging/production gets set at
# build time, so we don't need to pass the STAGING
# variable here, like we do in trigger_build
......@@ -116,7 +128,7 @@ def trigger_deploy(project, pid, server, token, production):
meta = load_meta_yaml(meta)
platforms = get_platform_ids(meta)
print(f'Triggering deploy-{platforms}-conda-package jobs '
print(f'{now()} Triggering deploy-{platforms}-conda-package jobs '
f'on {project} (deploying to {channel} channel)')
try:
......@@ -128,9 +140,11 @@ def trigger_deploy(project, pid, server, token, production):
trigger_job(project, j['id'], server, token)
status = wait_on_pipeline(project, pid, server, token)
print(f'Deploy job {project} has finished: {status}')
print(f'{now()} Deploy job {project} has finished: {status}')
return True
except Exception as e:
print(f'Error triggering deploy job on {project}: {e}')
return False
def parseArgs(argv=None):
......@@ -183,19 +197,22 @@ def main(argv=None):
server=args.server,
token=args.token,
production=args.production)
result = True
if args.sequential:
for project in projects:
pid = build(project)
deploy(project, pid)
if not deploy(project, pid):
result = False
else:
pool = mp.Pool(len(projects))
pids = pool.map(build, projects)
pool.starmap(deploy, zip(projects, pids))
result = all(pool.starmap(deploy, zip(projects, pids)))
pool.close()
pool.join()
return 0 if result else 1
if __name__ == '__main__':
main()
sys.exit(main())
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment