Commit a685dab1 authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

Merge branch 'mnt/auth' into 'master'

Mnt/auth

See merge request fsl/fsl-ci-rules!94
parents 72a01f51 c2ac5316
......@@ -6,20 +6,21 @@
#
import os.path as op
import os
import sys
import errno
import shlex
import time
import tempfile
import contextlib as ctxlib
import subprocess as sp
import os.path as op
import os
import sys
import errno
import shlex
import time
import tempfile
import urllib.parse as urlparse
import contextlib as ctxlib
import subprocess as sp
import yaml
__version__ = '0.16.9'
__version__ = '0.17.0'
"""Current version of the fsl-ci-rules."""
......@@ -133,6 +134,14 @@ def dumpyaml(o):
return yaml.dump(o, sort_keys=False)
def add_credentials(url, username, password):
"""Returns a URL with the username/password added."""
username = urlparse.quote_plus(username)
password = urlparse.quote_plus(password)
scheme, path = url.split('://')
return f'{scheme}://{username}:{password}@{path}'
class CaptureStdout:
"""Context manager which captures stdout and stderr. """
......
......@@ -13,7 +13,8 @@ import urllib.parse as urlparse
from fsl_ci import (sprun,
indir,
fprint)
fprint,
add_credentials)
from fsl_ci.versioning import generate_staging_version
from fsl_ci.platform import get_platform_shortcut_if_not_applicable
from fsl_ci.conda import load_meta_yaml
......@@ -119,6 +120,8 @@ def main():
token = os.environ['FSL_CI_API_TOKEN']
staging = os.environ['STAGING'].lower() == 'true'
internal = 'FSLCONDA_INTERNAL' in os.environ
username = os.environ.get('FSLCONDA_INTERNAL_CHANNEL_USERNAME', None)
password = os.environ.get('FSLCONDA_INTERNAL_CHANNEL_PASSWORD', None)
skip_platforms = os.environ.get('FSLCONDA_SKIP_PLATFORM', '')
skip_platforms = skip_platforms.split()
......@@ -145,11 +148,12 @@ def main():
# we output to _platform_, rather thah platform.
output_dir = op.join(os.getcwd(), 'conda_build', f'_{platform}_')
if internal:
channel_urls = [os.environ['FSLCONDA_INTERNAL_CHANNEL_URL'],
os.environ['FSLCONDA_PUBLIC_CHANNEL_URL']]
else:
channel_urls = [os.environ['FSLCONDA_PUBLIC_CHANNEL_URL']]
pubchan = os.environ['FSLCONDA_PUBLIC_CHANNEL_URL']
intchan = os.environ['FSLCONDA_INTERNAL_CHANNEL_URL']
if username is not None:
intchan = add_credentials(intchan, username, password)
if internal: channel_urls = [intchan, pubchan]
else: channel_urls = [pubchan]
os.makedirs(output_dir)
......
......@@ -12,7 +12,7 @@ import os
import sys
import glob
from fsl_ci import sprun
from fsl_ci import sprun, add_credentials
from fsl_ci.platform import get_platform_shortcut_if_not_applicable
from fsl_ci.conda import (load_meta_yaml,
get_project_repository_and_revision)
......@@ -55,6 +55,11 @@ def create_test_env(env_dir, build_dir):
internal = 'FSLCONDA_INTERNAL' in os.environ
intchannel = os.environ['FSLCONDA_INTERNAL_CHANNEL_URL']
pubchannel = os.environ['FSLCONDA_PUBLIC_CHANNEL_URL']
username = os.environ.get('FSLCONDA_INTERNAL_CHANNEL_USERNAME', None)
password = os.environ.get('FSLCONDA_INTERNAL_CHANNEL_PASSWORD', None)
if username is not None:
intchannel = add_credentials(intchannel, username, password)
if internal: pkgchannels = [intchannel, pubchannel]
else: pkgchannels = [pubchannel]
......
......@@ -7,10 +7,12 @@
#
import sys
import os.path as op
import functools as ft
import textwrap as tw
import argparse
import datetime
import multiprocessing.dummy as mp
from fsl_ci.recipe import get_recipe_variable
......@@ -32,6 +34,11 @@ SERVER_URL = 'https://git.fmrib.ox.ac.uk'
"""Default gitlab instance URL, if not specified on the command.line."""
def now():
"""Returns the current time as a string. """
return datetime.datetime.now().strftime('%H:%M:%S')
def get_revision(recipe_path, server, token):
"""Return the value of the FSLCONDA_REVISION variable on the given
conda recipe repository, or None if it is not set.
......@@ -85,7 +92,7 @@ def trigger_build(project, server, token, production):
pid = pipeline['id']
print(f'Pipeline triggered on {project} ({channel} build) '
print(f'{now()} Pipeline triggered on {project} ({channel} build) '
f'- see {pipeline["web_url"]}')
try:
......@@ -93,7 +100,7 @@ def trigger_build(project, server, token, production):
except Exception:
return None
print(f'Build pipeline for {project} has finished: {status}')
print(f'{now()} Build pipeline for {project} has finished: {status}')
if status != 'manual':
return None
......@@ -106,6 +113,11 @@ def trigger_deploy(project, pid, server, token, production):
and waits for it to complete.
"""
# trigger_build returns None
# if the build failed
if pid is None:
return False
# deployment to staging/production gets set at
# build time, so we don't need to pass the STAGING
# variable here, like we do in trigger_build
......@@ -116,7 +128,7 @@ def trigger_deploy(project, pid, server, token, production):
meta = load_meta_yaml(meta)
platforms = get_platform_ids(meta)
print(f'Triggering deploy-{platforms}-conda-package jobs '
print(f'{now()} Triggering deploy-{platforms}-conda-package jobs '
f'on {project} (deploying to {channel} channel)')
try:
......@@ -128,9 +140,11 @@ def trigger_deploy(project, pid, server, token, production):
trigger_job(project, j['id'], server, token)
status = wait_on_pipeline(project, pid, server, token)
print(f'Deploy job {project} has finished: {status}')
print(f'{now()} Deploy job {project} has finished: {status}')
return True
except Exception as e:
print(f'Error triggering deploy job on {project}: {e}')
return False
def parseArgs(argv=None):
......@@ -183,19 +197,22 @@ def main(argv=None):
server=args.server,
token=args.token,
production=args.production)
result = True
if args.sequential:
for project in projects:
pid = build(project)
deploy(project, pid)
if not deploy(project, pid):
result = False
else:
pool = mp.Pool(len(projects))
pids = pool.map(build, projects)
pool.starmap(deploy, zip(projects, pids))
pool = mp.Pool(len(projects))
pids = pool.map(build, projects)
result = all(pool.starmap(deploy, zip(projects, pids)))
pool.close()
pool.join()
return 0 if result else 1
if __name__ == '__main__':
main()
sys.exit(main())
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment