Merge branch 'mnt/auth' into 'master'

Mnt/auth

See merge request fsl/fsl-ci-rules!94

fsl_ci/__init__.py

@@ -13,13 +13,14 @@ import               errno
 import                 shlex
 import                 time
 import                 tempfile
+import urllib.parse as urlparse
 import contextlib   as ctxlib
 import subprocess   as sp
 
 import yaml
 
 
-__version__ = '0.16.9'
+__version__ = '0.17.0'
 """Current version of the fsl-ci-rules."""
 
 
@@ -133,6 +134,14 @@ def dumpyaml(o):
     return yaml.dump(o, sort_keys=False)
 
 
+def add_credentials(url, username, password):
+    """Returns a URL with the username/password added."""
+    username     = urlparse.quote_plus(username)
+    password     = urlparse.quote_plus(password)
+    scheme, path = url.split('://')
+    return f'{scheme}://{username}:{password}@{path}'
+
+
 class CaptureStdout:
     """Context manager which captures stdout and stderr. """
 

fsl_ci/scripts/build_conda_package.py

@@ -13,7 +13,8 @@ import urllib.parse as urlparse
 
 from fsl_ci            import (sprun,
                                indir,
-                               fprint)
+                               fprint,
+                               add_credentials)
 from fsl_ci.versioning import  generate_staging_version
 from fsl_ci.platform   import  get_platform_shortcut_if_not_applicable
 from fsl_ci.conda      import  load_meta_yaml
@@ -119,6 +120,8 @@ def main():
     token          = os.environ['FSL_CI_API_TOKEN']
     staging        = os.environ['STAGING'].lower() == 'true'
     internal       = 'FSLCONDA_INTERNAL' in os.environ
+    username       = os.environ.get('FSLCONDA_INTERNAL_CHANNEL_USERNAME', None)
+    password       = os.environ.get('FSLCONDA_INTERNAL_CHANNEL_PASSWORD', None)
     skip_platforms = os.environ.get('FSLCONDA_SKIP_PLATFORM', '')
     skip_platforms = skip_platforms.split()
 
@@ -145,11 +148,12 @@ def main():
     # we output to _platform_, rather thah platform.
     output_dir = op.join(os.getcwd(), 'conda_build', f'_{platform}_')
 
-    if internal:
-        channel_urls = [os.environ['FSLCONDA_INTERNAL_CHANNEL_URL'],
-                        os.environ['FSLCONDA_PUBLIC_CHANNEL_URL']]
-    else:
-        channel_urls = [os.environ['FSLCONDA_PUBLIC_CHANNEL_URL']]
+    pubchan = os.environ['FSLCONDA_PUBLIC_CHANNEL_URL']
+    intchan = os.environ['FSLCONDA_INTERNAL_CHANNEL_URL']
+    if username is not None:
+        intchan = add_credentials(intchan, username, password)
+    if internal: channel_urls = [intchan, pubchan]
+    else:        channel_urls = [pubchan]
 
     os.makedirs(output_dir)
 

fsl_ci/scripts/run_unit_tests.py

@@ -12,7 +12,7 @@ import            os
 import            sys
 import            glob
 
-from fsl_ci          import  sprun
+from fsl_ci          import  sprun, add_credentials
 from fsl_ci.platform import  get_platform_shortcut_if_not_applicable
 from fsl_ci.conda    import (load_meta_yaml,
                              get_project_repository_and_revision)
@@ -55,6 +55,11 @@ def create_test_env(env_dir, build_dir):
     internal   = 'FSLCONDA_INTERNAL' in os.environ
     intchannel = os.environ['FSLCONDA_INTERNAL_CHANNEL_URL']
     pubchannel = os.environ['FSLCONDA_PUBLIC_CHANNEL_URL']
+    username   = os.environ.get('FSLCONDA_INTERNAL_CHANNEL_USERNAME', None)
+    password   = os.environ.get('FSLCONDA_INTERNAL_CHANNEL_PASSWORD', None)
+
+    if username is not None:
+        intchannel = add_credentials(intchannel, username, password)
 
     if internal: pkgchannels = [intchannel, pubchannel]
     else:        pkgchannels = [pubchannel]

fsl_ci/utils/trigger_build.py

@@ -7,10 +7,12 @@
 #
 
 
+import                          sys
 import os.path               as op
 import functools             as ft
 import textwrap              as tw
 import                          argparse
+import                          datetime
 import multiprocessing.dummy as mp
 
 from   fsl_ci.recipe   import  get_recipe_variable
@@ -32,6 +34,11 @@ SERVER_URL = 'https://git.fmrib.ox.ac.uk'
 """Default gitlab instance URL, if not specified on the command.line."""
 
 
+def now():
+    """Returns the current time as a string. """
+    return datetime.datetime.now().strftime('%H:%M:%S')
+
+
 def get_revision(recipe_path, server, token):
     """Return the value of the FSLCONDA_REVISION variable on the given
     conda recipe repository, or None if it is not set.
@@ -85,7 +92,7 @@ def trigger_build(project, server, token, production):
 
     pid = pipeline['id']
 
-    print(f'Pipeline triggered on {project} ({channel} build) '
+    print(f'{now()} Pipeline triggered on {project} ({channel} build) '
           f'- see {pipeline["web_url"]}')
 
     try:
@@ -93,7 +100,7 @@ def trigger_build(project, server, token, production):
     except Exception:
         return None
 
-    print(f'Build pipeline for {project} has finished: {status}')
+    print(f'{now()} Build pipeline for {project} has finished: {status}')
 
     if status != 'manual':
         return None
@@ -106,6 +113,11 @@ def trigger_deploy(project, pid, server, token, production):
     and waits for it to complete.
     """
 
+    # trigger_build returns None
+    # if the build failed
+    if pid is None:
+        return False
+
     # deployment to staging/production gets set at
     # build time, so we don't need to pass the STAGING
     # variable here, like we do in trigger_build
@@ -116,7 +128,7 @@ def trigger_deploy(project, pid, server, token, production):
     meta      = load_meta_yaml(meta)
     platforms = get_platform_ids(meta)
 
-    print(f'Triggering deploy-{platforms}-conda-package jobs '
+    print(f'{now()} Triggering deploy-{platforms}-conda-package jobs '
           f'on {project} (deploying to {channel} channel)')
 
     try:
@@ -128,9 +140,11 @@ def trigger_deploy(project, pid, server, token, production):
             trigger_job(project, j['id'], server, token)
 
         status = wait_on_pipeline(project, pid, server, token)
-        print(f'Deploy job {project} has finished: {status}')
+        print(f'{now()} Deploy job {project} has finished: {status}')
+        return True
     except Exception as e:
         print(f'Error triggering deploy job on {project}: {e}')
+        return False
 
 
 def parseArgs(argv=None):
@@ -183,19 +197,22 @@ def main(argv=None):
                           server=args.server,
                           token=args.token,
                           production=args.production)
+    result   = True
 
     if args.sequential:
         for project in projects:
             pid = build(project)
-            deploy(project, pid)
+            if not deploy(project, pid):
+                result = False
     else:
         pool   = mp.Pool(len(projects))
         pids   = pool.map(build, projects)
-        pool.starmap(deploy, zip(projects, pids))
+        result = all(pool.starmap(deploy, zip(projects, pids)))
         pool.close()
         pool.join()
 
+    return 0 if result else 1
 
 
 if __name__  == '__main__':
-    main()
+    sys.exit(main())