RF,ENH: deploy_conda_package shell script replaced with python script;

implemented rudimentary concurrent access control for re-indexing channel
directory.

fsl_ci/__init__.py

@@ -6,9 +6,12 @@
 #
 
 
+import os.path    as op
 import               os
+import               errno
 import               shlex
 import               string
+import               time
 import               tempfile
 import contextlib as ctxlib
 import subprocess as sp
@@ -74,3 +77,35 @@ def is_valid_project_version(version):
         if not all([c in string.digits for c in part]):
             return False
     return True
+
+
+@ctxlib.contextmanager
+def lockdir(dirname):
+    """Primitive mechanism by which concurrent access to a directory can be
+    prevented. Attempts to create a semaphore file in the directory, but waits
+    if that file already exists. Removes the file when finished.
+    """
+
+    delay    = 10
+    lockfile = op.join(dirname, '.fsl_ci.lockdir')
+
+    while True:
+        try:
+            fprint(f'Attempting to lock {dirname} for exclusive access.')
+            fd = os.open(os.O_CREAT | os.O_EXCL | os.O_RDWR)
+            break
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise e
+            fprint(f'{dirname} is already locked - '
+                   f'trying again in {delay} seconds ...')
+        time.sleep(10)
+
+    fprint(f'Exclusive access acquired for {dirname} ...')
+    try:
+        yield
+
+    finally:
+        fprint(f'Relinquishing lock on {dirname}')
+        os.close( fd)
+        os.unlink(lockfile)

fsl_ci/scripts/deploy_conda_package.py

+#!/usr/bin/env python
+#
+# Deploy a built conda package to a FSL conda channel.
+#
+# Deploying a built conda package is simply a matter of copying the
+# built packages into the channel directory and running "conda index".
+#
+# Author: Paul McCarthy <pauldmccarthy@gmail.com>
+#
+
+
+import               glob
+import itertools  as it
+import os.path    as op
+import               os
+import               shutil
+import               sys
+
+from fsl_ci import fprint, sprun, lockdir
+
+
+def reindex_channel(channeldir):
+    """Run "conda index" on the conda channel directory.
+    """
+    with lockdir(channeldir):
+        # The conda index command doesn't seem to reliably
+        # refresh information about newly added packages,
+        # so we delete the old index metadata before running
+        # it.
+        sprun(f'find {channeldir} -name "*.html"     -delete')
+        sprun(f'find {channeldir} -name "*.json"     -delete')
+        sprun(f'find {channeldir} -name "*.json.bz2" -delete')
+        sprun(f'conda index {channeldir}')
+
+
+def copy_packages(builddir, channeldir):
+    """copy all package files from the conda-build output directory into
+    the conda channel directory. Returns the number of package files
+    found and copied.
+    """
+
+    count = 0
+
+    # The build_conda_package.py script stores its
+    # outputs in a platform-specific subdirectory -
+    # one of _linux_, _macos_, or _noarch_. Within
+    # each one of those is a conventional conda
+    # channel structure.
+    fslplats   = ('_linux_', '_macos_', '_noarch_')
+    condaplats = ('noarch', 'linux-64', 'osx-64', 'win-32', 'win64')
+
+    for fslplat, condaplat in it.product(fslplats, condaplats):
+        # copy any package files that we find in
+        # the build dir into the channel directory.
+        pkgs    = glob.glob(op.join(builddir, fslplat, condaplat, '*.tar.bz2'))
+        destdir = op.join(channeldir, condaplat)
+        count  += len(pkgs)
+
+        for pkg in pkgs:
+            fprint(f'Installing {pkg}...')
+            os.makedirs(destdir, exist_ok=True)
+            shutil.copy(pkg, destdir)
+
+    return count
+
+
+def main():
+
+    recipename = os.environ['CI_PROJECT_NAME']
+    recipeurl  = os.environ['CI_PROJECT_URL']
+    projrepo   = os.environ.get('FSLCONDA_=REPOSITORY', '')
+    projrev    = os.environ.get('FSLCONDA_REVISION',    '')
+    builddir   = op.join(os.getcwd(), 'conda_build')
+
+    if os.environ.get('STAGING', 'true'):
+        channeldir = os.environ['FSLCONDA_STAGING_CHANNEL_DIRECTORY']
+    else:
+        channeldir = os.environ['FSLCONDA_PRODUCTION_CHANNEL_DIRECTORY']
+
+    fprint( '************************************')
+    fprint(f'Deploying built conda package for:      {recipename}')
+    fprint(f'Recipe URL:                             {recipeurl}')
+    fprint( 'Project repository (empty means build ')
+    fprint(f'  is from repo specified in meta.yaml): {projrepo}')
+    fprint( 'Project revision (empty means build is ')
+    fprint(f'  from ref specified in meta.yaml):     {projrev}')
+    fprint(f'FSL conda channel:                      {channeldir}')
+    fprint( '************************************')
+
+    npkgs = copy_packages(builddir, channeldir)
+
+    if npkgs > 0:
+        reindex_channel(channeldir)
+    else:
+        print('No built conda packages found - aborting.')
+        return 1
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

scripts/deploy_conda_package.sh

-#!/usr/bin/env bash
-#
-# Deploy a built conda package to a FSL conda channel. See
-# fsl-ci-conda-rules.yml.
-#
-# Deploying a built conda package is simply a matter of copying the
-# built packages into the channel directory and running "conda index".
-
-pkgdir=./conda_build
-
-if [ "$STAGING" == "true" ]; then
-  channel_dir=$FSLCONDA_STAGING_CHANNEL_DIRECTORY
-else
-  channel_dir=$FSLCONDA_PRODUCTION_CHANNEL_DIRECTORY
-fi
-
-echo "************************************"
-echo "Deploying built conda package for:      $CI_PROJECT_NAME"
-echo "Recipe URL:                             $CI_PROJECT_URL"
-echo "Project repository (empty means build "
-echo "  is from repo specified in meta.yaml): $FSLCONDA_REPOSITORY"
-echo "Project revision (empty means build is "
-echo "  from ref specified in meta.yaml):     $FSLCONDA_REVISION"
-echo "FSL conda channel:                      $channel_dir"
-echo "************************************"
-
-if ! find $pkgdir -name "*.tar.bz2" | grep .; then
-  echo "No built conda packages found - aborting"
-  exit 1
-fi
-
-# copy all package files from the conda-build
-# output directory into the conda channel directory
-
-# the build_conda_package.py script stores its
-# outputs in a platform-specific subdirectory -
-# one of _linux_, _macos_, or _noarch_
-for fslplat in _linux_ _macos_ _noarch_ ; do
-
-  # within each one of those is a conventional
-  # conda channel structure - copy any package
-  # files that we find into the output channel
-  # directory.
-  for condaplat in noarch linux-64 osx-64 win-32 win64; do
-    if ls $pkgdir/$fslplat/$condaplat/*.tar.bz2 &> /dev/null; then
-
-      ls -l $pkgdir/$fslplat/$condaplat/*.tar.bz2
-
-      mkdir -p $channel_dir/$condaplat
-      cp $pkgdir/$fslplat/$condaplat/*.tar.bz2 $channel_dir/$condaplat
-
-    fi
-  done
-done
-
-# The conda index command doesn't seem to
-# reliably refresh information about newly
-# added packages, so we delete the old index
-# metadata before running it.
-find . -name "*.html"     -delete
-find . -name "*.json"     -delete
-find . -name "*.json.bz2" -delete
-
-# re-generate the channel index
-conda index $channel_dir