Commit 93b116bb authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

RF,ENH: deploy_conda_package shell script replaced with python script;

implemented rudimentary concurrent access control for re-indexing channel
directory.
parent 231555df
......@@ -6,9 +6,12 @@
#
import os.path as op
import os
import errno
import shlex
import string
import time
import tempfile
import contextlib as ctxlib
import subprocess as sp
......@@ -74,3 +77,35 @@ def is_valid_project_version(version):
if not all([c in string.digits for c in part]):
return False
return True
@ctxlib.contextmanager
def lockdir(dirname):
"""Primitive mechanism by which concurrent access to a directory can be
prevented. Attempts to create a semaphore file in the directory, but waits
if that file already exists. Removes the file when finished.
"""
delay = 10
lockfile = op.join(dirname, '.fsl_ci.lockdir')
while True:
try:
fprint(f'Attempting to lock {dirname} for exclusive access.')
fd = os.open(os.O_CREAT | os.O_EXCL | os.O_RDWR)
break
except OSError as e:
if e.errno != errno.EEXIST:
raise e
fprint(f'{dirname} is already locked - '
f'trying again in {delay} seconds ...')
time.sleep(10)
fprint(f'Exclusive access acquired for {dirname} ...')
try:
yield
finally:
fprint(f'Relinquishing lock on {dirname}')
os.close( fd)
os.unlink(lockfile)
#!/usr/bin/env python
#
# Deploy a built conda package to a FSL conda channel.
#
# Deploying a built conda package is simply a matter of copying the
# built packages into the channel directory and running "conda index".
#
# Author: Paul McCarthy <pauldmccarthy@gmail.com>
#
import glob
import itertools as it
import os.path as op
import os
import shutil
import sys
from fsl_ci import fprint, sprun, lockdir
def reindex_channel(channeldir):
"""Run "conda index" on the conda channel directory.
"""
with lockdir(channeldir):
# The conda index command doesn't seem to reliably
# refresh information about newly added packages,
# so we delete the old index metadata before running
# it.
sprun(f'find {channeldir} -name "*.html" -delete')
sprun(f'find {channeldir} -name "*.json" -delete')
sprun(f'find {channeldir} -name "*.json.bz2" -delete')
sprun(f'conda index {channeldir}')
def copy_packages(builddir, channeldir):
"""copy all package files from the conda-build output directory into
the conda channel directory. Returns the number of package files
found and copied.
"""
count = 0
# The build_conda_package.py script stores its
# outputs in a platform-specific subdirectory -
# one of _linux_, _macos_, or _noarch_. Within
# each one of those is a conventional conda
# channel structure.
fslplats = ('_linux_', '_macos_', '_noarch_')
condaplats = ('noarch', 'linux-64', 'osx-64', 'win-32', 'win64')
for fslplat, condaplat in it.product(fslplats, condaplats):
# copy any package files that we find in
# the build dir into the channel directory.
pkgs = glob.glob(op.join(builddir, fslplat, condaplat, '*.tar.bz2'))
destdir = op.join(channeldir, condaplat)
count += len(pkgs)
for pkg in pkgs:
fprint(f'Installing {pkg}...')
os.makedirs(destdir, exist_ok=True)
shutil.copy(pkg, destdir)
return count
def main():
recipename = os.environ['CI_PROJECT_NAME']
recipeurl = os.environ['CI_PROJECT_URL']
projrepo = os.environ.get('FSLCONDA_=REPOSITORY', '')
projrev = os.environ.get('FSLCONDA_REVISION', '')
builddir = op.join(os.getcwd(), 'conda_build')
if os.environ.get('STAGING', 'true'):
channeldir = os.environ['FSLCONDA_STAGING_CHANNEL_DIRECTORY']
else:
channeldir = os.environ['FSLCONDA_PRODUCTION_CHANNEL_DIRECTORY']
fprint( '************************************')
fprint(f'Deploying built conda package for: {recipename}')
fprint(f'Recipe URL: {recipeurl}')
fprint( 'Project repository (empty means build ')
fprint(f' is from repo specified in meta.yaml): {projrepo}')
fprint( 'Project revision (empty means build is ')
fprint(f' from ref specified in meta.yaml): {projrev}')
fprint(f'FSL conda channel: {channeldir}')
fprint( '************************************')
npkgs = copy_packages(builddir, channeldir)
if npkgs > 0:
reindex_channel(channeldir)
else:
print('No built conda packages found - aborting.')
return 1
return 0
if __name__ == '__main__':
sys.exit(main())
#!/usr/bin/env bash
#
# Deploy a built conda package to a FSL conda channel. See
# fsl-ci-conda-rules.yml.
#
# Deploying a built conda package is simply a matter of copying the
# built packages into the channel directory and running "conda index".
pkgdir=./conda_build
if [ "$STAGING" == "true" ]; then
channel_dir=$FSLCONDA_STAGING_CHANNEL_DIRECTORY
else
channel_dir=$FSLCONDA_PRODUCTION_CHANNEL_DIRECTORY
fi
echo "************************************"
echo "Deploying built conda package for: $CI_PROJECT_NAME"
echo "Recipe URL: $CI_PROJECT_URL"
echo "Project repository (empty means build "
echo " is from repo specified in meta.yaml): $FSLCONDA_REPOSITORY"
echo "Project revision (empty means build is "
echo " from ref specified in meta.yaml): $FSLCONDA_REVISION"
echo "FSL conda channel: $channel_dir"
echo "************************************"
if ! find $pkgdir -name "*.tar.bz2" | grep .; then
echo "No built conda packages found - aborting"
exit 1
fi
# copy all package files from the conda-build
# output directory into the conda channel directory
# the build_conda_package.py script stores its
# outputs in a platform-specific subdirectory -
# one of _linux_, _macos_, or _noarch_
for fslplat in _linux_ _macos_ _noarch_ ; do
# within each one of those is a conventional
# conda channel structure - copy any package
# files that we find into the output channel
# directory.
for condaplat in noarch linux-64 osx-64 win-32 win64; do
if ls $pkgdir/$fslplat/$condaplat/*.tar.bz2 &> /dev/null; then
ls -l $pkgdir/$fslplat/$condaplat/*.tar.bz2
mkdir -p $channel_dir/$condaplat
cp $pkgdir/$fslplat/$condaplat/*.tar.bz2 $channel_dir/$condaplat
fi
done
done
# The conda index command doesn't seem to
# reliably refresh information about newly
# added packages, so we delete the old index
# metadata before running it.
find . -name "*.html" -delete
find . -name "*.json" -delete
find . -name "*.json.bz2" -delete
# re-generate the channel index
conda index $channel_dir
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment