Commit 93b116bb authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

RF,ENH: deploy_conda_package shell script replaced with python script;

implemented rudimentary concurrent access control for re-indexing channel
directory.
parent 231555df
...@@ -6,9 +6,12 @@ ...@@ -6,9 +6,12 @@
# #
import os.path as op
import os import os
import errno
import shlex import shlex
import string import string
import time
import tempfile import tempfile
import contextlib as ctxlib import contextlib as ctxlib
import subprocess as sp import subprocess as sp
...@@ -74,3 +77,35 @@ def is_valid_project_version(version): ...@@ -74,3 +77,35 @@ def is_valid_project_version(version):
if not all([c in string.digits for c in part]): if not all([c in string.digits for c in part]):
return False return False
return True return True
@ctxlib.contextmanager
def lockdir(dirname):
"""Primitive mechanism by which concurrent access to a directory can be
prevented. Attempts to create a semaphore file in the directory, but waits
if that file already exists. Removes the file when finished.
"""
delay = 10
lockfile = op.join(dirname, '.fsl_ci.lockdir')
while True:
try:
fprint(f'Attempting to lock {dirname} for exclusive access.')
fd = os.open(os.O_CREAT | os.O_EXCL | os.O_RDWR)
break
except OSError as e:
if e.errno != errno.EEXIST:
raise e
fprint(f'{dirname} is already locked - '
f'trying again in {delay} seconds ...')
time.sleep(10)
fprint(f'Exclusive access acquired for {dirname} ...')
try:
yield
finally:
fprint(f'Relinquishing lock on {dirname}')
os.close( fd)
os.unlink(lockfile)
#!/usr/bin/env python
#
# Deploy a built conda package to a FSL conda channel.
#
# Deploying a built conda package is simply a matter of copying the
# built packages into the channel directory and running "conda index".
#
# Author: Paul McCarthy <pauldmccarthy@gmail.com>
#
import glob
import itertools as it
import os.path as op
import os
import shutil
import sys
from fsl_ci import fprint, sprun, lockdir
def reindex_channel(channeldir):
"""Run "conda index" on the conda channel directory.
"""
with lockdir(channeldir):
# The conda index command doesn't seem to reliably
# refresh information about newly added packages,
# so we delete the old index metadata before running
# it.
sprun(f'find {channeldir} -name "*.html" -delete')
sprun(f'find {channeldir} -name "*.json" -delete')
sprun(f'find {channeldir} -name "*.json.bz2" -delete')
sprun(f'conda index {channeldir}')
def copy_packages(builddir, channeldir):
"""copy all package files from the conda-build output directory into
the conda channel directory. Returns the number of package files
found and copied.
"""
count = 0
# The build_conda_package.py script stores its
# outputs in a platform-specific subdirectory -
# one of _linux_, _macos_, or _noarch_. Within
# each one of those is a conventional conda
# channel structure.
fslplats = ('_linux_', '_macos_', '_noarch_')
condaplats = ('noarch', 'linux-64', 'osx-64', 'win-32', 'win64')
for fslplat, condaplat in it.product(fslplats, condaplats):
# copy any package files that we find in
# the build dir into the channel directory.
pkgs = glob.glob(op.join(builddir, fslplat, condaplat, '*.tar.bz2'))
destdir = op.join(channeldir, condaplat)
count += len(pkgs)
for pkg in pkgs:
fprint(f'Installing {pkg}...')
os.makedirs(destdir, exist_ok=True)
shutil.copy(pkg, destdir)
return count
def main():
recipename = os.environ['CI_PROJECT_NAME']
recipeurl = os.environ['CI_PROJECT_URL']
projrepo = os.environ.get('FSLCONDA_=REPOSITORY', '')
projrev = os.environ.get('FSLCONDA_REVISION', '')
builddir = op.join(os.getcwd(), 'conda_build')
if os.environ.get('STAGING', 'true'):
channeldir = os.environ['FSLCONDA_STAGING_CHANNEL_DIRECTORY']
else:
channeldir = os.environ['FSLCONDA_PRODUCTION_CHANNEL_DIRECTORY']
fprint( '************************************')
fprint(f'Deploying built conda package for: {recipename}')
fprint(f'Recipe URL: {recipeurl}')
fprint( 'Project repository (empty means build ')
fprint(f' is from repo specified in meta.yaml): {projrepo}')
fprint( 'Project revision (empty means build is ')
fprint(f' from ref specified in meta.yaml): {projrev}')
fprint(f'FSL conda channel: {channeldir}')
fprint( '************************************')
npkgs = copy_packages(builddir, channeldir)
if npkgs > 0:
reindex_channel(channeldir)
else:
print('No built conda packages found - aborting.')
return 1
return 0
if __name__ == '__main__':
sys.exit(main())
#!/usr/bin/env bash
#
# Deploy a built conda package to a FSL conda channel. See
# fsl-ci-conda-rules.yml.
#
# Deploying a built conda package is simply a matter of copying the
# built packages into the channel directory and running "conda index".
pkgdir=./conda_build
if [ "$STAGING" == "true" ]; then
channel_dir=$FSLCONDA_STAGING_CHANNEL_DIRECTORY
else
channel_dir=$FSLCONDA_PRODUCTION_CHANNEL_DIRECTORY
fi
echo "************************************"
echo "Deploying built conda package for: $CI_PROJECT_NAME"
echo "Recipe URL: $CI_PROJECT_URL"
echo "Project repository (empty means build "
echo " is from repo specified in meta.yaml): $FSLCONDA_REPOSITORY"
echo "Project revision (empty means build is "
echo " from ref specified in meta.yaml): $FSLCONDA_REVISION"
echo "FSL conda channel: $channel_dir"
echo "************************************"
if ! find $pkgdir -name "*.tar.bz2" | grep .; then
echo "No built conda packages found - aborting"
exit 1
fi
# copy all package files from the conda-build
# output directory into the conda channel directory
# the build_conda_package.py script stores its
# outputs in a platform-specific subdirectory -
# one of _linux_, _macos_, or _noarch_
for fslplat in _linux_ _macos_ _noarch_ ; do
# within each one of those is a conventional
# conda channel structure - copy any package
# files that we find into the output channel
# directory.
for condaplat in noarch linux-64 osx-64 win-32 win64; do
if ls $pkgdir/$fslplat/$condaplat/*.tar.bz2 &> /dev/null; then
ls -l $pkgdir/$fslplat/$condaplat/*.tar.bz2
mkdir -p $channel_dir/$condaplat
cp $pkgdir/$fslplat/$condaplat/*.tar.bz2 $channel_dir/$condaplat
fi
done
done
# The conda index command doesn't seem to
# reliably refresh information about newly
# added packages, so we delete the old index
# metadata before running it.
find . -name "*.html" -delete
find . -name "*.json" -delete
find . -name "*.json.bz2" -delete
# re-generate the channel index
conda index $channel_dir
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment