Commit 40cb4d8c authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

Merge branch 'mnt/clear-channels' into 'master'

MNT: New clear_channel script for poking at the channel dirs - needed because

See merge request fsl/fsl-ci-rules!95
parents a685dab1 f502d051
......@@ -20,7 +20,7 @@ import subprocess as sp
import yaml
__version__ = '0.17.0'
__version__ = '0.18.0'
"""Current version of the fsl-ci-rules."""
......
#!/usr/bin/env python
#
# Remove packages from a conda channel. See the
# rules/fsl-ci-management-rules.yml:clear-public-channel and
# clear-private-channel jobs.
#
# Exactly one of the CLEAR_ALL, CLEAR_OLD, or PATTERN variables must be
# set - if not, or if more than one of these variables are set, the job
# exits without doing anything.
#
# If the CLEAR_ALL variable is set, *all* packages are removed.
#
# If the CLEAR_OLD variable is set, all except the most recent version
# of each package is removed.
#
# The PATTERN variable may be a semi-colon- separated list of glob-style
# patterns which indicate the packages to be deleted. For example, to
# remove all packages starting with "abc-", and all packages with
# "-misc" in their name:
#
# PATTERN="abc-*;*-misc*"
#
# Each pattern is matched against the conda package file names in each
# of the channel platform directories (i.e. noarch, linux-64, osx-64,
# etc), and all matching package files are removed.
#
# The DRY_RUN variable has a default value of "true". It must be
# manually set to "false" in order for packages to be removed. When
# DRY_RUN is "true", no package files are removed - instead, the package
# files that would be removed are printed to standard output.
#
# Author: Paul McCarthy <pauldmccarthy@gmail.com>
#
import os
import os.path as op
import itertools as it
import sys
import glob
from packaging.version import parse
from fsl_ci import sprun, lockdir
from fsl_ci.conda import (read_channel_repodata,
load_packages)
PLATFORMS = ('noarch', 'linux-64', 'osx-64', 'win-32', 'win64')
def remove(path):
dry_run = os.environ.get('DRY_RUN', 'true') == 'true'
if dry_run: print(path)
else: os.remove(path)
def clear_all_packages(channeldir):
for plat in PLATFORMS:
packages = glob.glob(op.join(channeldir, plat, '*.tar.bz2'))
for package in packages:
remove(package)
def clear_old_packages(channeldir):
def sortkey(pkg):
return (parse(pkg[1]['version']), int(pkg[1]['build']))
chandata, platdata = read_channel_repodata(channeldir)
packages = load_packages(chandata, platdata)
for pkg in packages.values():
for plat in pkg.platforms:
# get file name + package entry
pkgfiles = [(k, v) for k, v in platdata[plat]['packages'].items()
if v['name'] == pkg.name]
# sort so oldest is first
pkgfiles = sorted(pkgfiles, sortkey)
for filename, _ in pkgfiles[:-1]:
remove(filename)
def clear_packages_matching_pattern(channeldir, patterns):
patterns = patterns.split(';')
for plat, pattern in it.product(PLATFORMS, patterns):
pattern = op.join(channeldir, plat, pattern)
packages = glob.glob(pattern)
for package in packages:
remove(package)
def main():
pattern = os.environ.get('PATTERN', None)
clear_all = os.environ.get('CLEAR_ALL', None)
clear_old = os.environ.get('CLEAR_OLD', None)
jobname = os.environ['CI_JOB_NAME']
if jobname == 'clear-public-channel':
channeldir = os.environ['FSLCONDA_INTERNAL_CHANNEL_DIRECTORY']
else:
channeldir = os.environ['FSLCONDA_PUBLIC_CHANNEL_DIRECTORY']
nset = sum((pattern is not None,
clear_all is not None,
clear_old is not None))
if nset != 1:
print('Exactly one of PATTERN, CLEAR_OLD, or CLEAR_ALL must be set')
return 1
with lockdir(channeldir):
if clear_all:
clear_all_packages(channeldir)
if clear_old:
clear_old_packages(channeldir)
if pattern:
clear_packages_matching_pattern(channeldir, pattern)
sprun(f'conda index {channeldir}')
return 0
if __name__ == '__main__':
sys.exit(main())
......@@ -8,6 +8,7 @@ def test_import():
import fsl_ci.recipe
import fsl_ci.platform
import fsl_ci.scripts.build_conda_package
import fsl_ci.scripts.clear_channel
import fsl_ci.scripts.deploy_conda_package
import fsl_ci.scripts.purge_channel_indexes
import fsl_ci.scripts.run_unit_tests
......
......@@ -17,7 +17,6 @@
# the gitlab UI.
.deploy-conda-package:
stage: fsl-ci-deploy
image: $FSL_CI_IMAGE_LINUX_64
extends: .fsl-ci-rules-job
tags:
- fslconda-channel-host
......
......@@ -25,6 +25,7 @@ test-ci-rules:
- pip install pytest
- pytest -v fsl_ci/tests
# This job is only executed on the
# fsl/fsl-ci-rules repository, on
# non-master branches (i.e. merge request
......@@ -130,7 +131,6 @@ build-docker-image-linux-64-cuda-11.3:
# as they can sometimes be corrupted.
purge-channel-indexes:
stage: fsl-ci-pre
image: $FSL_CI_IMAGE_LINUX_64
extends: .fsl-ci-rules-job
tags:
- fslconda-channel-host
......@@ -142,3 +142,38 @@ purge-channel-indexes:
when: manual
script:
- purge_channel_indexes
# USE WITH CAUTION
#
# This job can be manually invoked to clear
# the contents of the public conda channel.
# See the clear_channel script for details.
.clear-channel:
stage: fsl-ci-pre
image: $FSL_CI_IMAGE_LINUX_64
extends: .fsl-ci-rules-job
tags:
- fslconda-channel-host
variables:
DRY_RUN: "true"
rules:
- if: '$CI_PROJECT_PATH == "fsl/fsl-ci-rules" &&
$CI_COMMIT_TAG == null &&
$CI_COMMIT_REF_NAME == "master" &&
$CI_PIPELINE_SOURCE != "merge_request_event"'
when: manual
script:
- clear_channel
# The clear_channel script looks at the
# CI job name to determine which channel
# to clear.
clear-public-channel:
extends: .clear-channel
clear-private-channel:
extends: .clear-channel
\ No newline at end of file
......@@ -36,6 +36,7 @@ setup(
include_package_data=True,
entry_points={'console_scripts' : [
'build_conda_package = fsl_ci.scripts.build_conda_package:main',
'clear_channel = fsl_ci.scripts.clear_channel:main',
'configure_repositories = fsl_ci.utils.configure_repsitories:main',
'create_conda_recipe = fsl_ci.utils.create_conda_recipe:main',
'deploy_conda_package = fsl_ci.scripts.deploy_conda_package:main',
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment