Commit 60c94e64 authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

RF: Simplify clear_channel script

parent 7a16dc15
......@@ -4,42 +4,49 @@
# rules/fsl-ci-management-rules.yml:clear-public-channel and
# clear-private-channel jobs.
#
# Exactly one of the CLEAR_ALL, CLEAR_OLD, or PATTERN variables must be
# set - if not, or if more than one of these variables are set, the job
# exits without doing anything.
# The following variables control the behaviour of this script:
# - DRY_RUN: If "true", don't delete anything, just print out what
# would be deleted
# - CLEAR: Either "all" or "old", indicating whether all packages,
# or all but the most recent, should be deleted.
# - PATTERN: Delete packages matching one or more glob-style patterns.
#
# If the CLEAR_ALL variable is set, *all* packages are removed.
# The CLEAR variable must be set - if not, the job exits without doing
# anything.
#
# If the CLEAR_OLD variable is set, all except the most recent version
# of each package is removed.
# If CLEAR is set to "all", *all* packages (or all packages matching PATTERN,
# if it is set) are removed.
#
# The PATTERN variable may be a semi-colon- separated list of glob-style
# patterns which indicate the packages to be deleted. For example, to
# remove all packages starting with "abc-", and all packages with
# "-misc" in their name:
# If CLEAR is set to "old", all except the most recent version
# of each package (or all packages matching PATTERN, if it is set).
#
# PATTERN="abc-*;*-misc*"
# The PATTERN variable can be used to limit which packages are removed. It
# may be a semi-colon- separated list of glob-style patterns which indicate
# the packages to be deleted. For example, to remove all packages starting
# with "abc-", and all packages with "-misc" in their name:
#
# Each pattern is matched against the conda package file names in each
# of the channel platform directories (i.e. noarch, linux-64, osx-64,
# etc), and all matching package files are removed.
# PATTERN="abc-*;*-misc*"
#
# The DRY_RUN variable has a default value of "true". It must be
# manually set to "false" in order for packages to be removed. When
# DRY_RUN is "true", no package files are removed - instead, the package
# files that would be removed are printed to standard output.
# Each pattern is matched against the conda package file names in each of the
# channel platform directories (i.e. noarch, linux-64, osx-64, etc), and all
# matching package files (according to the CLEAR variable) are removed.
#
# Author: Paul McCarthy <pauldmccarthy@gmail.com>
# The DRY_RUN variable has a default value of "true". It must be explicitly
# set to "false" in order for packages to be removed. When DRY_RUN is not
# "false", no package files are removed - instead, the package files that
# would be removed are printed to standard output.
#
import os
import os.path as op
import itertools as it
import sys
import glob
import pathlib
import fnmatch
import packaging.version as pkgver
from packaging.version import parse
from fsl_ci import sprun, lockdir
from fsl_ci.conda import (read_channel_repodata,
......@@ -49,26 +56,28 @@ from fsl_ci.conda import (read_channel_repodata,
PLATFORMS = ('noarch', 'linux-64', 'osx-64', 'win-32', 'win64')
def remove(path):
dry_run = os.environ.get('DRY_RUN', 'true') == 'true'
if dry_run: print(path)
else: os.remove(path)
def remove(path, dry_run):
print(f'Removing: {path}')
if not dry_run:
os.remove(path)
def clear_all_packages(channeldir):
for plat in PLATFORMS:
packages = glob.glob(op.join(channeldir, plat, '*.tar.bz2'))
for package in packages:
remove(package)
def clear_packages(channeldir, clear, patterns, dry_run):
if patterns is not None:
patterns = patterns.split(';')
def clear_old_packages(channeldir):
channelurl = pathlib.Path(channeldir).as_uri()
chandata, platdata = read_channel_repodata(channelurl)
packages = load_packages(chandata, platdata)
def sortkey(pkg):
return (parse(pkg[1]['version']), int(pkg[1]['build']))
return (pkgver.parse(pkg[1]['version']), int(pkg[1]['build']))
chandata, platdata = read_channel_repodata(channeldir)
packages = load_packages(chandata, platdata)
def match(filename):
if patterns is None:
return True
return any(fnmatch.fnmatch(filename, p) for p in patterns)
for pkg in packages.values():
for plat in pkg.platforms:
......@@ -79,51 +88,41 @@ def clear_old_packages(channeldir):
# sort so oldest is first
pkgfiles = sorted(pkgfiles, sortkey)
pkgfiles = [pf[0] for pf in pkgfiles]
pkgfiles = [op.join(channeldir, plat, pf) for pf in pkgfiles]
for filename, _ in pkgfiles[:-1]:
remove(filename)
if clear == 'old':
pkgfiles = pkgfiles[:-1]
def clear_packages_matching_pattern(channeldir, patterns):
patterns = patterns.split(';')
for plat, pattern in it.product(PLATFORMS, patterns):
pattern = op.join(channeldir, plat, pattern)
packages = glob.glob(pattern)
for package in packages:
remove(package)
for filename in pkgfiles:
if match(op.basename(filename)):
remove(filename, dry_run)
def main():
pattern = os.environ.get('PATTERN', None)
clear_all = os.environ.get('CLEAR_ALL', None)
clear_old = os.environ.get('CLEAR_OLD', None)
pattern = os.environ.get('PATTERN', None)
clear = os.environ.get('CLEAR', None)
dry_run = os.environ.get('DRY_RUN', 'true') != 'false'
jobname = os.environ['CI_JOB_NAME']
if jobname == 'clear-public-channel':
if jobname == 'clear-public-channel':
channeldir = os.environ['FSLCONDA_INTERNAL_CHANNEL_DIRECTORY']
else:
elif jobname == 'clear-internal-channel':
channeldir = os.environ['FSLCONDA_PUBLIC_CHANNEL_DIRECTORY']
nset = sum((pattern is not None,
clear_all is not None,
clear_old is not None))
if nset != 1:
print('Exactly one of PATTERN, CLEAR_OLD, or CLEAR_ALL must be set')
if clear is None or clear not in ('all', 'old'):
print('CLEAR must be set to "all" or "old"')
return 1
print(f'Clearing {clear} packages')
print(f' pattern: {pattern}')
print(f' dry run: {dry_run}')
with lockdir(channeldir):
if clear_all:
clear_all_packages(channeldir)
if clear_old:
clear_old_packages(channeldir)
if pattern:
clear_packages_matching_pattern(channeldir, pattern)
sprun(f'conda index {channeldir}')
clear_packages(channeldir, clear, pattern, dry_run)
if not dry_run:
sprun(f'conda index {channeldir}')
return 0
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment