Commit 97240ebe authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

BF: Various fixes, rerender script seems to be working

parent 6560197d
......@@ -8,7 +8,6 @@
import os.path as op
import os
import io
import sys
import errno
import shlex
......@@ -19,7 +18,7 @@ import contextlib as ctxlib
import subprocess as sp
__version__ = '0.6.2'
__version__ = '0.6.3'
"""Current version of the fsl-ci-rules."""
......@@ -69,8 +68,13 @@ def sprun(cmd, **kwargs):
fprint(f'Running {cmd}')
if not kwargs.get('shell', False):
cmd = shlex.split(cmd)
if 'check' not in kwargs:
kwargs['check'] = True
# we set stdout/stderr for compatibility
# with CaptureStdout (see below)
if 'check' not in kwargs: kwargs['check'] = True
if 'stdout' not in kwargs: kwargs['stdout'] = sys.stdout
if 'stderr' not in kwargs: kwargs['stderr'] = sys.stderr
return, **kwargs)
......@@ -124,35 +128,53 @@ class CaptureStdout:
"""Context manager which captures stdout and stderr. """
def __init__(self):
self.tmpdir = tempfile.TemporaryDirectory()
self.__mock_stdout = open(op.join(, 'stdout'), 'wt')
self.__mock_stderr = open(op.join(, 'stderr'), 'wt')
def reset(self):
self.__mock_stdout = io.StringIO('')
self.__mock_stderr = io.StringIO('')
self.__mock_stdout.mode = 'w'
self.__mock_stderr.mode = 'w'
return self
def __enter__(self):
if self.tmpdir is None:
raise RuntimeError('CaptureStdout can only be used once')
self.__real_stdout = sys.stdout
self.__real_stderr = sys.stderr
sys.stdout = self.__mock_stdout
sys.stderr = self.__mock_stderr
sys.stdout = self.__mock_stdout
sys.stderr = self.__mock_stderr
return self
def __exit__(self, *args, **kwargs):
sys.stdout = self.__real_stdout
sys.stderr = self.__real_stderr
with open(, 'rt') as f:
self.__mock_stdout =
with open(, 'rt') as f:
self.__mock_stderr =
self.tmpdir = None
def real_stdout(self):
return self.__real_stdout
def real_stderr(self):
return self.__real_stderr
def stdout(self):
return self.__mock_stdout
def stderr(self):
return self.__mock_stderr
def health_check():
......@@ -149,10 +149,12 @@ def get_project_repository_and_revision(filename, ignore_env=False):
rev = source.get('git_rev', None)
build = build .get('number', None)
if repo is not None: repo = repo .strip()
if rev is not None: rev = rev .strip()
if repo is not None: repo = str(repo) .strip()
if rev is not None: rev = str(rev) .strip()
if build is not None: build = str(build).strip()
if repo == '': repo = None
if rev == '': rev = None
if build == '': build = None
return repo, rev, build
......@@ -32,6 +32,7 @@ def gen_repository_url(project_path, server, token=None):
return project_path
if token is not None:
token = token.strip()
prefix, suffix = server.split('://')
server = f'{prefix}://gitlab-ci-token:{token}@{suffix}'
......@@ -32,7 +32,7 @@ SERVER_URL = ''
"""Default gitlab instance URL, if not specified on the command.line."""
DEFAULT_EXCLUDES = ['docs', 'fsl-base']
"""List of repository names in fsl/conda/ that will always be skipped."""
......@@ -55,19 +55,32 @@ def filter_projects(projects, exclude):
return filtered
def get_project_url(recipe_path, server, token):
"""Returns the URL, revision, and build number of the project for
the specfied recipe.
def get_project_info(recipe_path, server, token):
"""Returns the project URL, path, revision, and build number of the
project for the specfied recipe.
metayaml = download_file(recipe_path, 'meta.yaml', server, token)
return get_project_repository_and_revision(metayaml)
url, version, build = get_project_repository_and_revision(metayaml)
path = None
if url is not None:
path = urlparse.urlparse(url).path
def rerender_recipe(project_url, project_ver, build_number, server, token):
if path.endswith('.git'):
path = path[:-4]
return url, path, version, build
def rerender_recipe(project_url,
"""Re-renders the recipe(s) associated with the specified project.
Essentially just calls update_conda_recipe.
project_path = urlparse.urlparse(project_url).path
project_name = project_path.rsplit('/')[-1]
if build_number is None: build_number = 0
......@@ -106,29 +119,44 @@ def rerender_all_recipes(recipes, server, token):
projects = set()
errored = {}
for recipe in recipes:
maxlen = max([len(r) for r in recipes])
for i, recipe in enumerate(recipes):
with CaptureStdout() as cap:
purl, pver, pbuild = get_project_url(recipe, server, token)
if purl in projects:
url, path, ver, build = get_project_info(recipe, server, token)
fmt = recipe + ' ' * (maxlen - len(recipe))
print(f'{i+1:4d} / {len(recipes)} {fmt} - {path}: ',
end='', file=cap.real_stdout)
if (url is None) or (path is None):
print('no url or path - skipping', file=cap.real_stdout)
if path in projects:
print('already visited - skipping', file=cap.real_stdout)
# Re-render is restricted to
# internally hosted projects
if '' not in purl:
if (url is None) or ('' not in url):
print('external URL - skipping', file=cap.real_stdout)
print(f'{recipe}: ', end='')
mrurl = rerender_recipe(purl, pver, pbuild, server, token)
mrurl = rerender_recipe(url, path, ver, build, server, token)
print(mrurl, file=cap.real_stdout)
except Exception as e:
errored[recipe] = cap
print(f'errored - {e}', file=cap.real_stdout)
errored[recipe] = cap.stdout, cap.stderr
for recipe, output in errored.items():
stdout = tw.indent(output.stdout, ' ')
stderr = tw.indent(output.stderr, ' ')
for recipe, (stdout, stderr) in errored.items():
print(f'{recipe} rerender errored!')
stdout = tw.indent(stdout, ' ')
stderr = tw.indent(stderr, ' ')
print('Standard output')
print('Standard error')
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment