Commit 97240ebe authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

BF: Various fixes, rerender script seems to be working

parent 6560197d
...@@ -8,7 +8,6 @@ ...@@ -8,7 +8,6 @@
import os.path as op import os.path as op
import os import os
import io
import sys import sys
import errno import errno
import shlex import shlex
...@@ -19,7 +18,7 @@ import contextlib as ctxlib ...@@ -19,7 +18,7 @@ import contextlib as ctxlib
import subprocess as sp import subprocess as sp
__version__ = '0.6.2' __version__ = '0.6.3'
"""Current version of the fsl-ci-rules.""" """Current version of the fsl-ci-rules."""
...@@ -69,8 +68,13 @@ def sprun(cmd, **kwargs): ...@@ -69,8 +68,13 @@ def sprun(cmd, **kwargs):
fprint(f'Running {cmd}') fprint(f'Running {cmd}')
if not kwargs.get('shell', False): if not kwargs.get('shell', False):
cmd = shlex.split(cmd) cmd = shlex.split(cmd)
if 'check' not in kwargs:
kwargs['check'] = True # we set stdout/stderr for compatibility
# with CaptureStdout (see below)
if 'check' not in kwargs: kwargs['check'] = True
if 'stdout' not in kwargs: kwargs['stdout'] = sys.stdout
if 'stderr' not in kwargs: kwargs['stderr'] = sys.stderr
return sp.run(cmd, **kwargs) return sp.run(cmd, **kwargs)
...@@ -124,35 +128,53 @@ class CaptureStdout: ...@@ -124,35 +128,53 @@ class CaptureStdout:
"""Context manager which captures stdout and stderr. """ """Context manager which captures stdout and stderr. """
def __init__(self): def __init__(self):
self.reset() self.tmpdir = tempfile.TemporaryDirectory()
self.__mock_stdout = open(op.join(self.tmpdir.name, 'stdout'), 'wt')
self.__mock_stderr = open(op.join(self.tmpdir.name, 'stderr'), 'wt')
def reset(self):
self.__mock_stdout = io.StringIO('')
self.__mock_stderr = io.StringIO('')
self.__mock_stdout.mode = 'w'
self.__mock_stderr.mode = 'w'
return self
def __enter__(self): def __enter__(self):
if self.tmpdir is None:
raise RuntimeError('CaptureStdout can only be used once')
self.__real_stdout = sys.stdout self.__real_stdout = sys.stdout
self.__real_stderr = sys.stderr self.__real_stderr = sys.stderr
sys.stdout = self.__mock_stdout
sys.stderr = self.__mock_stderr
sys.stdout = self.__mock_stdout return self
sys.stderr = self.__mock_stderr
def __exit__(self, *args, **kwargs): def __exit__(self, *args, **kwargs):
sys.stdout = self.__real_stdout sys.stdout = self.__real_stdout
sys.stderr = self.__real_stderr sys.stderr = self.__real_stderr
self.__mock_stdout.close()
self.__mock_stderr.close()
with open(self.__mock_stdout.name, 'rt') as f:
self.__mock_stdout = f.read()
with open(self.__mock_stderr.name, 'rt') as f:
self.__mock_stderr = f.read()
self.tmpdir.cleanup()
self.tmpdir = None
@property
def real_stdout(self):
return self.__real_stdout
@property
def real_stderr(self):
return self.__real_stderr
@property @property
def stdout(self): def stdout(self):
self.__mock_stdout.seek(0) return self.__mock_stdout
return self.__mock_stdout.read()
@property @property
def stderr(self): def stderr(self):
self.__mock_stderr.seek(0) return self.__mock_stderr
return self.__mock_stderr.read()
def health_check(): def health_check():
......
...@@ -149,10 +149,12 @@ def get_project_repository_and_revision(filename, ignore_env=False): ...@@ -149,10 +149,12 @@ def get_project_repository_and_revision(filename, ignore_env=False):
rev = source.get('git_rev', None) rev = source.get('git_rev', None)
build = build .get('number', None) build = build .get('number', None)
if repo is not None: repo = repo .strip() if repo is not None: repo = str(repo) .strip()
if rev is not None: rev = rev .strip() if rev is not None: rev = str(rev) .strip()
if build is not None: build = str(build).strip()
if repo == '': repo = None if repo == '': repo = None
if rev == '': rev = None if rev == '': rev = None
if build == '': build = None
return repo, rev, build return repo, rev, build
......
...@@ -32,6 +32,7 @@ def gen_repository_url(project_path, server, token=None): ...@@ -32,6 +32,7 @@ def gen_repository_url(project_path, server, token=None):
return project_path return project_path
if token is not None: if token is not None:
token = token.strip()
prefix, suffix = server.split('://') prefix, suffix = server.split('://')
server = f'{prefix}://gitlab-ci-token:{token}@{suffix}' server = f'{prefix}://gitlab-ci-token:{token}@{suffix}'
......
...@@ -32,7 +32,7 @@ SERVER_URL = 'https://git.fmrib.ox.ac.uk' ...@@ -32,7 +32,7 @@ SERVER_URL = 'https://git.fmrib.ox.ac.uk'
"""Default gitlab instance URL, if not specified on the command.line.""" """Default gitlab instance URL, if not specified on the command.line."""
DEFAULT_EXCLUDES = ['docs'] DEFAULT_EXCLUDES = ['docs', 'fsl-base']
"""List of repository names in fsl/conda/ that will always be skipped.""" """List of repository names in fsl/conda/ that will always be skipped."""
...@@ -55,19 +55,32 @@ def filter_projects(projects, exclude): ...@@ -55,19 +55,32 @@ def filter_projects(projects, exclude):
return filtered return filtered
def get_project_url(recipe_path, server, token): def get_project_info(recipe_path, server, token):
"""Returns the URL, revision, and build number of the project for """Returns the project URL, path, revision, and build number of the
the specfied recipe. project for the specfied recipe.
""" """
metayaml = download_file(recipe_path, 'meta.yaml', server, token) metayaml = download_file(recipe_path, 'meta.yaml', server, token)
return get_project_repository_and_revision(metayaml) url, version, build = get_project_repository_and_revision(metayaml)
path = None
if url is not None:
path = urlparse.urlparse(url).path
def rerender_recipe(project_url, project_ver, build_number, server, token): if path.endswith('.git'):
path = path[:-4]
return url, path, version, build
def rerender_recipe(project_url,
project_path,
project_ver,
build_number,
server,
token):
"""Re-renders the recipe(s) associated with the specified project. """Re-renders the recipe(s) associated with the specified project.
Essentially just calls update_conda_recipe. Essentially just calls update_conda_recipe.
""" """
project_path = urlparse.urlparse(project_url).path
project_name = project_path.rsplit('/')[-1] project_name = project_path.rsplit('/')[-1]
if build_number is None: build_number = 0 if build_number is None: build_number = 0
...@@ -106,29 +119,44 @@ def rerender_all_recipes(recipes, server, token): ...@@ -106,29 +119,44 @@ def rerender_all_recipes(recipes, server, token):
projects = set() projects = set()
errored = {} errored = {}
for recipe in recipes: maxlen = max([len(r) for r in recipes])
for i, recipe in enumerate(recipes):
with CaptureStdout() as cap: with CaptureStdout() as cap:
try: try:
purl, pver, pbuild = get_project_url(recipe, server, token) url, path, ver, build = get_project_info(recipe, server, token)
if purl in projects:
fmt = recipe + ' ' * (maxlen - len(recipe))
print(f'{i+1:4d} / {len(recipes)} {fmt} - {path}: ',
end='', file=cap.real_stdout)
if (url is None) or (path is None):
print('no url or path - skipping', file=cap.real_stdout)
continue
if path in projects:
print('already visited - skipping', file=cap.real_stdout)
continue continue
# Re-render is restricted to # Re-render is restricted to
# internally hosted projects # internally hosted projects
if 'git.fmrib.ox.ac.uk' not in purl: if (url is None) or ('git.fmrib.ox.ac.uk' not in url):
print('external URL - skipping', file=cap.real_stdout)
continue continue
projects.add(purl) projects.add(path)
print(f'{recipe}: ', end='')
mrurl = rerender_recipe(purl, pver, pbuild, server, token) mrurl = rerender_recipe(url, path, ver, build, server, token)
print(mrurl) print(mrurl, file=cap.real_stdout)
except Exception as e: except Exception as e:
print('ERROR') print(f'errored - {e}', file=cap.real_stdout)
errored[recipe] = cap errored[recipe] = cap.stdout, cap.stderr
for recipe, output in errored.items(): for recipe, (stdout, stderr) in errored.items():
stdout = tw.indent(output.stdout, ' ')
stderr = tw.indent(output.stderr, ' ')
print(f'{recipe} rerender errored!') print(f'{recipe} rerender errored!')
stdout = tw.indent(stdout, ' ')
stderr = tw.indent(stderr, ' ')
print('Standard output') print('Standard output')
print(stdout) print(stdout)
print('Standard error') print('Standard error')
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment