BF: Various fixes, rerender script seems to be working

fsl_ci/__init__.py

@@ -8,7 +8,6 @@
 
 import os.path    as op
 import               os
-import               io
 import               sys
 import               errno
 import               shlex
@@ -19,7 +18,7 @@ import contextlib as ctxlib
 import subprocess as sp
 
 
-__version__ = '0.6.2'
+__version__ = '0.6.3'
 """Current version of the fsl-ci-rules."""
 
 
@@ -69,8 +68,13 @@ def sprun(cmd, **kwargs):
     fprint(f'Running {cmd}')
     if not kwargs.get('shell', False):
         cmd = shlex.split(cmd)
-    if 'check' not in kwargs:
-        kwargs['check'] = True
+
+    # we set stdout/stderr for compatibility
+    # with CaptureStdout (see below)
+    if 'check'  not in kwargs: kwargs['check']  = True
+    if 'stdout' not in kwargs: kwargs['stdout'] = sys.stdout
+    if 'stderr' not in kwargs: kwargs['stderr'] = sys.stderr
+
     return sp.run(cmd, **kwargs)
 
 
@@ -124,35 +128,53 @@ class CaptureStdout:
     """Context manager which captures stdout and stderr. """
 
     def __init__(self):
-        self.reset()
+        self.tmpdir        = tempfile.TemporaryDirectory()
+        self.__mock_stdout = open(op.join(self.tmpdir.name, 'stdout'), 'wt')
+        self.__mock_stderr = open(op.join(self.tmpdir.name, 'stderr'), 'wt')
 
-    def reset(self):
-        self.__mock_stdout = io.StringIO('')
-        self.__mock_stderr = io.StringIO('')
-        self.__mock_stdout.mode = 'w'
-        self.__mock_stderr.mode = 'w'
-        return self
 
     def __enter__(self):
+        if self.tmpdir is None:
+            raise RuntimeError('CaptureStdout can only be used once')
+
         self.__real_stdout = sys.stdout
         self.__real_stderr = sys.stderr
-
         sys.stdout         = self.__mock_stdout
         sys.stderr         = self.__mock_stderr
 
+        return self
+
     def __exit__(self, *args, **kwargs):
+
         sys.stdout = self.__real_stdout
         sys.stderr = self.__real_stderr
 
+        self.__mock_stdout.close()
+        self.__mock_stderr.close()
+
+        with open(self.__mock_stdout.name, 'rt') as f:
+            self.__mock_stdout = f.read()
+        with open(self.__mock_stderr.name, 'rt') as f:
+            self.__mock_stderr = f.read()
+
+        self.tmpdir.cleanup()
+        self.tmpdir = None
+
+    @property
+    def real_stdout(self):
+        return self.__real_stdout
+
+    @property
+    def real_stderr(self):
+        return self.__real_stderr
+
     @property
     def stdout(self):
-        self.__mock_stdout.seek(0)
-        return self.__mock_stdout.read()
+        return self.__mock_stdout
 
     @property
     def stderr(self):
-        self.__mock_stderr.seek(0)
-        return self.__mock_stderr.read()
+        return self.__mock_stderr
 
 
 def health_check():

fsl_ci/conda.py

@@ -149,10 +149,12 @@ def get_project_repository_and_revision(filename, ignore_env=False):
     rev    = source.get('git_rev', None)
     build  = build .get('number',  None)
 
-    if repo   is not None: repo  = repo .strip()
-    if rev    is not None: rev   = rev  .strip()
+    if repo   is not None: repo  = str(repo) .strip()
+    if rev    is not None: rev   = str(rev)  .strip()
+    if build  is not None: build = str(build).strip()
     if repo   == '':       repo  = None
     if rev    == '':       rev   = None
+    if build  == '':       build = None
 
     return repo, rev, build
 

fsl_ci/gitlab.py

@@ -32,6 +32,7 @@ def gen_repository_url(project_path, server, token=None):
         return project_path
 
     if token is not None:
+        token          = token.strip()
         prefix, suffix = server.split('://')
         server         = f'{prefix}://gitlab-ci-token:{token}@{suffix}'
 

fsl_ci/scripts/rerender_all_recipes.py

@@ -32,7 +32,7 @@ SERVER_URL = 'https://git.fmrib.ox.ac.uk'
 """Default gitlab instance URL, if not specified on the command.line."""
 
 
-DEFAULT_EXCLUDES = ['docs']
+DEFAULT_EXCLUDES = ['docs', 'fsl-base']
 """List of repository names in fsl/conda/ that will always be skipped."""
 
 
@@ -55,19 +55,32 @@ def filter_projects(projects, exclude):
     return filtered
 
 
-def get_project_url(recipe_path, server, token):
-    """Returns the URL, revision, and build number of the project for
-    the specfied recipe.
+def get_project_info(recipe_path, server, token):
+    """Returns the project URL, path, revision, and build number of the
+    project for the specfied recipe.
     """
     metayaml = download_file(recipe_path, 'meta.yaml', server, token)
-    return get_project_repository_and_revision(metayaml)
+    url, version, build = get_project_repository_and_revision(metayaml)
+    path = None
 
+    if url is not None:
+        path = urlparse.urlparse(url).path
 
-def rerender_recipe(project_url, project_ver, build_number, server, token):
+        if path.endswith('.git'):
+            path = path[:-4]
+
+    return url, path, version, build
+
+
+def rerender_recipe(project_url,
+                    project_path,
+                    project_ver,
+                    build_number,
+                    server,
+                    token):
     """Re-renders the recipe(s) associated with the specified project.
     Essentially just calls update_conda_recipe.
     """
-    project_path = urlparse.urlparse(project_url).path
     project_name = project_path.rsplit('/')[-1]
 
     if build_number is None: build_number = 0
@@ -106,29 +119,44 @@ def rerender_all_recipes(recipes, server, token):
     projects = set()
     errored  = {}
 
-    for recipe in recipes:
+    maxlen = max([len(r) for r in recipes])
+
+    for i, recipe in enumerate(recipes):
         with CaptureStdout() as cap:
             try:
-                purl, pver, pbuild = get_project_url(recipe, server, token)
-                if purl in projects:
+                url, path, ver, build = get_project_info(recipe, server, token)
+
+                fmt = recipe + ' ' * (maxlen - len(recipe))
+                print(f'{i+1:4d} / {len(recipes)} {fmt} - {path}: ',
+                      end='', file=cap.real_stdout)
+
+                if (url is None) or (path is None):
+                    print('no url or path - skipping', file=cap.real_stdout)
+                    continue
+
+                if path in projects:
+                    print('already visited - skipping', file=cap.real_stdout)
                     continue
                 # Re-render is restricted to
                 # internally hosted projects
-                if 'git.fmrib.ox.ac.uk' not in purl:
+                if (url is None) or ('git.fmrib.ox.ac.uk' not in url):
+                    print('external URL - skipping', file=cap.real_stdout)
                     continue
-                projects.add(purl)
-                print(f'{recipe}: ', end='')
-                mrurl = rerender_recipe(purl, pver, pbuild, server, token)
-                print(mrurl)
+                projects.add(path)
+
+                mrurl = rerender_recipe(url, path, ver, build, server, token)
+                print(mrurl, file=cap.real_stdout)
 
             except Exception as e:
-                print('ERROR')
-                errored[recipe] = cap
+                print(f'errored - {e}', file=cap.real_stdout)
+                errored[recipe] = cap.stdout, cap.stderr
 
-    for recipe, output in errored.items():
-        stdout = tw.indent(output.stdout, '    ')
-        stderr = tw.indent(output.stderr, '    ')
+    for recipe, (stdout, stderr) in errored.items():
         print(f'{recipe} rerender errored!')
+
+        stdout = tw.indent(stdout, '    ')
+        stderr = tw.indent(stderr, '    ')
+
         print('Standard output')
         print(stdout)
         print('Standard error')