Commit 3b84fc82 authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

RF: Move project introspection stuff into a separate "fsl" module

parent 64c5e303
...@@ -82,6 +82,16 @@ def sprun(cmd, **kwargs): ...@@ -82,6 +82,16 @@ def sprun(cmd, **kwargs):
return sp.run(cmd, **kwargs) return sp.run(cmd, **kwargs)
def spcap(cmd, **kwargs):
"""Call sprun(..., capture_output=True), and return stdout as a string. """
return sprun(cmd, capture_output=True, **kwargs).stdout.decode()
def which(command):
"""Run "which command". """
return spcap(f'which {command}').strip()
def is_valid_project_version(version): def is_valid_project_version(version):
"""Return True if the given version/tag is "valid" - it must """Return True if the given version/tag is "valid" - it must
be a sequence of integers, separated by periods, with an optional be a sequence of integers, separated by periods, with an optional
......
#!/usr/bin/env python
#
# fsl.py - Functions for introspecting information (dependencies and
# executables) about a FSL project.
#
# Author: Paul McCarthy <pauldmccarthy@gmail.com>
#
import os.path as op
import os
import re
import glob
import shutil
import itertools as it
from unittest import mock
from fsl_ci import (indir,
tempdir,
sprun,
spcap,
which)
from fsl_ci.gitlab import get_project_version
from fsl_ci.conda import gen_recipe_path
EXCLUDE = [
'boost-cpp',
'sgeutils',
'glibmm',
'libxmlpp',
'libxml2',
'fslio',
'zlib',
'tk',
'libsqlite',
'libpng',
]
VERSION_OVERRIDE = {
'libgd' : '2.2.5',
'libnlopt' : '2.6.2',
'libpng' : '1.6.37',
'libsqlite' : '3.33.0',
'tk' : '8.6.10',
'zlib' : '1.2.11',
'ciftilib' : '1.5.3',
}
"""Some hard-coded version numbers for external ("extras") FSL dependencies.
"""
DEPENDENCY_REPLACEMENTS = {
'boost' : 'boost-cpp',
'libprob' : 'cprob',
'ciftiio' : 'CiftiLib',
'CiftiLib-master' : 'CiftiLib',
'CiftiLib-master/Nifti' : 'CiftiLib',
'flirt/flirtsch' : 'flirt',
'miscvis/luts+pics' : 'miscvis',
'libxml++-2.6' : 'libxmlpp',
}
"""Used by get_binary_dependencies to replace some common false positives.
"""
EXCLUDE_FROM_BINARY_DEPOENDENCIES = [
'x86_64-linux-gnu',
'linux'
]
"""Any paths which are listed as binary (C/C++) dependencies of a project,
and which contain any of the items in this list as sub-strings, are excluded.
See get_binary_dependencies.
"""
# These identifiers are returned by the filetype function.
SHELL = 'shell'
PYTHON = 'python'
BINARY = 'binary'
DATA = 'data'
UNKNOWN = 'unknown'
def filetype(filename):
"""Attempt to identify the type of the file, either a shell script,
compiled executable, python script, or unknown.
"""
filename = op.expandvars(filename)
ftype = spcap(f'file -L "{filename}"').lower()
if 'shell' in ftype: return SHELL
elif 'python' in ftype: return PYTHON
elif 'x86-64' in ftype: return BINARY
else: return UNKNOWN
def get_fsl_project_type(project_dir):
"""Use dodgy heuristics to figure out the type of a FSL project.
Return:
- "python" if the project looks like a Python based project
- "cpp" if the project looks like a C/C++ Makefile-based project
- "cuda" if the project looks like a CUDA Makefile-based project
- "other" if the project looks like a non-C/C++ Makefile based project
- "unwknown" if the project doesn't look like anything we can work with.
"""
if op.exists(op.join(project_dir, 'setup.py')):
return 'python'
elif op.exists(op.join(project_dir, 'Makefile')):
# Check for nvcc in the makefile -
# if present, assume CUDA.
with open(op.join(project_dir, 'Makefile'), 'rt') as f:
makefile = f.read()
if ('nvcc' in makefile) or ('NVCC' in makefile):
return 'cuda'
# Check for c/c++ source files -
# if present, assume C/C++
for pattern in ['*.cc', '*.h', '*.cpp', '*.cxx', '*.c']:
if glob.glob(op.join(project_dir, '**', pattern), recursive=True):
return 'cpp'
# Otherwise assume some other Makefile
# project (tcl, sh, python, etc)
return 'other'
else:
return 'unknown'
def find_source_project(cmdname):
"""Try and identify the FSL project which provides the given command. """
cmdname = op.basename(cmdname)
srcdir = op.join(os.environ['FSLDIR'], 'src')
results = spcap(f'find {srcdir} -name {cmdname} -or -name "{cmdname}.*"')
results = [r.strip() for r in results.split()]
results = [op.relpath(r, srcdir) for r in results]
results = [op.dirname(r) for r in results]
results = sorted(set(results))
results = [r for r in results if r != '']
if len(results) > 0: return results[0]
else: return ''
def get_project_name(project_dir):
"""Return the name of the given FSL project, as specified in its Makefile.
"""
makefile = op.join(project_dir, 'Makefile')
if not op.exists(makefile):
return None
with open(makefile) as f:
for line in f:
line = line.strip()
if line.startswith('PROJNAME'):
return line.split('=')[1].strip()
return None
def get_project_executables(project_dir):
"""Return a list of executables provided by the given FSL project."""
makefile = op.join(project_dir, 'Makefile')
if not op.exists(makefile):
return []
lines = []
concat = False
with open(makefile) as f:
for line in f:
line = line.strip()
if not concat:
lines.append(line)
else:
lines[-1] = lines[-1][:-2] + ' ' + line
concat = line.endswith('\\')
patterns = [
r'XFILES *=',
r'SCRIPTS *='
]
xlines = [l for l in lines if any([re.match(p, l) for p in patterns])]
xfiles = list(it.chain(*[l.split('=')[1].split() for l in xlines]))
xfiles = [op.basename(x) for x in xfiles]
xfiles = [x.strip() for x in xfiles]
return sorted(set(xfiles))
def _get_shell_dependencies(filename):
"""Return the FSL dependencies of the given shell script. Used by
get_shell_dependencies.
"""
filename = op.expandvars(filename)
ftype = filetype(filename)
if ftype != SHELL:
return []
with open(filename, 'rt', encoding='UTF-8') as f:
lines = f.readlines()
lines = [l.strip() for l in lines]
pats = [
r'(\$FSLDIR/[^ \'"`\)]+)',
r'(\$\{FSLDIR\}/[^ \'"`\)]+)']
pats = [re.compile(p) for p in pats]
deps = []
for line in lines:
for pat in pats:
match = re.search(pat, line)
if match:
dep = match.groups(1)[0].strip('\\')
if op.expandvars(dep) != filename:
deps.append(dep)
break
return list(sorted(set(deps)))
def get_shell_dependencies(filename):
"""Recursively identify the FSL dependencies of the given shell script.
"""
cmddeps = _get_shell_dependencies(filename)
subdeps = []
for dep in cmddeps:
subdeps.extend(get_shell_dependencies(dep))
return cmddeps + subdeps
def get_binary_dependencies(project_dir):
"""Attempt to identify binary (C/C++) dependencies of the given
FSL project.
"""
with tempdir():
shutil.copytree(project_dir, 'project')
with indir('project'):
sprun('rm -f depend.mk')
sprun('make depend.mk')
with open('depend.mk') as f:
lines = f.readlines()
lines = [l.strip(' \\\n') for l in lines]
lines = [l for l in lines if '/include/' in l]
lines = [l.split('/include/')[1] for l in lines]
projects = [l.split('/')[0].strip() for l in lines]
projects = [p for p in projects if not p.endswith('.h')]
projects = [p for p in projects
if p not in EXCLUDE_FROM_BINARY_DEPOENDENCIES]
return sorted(set(projects))
def get_project_dependencies(project_dir, verbose=False):
"""Attempt to identify all of the dependencies of the given FSL project.
"""
exes = get_project_executables(project_dir)
deps = get_binary_dependencies(project_dir)
if verbose:
print('Binary dependencies')
for d in deps:
print(' ', d)
for exe in exes:
exe = op.join(project_dir, exe)
print(exe, filetype(exe))
if filetype(exe) == SHELL:
edeps = get_shell_dependencies(exe)
eprojs = [find_source_project(e) for e in edeps]
edeps = [(ed, ep) for (ed, ep) in zip(edeps, eprojs)
if ed.strip() != '']
if verbose:
print(f'{op.basename(exe)} dependencies')
for ed, ep in sorted(set(edeps)):
print(f' {ep} [{ed}]')
deps.extend([ep for (ed, ep) in edeps])
name = get_project_name(project_dir)
deps = [DEPENDENCY_REPLACEMENTS.get(d, d) for d in deps]
deps = [d for d in deps if d != name]
return sorted(set(deps))
def get_fsl_project_dependencies(project_path, project_dir, server, token):
"""Returns a list of all FSL dependencies of the project as (project,
version) tuples.
"""
if ('FSLDIR' not in os.environ) or ('FSLDEVDIR' not in os.environ):
print('FSLDIR/FSLDEVDIR are not set - cannot identify '
f'dependencies for project {project_path}')
return []
deps = get_project_dependencies(project_dir)
deps = [d.lower() for d in deps if d not in EXCLUDE]
vers = []
for i, dep in enumerate(deps):
if dep in VERSION_OVERRIDE:
dver = VERSION_OVERRIDE[dep]
else:
try:
# assume that the dependency has
# a repo at "server/fsl/dep"
dver = get_project_version(f'fsl/{dep}', server, token)
dep = gen_recipe_path(dep).rsplit('/')[-1]
except Exception:
dver = 'UNKNOWN'
vers.append(dver)
deps[i] = dep
deps = list(zip(deps, vers))
deps = sorted([d for d in deps if d[0].startswith('fsl')]) + \
sorted([d for d in deps if not d[0].startswith('fsl')])
return deps
def get_setup_py_metadata(project_dir):
"""Attempts to extract project metadata from its setup.py file. """
setup_py = op.abspath(op.join(project_dir, 'setup.py'))
if not op.exists(setup_py):
return {}
setup_meta = {}
def setup(**kwargs):
setup_meta.update(kwargs)
with open(setup_py) as f:
code = f.read()
ns = {
'__name__' : '__main__',
'__doc__' : None,
'__file__' : setup_py,
}
with mock.patch('setuptools.setup', setup), \
mock.patch('distutils.core.setup', setup), \
mock.patch.dict('sys.modules', {'versioneer' : mock.MagicMock()}), \
indir(project_dir):
try:
code = compile(code, setup_py, 'exec', dont_inherit=True)
exec(code, ns, ns)
except Exception as e:
print(f'Could not extract project metadata from {setup_py}: {e}')
return setup_meta
def get_python_entrypoints(project_dir):
"""Return entry points provided by a python project managed by the
given setup.py file. Will only work for simple setup.py files which
don't try to do anything too complicated.
"""
setup_meta = get_setup_py_metadata(project_dir)
if 'entry_points' in setup_meta:
entry_points = setup_meta['entry_points']
entry_points = entry_points.get('console_scripts', []) + \
entry_points.get('gui_scripts', [])
else:
entry_points = []
return sorted(entry_points)
def get_python_executables(project_dir):
"""Return a list of all executables provided by the project."""
setup_meta = get_setup_py_metadata(project_dir)
entrypoints = get_python_entrypoints(project_dir)
entrypoints = [ep.split('=')[0].strip() for ep in entrypoints]
scripts = setup_meta.get('scripts', [])
scripts = [op.basename(s) for s in scripts]
return entrypoints + scripts
...@@ -13,13 +13,17 @@ import sys ...@@ -13,13 +13,17 @@ import sys
import glob import glob
import shutil import shutil
import argparse import argparse
from unittest import mock
from fsl_ci import (USERNAME, from fsl_ci import (USERNAME,
EMAIL, EMAIL,
indir, indir,
tempdir, tempdir,
sprun) sprun)
from fsl_ci.fsl import (get_fsl_project_type,
get_fsl_project_dependencies,
get_project_executables,
get_python_executables,
get_python_entrypoints)
from fsl_ci.gitlab import (gen_repository_url, from fsl_ci.gitlab import (gen_repository_url,
project_exists, project_exists,
create_repository, create_repository,
...@@ -30,8 +34,6 @@ from fsl_ci.conda import gen_recipe_path ...@@ -30,8 +34,6 @@ from fsl_ci.conda import gen_recipe_path
from fsl_ci.recipe import (create_python_recipe_template, from fsl_ci.recipe import (create_python_recipe_template,
create_cpp_recipe_template) create_cpp_recipe_template)
import fsl_ci.utils.fsl_project_dependencies as fsldeps
SERVER_URL = 'https://git.fmrib.ox.ac.uk' SERVER_URL = 'https://git.fmrib.ox.ac.uk'
"""Default gitlab instance URL, if not specified on the command.line. """ """Default gitlab instance URL, if not specified on the command.line. """
...@@ -66,43 +68,6 @@ VERSION_OVERRIDE = { ...@@ -66,43 +68,6 @@ VERSION_OVERRIDE = {
""" """
def project_type(project_dir):
"""Use dodgy heuristics to figure out the type of a FSL project.
Return:
- "python" if the project looks like a Python based project
- "cpp" if the project looks like a C/C++ Makefile-based project
- "cuda" if the project looks like a CUDA Makefile-based project
- "other" if the project looks like a non-C/C++ Makefile based project
- "unwknown" if the project doesn't look like anything we can work with.
"""
if op.exists(op.join(project_dir, 'setup.py')):
return 'python'
elif op.exists(op.join(project_dir, 'Makefile')):
# Check for nvcc in the makefile -
# if present, assume CUDA.
with open(op.join(project_dir, 'Makefile'), 'rt') as f:
makefile = f.read()
if ('nvcc' in makefile) or ('NVCC' in makefile):
return 'cuda'
# Check for c/c++ source files -
# if present, assume C/C++
for pattern in ['*.cc', '*.h', '*.cpp', '*.cxx', '*.c']:
if glob.glob(op.join(project_dir, '**', pattern), recursive=True):
return 'cpp'
# Otherwise assume some other Makefile
# project (tcl, sh, python, etc)
return 'other'
else:
return 'unknown'
def clone_repository(path, server, token, destdir, branch='master'): def clone_repository(path, server, token, destdir, branch='master'):
"""Clone the specified gitlab repository into destdir. """ """Clone the specified gitlab repository into destdir. """
url = gen_repository_url(path, server, token) url = gen_repository_url(path, server, token)
...@@ -152,105 +117,6 @@ def commit_and_push_recipe_repository(recipe_dir, ...@@ -152,105 +117,6 @@ def commit_and_push_recipe_repository(recipe_dir,
sprun(f'git push origin {branch}') sprun(f'git push origin {branch}')
def get_fsl_project_dependencies(project_path, project_dir, server, token):
"""Returns a list of all FSL dependencies of the project as (project,
version) tuples.
"""
if ('FSLDIR' not in os.environ) or ('FSLDEVDIR' not in os.environ):
print('FSLDIR/FSLDEVDIR are not set - cannot identify '
f'dependencies for project {project_path}')
return []
deps = fsldeps.get_project_dependencies(project_dir)
deps = [d.lower() for d in deps if d not in EXCLUDE]
vers = []
for i, dep in enumerate(deps):
if dep in VERSION_OVERRIDE:
dver = VERSION_OVERRIDE[dep]
else:
try:
# assume that the dependency has
# a repo at "server/fsl/dep"
dver = get_project_version(f'fsl/{dep}', server, token)
dep = gen_recipe_path(dep).rsplit('/')[-1]
except Exception:
dver = 'UNKNOWN'
vers.append(dver)
deps[i] = dep
deps = list(zip(deps, vers))
deps = sorted([d for d in deps if d[0].startswith('fsl')]) + \
sorted([d for d in deps if not d[0].startswith('fsl')])
return deps
def get_setup_py_metadata(project_dir):
"""Attempts to extract project metadata from its setup.py file. """
setup_py = op.abspath(op.join(project_dir, 'setup.py'))
if not op.exists(setup_py):
return {}
setup_meta = {}
def setup(**kwargs):
setup_meta.update(kwargs)
with open(setup_py) as f:
code = f.read()
ns = {
'__name__' : '__main__',
'__doc__' : None,
'__file__' : setup_py,
}
with mock.patch('setuptools.setup', setup), \
mock.patch('distutils.core.setup', setup), \
mock.patch.dict('sys.modules', {'versioneer' : mock.MagicMock()}), \
indir(project_dir):
try:
code = compile(code, setup_py, 'exec', dont_inherit=True)
exec(code, ns, ns)
except Exception as e:
print(f'Could not extract project metadata from {setup_py}: {e}')
return setup_meta
def get_python_entrypoints(project_dir):
"""Return entry points provided by a python project managed by the
given setup.py file. Will only work for simple setup.py files which
don't try to do anything too complicated.
"""
setup_meta = get_setup_py_metadata(project_dir)
if 'entry_points' in setup_meta:
entry_points = setup_meta['entry_points']
entry_points = entry_points.get('console_scripts', []) + \
entry_points.get('gui_scripts', [])
else:
entry_points = []
return sorted(entry_points)
def get_python_executables(project_dir):
"""Return a list of all executables provided by the project."""
setup_meta = get_setup_py_metadata(project_dir)
entrypoints = get_python_entrypoints(project_dir)
entrypoints = [ep.split('=')[0].strip() for ep in entrypoints]
scripts = setup_meta.get('scripts', [])
scripts = [op.basename(s) for s in scripts]
return entrypoints + scripts
def create_recipe_template(project_path, def create_recipe_template(project_path,
...@@ -267,10 +133,10 @@ def create_recipe_template(project_path, ...@@ -267,10 +133,10 @@ def create_recipe_template(project_path,
fslbase = get_project_version('fsl/base', server, token) fslbase = get_project_version('fsl/base', server, token)
if ptype is None: