Commit d040c4a5 authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

Merge branch 'rf/cuda' into 'master'

RF: We are no longer creating separate environment specs for different CUDA versions

See merge request fsl/conda/installer!35
parents b7356b69 8a54e7d7
# FSL installer script release history
# 1.9.0
- Removed/disabled the `--update` option, for updating an existing FSL
installation. This option may be re-enabled in the future.
- Removed the hidden `--environment` option.
- Update the `fslinstaller.py` script to work with the new CUDA package
arrangement - FSL environment specifications are no longer provided
for each supported CUDA version. Instead, all CUDA packages are included
as part of the `linux-64` environment. The `--cuda` option can be used
to select one set of packages to be installed, and the `--no_cuda` option
can be used to exclude all CUDA packages from the installation.
# 1.8.0
- The default FSL installation directory has been changed from `/usr/local/fsl/`
......
......@@ -50,7 +50,7 @@ log = logging.getLogger(__name__)
__absfile__ = op.abspath(__file__).rstrip('c')
__version__ = '1.8.0'
__version__ = '1.9.0'
"""Installer script version number. This must be updated
whenever a new version of the installer script is released.
"""
......@@ -116,8 +116,8 @@ class Version(object):
class Context(object):
"""Bag of information and settings created in main, and passed around
this script.
"""Bag of information and settings created in the main function, and passed
around this script.
Several settings are lazily evaluated on first access, but once evaluated,
their values are immutable.
......@@ -135,7 +135,6 @@ class Context(object):
# the property accessors defined below, or all
# all updated via the finalise-settings method.
self.__platform = None
self.__cuda = None
self.__manifest = None
self.__build = None
self.__destdir = None
......@@ -175,7 +174,6 @@ class Context(object):
"""
self.manifest
self.platform
self.cuda
self.build
self.destdir
self.need_admin
......@@ -190,24 +188,10 @@ class Context(object):
return self.__platform
@property
def cuda(self):
"""The available CUDA version, or a CUDA version requested by the user.
"""
if self.__cuda is not None:
return self.__cuda
if self.args.cuda is not None:
self.__cuda = self.args.cuda
if self.__cuda is None:
self.__cuda = Context.identify_cuda()
return self.__cuda
@property
def build(self):
"""Returns a suitable FSL build (a dictionary entry from the FSL
installer manifest) for the target platform and requested FSL/CUDA
versions.
installer manifest) for the target platform.
The returned dictionary has the following elements:
- 'version' FSL version.
......@@ -216,8 +200,6 @@ class Context(object):
- 'sha256': Checksum of environment file
- 'output': Number of lines of expected output, for reporting
progress
- 'cuda': X.Y CUDA version, if a CUDA-enabled version of FSL
is to be installed.
"""
if self.__build is not None:
......@@ -236,53 +218,27 @@ class Context(object):
if fslversion == 'latest':
fslversion = self.manifest['versions']['latest']
# Find refs to all compatible builds,
# separating the default (no CUDA) build
# from CUDA-enabled builds. We assume
# that there is only one default build
# for each platform.
default = None
candidates = []
for build in self.manifest['versions'][fslversion]:
if build['platform'] == self.platform:
if build.get('cuda', None) is None:
default = build
else:
candidates.append(build)
# Find refs to a suitable build for this
# platform. We assume that there is only
# one default build for each platform.
# in the list of builds for a given FSL
# version.
build = None
for candidate in self.manifest['versions'][fslversion]:
if candidate['platform'] == self.platform:
build = candidate
break
if (default is None) and (len(candidates) == 0):
if build is None:
raise Exception(
'Cannot find a version of FSL matching platform '
'{} and CUDA {}'.format(self.platform, self.cuda))
# If we have CUDA (or the user has
# specifically requested a CUDA build),
# try and find a suitable build
match = default
if self.cuda is not None:
candidates = sorted(candidates, key=lambda b: float(b['cuda']))
for build in reversed(candidates):
if self.cuda >= float(build['cuda']):
match = build
break
else:
available = [b['cuda'] for b in candidates]
printmsg('Could not find a suitable FSL CUDA '
'build for CUDA version {} (available: '
'{}. Installing default (non-CUDA) '
'FSL build.'.format(self.cuda, available),
WARNING)
printmsg('You can use the --cuda command-line option '
'to install a FSL build that is compatible '
'with a specific CUDA version', INFO)
'Cannot find a version of FSL matching '
'platform {}'.format(self.platform))
printmsg('FSL {} [CUDA: {}] selected for installation'.format(
match['version'], match.get('cuda', 'n/a')))
printmsg('FSL {} selected for installation'.format(build['version']))
self.__build = match
return match
self.__build = build
return build
@property
......@@ -393,41 +349,6 @@ class Context(object):
return platforms[key]
@staticmethod
def identify_cuda():
"""Identifies the CUDA version supported on the platform. Returns a
float representing the X.Y CUDA version, or None if CUDA is not
available on the platform.
"""
# see below - no_cuda is set to prevent unnecessary
# attempts to call nvidia-smi more than once
if getattr(Context.identify_cuda, 'no_cuda', False):
return None
try:
output = Process.check_output('nvidia-smi')
except Exception:
Context.identify_cuda.no_cuda = True
return None
pat = r'CUDA Version: (\S+)'
lines = output.split('\n')
for line in lines:
match = re.search(pat, line)
if match:
cudaver = match.group(1)
break
else:
# message for debugging - the output
# will be present in the logfile
log.debug('Could not parse nvidia-smi output')
Context.identify_cuda.no_cuda = True
return None
return float(cudaver)
@staticmethod
def check_need_admin(dirname):
"""Returns True if dirname needs administrator privileges to write to,
......@@ -706,14 +627,20 @@ class Progress(object):
if it cannot be determined.
"""
# os.get_terminal_size added in python
# 3.3, so we try it but fall back to tput
# 3.3, so we try it but fall back to
# COLUMNS, or tput as a last resort.
try:
return os.get_terminal_size()[0]
except Exception:
pass
try:
result = sp.check_output(('tput', 'cols'))
return int(os.environ['COLUMNS'])
except Exception:
pass
try:
result = Process.check_output('tput cols')
return int(result.strip())
except Exception:
return fallback
......@@ -727,6 +654,16 @@ def isstr(s):
except Exception: return isinstance(s, str)
def match_any(s, patterns):
"""Test if the string s matches any of the fnmatch-style patterns.
Returns the matched pattern, or None.
"""
for pat in patterns:
if fnmatch.fnmatch(s, pat):
return pat
return None
@contextlib.contextmanager
def tempdir(override_dir=None):
"""Returns a context manager which creates, changes into, and returns a
......@@ -1150,11 +1087,7 @@ def list_available_versions(manifest):
continue
printmsg(version, IMPORTANT, EMPHASIS)
for build in manifest['versions'][version]:
if build.get('cuda', '').strip() != '':
template = ' {platform} [CUDA {cuda}]'
else:
template = ' {platform}'
printmsg(template.format(**build), EMPHASIS, end=' ')
printmsg(' {}'.format(build['platform']), EMPHASIS, end=' ')
printmsg(build['environment'], INFO)
......@@ -1162,9 +1095,6 @@ def download_fsl_environment(ctx):
"""Downloads the environment specification file for the selected FSL
version.
If the (hidden) --environment option is provided, the specified file
is used instead.
Internal/development FSL versions may source packages from the internal
FSL conda channel, which requires a username+password to authenticate.
......@@ -1173,21 +1103,16 @@ def download_fsl_environment(ctx):
If the user has not provided a username+password on the command-line, they
are prompted for them.
"""
if ctx.args.environment is None:
build = ctx.build
url = build['environment']
checksum = build.get('sha256', None)
basepkgnames = build.get('base_packages', [])
The downloaded environment file may be modified - if the user has requested
a specific CUDA version, or no CUDA packages (--cuda or --no_cuda), all
CUDA packages are removed from the environment file.
"""
# disable checksum if env file is passed
# via --environment cli option
else:
build = {}
url = ctx.args.environment
checksum = None
basepkgnames = []
build = ctx.build
url = build['environment']
checksum = build.get('sha256', None)
basepkgnames = build.get('base_packages', [])
printmsg('Downloading FSL environment specification '
'from {}...'.format(url))
......@@ -1273,13 +1198,13 @@ def download_fsl_environment(ctx):
pkgver = line.strip().split(' ', 2)[2]
basepkgs[pkg] = pkgver.replace(' ', '=')
# Exclude packages upon user request
# Include/exclude packages upon user request
pkgname = line.strip(' -').split()[0]
for pattern in ctx.args.exclude_package:
if fnmatch.fnmatch(pkgname, pattern):
log.debug('Excluding package %s (matched '
'--exclude_package %s)', line, pattern)
break
exclude = match_any(pkgname, ctx.args.exclude_package)
include = match_any(pkgname, ctx.args.include_package)
if exclude and not include:
log.debug('Excluding package %s (matched '
'--exclude_package %s)', line, exclude)
else:
outf.write(line)
......@@ -1697,6 +1622,9 @@ def update_destdir(ctx):
Returns the old FSL version string if the existing FSL installation
should be updated, or None if it should be overwritten.
Note: This functionality is not currently supported - the --update
command-line option is hidden for the time being.
"""
installed = read_fslversion(ctx.destdir)
......@@ -1816,8 +1744,6 @@ def parse_args(argv=None):
'listversions' : 'List available FSL versions and exit',
'dest' : 'Install FSL into this folder (default: '
'{})'.format(DEFAULT_INSTALLATION_DIRECTORY),
'update' : 'Update existing FSL installation if possible, '
'without asking',
'overwrite' : 'Delete existing destination directory if it exists, '
'without asking',
'no_env' : 'Do not modify your shell or MATLAB configuration '
......@@ -1825,8 +1751,15 @@ def parse_args(argv=None):
'no_shell' : 'Do not modify your shell configuration',
'no_matlab' : 'Do not modify your MATLAB configuration',
'fslversion' : 'Install this specific version of FSL',
'cuda' : 'Install FSL for this CUDA version (default: '
'automatically detected)',
'cuda' : 'Install FSL packages for this CUDA version only '
'(default: install packages for all CUDA versions)',
'no_cuda' : 'Do not install any FSL CUDA packages',
# Update existing FSL installation if
# possible, without asking. This
# option is hidden/unsupported at the
# moment, but may be added in the future.
'update' : argparse.SUPPRESS,
# Username / password for accessing
# internal FSL conda channel, if an
......@@ -1843,14 +1776,6 @@ def parse_args(argv=None):
# Path to alternative FSL release manifest.
'manifest' : argparse.SUPPRESS,
# Path to FSL conda environment.yml file.
# Using this option will cause the
# --fslversion and --cuda options to be
# ignored. It is assumed that the
# environment file is compatible with the
# host platform.
'environment' : argparse.SUPPRESS,
# Print debugging messages
'debug' : argparse.SUPPRESS,
......@@ -1897,7 +1822,14 @@ def parse_args(argv=None):
help=helps['no_matlab'])
parser.add_argument('-V', '--fslversion', default='latest',
help=helps['version'])
parser.add_argument('-c', '--cuda', help=helps['cuda'], type=float)
# CUDA packages are currently
# only built for linux-64
if Context.identify_platform() == 'linux-64':
parser.add_argument('-c', '--cuda', help=helps['cuda'],
type=float, metavar='X.Y')
parser.add_argument('-nc', '--no_cuda', help=helps['no_cuda'],
action='store_true')
# hidden options
parser.add_argument('--username', help=helps['username'])
......@@ -1911,7 +1843,6 @@ def parse_args(argv=None):
default=op.expanduser('~'))
parser.add_argument('--manifest', default=FSL_INSTALLER_MANIFEST,
help=helps['manifest'])
parser.add_argument('--environment', help=helps['environment'])
parser.add_argument('--no_self_update', action='store_true',
help=helps['no_self_update'])
parser.add_argument('--exclude_package', action='append',
......@@ -1948,11 +1879,25 @@ def parse_args(argv=None):
if args.exclude_package is None:
args.exclude_package = []
# The download_fsl_environment function also checks
# package names against this "include_package" list,
# although it is not exposed on the command
# line. This is used to override any patterns in
# exclude_package, and is currently used to select
# packages for a specific CUDA version.
#
# All FSL cuda packages have a name ending with
# "-cuda-X.Y".
args.include_package = []
if args.cuda is not None:
args.include_package.append('fsl-*-cuda-{:0.1f}'.format(args.cuda))
if args.no_cuda or (args.cuda is not None):
args.exclude_package.append('fsl-*-cuda-*')
# accept local path for manifest and environment
if args.manifest is not None and op.exists(args.manifest):
args.manifest = op.abspath(args.manifest)
if args.environment is not None and op.exists(args.environment):
args.environment = op.abspath(args.environment)
return args
......@@ -2060,7 +2005,8 @@ def main(argv=None):
ctx.exists = op.exists(ctx.destdir)
if ctx.exists:
ctx.update = update_destdir(ctx)
# The update facility is currently disabled.
# ctx.update = update_destdir(ctx)
if not ctx.update:
overwrite_destdir(ctx)
......
......@@ -36,47 +36,6 @@ def test_Context_identify_plaform():
assert inst.Context.identify_platform() == expected
def test_Context_identify_cuda():
with inst.tempdir() as cwd:
with onpath(cwd):
nvidia_smi = tw.dedent("""
#!/usr/bin/env bash
echo "{stdout}"
exit {retcode}
""").strip()
# test when nvidia-smi doesn't exist
# (assuming that it won't be present
# in any of the mock paths)
path = op.pathsep.join(('/usr/sbin', '/usr/bin', '/sbin', '/bin'))
with mock.patch.dict(os.environ, PATH=path):
assert inst.Context.identify_cuda() is None
if hasattr(inst.Context.identify_cuda, 'no_cuda'):
delattr(inst.Context.identify_cuda, 'no_cuda')
with open('nvidia-smi', 'wt') as f:
f.write(nvidia_smi.format(stdout='CUDA Version: 10.1', retcode=0))
os.chmod('nvidia-smi', 0o755)
assert inst.Context.identify_cuda() == 10.1
if hasattr(inst.Context.identify_cuda, 'no_cuda'):
delattr(inst.Context.identify_cuda, 'no_cuda')
with open('nvidia-smi', 'wt') as f:
f.write(nvidia_smi.format(stdout='CUDA Version: 11.2', retcode=0))
os.chmod('nvidia-smi', 0o755)
assert inst.Context.identify_cuda() == 11.2
if hasattr(inst.Context.identify_cuda, 'no_cuda'):
delattr(inst.Context.identify_cuda, 'no_cuda')
with open('nvidia-smi', 'wt') as f:
f.write(nvidia_smi.format(stdout='CUDA Version: 11.2', retcode=1))
os.chmod('nvidia-smi', 0o755)
assert inst.Context.identify_cuda() is None
if hasattr(inst.Context.identify_cuda, 'no_cuda'):
delattr(inst.Context.identify_cuda, 'no_cuda')
def test_Context_get_admin_password():
sudo = tw.dedent("""
#!/usr/bin/env bash
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment