Merge branch 'enh/devreleases' into 'master'

Ability to install development releases

See merge request fsl/conda/installer!36

fslinstaller.py

@@ -36,6 +36,10 @@ import                   traceback
 try:                import urllib.request as urlrequest
 except ImportError: import urllib         as urlrequest
 
+
+try:                import urllib.parse as urlparse
+except ImportError: import                 urlparse
+
 try:                import queue
 except ImportError: import Queue as queue
 
@@ -50,7 +54,7 @@ log = logging.getLogger(__name__)
 __absfile__ = op.abspath(__file__).rstrip('c')
 
 
-__version__ = '1.9.0'
+__version__ = '1.10.0'
 """Installer script version number. This must be updated
 whenever a new version of the installer script is released.
 """
@@ -60,7 +64,8 @@ DEFAULT_INSTALLATION_DIRECTORY = op.join(op.expanduser('~'), 'fsl')
 """Default FSL installation directory. """
 
 
-FSL_INSTALLER_MANIFEST = 'https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/releases/manifest.json'
+FSL_RELEASE_MANIFEST = 'https://fsl.fmrib.ox.ac.uk/fsldownloads/' \
+                       'fslconda/releases/manifest.json'
 """URL to download the FSL installer manifest file from. The installer
 manifest file is a JSON file which contains information about available FSL
 versions.
@@ -73,6 +78,13 @@ option.
 """
 
 
+FSL_DEV_RELEASES = 'https://fsl.fmrib.ox.ac.uk/fsldownloads/' \
+                   'fslconda/releases/devreleases.txt'
+"""URL to the devreleases.txt file, which contains a list of available
+internal/development FSL releases.
+"""
+
+
 FIRST_FSL_CONDA_RELEASE = '6.0.6'
 """Oldest conda-based FSL version that can be updated in-place by this
 installer script. Versions older than this will need to be overwritten.
@@ -136,6 +148,7 @@ class Context(object):
         # all updated via the finalise-settings method.
         self.__platform       = None
         self.__manifest       = None
+        self.__devmanifest    = None
         self.__build          = None
         self.__destdir        = None
         self.__need_admin     = None
@@ -313,11 +326,104 @@ class Context(object):
     def manifest(self):
         """Returns the FSL installer manifest as a dictionary. """
         if self.__manifest is None:
+            if self.devmanifest is not None:
+                self.args.manifest = self.devmanifest
+
             self.__manifest = Context.download_manifest(self.args.manifest,
                                                         self.args.workdir)
         return self.__manifest
 
 
+    @property
+    def devmanifest(self):
+        """Returns a URL to a development manifest to use for installation.
+        This will only return a value if the --devrelease or --devlatest
+        options are active.
+
+        If this is the case, the FSL_DEV_RELEASES file is downloaded - this
+        file contains a list of available development manifest URLS. The
+        user is then prompted to choose which development manifest to use
+        for the installation, unless --devlatest is active, in which case
+        the newest manifest is selected.
+        """
+        if not self.args.devrelease:
+            return None
+        if self.__devmanifest == 'na':
+            return None
+        elif self.__devmanifest is not None:
+            return self.__devmanifest
+
+        # parse a dev manifest file name, returning
+        # a sequence containing the tage, date, commit
+        # hash, and branch name. Dev manifest files
+        # are named like so:
+        #
+        #   manifest-<tag>.<date>.<commit>.<branch>.json
+        #
+        # where <tag> is the tag of the most recent
+        # public FSL release, and everything else is
+        # self-explanatory.
+        def parse_devrelease_name(url):
+            name = urlparse.urlparse(url).path
+            name = op.basename(name)
+            name = name.lstrip('manifest-').rstrip('.json')
+            # Awkward - the tag may have periods in it
+            name = name.rsplit('.', 3)
+            return name
+
+        # list of (url, tag, date, commit, branch),
+        # sorted by date
+        devreleases = []
+
+        with tempdir(self.args.workdir):
+
+            try:
+                download_file(FSL_DEV_RELEASES, 'devreleases.txt')
+            except Exception as e:
+                log.debug('Error downloading devreleases.txt from %s',
+                          FSL_DEV_RELEASES, exc_info=True)
+                raise Exception('Unable to download development manifest '
+                                'list from {}!'.format(FSL_DEV_RELEASES))
+
+            with open('devreleases.txt', 'rt') as f:
+                urls = f.readlines()
+                urls = [l.strip() for l in urls]
+
+            for url in urls:
+                devreleases.append([url] + parse_devrelease_name(url))
+
+        devreleases = sorted(devreleases, key=lambda r: r[2], reverse=True)
+
+        if len(devreleases) == 0:
+            self.__devmanifest = 'na'
+            return None
+
+        # automatically choose latest dev manifest?
+        if self.args.devlatest:
+            devrelease = devreleases[0][0]
+        # show the user a list, ask them which one they want
+        else:
+            printmsg('Available development releases:', EMPHASIS)
+            for i, (url, tag, date, commit, branch) in enumerate(devreleases):
+                printmsg('  [{}]: {} [{} commit {}]'.format(
+                    i + 1, date, branch, commit), IMPORTANT)
+            while True:
+                selection = prompt('Which release would you like to '
+                                   'install? [1]:', PROMPT)
+                if selection == '':
+                    selection = '1'
+                try:
+                    selection = int(selection) - 1
+                except Exception:
+                    continue
+                if selection >= 0 and selection < len(devreleases):
+                    break
+            devrelease = devreleases[selection][0]
+
+        self.__devmanifest = devrelease
+        return self.__devmanifest
+
+
     @staticmethod
     def identify_platform():
         """Figures out what platform we are running on. Returns a platform
@@ -408,7 +514,15 @@ class Context(object):
         log.debug('Downloading FSL installer manifest from %s', url)
 
         with tempdir(workdir):
-            download_file(url, 'manifest.json')
+
+            try:
+                download_file(url, 'manifest.json')
+            except Exception:
+                log.debug('Error downloading FSL release manifest from %s',
+                          url, exc_info=True)
+                raise Exception('Unable to download FSL release manifest '
+                                'from {}!'.format(url))
+
             with open('manifest.json') as f:
                 lines = f.readlines()
 
@@ -1773,6 +1887,19 @@ def parse_args(argv=None):
         # Do not automatically update the installer script,
         'no_self_update'  : argparse.SUPPRESS,
 
+        # Install a development release. This
+        # option will cause the installer to
+        # download the devrelreases.txt file,
+        # which contains a list of available
+        # internal/development manifests. The
+        # user will be prompted to choose one,
+        # which will be propagated on to the
+        # --manifest option. If --devlatest
+        # is used, the most recent developmet
+        # release is automatically selected.
+        'devrelease'      : argparse.SUPPRESS,
+        'devlatest'       : argparse.SUPPRESS,
+
         # Path to alternative FSL release manifest.
         'manifest'        : argparse.SUPPRESS,
 
@@ -1841,7 +1968,11 @@ def parse_args(argv=None):
     parser.add_argument('--workdir', help=helps['workdir'])
     parser.add_argument('--homedir', help=helps['homedir'],
                         default=op.expanduser('~'))
-    parser.add_argument('--manifest', default=FSL_INSTALLER_MANIFEST,
+    parser.add_argument('--devrelease', help=helps['devrelease'],
+                        action='store_true')
+    parser.add_argument('--devlatest', help=helps['devlatest'],
+                        action='store_true')
+    parser.add_argument('--manifest', default=FSL_RELEASE_MANIFEST,
                         help=helps['manifest'])
     parser.add_argument('--no_self_update', action='store_true',
                         help=helps['no_self_update'])
@@ -1876,6 +2007,9 @@ def parse_args(argv=None):
         if not op.exists(args.workdir):
             os.mkdir(args.workdir)
 
+    if args.devlatest:
+        args.devrelease = True
+
     if args.exclude_package is None:
         args.exclude_package = []
 
@@ -1976,6 +2110,10 @@ def main(argv=None):
     configures the user's environment.
     """
 
+    printmsg('FSL installer version:', EMPHASIS, UNDERLINE, end='')
+    printmsg(' {}'.format(__version__))
+    printmsg('Press CTRL+C at any time to cancel installation', INFO)
+
     args = parse_args(argv)
     ctx  = Context(args)
 
@@ -1986,16 +2124,17 @@ def main(argv=None):
     if not args.no_self_update:
         self_update(ctx.manifest, args.workdir, not args.no_checksum)
 
-    printmsg('FSL installer version:', EMPHASIS, UNDERLINE, end='')
-    printmsg(' {}'.format(__version__))
-    printmsg('Press CTRL+C at any time to cancel installation', INFO)
     printmsg('Installation log file: {}\n'.format(ctx.logfile), INFO)
 
     if args.listversions:
         list_available_versions(ctx.manifest)
         sys.exit(0)
 
-    ctx.finalise_settings()
+    try:
+        ctx.finalise_settings()
+    except Exception as e:
+        printmsg('An error has occurred: {}'.format(e), ERROR)
+        sys.exit(1)
 
     with tempdir(args.workdir):
 

test/test_installer.py

@@ -4,6 +4,7 @@ import os
 import os.path  as op
 import contextlib
 import shutil
+import json
 
 import fslinstaller as inst
 
@@ -96,6 +97,18 @@ packages:
 
 
 
+def patch_manifest(src, dest, latest):
+    with open(src, 'rt') as f:
+        manifest = json.loads(f.read())
+
+    prev                           = manifest['versions']['latest']
+    manifest['versions']['latest'] = latest
+    manifest['versions'][latest]   = manifest['versions'][prev]
+
+    with open(dest, 'wt') as f:
+        f.write(json.dumps(manifest))
+
+
 @contextlib.contextmanager
 def installer_server(cwd=None):
     if cwd is None:
@@ -128,7 +141,19 @@ def installer_server(cwd=None):
         yield srv
 
 
-def check_install(homedir, destdir, version):
+def check_install(homedir, destdir, version, envver=None):
+    # the devrelease test patches the manifest
+    # file with devrelease versions, but leaves
+    # the env files untouched, and referring to
+    # the hard- coded versions in the temlates
+    # above. So the "version" argument specifies
+    # the actual version (which should be written
+    # to $FSLDIR/etc/fslversion), and the
+    # "envver" argument gives the version that
+    # the yml file should refer to.
+
+    if envver is None:
+        envver = version
 
     destdir = op.abspath(destdir)
     etc     = op.join(destdir, 'etc')
@@ -137,8 +162,8 @@ def check_install(homedir, destdir, version):
 
     with indir(destdir):
         # added by our mock conda env creeate call
-        with open(op.join(destdir, 'env-{}.yml'.format(version)), 'rt') as f:
-            exp = mock_env_yml_template.format(version=version)
+        with open(op.join(destdir, 'env-{}.yml'.format(envver)), 'rt') as f:
+            exp = mock_env_yml_template.format(version=envver)
             assert f.read().strip() == exp
 
         # added by our mock conda install call
@@ -152,7 +177,7 @@ def check_install(homedir, destdir, version):
         with open(op.join(etc, 'fslversion'), 'rt') as f:
             assert f.read().strip() == version
         assert op.exists(op.join(etc,     'fslinstaller.py'))
-        assert op.exists(op.join(etc,     'env-{}.yml'.format(version)))
+        assert op.exists(op.join(etc,     'env-{}.yml'.format(envver)))
         assert op.exists(op.join(homedir, 'Documents', 'MATLAB'))
 
         if profile is not None:
@@ -162,7 +187,7 @@ def check_install(homedir, destdir, version):
 def test_installer_normal_interactive_usage():
     with inst.tempdir():
         with installer_server() as srv:
-            with mock.patch('fslinstaller.FSL_INSTALLER_MANIFEST',
+            with mock.patch('fslinstaller.FSL_RELEASE_MANIFEST',
                             '{}/manifest.json'.format(srv.url)):
                 # accept rel/abs paths
                 for i in range(3):
@@ -181,7 +206,7 @@ def test_installer_list_versions():
     platform = inst.Context.identify_platform()
     with inst.tempdir():
         with installer_server() as srv:
-            with mock.patch('fslinstaller.FSL_INSTALLER_MANIFEST',
+            with mock.patch('fslinstaller.FSL_RELEASE_MANIFEST',
                             '{}/manifest.json'.format(srv.url)):
                 with inst.tempdir() as cwd:
                     with CaptureStdout() as cap:
@@ -201,7 +226,7 @@ def test_installer_list_versions():
 def test_installer_normal_cli_usage():
     with inst.tempdir():
         with installer_server() as srv:
-            with mock.patch('fslinstaller.FSL_INSTALLER_MANIFEST',
+            with mock.patch('fslinstaller.FSL_RELEASE_MANIFEST',
                             '{}/manifest.json'.format(srv.url)):
 
                 # accept rel/abs paths
@@ -220,3 +245,47 @@ def test_installer_normal_cli_usage():
                                '--fslversion', '6.1.0'])
                     check_install(cwd, 'fsl', '6.1.0')
                     shutil.rmtree('fsl')
+
+
+def test_installer_devrelease():
+    with inst.tempdir():
+        with installer_server() as srv:
+            with mock.patch('fslinstaller.FSL_DEV_RELEASES',
+                            '{}/devreleases.txt'.format(srv.url)):
+                patch_manifest('manifest.json',
+                               'manifest-6.1.0.20220518.abcdefg.master.json',
+                               '6.1.0.20220518')
+                patch_manifest('manifest.json',
+                               'manifest-6.1.0.20220519.asdjeia.master.json',
+                               '6.1.0.20220519')
+                patch_manifest('manifest.json',
+                               'manifest-6.1.0.20220520.rkjlvis.master.json',
+                               '6.1.0.20220520')
+
+                # the installer should order
+                # entries by date, newest first
+                with open('devreleases.txt', 'wt') as f:
+                    f.write('{}/manifest-6.1.0.20220518.abcdefg.master.json\n'.format(srv.url))
+                    f.write('{}/manifest-6.1.0.20220520.rkjlvis.master.json\n'.format(srv.url))
+                    f.write('{}/manifest-6.1.0.20220519.asdjeia.master.json\n'.format(srv.url))
+
+                with inst.tempdir() as cwd:
+                    dest = 'fsl'
+                    with mock_input('2', dest):
+                        inst.main(['--homedir', cwd, '--devrelease'])
+                    check_install(cwd, dest, '6.1.0.20220519', '6.2.0')
+                    shutil.rmtree(dest)
+                # default option is newest devrelease
+                with inst.tempdir() as cwd:
+                    dest = 'fsl'
+                    with mock_input('', dest):
+                        inst.main(['--homedir', cwd, '--devrelease'])
+                    check_install(cwd, dest, '6.1.0.20220520', '6.2.0')
+                    shutil.rmtree(dest)
+
+                with inst.tempdir() as cwd:
+                    dest = 'fsl'
+                    with mock_input(dest):
+                        inst.main(['--homedir', cwd, '--devlatest'])
+                    check_install(cwd, dest, '6.1.0.20220520', '6.2.0')
+                    shutil.rmtree(dest)