generate_manifest_file.py 8.84 KB
Newer Older
Paul McCarthy's avatar
Paul McCarthy committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
#!/usr/bin/env python
#
# Generate FSL release manifest file from fsl-release.yml


import os.path      as     op
import                     os
import                     sys
import                     glob
import                     json
import urllib.parse as     urlparse
from   collections  import defaultdict

from manifest_rules.utils import (load_release_info,
                                  tempdir,
                                  download_file,
                                  sha256,
                                  parse_environment_file_name,
                                  generate_development_version_identifier)


def load_test_install_info(install_dir):
    """Returns information (number of output lines produced) about
    the test installations. Returns them in a dict.
    """
    inst_info  = defaultdict(dict)
    inst_files = glob.glob(op.join(install_dir, '*.install.txt'))

    for fname in inst_files:
        with open(fname, 'rt') as f:
            lines = f.readlines()
            info = lines[-1].strip()

        envfile, inst_lines, env_lines = info.split()
        platform        = parse_environment_file_name(envfile)[1]
        inst_lines      = int(inst_lines)
        env_lines       = int(env_lines)
        prev_inst_lines = inst_info['miniconda'].get('platform', 0)
        inst_lines      = max((inst_lines, prev_inst_lines))

        inst_info['miniconda'][platform] = inst_lines
        inst_info[envfile]               = env_lines

    return inst_info


def load_previous_manifest(release_info):
    """Downloads the previous official FSL manifest, returning it
    as a dict. Returns None if the manifest cannot be downloaded.
    """
51
    url = urlparse.urljoin(release_info['release-url'], 'manifest.json')
Paul McCarthy's avatar
Paul McCarthy committed
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
    with tempdir():

        try:
            download_file(url, 'manifest.json')
        except Exception:
            return None

        with open('manifest.json', 'rt') as f:
            lines = f.readlines()

    # Drop comments
    lines = [l for l in lines if not l.lstrip().startswith('//')]

    manifest = json.loads('\n'.join(lines))

    # Filter out any releases/builds for which
    # the environment file is not available
    versions = dict(manifest.get('versions', {}))
    for version, builds in versions.items():
71
72
        if version == 'latest':
            continue
Paul McCarthy's avatar
Paul McCarthy committed
73
74
75
        for i, build in enumerate(list(builds)):
            try:
                with tempdir():
76
                    download_file(build['environment'], 'env.yml')
Paul McCarthy's avatar
Paul McCarthy committed
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
            except Exception:
                print(f'WARNING: Build environment file '
                      f'is not available: {build}')
                builds.pop(i)

        if len(builds) == 0:
            versions.pop(version)
        else:
            versions[version] = builds

    latest = versions.get('latest', None)
    if (latest is not None) and (latest not in versions):
        print(f'WARNING: Build for latest FSL '
              f'version {latest} is not available')
        versions.pop('latest')

    manifest['versions'] = versions

    return manifest


def generate_installer_section(release_info):
    """Generates the "installer" manifest section, returning a dict. """
    version  = release_info['installer']
101
    url      = urlparse.urljoin(release_info['release-url'], 'fslinstaller.py')
Paul McCarthy's avatar
Paul McCarthy committed
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165

    try:
        with tempdir():
            download_file(url, 'fslinstaller.py')
            checksum = sha256('fslinstaller.py')
    except Exception:
        print(f'WARNING: fslinstaller.py script not available at {url}!')
        checksum = ''

    return {'version' : version,
            'url'     : url,
            'sha256'  : checksum}


def generate_miniconda_section(release_info, install_info):
    """Generates the "miniconda" manifest section, returning a dict. """
    section = {}

    outputs = install_info.get('miniconda', {})

    for platform, url in release_info['miniconda'].items():
        output = outputs.get(platform, None)
        with tempdir():
            download_file(url, 'miniconda.sh')
            checksum = sha256('miniconda.sh')

            section[platform] = {
                'url'    : url,
                'sha256' : checksum,
            }
            if output is not None:
                section[platform]['output'] = str(output)

    return section


def generate_version_section(version,
                             include_current_release,
                             include_past_releases,
                             envdir,
                             release_info,
                             install_info):
    """Generates the "versions" manifest section, returning a dict.
    Does not add the "latest" entry.
    """

    section = {}

    if include_past_releases:
        prev_manifest = load_previous_manifest(release_info)
        if prev_manifest is not None:
            section = prev_manifest['versions']

    if not include_current_release:
        return section

    versions = []
    section  = {version : versions}

    for envfile in glob.glob(op.join(envdir, '*.yml')):

        checksum = sha256(envfile)
        envfile  = op.basename(envfile)
        output   = install_info.get(envfile, None)
166
        url      = urlparse.urljoin(release_info['release-url'], envfile)
Paul McCarthy's avatar
Paul McCarthy committed
167
168
169
        version, platform, cuda = parse_environment_file_name(envfile)

        build = {
170
171
172
            'platform'      : platform,
            'environment'   : url,
            'sha256'        : checksum,
173
            'base_packages' : release_info['base-packages']
Paul McCarthy's avatar
Paul McCarthy committed
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
        }
        if cuda is not None:
            build['cuda'] = cuda

        versions.append(build)
        if output is not None:
            versions[-1]['output'] = {'install' : str(output)}

    return section


def main():
    # This script is run in the following scenarios:
    #   - New public FSL release - generate a new official
    #     manifest, describing the new release, and all past
    #     public releases
    #   - New internal/development release - generate a
    #     development manifest describing just the development
    #     release
    #   - Update to official manifest - re-generate the
    #     official manifest, describing all past public
    #     releases
    # We use the "official" and "add_current_release"
    # variables to figure out the scenario we are in

    release_file = op.abspath(sys.argv[1])

    # See .gitlab-ci.yml - we are passed "true" or "false"
    # to determine whether we are to generate an official
    # or development manifest
    official = sys.argv[2] == 'true'

    # Directory to save the generated manifest file
    outdir = op.abspath(sys.argv[3])

    # Directory containing the environment files
    # to be added to/described by the manifest
    envdir = op.abspath(sys.argv[4])

    # Directory containing outputs of the test
    # installation
    installdir = op.abspath(sys.argv[5])

    # FSL version identifier
    version = os.environ.get('CI_COMMIT_TAG', None)

    # Add information about the current release
    # (tag or commit) to the generated manifest?
    # We do this for new official releases, or
    # for development releases
    add_current_release = (version is not None) or (not official)

226
227
228
229
230
231
    # The "official" variable controls whether we
    # are generating the official "manifest.json",
    # which typically includes information about
    # past official FSL releases (which is done by
    # downloading the currently available official
    # release manifest).  This can be overridden by
232
    # setting the NO_PAST_RELEASES variable.
233
    add_past_releases = official and ('NO_PAST_RELEASES' not in os.environ)
234

Paul McCarthy's avatar
Paul McCarthy committed
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
    # Generate dev version identnfier
    if version is None:
        version = generate_development_version_identifier()

    # Load fsl-release.yml, and load
    # outputs of test install stage
    release_info = load_release_info(release_file)
    install_info = load_test_install_info(installdir)

    # Generate the manifest
    manifest = {
        'installer' : generate_installer_section(release_info),
        'miniconda' : generate_miniconda_section(release_info, install_info),
        'versions'  : generate_version_section(version,
                                               add_current_release,
250
                                               add_past_releases,
Paul McCarthy's avatar
Paul McCarthy committed
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
                                               envdir,
                                               release_info,
                                               install_info),
    }

    # Set the "latest" field for for
    # new public or developmenmt releases
    if add_current_release:
        manifest['versions']['latest'] = version

    if official:  fname = 'manifest.json'
    else:         fname = f'manifest-{version}.json'

    manifest = json.dumps(manifest, indent=4, sort_keys=True)

    with open(op.join(outdir, fname), 'wt') as f:
        f.write(manifest)

    print(f'Generated manifest file {fname}:')
    print(manifest)


if __name__ == '__main__':
    sys.exit(main())