utils.py 7.06 KB
Newer Older
Paul McCarthy's avatar
Paul McCarthy committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
#!/usr/bin/env python
# TODO change #output lines to characters?
#
#
# utils.py - Miscellaneous utility functions.


import functools       as ft
import os.path         as op
import subprocess      as sp
import                    os
import                    re
import                    shlex
import                    shutil
import                    hashlib
import                    tempfile
import                    contextlib
import urllib.request  as urlrequest

import                    yaml
import dateutil.parser as dateutil

23
24
from fsl_ci.gitlab import lookup_project_tags

Paul McCarthy's avatar
Paul McCarthy committed
25
26
27

def sprun(cmd, **kwargs):
    """Runs the given command with subprocess.run, returning its standard
28
    output as a byte string (this can be overridden by passing text=True).
Paul McCarthy's avatar
Paul McCarthy committed
29
30
31
    """
    print(f'Running: {cmd}')
    cmd    = shlex.split(cmd)
32
    result = sp.run(cmd, capture_output=True, check=False, **kwargs)
Paul McCarthy's avatar
Paul McCarthy committed
33
34

    if result.returncode != 0:
35
36
37
38
39
40
        if kwargs.get('text', False):
            stdout = result.stdout
            stderr = result.stderr
        else:
            stdout = result.stdout.decode('utf-8')
            stderr = result.stderr.decode('utf-8')
Paul McCarthy's avatar
Paul McCarthy committed
41
        err  = f'Called process returned {result.returncode}: {cmd}\n'
42
43
        err += f'standard output:\n{stdout}\n'
        err += f'standard error:\n{stderr}'
Paul McCarthy's avatar
Paul McCarthy committed
44
45
        print(err, flush=True)
        raise RuntimeError(err)
46
    return result.stdout
Paul McCarthy's avatar
Paul McCarthy committed
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84


def get_most_recent_release():
    """Returns the version identifier of the most recent public FSL release.
    """

    @ft.total_ordering
    class Version:
        def __init__(self, verstr):
            # Version identifiers for official FSL
            # releases will have up to four
            # components (X.Y.Z.W), but We accept
            # any number of (integer) components,
            # as internal releases may have more.
            components = []

            for comp in verstr.split('.'):
                try:              components.append(int(comp))
                except Exception: break

            self.components = components
            self.verstr     = verstr

        def __str__(self):
            return self.verstr

        def __eq__(self, other):
            for sn, on in zip(self.components, other.components):
                if sn != on:
                    return False
            return len(self.components) == len(other.components)

        def __lt__(self, other):
            for p1, p2 in zip(self.components, other.components):
                if p1 < p2: return True
                if p1 > p2: return False
            return len(self.components) < len(other.components)

85
    # get tag list from gitlab if possible
86
87
88
    project = os.environ.get('CI_PROJECT_PATH',  None)
    server  = os.environ.get('CI_SERVER_URL',    None)
    token   = os.environ.get('FSL_CI_API_TOKEN', None)
89
90
91
92
93
94
95
96
97
98

    if all ((project is not None,
             server  is not None,
             token   is not None)):
        tags = lookup_project_tags(project, server, token)

    else:
        tags = sprun('git tag --list', text=True).split('\n')
        tags = [t.strip() for t in tags]

Paul McCarthy's avatar
Paul McCarthy committed
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
    tags = [Version(t) for t in tags if t != '']
    tags = sorted(tags)

    # 6.0.5 was the last non-conda-based FSL release
    if len(tags) == 0: return '6.0.5'
    else:              return str(tags[-1])


def generate_development_version_identifier():
    """Generate a version string for a development/internal FSL release.

    The version string incorporates the current git commit hash, branch name,
    and date.
    """

    # get info about the current branch/commit/date.
    # Use gitlab CI vars if possible, fall back to
    # git call-out
    tag    = get_most_recent_release()
    commit = os.environ.get('CI_COMMIT_SHORT_SHA', None)
    branch = os.environ.get('CI_COMMIT_BRANCH',    None)
    date   = os.environ.get('CI_COMMIT_TIMESTAMP', None)

    if commit is None:
123
        commit = sprun('git rev-parse --short      HEAD', text=True)
Paul McCarthy's avatar
Paul McCarthy committed
124
    if branch is None:
125
        branch = sprun('git rev-parse --abbrev-ref HEAD', text=True)
Paul McCarthy's avatar
Paul McCarthy committed
126
127
128
129
    if date is not None:
        date = dateutil.isoparse(date)
        date = date.strftime('%Y%m%d')
    else:
130
131
        date = sprun('git show -s --format=%cd --date=format:"%Y%m%d" HEAD',
                     text=True)
Paul McCarthy's avatar
Paul McCarthy committed
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176

    # make usable as part of a file name,
    # and make sure no underscores, as we
    # use them as separators in env file
    # names
    branch = re.sub(r'[^a-zA-Z0-9]', '-', branch.lower())

    return f'{tag}.{date}.{commit}.{branch}'


def generate_environment_file_name(version, platform, cudaver):
    """Generate a file name for the FSL conda environment file
    for version, platform and CUDA version. If version is None,
    a development version identifier is generated.
    """

    if version is None:
        version = generate_development_version_identifier()

    base = f'fsl-{version}_{platform}'

    if cudaver is None: return f'{base}.yml'
    else:               return f'{base}_cuda{cudaver}.yml'


def parse_environment_file_name(filename):
    """Parses the given FSL environment file name, returning the
    version, platform, and CUDA version (None if the environment
    file does not correspond to a CUDA build).
    """

    pat      = r'fsl-([^_]+)_([^_]+)(?:_cuda([\d]+\.[\d]+))?\.yml'
    filename = op.basename(filename)
    match    = re.fullmatch(pat, filename)

    if match is not None: return match.groups()
    else:                 return (None, None, None)


def load_release_info(fsl_release_file):
    """Loads fsl-release.yml, returning its contents as a dict. """
    with open(fsl_release_file, 'rt') as f:
        return yaml.load(f.read(), Loader=yaml.Loader)


177
178
179
180
181
182
183
184
185
186
187
188
def get_platform_identifiers(release_info):
    """Returns a list of all platform identifiers for which FSL environment
    files are being generated, e.g. ['linux-64', 'macos-64', 'macos-M1'].
    """
    platforms  = os.environ.get('FSLCONDA_PLATFORMS', None)
    if platforms is None:
        platforms = list(release_info['miniconda'].keys())
    else:
        platforms = platforms.split(',')
    return platforms


Paul McCarthy's avatar
Paul McCarthy committed
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
@contextlib.contextmanager
def tempdir():
    """Returns a context manager which creates, changes into, and returns a
    temporary directory, and then deletes it on exit.
    """

    tmpdir  = tempfile.mkdtemp()
    prevdir = os.getcwd()

    try:
        os.chdir(tmpdir)
        yield tmpdir

    finally:
        os.chdir(prevdir)
        shutil.rmtree(tmpdir)


def download_file(url, destination, blocksize=1048576):
    """Download a file from url, saving it to destination. """
    print(f'Downloading {url}')
    with urlrequest.urlopen(url) as req, \
         open(destination, 'wb') as outf:
        while True:
            block = req.read(blocksize)
            if len(block) == 0:
                break
            outf.write(block)


def sha256(filename):
    """Calculate the SHA256 checksum of the given file. """
    blocksize = 1048576
    hashobj   = hashlib.sha256()
    with open(filename, 'rb') as f:
        while True:
            block = f.read(blocksize)
            if len(block) == 0:
                break
            hashobj.update(block)
    return hashobj.hexdigest()