__init__.py 5.57 KB
Newer Older
1
2
#!/usr/bin/env python
#
3
# __init__.py - Miscellaneous functions used throughout fsl_ci.
4
5
6
7
8
#
# Author: Paul McCarthy <pauldmccarthy@gmail.com>
#


9
import os.path    as op
10
import               os
11
import               io
12
import               sys
13
import               errno
14
import               shlex
15
import               string
16
import               time
17
18
19
20
import               tempfile
import contextlib as ctxlib
import subprocess as sp

21
import yaml
22

23

Paul McCarthy's avatar
Paul McCarthy committed
24
__version__ = '0.7.0'
25
26
27
"""Current version of the fsl-ci-rules."""


28
29
30
31
32
33
34
35
USERNAME = 'fsl-ci-rules'
"""Username to be used for all git interactions which require one. """


EMAIL = 'fsl-ci-rules@git.fmrib.ox.ac.uk'
"""Password to be used for all git interactions which require one. """


36
37
38
39
40
def fprint(*args, **kwargs):
    """Print with flush=True. """
    print(*args, **kwargs, flush=True)


41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
@ctxlib.contextmanager
def tempdir():
    """Context manager to create, and change into, a temporary directory, and
    then afterwards delete it and change back to the original working
    directory.
    """
    with tempfile.TemporaryDirectory() as td:
        prevdir = os.getcwd()
        os.chdir(td)
        try:
            yield td
        finally:
            os.chdir(prevdir)


@ctxlib.contextmanager
def indir(dirname):
    """Context manager to change into a directory, and then afterwards
    change back to the original working directory.
    """
    prevdir = os.getcwd()
    os.chdir(dirname)
    try:
        yield
    finally:
        os.chdir(prevdir)


def sprun(cmd, **kwargs):
    """Runs the given command with subprocess.run. """
71
    fprint(f'Running {cmd}')
72
73
    if not kwargs.get('shell', False):
        cmd = shlex.split(cmd)
74
75
76
77

    # we set stdout/stderr for compatibility
    # with CaptureStdout (see below)
    if 'check'  not in kwargs: kwargs['check']  = True
78
79
80
81

    if 'capture_output' not in kwargs:
        if 'stdout' not in kwargs: kwargs['stdout'] = sys.stdout
        if 'stderr' not in kwargs: kwargs['stderr'] = sys.stderr
82

83
    return sp.run(cmd, **kwargs)
84
85
86
87
88
89
90
91
92
93
94
95
96
97


def is_valid_project_version(version):
    """Return True if the given version/tag is "valid" - it must
    be a sequence of integers, separated by periods, with an optional
    leading 'v'.
    """
    if version.lower().startswith('v'):
        version = version[1:]

    for part in version.split('.'):
        if not all([c in string.digits for c in part]):
            return False
    return True
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112


@ctxlib.contextmanager
def lockdir(dirname):
    """Primitive mechanism by which concurrent access to a directory can be
    prevented. Attempts to create a semaphore file in the directory, but waits
    if that file already exists. Removes the file when finished.
    """

    delay    = 10
    lockfile = op.join(dirname, '.fsl_ci.lockdir')

    while True:
        try:
            fprint(f'Attempting to lock {dirname} for exclusive access.')
Paul McCarthy's avatar
Paul McCarthy committed
113
            fd = os.open(lockfile, os.O_CREAT | os.O_EXCL | os.O_RDWR)
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
            break
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise e
            fprint(f'{dirname} is already locked - '
                   f'trying again in {delay} seconds ...')
        time.sleep(10)

    fprint(f'Exclusive access acquired for {dirname} ...')
    try:
        yield

    finally:
        fprint(f'Relinquishing lock on {dirname}')
        os.close( fd)
        os.unlink(lockfile)
130
131


132
133
def loadyaml(s):
    """Loads a YAML string, returning a dict-like. """
134
    return yaml.load(s)
135
136
137
138


def dumpyaml(o):
    """Dumps the given YAML to a string. """
139
    return yaml.dump(o, sort_keys=False)
140
141


142
143
144
145
class CaptureStdout:
    """Context manager which captures stdout and stderr. """

    def __init__(self):
146
147
148
        self.tmpdir        = tempfile.TemporaryDirectory()
        self.__mock_stdout = open(op.join(self.tmpdir.name, 'stdout'), 'wt')
        self.__mock_stderr = open(op.join(self.tmpdir.name, 'stderr'), 'wt')
149
150
151


    def __enter__(self):
152
153
154
        if self.tmpdir is None:
            raise RuntimeError('CaptureStdout can only be used once')

155
156
        self.__real_stdout = sys.stdout
        self.__real_stderr = sys.stderr
157
158
        sys.stdout         = self.__mock_stdout
        sys.stderr         = self.__mock_stderr
159

160
        return self
161
162

    def __exit__(self, *args, **kwargs):
163

164
165
166
        sys.stdout = self.__real_stdout
        sys.stderr = self.__real_stderr

167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
        self.__mock_stdout.close()
        self.__mock_stderr.close()

        with open(self.__mock_stdout.name, 'rt') as f:
            self.__mock_stdout = f.read()
        with open(self.__mock_stderr.name, 'rt') as f:
            self.__mock_stderr = f.read()

        self.tmpdir.cleanup()
        self.tmpdir = None

    @property
    def real_stdout(self):
        return self.__real_stdout

    @property
    def real_stderr(self):
        return self.__real_stderr

186
187
    @property
    def stdout(self):
188
        return self.__mock_stdout
189
190
191

    @property
    def stderr(self):
192
        return self.__mock_stderr
193
194


195
196
197
198
199
200
201
202
203
def health_check():
    """Rudimentary health check of the fsl-ci-rules codebase. """
    print(f'fsl-ci-rules version {__version__}')
    import fsl_ci.conda
    import fsl_ci.gitlab
    import fsl_ci.scripts.build_conda_package
    import fsl_ci.scripts.deploy_conda_package
    import fsl_ci.scripts.trigger_package_build
    import fsl_ci.scripts.update_conda_recipe
Paul McCarthy's avatar
Paul McCarthy committed
204
    import fsl_ci.scripts.run_unit_tests
205
206
207
208
209
210
211
    import fsl_ci.utils.configure_repositories
    import fsl_ci.utils.create_conda_recipe
    import fsl_ci.utils.fsl_project_dependencies
    import fsl_ci.utils.package_status
    import fsl_ci.utils.set_gitlab_variables
    import fsl_ci.utils.trigger_build
    import fsl_ci.utils.trigger_pipeline