trigger_build.py 5.98 KB
Newer Older
1
2
3
4
5
6
7
8
9
#!/usr/bin/env python
#
# trigger_build.py - Trigger a package build and deployment on one or more FSL
# conda recipe repositories.
#
# Author: Paul McCarthy <pauldmccarthy@gmail.com>
#


10
11
12
13
14
import os.path               as op
import functools             as ft
import textwrap              as tw
import                          argparse
import multiprocessing.dummy as mp
15

16
17
18
19
20
21
22
23
24
25
from   fsl_ci.recipe   import  get_recipe_variable
from   fsl_ci.platform import  get_platform_ids
from   fsl_ci.conda    import  load_meta_yaml
import fsl_ci.gitlab   as      gitlab
from   fsl_ci.gitlab   import (trigger_job,
                               get_variables,
                               download_file,
                               find_latest_job,
                               trigger_pipeline,
                               wait_on_pipeline)
26
27
28
29
30
31
32
33
34


gitlab.VERBOSE = False


SERVER_URL = 'https://git.fmrib.ox.ac.uk'
"""Default gitlab instance URL, if not specified on the command.line."""


35
def get_revision(recipe_path, server, token):
36
37
38
39
    """Return the value of the FSLCONDA_REVISION variable on the given
    conda recipe repository, or None if it is not set.

    The returned revision is only used for staging builds.
40
    """
41
42
43

    meta         = download_file(recipe_path, 'meta.yaml', server, token)
    project_repo = get_recipe_variable(meta, 'repository')
44
    variables    = get_variables(recipe_path, server, token)
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

    # meta.yaml not parseable
    if project_repo is None:
        return None

    # externally hosted project
    if SERVER_URL not in project_repo:
        return None

    # Build off FSLCONDA_REVISION if set, otherwise
    # build off master (remember this only affects
    # staging builds)
    rev = variables.get('FSLCONDA_REVISION', None)

    if rev is not None: return rev
    else:               return 'master'
61
62


63
64
65
def trigger_build(project, server, token, production):
    """Triggers a pipeline on the master branch of project and waits for it
    to complete.
66
67
    """

68
69
    rev = get_revision(project, server, token)

70
71
72
73
74
    if production:
        channel   = 'production'
        variables = {}
    else:
        channel   = 'staging'
75
76
77
78
        variables = {'STAGING' : 'true'}

        if rev is not None:
            variables['FSLCONDA_REVISION'] = rev
79

80
    try:
81
82
83
        pipeline = trigger_pipeline(
            project, 'master', server, token, variables)
    except Exception:
84
85
86
        return None

    pid = pipeline['id']
87

88
89
    print(f'Pipeline triggered on {project} ({channel} build) '
          f'- see {pipeline["web_url"]}')
90

91
92
    try:
        status = wait_on_pipeline(project, pid, server, token)
93
    except Exception:
94
        return None
95

96
    print(f'Build pipeline for {project} has finished: {status}')
97
98

    if status != 'manual':
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
        return None

    return pid


def trigger_deploy(project, pid, server, token, production):
    """Triggers the most recently created manual 'deploy-conda-package' job,
    and waits for it to complete.
    """

    # deployment to staging/production gets set at
    # build time, so we don't need to pass the STAGING
    # variable here, like we do in trigger_build
    if production: channel = 'production'
    else:          channel = 'staging'
114

115
116
117
118
119
    meta      = download_file(project, 'meta.yaml', server, token)
    meta      = load_meta_yaml(meta)
    platforms = get_platform_ids(meta)

    print(f'Triggering deploy-{platforms}-conda-package jobs '
120
          f'on {project} (deploying to {channel} channel)')
121

122
    try:
123
124
        jids = []
        for platform in platforms:
125
            jids.extend(find_latest_job(
126
                project, server, token, f'deploy-{platform}-conda-package', 1))
127
128
        for j in jids:
            trigger_job(project, j['id'], server, token)
129

130
131
132
133
        status = wait_on_pipeline(project, pid, server, token)
        print(f'Deploy job {project} has finished: {status}')
    except Exception as e:
        print(f'Error triggering deploy job on {project}: {e}')
134

135

136
def parseArgs(argv=None):
137
138
139
140
141
142
143
144
145
146
    """Parses and returns command line arguments. """

    name   = op.basename(__file__)
    usage  = f'Usage: {name} -t <token> [options] project [project ...]'
    desc  = tw.dedent("""
    Trigger a package build and deployment on
    one or more FSL conda recipe repositories.
    """).strip()

    helps = {
147
148
149
        'token'      : 'Gitlab API access token with read+write access',
        'server'     :  f'Gitlab server (default: {SERVER_URL})',
        'project'    : 'Project(s) to build',
150
151
        'production' : 'Build production/stable version '
                       '(default: label built package as staging/development)'
152
153
154
155
156
157
158
159
160
    }

    parser = argparse.ArgumentParser(usage=usage, description=desc)
    parser.add_argument('project', nargs='+',
                        help=helps['project'])
    parser.add_argument('-s', '--server', default=SERVER_URL,
                        help=helps['server'])
    parser.add_argument('-t', '--token', required=True,
                        help=helps['token'])
161
162
    parser.add_argument('-p', '--production', action='store_true',
                        help=helps['production'])
163

164
    return parser.parse_args(argv)
165
166


167
def main(argv=None):
168
169
170
171
    """Trigger builds on all listed projects concurrently, and wait for them
    all to complete or fail.
    """

172
    args     = parseArgs(argv)
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
    projects = args.project
    pool     = mp.Pool(len(projects))

    build  = ft.partial(trigger_build,
                        server=args.server,
                        token=args.token,
                        production=args.production)
    deploy = ft.partial(trigger_deploy,
                        server=args.server,
                        token=args.token,
                        production=args.production)

    # Build packages in parallel
    pids = pool.map(build, projects)
    pool.close()
    pool.join()

    # Deploy sequentially, to avoid conflicts
    # arising from simultaneous access to the
    # channel directory
    for project, pid in zip(projects, pids):
        if pid is not None:
            deploy(project, pid)
196
197
198
199


if __name__  == '__main__':
    main()