gitlab.py 20.1 KB
Newer Older
1
2
#!/usr/bin/env python
#
3
# Functions for interacting with Gitlab over its HTTP REST API.
4
5
6
7
8
#
# Author: Paul McCarthy <pauldmccarthy@gmail.com>
#

import                    json
9
import                    time
10
import                    datetime
11
import                    fnmatch
12
import base64          as b64
13
import functools       as ft
14
15
16
import urllib.parse    as urlparse
import urllib.request  as urlrequest

17
18
19
20
from fsl_ci.versioning import  is_valid_project_version
from fsl_ci            import (USERNAME,
                               EMAIL)

21

22
23
VERBOSE = True

24

25
26
27
28
def _print(*args, **kwargs):
    if VERBOSE:
        print(*args, **kwargs)

29

30
31
32
33
34
35
36
37
def gen_repository_url(project_path, server, token=None):
    """Generates a URL for the given project. """

    # we've been given a full URL to a git repo
    if project_path.startswith('http') or project_path.startswith('git'):
        return project_path

    if token is not None:
38
        token          = token.strip()
39
40
41
42
43
44
        prefix, suffix = server.split('://')
        server         = f'{prefix}://gitlab-ci-token:{token}@{suffix}'

    return f'{server}/{project_path}.git'


45
46
47
48
49
50
51
52
def http_request(
        url,
        token=None,
        data=None,
        method=None,
        header=False,
        username=None,
        password=None):
53
54
55
56
57
58
    """Submit a HTTP request to the given URL. """

    if method is None:
        if data is None: method = 'GET'
        else:            method = 'POST'

59
    _print(f'{method} {url} ...')
60

61
62
63
64
    headers = {}

    if token is not None:
        headers['PRIVATE-TOKEN'] = token
65
66
67
68
69

    if data is not None:
        headers['Content-Type'] = 'application/json'
        data                    = json.dumps(data).encode('utf-8')

70
        _print(f'    payload: {data}')
71

72
73
74
75
76
77
78
79
80
    if username is not None:
        urlbase = urlparse.urlparse(url).netloc
        pwdmgr = urlrequest.HTTPPasswordMgrWithDefaultRealm()
        pwdmgr.add_password(None, urlbase, username, password)
        handler = urlrequest.HTTPBasicAuthHandler(pwdmgr)
        opener = urlrequest.build_opener(handler)
        opener.open(url)
        urlrequest.install_opener(opener)

81
82
    request  = urlrequest.Request(
        url, headers=headers, data=data, method=method)
83
84
    response = urlrequest.urlopen(request)
    payload  = response.read()
85

86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
    if len(payload) == 0: payload = {}
    else:                 payload = json.loads(payload)

    if header: return payload, response.info()
    else:      return payload


def parse_link_header(header, get='next'):
    """Parses a linmk header, returning the URL corresponding to "get", or
    None if there is no such link/url.

    See https://docs.gitlab.com/ee/api/README.html#pagination
    """

    link = header.get('Link', None)

    if link is None:
        return None

    try:
        links = link.split(', ')

        for link in links:

            url, what = link.split('; ')

            if what == f'rel="{get}"':
                url = url.strip('<>')
                return url

116
    except Exception:
117
118
        pass
    return None
119
120


Paul McCarthy's avatar
Paul McCarthy committed
121
@ft.lru_cache()
122
def lookup_project_id(project_path, server, token):
123
124
    """Look up the integer ID of a gitlab project from its fully qualified
    path.
125
126
127
128
129
130
    """
    project_path = urlparse.quote_plus(project_path)
    url          = f'{server}/api/v4/projects/{project_path}'
    return http_request(url, token)['id']


131
132
133
134
135
136
137
138
139
def project_exists(project_path, server, token):
    """Returns true if the given project exists, false othewrise. """
    try:
        lookup_project_id(project_path, server, token)
        return True
    except Exception:
        return False


140
141
142
143
144
145
146
147
def lookup_namespace_id(namespace_path, server, token):
    """Look up the integer ID of a gitlab namespace from its fully qualified
    path.
    """
    url        = f'{server}/api/v4/namespaces'
    namespaces = http_request(url, token)

    for n in namespaces:
148
        if n['full_path'] == namespace_path:
149
150
151
152
153
            return n['id']

    raise ValueError(f'No namespace matching {namespace_path}')


154
155
156
157
def get_projects_in_namespace(namespace_path, server, token):
    """Returns a list of the paths of all projects in the given namespace.
    """
    nid      = lookup_namespace_id(namespace_path, server, token)
Paul McCarthy's avatar
Paul McCarthy committed
158
159
    url      = f'{server}/api/v4/groups/{nid}/projects'\
                '?per_page=100&archived=False'
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
    projects = []

    # each request gives us max 100 projects
    while True:

        page, header = http_request(url, token=token, header=True)

        projects.extend([p['path_with_namespace'] for p in page])

        url = parse_link_header(header, 'next')
        if url is None:
            break

    return projects


176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
def get_available_runners(project_path, server, token):
    """Returns a list of the IDs of all specific runners
    which are available to be used for the given project,
    and are not already enabled for it.
    """
    # get all specific runners available to the user
    url     = f'{server}/api/v4/runners'
    runners = http_request(url, token)

    # remove inactive ones, and ones that are already
    # enabled
    runners = [r for r in runners if r['online'] and r['active']]
    runners = [r for r in runners if not runner_is_enabled(
        project_path, r['id'], server, token)]

191

192
193
194
195
196
197
198
199
200
201
    return [r['id'] for r in runners]


def get_runner_metadata(runner_id, server, token):
    """Returns metadata for the specified runner. """
    url      = f'{server}/api/v4/runners/{runner_id}'
    response = http_request(url, token)
    return response


202
def get_runner_tags(runner_id, server, token, lower=True):
203
    """Returns the list of tags for the specified runner. """
204
205
206
207
208
209
210

    tags = get_runner_metadata(runner_id, server, token)['tag_list']

    if lower:
        tags = [t.lower() for t in tags]

    return tags
211
212
213
214
215
216


def find_suitable_runners(project_path, tags, server, token):
    """Identifies runners with the specified set of tags which
    are available to be used on the given project.
    """
217
218
219

    tags = [t.lower() for t in tags]

220
    def match(runner_tags):
221
        return all([t in runner_tags for t in tags])
222
223
224
225
226
227
228

    rids  = get_available_runners(project_path, server, token)
    rtags = [get_runner_tags(r, server, token) for r in rids]

    return [r for r, t in zip(rids, rtags) if match(t)]


229
230
def lookup_project_tags(project_path, server, token):
    """Return the a list of tags for the given project, or an empty list
231
    if the project has no tags.
232
233
234

    The tags are sorted such that the most recently updated tag is first
    in the list.
235
236
237
238
    """
    pid  = lookup_project_id(project_path, server, token)
    url  = f'{server}/api/v4/projects/{pid}/repository/tags'
    tags = http_request(url, token)
239
240
241
    tags = [t['name'] for t in tags]

    return tags
242
243


244
def get_project_version(project_path, server, token):
245
246
247
    """Return the most recent version of the specified project, or None if
    the project has no tags, or if there are no tags that are a valid project
    version.
248
    """
249

250
251
    # tags ordered from newest to oldest
    tags = lookup_project_tags(project_path, server, token)
252

253
254
255
    for tag in tags:
        if is_valid_project_version(tag):
            return tag
256

257
    return None
258
259


260
261
262
263
264
265
266
267
def list_project_branches(project_path, server, token):
    """Returns a list of all branches of the project. """
    pid      = lookup_project_id(project_path, server, token)
    url      = f'{server}/api/v4/projects/{pid}/repository/branches'
    response = http_request(url, token)
    return [r['name'] for r in response]


268
269
270
271
def get_revision_hash(project_path, server, token, rev):
    """Returns the commit hash for the specified project revision, assumed to
    be a branch or tag name.
    """
Paul McCarthy's avatar
Paul McCarthy committed
272
    rev      = urlparse.quote_plus(rev)
273
274
275
276
277
278
    pid      = lookup_project_id(project_path, server, token)
    url      = f'{server}/api/v4/projects/{pid}/repository/commits/{rev}'
    response = http_request(url, token)
    return response['id']


279
280
281
282
283
284
285
286
287
288
289
290
291
def get_project_metadata(project_path, server, token):
    """Returns metadata for the specifie project."""
    pid = lookup_project_id(project_path, server, token)
    url = f'{server}/api/v4/projects/{pid}/'
    return http_request(url, token)


def set_project_metadata(project_path, server, token, data):
    """Sets metadata for the specifie project."""
    pid = lookup_project_id(project_path, server, token)
    url = f'{server}/api/v4/projects/{pid}/'
    http_request(url, token, data, method='PUT')

292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
def get_protected_branches(project_path, server, token):
    """Returns a list of all protected branches for the given project. """
    pid      = lookup_project_id(project_path, server, token)
    url      = f'{server}/api/v4/projects/{pid}/protected_branches'
    response = http_request(url, token)
    return [r['name'] for r in response]


def protect_branch(project_path, branch, server, token):
    """Protects the specified branch, so that commits cannot be pushed
    directly to it.
    """
    pid = lookup_project_id(project_path, server, token)

    # GItlab does not let you change the settings
    # on a branch that is already protected, so
    # we first have to unprotect the branch.
    if branch in get_protected_branches(project_path, server, token):
        url = f'{server}/api/v4/projects/{pid}/protected_branches/{branch}'
        http_request(url, token, method='DELETE')

    url  = f'{server}/api/v4/projects/{pid}/protected_branches'
    data = {
        'name'                   : branch,
        'push_access_level'      : '0',
        'merge_access_level'     : '40',
        'unprotect_access_level' : '40',
    }
    http_request(url, token, data=data, method='POST')

322

323
324
325
326
327
328
329
330
331
332
333
334
335
def create_repository(project_path, server, token):
    """Create a repository on gitlab. """
    namespace, name = project_path.rsplit('/', 1)
    namespace       = lookup_namespace_id(namespace, server, token)
    url             = f'{server}/api/v4/projects'
    data            = {
        'name'         : name,
        'visibility'   : 'internal',
        'namespace_id' : namespace
    }
    http_request(url, token, data)


336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
@ft.lru_cache()
def download_file(
        project_path, filename, server, token, ref='master', text=True):
    """Download the specified from the specified branch/ref of the project. """
    pid      = lookup_project_id(project_path, server, token)
    filename = urlparse.quote_plus(filename)
    url      = f'{server}/api/v4/projects/{pid}/repository/files/'
    url      = f'{url}/{filename}?ref={ref}'
    contents = http_request(url, token)['content']

    # contents are base64 encoded
    contents = b64.b64decode(contents)

    # bytes or str
    if text:
        contents = contents.decode('utf-8')

    return contents


def update_file(project_path,
                filename,
                contents,
                message,
                server,
                token,
                branch='master'):
    """Update a file on the specified branch of the project. """

    pid      = lookup_project_id(project_path, server, token)
    filename =  urlparse.quote_plus(filename)
    url      = f'{server}/api/v4/projects/{pid}/repository/files/{filename}'
    data     = {
369
        'file_path'      : urlparse.quote_plus(filename),
370
371
        'content'        : contents,
        'branch'         : branch,
372
373
374
        'commit_message' : message,
        'author_name'    : USERNAME,
        'author_email'   : EMAIL,
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
    }
    http_request(url,  token, data=data, method='PUT')


def runner_is_enabled(project_path, runner_id, server, token):
    """Return True if the given runner is enabled for the given project,
    False otherwise.
    """
    pid     = lookup_project_id(project_path, server, token)
    url     = f'{server}/api/v4/projects/{pid}/runners'
    runners = http_request(url, token)
    rids    = [r['id'] for r in runners]

    return runner_id in rids


def enable_runner(project_path, runner_id, server, token):
    """Enables the specified runner on the specified project."""

    pid  = lookup_project_id(project_path, server, token)
    url  = f'{server}/api/v4/projects/{pid}/runners'
    data = {'runner_id' : runner_id}

    if not runner_is_enabled(project_path, runner_id, server, token):
        http_request(url, token, data)


402
403
404
405
406
407
408
409
410
411
def get_variables(project_path, server, token):
    """Returns a dict containing all environment variables set for the
    project.
    """
    pid      = lookup_project_id(project_path, server, token)
    url      = f'{server}/api/v4/projects/{pid}/variables'
    response = http_request(url, token)
    return {r['key'] : r['value'] for r in response}


412
413
414
415
416
417
def create_variable(project_path,
                    server,
                    token,
                    key,
                    value,
                    masked=False):
418
419
420
    """Creates a new variable on the project. """
    pid  = lookup_project_id(project_path, server, token)
    url  = f'{server}/api/v4/projects/{pid}/variables'
421
    data = dict(key=key, value=value, masked=masked)
422
423
424
    http_request(url, token, data=data)


425
426
427
428
429
430
def update_variable(project_path,
                    server,
                    token,
                    key,
                    value,
                    masked=False):
431
432
433
    """Updates the value of a variable on the project. """
    pid  = lookup_project_id(project_path, server, token)
    url  = f'{server}/api/v4/projects/{pid}/variables/{key}'
434
    data = {'value' : value, 'masked' : masked}
435
436
437
    http_request(url, token, data=data, method='PUT')


438
439
440
441
442
443
def create_or_update_variable(project_path,
                              server,
                              token,
                              key,
                              value,
                              **kwargs):
444
445
446
447
448
    """Create or update a variable on the project. """
    if key in get_variables(project_path, server, token):
        method = update_variable
    else:
        method = create_variable
449
    method(project_path, server, token, key, value, **kwargs)
450
451
452
453
454
455
456


def delete_variable(project_path, server, token, key):
    """Delete a variable from the project. """
    pid  = lookup_project_id(project_path, server, token)
    url  = f'{server}/api/v4/projects/{pid}/variables/{key}'
    http_request(url, token, method='DELETE')
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492


def create_branch(project_path,
                  branch_name,
                  source_branch,
                  server,
                  token):
    """Create a new branch on the project, from the source branch/ref. """
    pid  = lookup_project_id(project_path, server, token)
    url  = f'{server}/api/v4/projects/{pid}/repository/branches'
    data = {
        'branch' : branch_name,
        'ref'    : source_branch,
    }
    http_request(url, token, data)


def open_merge_request(project_path,
                       source_branch,
                       message,
                       server,
                       token,
                       target_branch='master'):
    """Opens a merge request from the source brqanch to the target branch,
    on the given project.
    """
    pid  = lookup_project_id(project_path, server, token)
    url  = f'{server}/api/v4/projects/{pid}/merge_requests'
    data = {
        'id'                   : pid,
        'source_branch'        : source_branch,
        'target_branch'        : target_branch,
        'remove_source_branch' : 'true',
        'title'                : f'WIP: {source_branch}',
        'description'          : message,
    }
493
    return http_request(url, token, data=data)
494
495
496


def trigger_pipeline(project_path,
497
                     ref,
498
                     server,
499
500
                     token,
                     variables=None):
501
502
    """Triggers a CI pipeline on the given project/ref. Returns the pipeline
    integer ID."""
503
    pid  = lookup_project_id(project_path, server, token)
Paul McCarthy's avatar
Paul McCarthy committed
504
    url  = f'{server}/api/v4/projects/{pid}/pipeline'
505
    data = {'ref' : ref}
506

507
    if variables is not None:
508
509
510
        data['variables'] = [{'key'           : k,
                              'variable_type' : 'env_var',
                              'value'         : v}
511
                             for k, v in variables.items()]
512
    return http_request(url, token, data)
513
514


515
516
517
518
519
def get_pipeline_status(project_path, pipeline_id, server, token):
    """Returns information about the given pipeline. """
    pid = lookup_project_id(project_path, server, token)
    url = f'{server}/api/v4/projects/{pid}/pipelines/{pipeline_id}'
    return http_request(url, token)
520

521
522
523
524
525
526
527
528
529

def wait_on_pipeline(project_path, pipeline_id, server, token):
    """Waits until the given pipeline finishes, returning its final status. """

    start = time.time()

    while True:

        secs = int(time.time() - start)
530
        _print(f'Waiting {secs} seconds...', flush=True)
531
532
533
534
535
536
537
        time.sleep(30)

        status = get_pipeline_status(
            project_path, pipeline_id, server, token)['status']

        if status not in ('created', 'waiting_for_resource',
                          'preparing', 'pending', 'running'):
538
            _print(f'Pipeline has stopped - final status: {status}')
539
540
            break

541
        _print(f'Pipeline status: {status}')
542
543

    return status
544
545
546
547
548
549
550
551
552
553
554
555
556
557


def get_project_jobs(project_path, server, token, scope=None, page=1):
    """Returns a list of pipeline jobs that have submitted for the given
    project.
    """
    pid = lookup_project_id(project_path, server, token)
    url = f'{server}/api/v4/projects/{pid}/jobs?page={page}'

    if scope is not None:
        url = f'{url}&scope={scope}'
    return http_request(url, token)


558
559
560
561
def find_latest_job(project_path, server, token, jobpat, age):
    """Return info about the most recent CI jobs with a name matching the
    specified ``jobpat`` (a fnmatch-style wildcard), which was submitted at
    most ``age`` hours ago.
562
563
564
    """
    now   = datetime.datetime.now().astimezone()
    jobs  = get_project_jobs(project_path, server, token)
565
    found = {}
566
567
568
569

    for job in jobs:
        created  = parse_gitlab_date(job['created_at'])
        timediff = (now - created).total_seconds() / 3600
570
571
        name     = job['name']
        match    = fnmatch.fnmatch(name, jobpat)
572

573
574
575
576
577
578
579
580
        # only return the most recent of each
        # uniquely named job (get_project_jobs
        # returns jobs sorted by ID, meaning
        # that they are also sorted by creation
        # time)
        if name in found:
            continue

581
        if all((match, timediff <= age)):
582
            jid = job['id']
583
            _print(f'Found job {name} [ID {jid}] '
584
                   f'scheduled {timediff} hours ago')
585

586
            found[name] = job
587

588
    return list(found.values())
589
590


591
592
593
594
595
596
597
598
599
600
601
def trigger_job(project_path, job_id, server, token):
    """Start the specified CI job."""
    pid = lookup_project_id(project_path, server, token)
    url = f'{server}/api/v4/projects/{pid}/jobs/{job_id}/play'
    return http_request(url, token, method='POST')


def parse_gitlab_date(datestr):
    """2020-11-13T15:13:48.649Z
    """
    timestamp = datestr.split('.')[0]
602
    offset    = datestr.split('+')[1]
603

604
605
606
607
608
    if offset == '':
        offset = 0
    else:
        offset = (datetime.datetime.strptime(offset, '%H:%M') -
                  datetime.datetime(1900, 1, 1, 0, 0)).seconds / 3600.0
609
610
611
612

    timestamp = datetime.datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S')
    offset    = datetime.timezone(datetime.timedelta(hours=offset))
    return timestamp.replace(tzinfo=offset)
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635


def get_divergence(project_path, server, token, branch1, branch2):
    """Get the commits that branch2 is ahead and behind branch1. """

    branches = list_project_branches(project_path, server, token)

    if (branch1 not in branches) or (branch2 not in branches):
        return None, None

    pid     = lookup_project_id(project_path, server, token)
    branch1 = urlparse.quote_plus(branch1)
    branch2 = urlparse.quote_plus(branch2)

    url1    = f'{server}/api/v4/projects/{pid}/repository/compare' + \
              f'?from={branch1}&to={branch2}'
    url2    = f'{server}/api/v4/projects/{pid}/repository/compare' + \
              f'?from={branch2}&to={branch1}'

    ahead   = http_request(url1, token)['commits']
    behind  = http_request(url2, token)['commits']

    return ahead, behind
636
637
638
639
640
641
642
643
644
645
646
647
648
649


def gen_branch_name(branch_name_base, project_path, server, token):
    """Generates a unique branch name on a project repository. """

    allbranches = list_project_branches(project_path, server, token)
    count       = 0
    branch      = branch_name_base

    while branch in allbranches:
        count += 1
        branch = f'{branch_name_base}-{count}'

    return branch