build_conda_package.py 7.01 KB
Newer Older
1
2
3
4
5
6
#!/usr/bin/env python
#
# Build a conda package from a FSL recipe repository.
#
# Author: Paul McCarthy <pauldmccarthy@gmail.com>
#
7
# (See rules/fsl-ci-build-rules.yml)
8
9


10
11
12
import                 os
import os.path      as op
import urllib.parse as urlparse
13

14
from fsl_ci            import (sprun,
15
                               indir,
16
17
                               fprint,
                               add_credentials)
18
from fsl_ci.versioning import  generate_staging_version
19
20
from fsl_ci.platform   import  get_platform_shortcut_if_not_applicable
from fsl_ci.conda      import  load_meta_yaml
21
22
23
from fsl_ci.gitlab     import  get_revision_hash
from fsl_ci.recipe     import (get_recipe_variable,
                               patch_recipe_version)
24
25


26
def build_recipe(recipe_dir, repo, ref, output_dir, *channels):
27
28
29
30
    """Build the conda recipe in the given directory. """

    env = dict(os.environ)

31
32
33
34
    if repo in ('', None): env.pop('FSLCONDA_REPOSITORY', None)
    else:                  env['FSLCONDA_REPOSITORY'] = repo
    if ref  in ('', None): env.pop('FSLCONDA_REVISION', None)
    else:                  env['FSLCONDA_REVISION'] = ref
35

36
37
38
    cmd = 'conda build -c conda-forge -c defaults'
    for chan in channels:
        cmd = f'{cmd} -c {chan}'
39

40
    sprun(f'{cmd} --output-folder={output_dir} {recipe_dir}', env=env)
41
42


43
44
45
46
47
48
49
50
51
def build_report(output_dir):
    """Lists all files produced by build_recipe."""
    for dirpath, _, filenames in os.walk(output_dir):
        for filename in filenames:
            filepath = op.join(dirpath, filename)
            size     = op.getsize(filepath) / 1048576
            fprint(f'{size:8.2f}MB {filepath}')


52
def patch_recipe(recipe_dir, project_ref, server, token):
53
54
    """Patches the package version number in the recipe metadata.

55
56
    For staging builds, a development suffix is added to the most recent
    stable project version number.
57

58
59
60
61
62
    :arg recipe_dir:  Directory to find the recipe meta.yaml
    :arg project_ref: Git reference of project being built
    :arg server:      GitLab server URL
    :arg token:       GitLab API access token
    """
63

64
65
    with open(op.join(recipe_dir, 'meta.yaml')) as f:
        meta = f.read()
66

67
68
69
70
71
    # Get project repository and version string
    # for most recent production release
    repo   = get_recipe_variable(meta, 'repository')
    ver    = get_recipe_variable(meta, 'version')
    gitref = None
72

73
74
75
76
77
78
79
80
81
    # if this is a FSL project, retrieve the git
    # revision and bake it into the version string
    # Staging packages are generally not built for
    # externally hosted projects, but the option
    # is there if we need it. See the
    # generate_staging_version function.
    if repo is not None and server in repo:
        project_path = urlparse.urlparse(repo).path.replace('.git', '')[1:]
        gitref = get_revision_hash(project_path, server, token, project_ref)
82

83
84
    ver  = generate_staging_version(ver, gitref)
    meta = patch_recipe_version(meta, ver)
Paul McCarthy's avatar
Paul McCarthy committed
85

86
87
    with open(op.join(recipe_dir, 'meta.yaml'), 'wt') as f:
        f.write(meta)
88
89


90
def main():
91
92
    """Build a conda package from a FSL recipe repository.

93
94
    This script assumes that the following environment variables
    are set, in addition to the Gitlab CI predefined environment variables:
95

96
     - FSLCONDA_REVISION: Name of the git ref (e.g. tag, branch) to build the
97
98
       recipe from. If empty or unset, ref specified in the recipe meta.yaml
       file is used.
99
100

     - FSLCONDA_REPOSITORY: URL of the git repository to build the recipe
101
102
       from. If empty or unset, the repo specified in the recipe meta.yaml
       file is used.
103

104
105
106
107
108
109
     - STAGING: If equal to "true", the package is marked as a staging/
       development package.

     - FSLCONDA_INTERNAL: Marks this package as internal - if set, the
       FSLCONDA_INTENRAL_CHANNEL_URL will be added to the list of conda
       channels to source from, when building the package.
110
111

     - FSL_CI_API_TOKEN: GitLab API access token
112
    """
113

114
115
116
117
118
    project_ref    = os.environ.get('FSLCONDA_REVISION',   '')
    project_repo   = os.environ.get('FSLCONDA_REPOSITORY', '')
    recipe_url     = os.environ['CI_PROJECT_URL']
    recipe_ref     = os.environ['CI_COMMIT_REF_NAME']
    job_name       = os.environ['CI_JOB_NAME']
119
120
    server_url     = os.environ['CI_SERVER_URL']
    token          = os.environ['FSL_CI_API_TOKEN']
121
    staging        = os.environ['STAGING'].lower() == 'true'
122
    internal       = 'FSLCONDA_INTERNAL' in os.environ
123
124
    username       = os.environ.get('FSLCONDA_INTERNAL_CHANNEL_USERNAME', None)
    password       = os.environ.get('FSLCONDA_INTERNAL_CHANNEL_PASSWORD', None)
125
126
127
    skip_platforms = os.environ.get('FSLCONDA_SKIP_PLATFORM', '')
    skip_platforms = skip_platforms.split()

128
129
130
131
132
133
134
    # get the package name from the un-rendered metayaml text
    with open('meta.yaml', 'rt') as f:
        meta = f.read()
    package_name = get_recipe_variable(meta, 'name')

    # then load the rendered meta.yaml for use
    # by the get_platform_shortcut.. function
135
    meta     = load_meta_yaml('meta.yaml')
136
    platform = get_platform_shortcut_if_not_applicable(
137
        meta, package_name, job_name, skip_platforms)
138

139
140
141
142
143
    # We direct builds for each platform type into a
    # separate sub-directory, because the outputs from
    # all builds will be forwarded to the deploy-package
    # job, and we want to avoid file collisions.
    #
144
145
146
147
148
    # conda has some seriously fucked up behaviour -
    # if you try to direct the build into a directory
    # called "noarch", it thinks that you are directing
    # it to a channel subdirectory, and explodes. So
    # we output to _platform_, rather thah platform.
Paul McCarthy's avatar
Paul McCarthy committed
149
    output_dir = op.join(os.getcwd(), 'conda_build', f'_{platform}_')
150

151
152
153
154
155
156
    pubchan = os.environ['FSLCONDA_PUBLIC_CHANNEL_URL']
    intchan = os.environ['FSLCONDA_INTERNAL_CHANNEL_URL']
    if username is not None:
        intchan = add_credentials(intchan, username, password)
    if internal: channel_urls = [intchan, pubchan]
    else:        channel_urls = [pubchan]
157

158
    os.makedirs(output_dir)
159

160
    fprint('************************************')
161
    fprint(f'Building conda recipe for:                 {package_name}')
162
163
164
165
    fprint(f'Recipe URL:                                {recipe_url}')
    fprint(f'Recipe revision:                           {recipe_ref}')
    fprint( 'Project repository (empty means to ')
    fprint(f'  build from repo specified in meta.yaml): {project_repo}')
166
    fprint( 'Project revision (empty means to ')
167
    fprint(f'  build release specified in meta.yaml):   {project_ref}')
168
    fprint(f'FSL conda channel URLs:                    {channel_urls}')
169
    fprint('************************************')
170
171
172
173

    if project_repo == '': project_repo = None
    if project_ref  == '': project_ref  = None

174
    # patch the recipe so it has a staging version number
175
    if staging and project_ref is not None:
176
        patch_recipe('.', project_ref, server_url, token)
177

178
179
180
181
    build_recipe('.',
                 project_repo,
                 project_ref,
                 output_dir,
182
                 *channel_urls)
183
184
    with indir(output_dir):
        build_report('.')
185
186
187
188


if __name__ == '__main__':
    main()