test_environment.py 4.72 KB
Newer Older
Paul McCarthy's avatar
Paul McCarthy committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
#!/usr/bin/env python
#
# Performs a test installation for a FSL conda environment file. Counts
# the number of output lines produced when installing miniconda, and
# when installing the FSL environment.
#


import os.path  as op
import             os
import             sys
import             shutil

import yaml

from manifest_rules.utils import (sprun,
                                  tempdir,
                                  download_file,
                                  load_release_info,
                                  parse_environment_file_name)


def preprocess_environment(envfile, condadir, regen_envfile=False):
    with open(envfile, 'rt') as f:
        env = f.read()
    env = yaml.load(env, Loader=yaml.Loader)

    packages = env['dependencies']
    channels = env['channels']
    with open(op.join(condadir, 'condarc'), 'wt') as f:
        f.write('channels:\n')
        for c in channels:
            f.write(f' - {c}\n')

    if regen_envfile:
        with open(envfile, 'wt') as f:
            f.write('channels:\n')
            for c in channels:
                f.write(f' - {c}\n')
            f.write('dependencies:\n')
            for p in packages:
                f.write(f' - {p}\n')

44
    for i, pkg in enumerate(packages):
45
46
47
48
49
50
51
52
53
        # package specs are of the form
        #     <package> <version>
        # or
        #     <package> <version> <variant>
        #
        # and are transformed into the form
        #     <package>=<version>
        # or
        #     <package>=<version>=<variant>
54
        pkg, ver    = pkg.split(maxsplit=1)
55
        ver         = ver.replace(' ', '=')
56
57
        packages[i] = f'{pkg}={ver}'

Paul McCarthy's avatar
Paul McCarthy committed
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
    return packages


def fast_test(envfile, release_info):

    install_out = sprun('conda create -y -p fsl')

    packages = preprocess_environment(envfile, 'fsl')

    env = os.environ.copy()
    env['CONDARC'] = op.abspath(op.join('fsl', 'condarc'))

    cmd = 'conda install -p fsl -y --dry-run ' + ' '.join(packages)
    env_out = sprun(cmd, env=env)

    return install_out, env_out


def full_test(envfile, release_info):

    platform      = parse_environment_file_name(envfile)[1]
    miniconda_url = release_info['miniconda'][platform]
80
    base_packages = release_info['base-packages']
81
    base_packages = sorted(base_packages, key=len, reverse=True)
Paul McCarthy's avatar
Paul McCarthy committed
82
83
84
85
86
87
88

    download_file(miniconda_url, 'miniconda.sh')

    install_out = sprun('sh ./miniconda.sh -b -p ./fsl')

    preprocess_environment(envfile, 'fsl', True)

89
    basepkgs = {}
90
91
    with open(envfile, 'rt') as f:
        for line in f:
92
            for pkg in base_packages:
93
                if line.strip().startswith(f'- {pkg} '):
94
95
                    pkgver        = line.strip().split(maxsplit=2)[2]
                    basepkgs[pkg] = pkgver.replace(' ', '=')
96

97
    env            = os.environ.copy()
Paul McCarthy's avatar
Paul McCarthy committed
98
99
    env['CONDARC'] = op.abspath(op.join('fsl', 'condarc'))

100
    if len(basepkgs) == 0:
101
102
        env_out = ''
    else:
103
104
105
106
        basepkgs = [f'{pkg}={ver}' for pkg, ver in basepkgs.items()]
        basepkgs = ' '.join(basepkgs)
        env_out  = sprun(f'./fsl/bin/conda install -y -n base {basepkgs}',
                         env=env)
107
108

    env_out += sprun(f'./fsl/bin/conda env update -n base -f {envfile}',
Paul McCarthy's avatar
Paul McCarthy committed
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
                     env=env)

    return install_out, env_out


def main():
    release_file  = op.abspath(sys.argv[1])
    envfile       = op.abspath(sys.argv[2])
    release_info  = load_release_info(release_file)

    publish_from_branches = os.environ.get('PUBLISH_FROM_BRANCHES', '')
    branch_name           = os.environ.get('CI_COMMIT_BRANCH', None)
    is_tag                = 'CI_COMMIT_TAG' in os.environ

    run_full_test = is_tag or ((branch_name is not None) and
                               (branch_name in publish_from_branches))

    username = os.environ.get('FSLCONDA_USERNAME', 'username')
    password = os.environ.get('FSLCONDA_PASSWORD', 'password')

    with tempdir():

        shutil.copy(envfile, '.environment.yml')

        with open('.environment.yml', 'rt') as inf, \
             open(envfile,            'wt') as outf:
            contents = inf.read()
            contents = contents.replace('${FSLCONDA_USERNAME}', username)
            contents = contents.replace('${FSLCONDA_PASSWORD}', password)
            outf.write(contents)

        if run_full_test:
            install_out, env_out = full_test(envfile, release_info)
        else:
            install_out, env_out = fast_test(envfile, release_info)

145
146
        print(install_out.decode('utf-8'))
        print(env_out    .decode('utf-8'))
Paul McCarthy's avatar
Paul McCarthy committed
147

148
149
        install_lines = len(install_out.split(b'\n'))
        env_lines     = len(env_out    .split(b'\n'))
Paul McCarthy's avatar
Paul McCarthy committed
150
151
152
153
154
155
156
        envfile       = op.basename(envfile)

        print(f'{envfile} {install_lines} {env_lines}')


if __name__ == '__main__':
    sys.exit(main())