.gitlab-ci.yml 7.63 KB
Newer Older
Paul McCarthy's avatar
Paul McCarthy committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
# CI / CD for the fsl/conda/manifest project
#
# The fsl/conda/manifest project is used to manage
# internal and public FSL releases.  The FSL release
# manifest and environment files are generated
# whenever changes are made to this project.
#
# The scripts used to generate and test files files
# are contained in the fsl/conda/manifest-rules
# project. All jobs are executed in an environment
# in which conda is available.
default:
  before_script:
    - conda create -c conda-forge -p ./env python=3.9
    - source activate ./env
    - python -m pip install git+https://git.fmrib.ox.ac.uk/fsl/conda/manifest-rules.git


# Don't run pipeline on merge requests
workflow:
  rules:
    - if: '$CI_COMMIT_TAG != null || $CI_COMMIT_BRANCH != null'


# Manifest and environment files can only be published
# from tags, or from these branches. Attempts to run a
# publish job from a different branch will fail.

# Additionally, the test stages will only perform a
# full test on these branches - a fast test is
# performed on other branches. See these files for more
# details:
#
#   - fsl/conda/manifest-rules:manifest_rules/test_environment.py
#   - fsl/conda/manifest-rules:manifest_rules/deploy_files.py
variables:
  PUBLISH_FROM_BRANCHES: "master internal"


stages:
  # Generate FSL conda environment specification
  # files for all platforms and CUDA versions
  - generate-environment-files

  # Do a test installation for all environment
  # files, and calculate the number of expected
  # output lines so that the fslinstaller
  # script can report progress to the user.
  - test-environments

  # Generate release or development manifest.json
  # files
  - generate-manifest-files

  # Run the fslinstaller.py script against
  # the generated manifest files
  - test-manifests

  # Publish the manifest and/or environment files.
  # All jobs in this stage must be manually triggered.
  - publish


64
65
66
67
# Generate FSL conda environment.yml files, for
# all supported platforms and CUDA versions,
# from fsl-release.yml. Refer to the
# manifest_rules//generate_environment_files.py
Paul McCarthy's avatar
Paul McCarthy committed
68
69
70
71
72
73
74
75
# script for details.
generate-environment-files:
  stage: generate-environment-files
  image: continuumio/miniconda3
  tags:
    - fsl-ci
    - docker
  script:
76
    - generate_environment_files fsl-release.yml ./environments
Paul McCarthy's avatar
Paul McCarthy committed
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
  artifacts:
    expire_in: 2 hrs
    paths:
      - environments


# Do a test install of all environment files on
# all platforms.  Information about the
# installation of each environment file is saved
# to test_install/<environment_file>.install.txt.
# The macos-64 test job is run on a macOS shell
# runner - it is assumed that that conda is
# installed in the environment.

# This is a template. Concrete test jobs (below)
# specify a SUFFIX variable, denoting the
# environment file to test.
.test-environments:
  stage: test-environments
  dependencies:
    - generate-environment-files
  # tags and image is overridden by the macOS job
  tags:
    - fsl-ci
    - docker
  script:
    - mkdir ./test_environments
104
    - test_environment ./fsl-release.yml ./environments/*${SUFFIX}.yml 2>&1 | tee test_environments/${SUFFIX}.install.txt
Paul McCarthy's avatar
Paul McCarthy committed
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
  artifacts:
    expire_in: 2 hrs
    when: always
    paths:
      - test_environments

test-environment:macos-64:
  extends: .test-environments
  tags:
    - fsl-ci
    - macOS
  variables:
    SUFFIX: "macos-64"

test-environment:linux-64:
  extends: .test-environments
121
  image:   fsldevelopment/fsl-linux-64
Paul McCarthy's avatar
Paul McCarthy committed
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
  variables:
    SUFFIX: "linux-64"


test-environment:linux-64_cudaX.Y:
  extends: .test-environments
  image:   fsldevelopment/fsl-linux-64
  parallel:
    matrix:
      - CUDA: ["9.2", "10.2", "11.0"]
  variables:
    SUFFIX: "linux-64_cuda${CUDA}"


# Generate release manifest files. Both release
# and development manifest files are generated
# on every commit, but as the deploy jobs must
# be manually invoked, the user can choose which
# one they want to publish.

.generate-manifest-file:
  stage: generate-manifest-files
  image: continuumio/miniconda3
  tags:
    - fsl-ci
    - docker
  dependencies:
    - generate-environment-files
    - test-environment:linux-64
    - test-environment:macos-64
    - test-environment:linux-64_cudaX.Y
  script:
    - mkdir -p ./manifest
155
    - generate_manifest_file ./fsl-release.yml "$OFFICIAL" ./manifest ./environments ./test_environments
Paul McCarthy's avatar
Paul McCarthy committed
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
  artifacts:
    expire_in: 2 hrs
    paths:
      - manifest

generate-release-manifest-file:
  extends: .generate-manifest-file
  variables:
    OFFICIAL: "true"

generate-development-manifest-file:
  extends: .generate-manifest-file
  variables:
    OFFICIAL: "false"


# Run the latest fslinstaller.py script
# on the generated manifest and environment
# files.
#
# This is a template. Each concrete job
# specifies a variable SUFFIX which denotes
# the environment file to be installed.
.test-manifest:
  stage: test-manifests
  script:
    # there should only be one manifest file, so
    # manifest/*.json will resolve to a single
    # file. Concrete jobs should define SUFFIX
    # so that environments/*${SUFFIX}.yml should
    # also resolve to one file.
    - test_manifest manifest/*.json environments/*${SUFFIX}.yml


test-release-manifest:linux-64:
  stage:   test-manifests
  extends: .test-manifest
  image:   fsldevelopment/fsl-linux-64
  variables:
    SUFFIX: "linux-64"
  tags:
    - fsl-ci
    - docker
  dependencies:
    - generate-environment-files
    - generate-release-manifest-file


test-development-manifest:linux-64:
  stage:   test-manifests
  extends: .test-manifest
  image:   fsldevelopment/fsl-linux-64
  variables:
    SUFFIX: "linux-64"
  tags:
    - fsl-ci
    - docker
  dependencies:
    - generate-environment-files
    - generate-development-manifest-file

test-release-manifest:linux-64_cudaX.Y:
  stage:   test-manifests
  extends: .test-manifest
  image:   fsldevelopment/fsl-linux-64
  parallel:
    matrix:
      - CUDA: ["9.2", "10.2", "11.0"]
  variables:
    SUFFIX: "linux-64_cuda$CUDA"
  tags:
    - fsl-ci
    - docker
  dependencies:
    - generate-environment-files
    - generate-release-manifest-file


test-development-manifest:linux-64_cudaX.Y:
  stage:   test-manifests
  extends: .test-manifest
  image:   fsldevelopment/fsl-linux-64
  parallel:
    matrix:
      - CUDA: ["9.2", "10.2", "11.0"]
  variables:
    SUFFIX: "linux-64_cuda${CUDA}"
  tags:
    - fsl-ci
    - docker
  dependencies:
    - generate-environment-files
    - generate-development-manifest-file


test-release-manifest:macos-64:
  stage:   test-manifests
  extends: .test-manifest
  variables:
    SUFFIX: "macos-64"
  tags:
    - fsl-ci
    - macOS
  dependencies:
    - generate-environment-files
    - generate-release-manifest-file


test-development-manifest:macos-64:
  stage:   test-manifests
  extends: .test-manifest
  variables:
    SUFFIX: "macos-64"
  tags:
    - fsl-ci
    - macOS
  dependencies:
    - generate-environment-files
    - generate-development-manifest-file


# Deploy/publish environment and manifest files.
# All deploy jobs must be manually invoked.
# Deployment destination is assumed to be
# available as a mount point at /releases/

.deploy-job-template:
  stage: publish
  image: continuumio/miniconda3
  tags:
    - fsl-ci
    - fslconda-channel-host
  when: manual

deploy-environment-files:
  extends: .deploy-job-template
  dependencies:
    - generate-environment-files
  script:
    - deploy_files /releases/ ./environments/*.yml

deploy-development-manifest:
  extends: .deploy-job-template
  dependencies:
    - generate-development-manifest-file
  script:
    - deploy_files /releases/ ./manifest/*.json

deploy-release-manifest:
  extends: .deploy-job-template
  dependencies:
    - generate-release-manifest-file
  script:
    - deploy_files /releases/ ./manifest/*.json