manifest-gitlab-ci.yml 7.8 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
# CI / CD for the fsl/conda/manifest project
#
# The fsl/conda/manifest project is used to manage
# internal and public FSL releases.  The FSL release
# manifest and environment files are generated
# whenever changes are made to that project.
#
# The CI configuration and scripts used to generate
# and test files are contained in the
# fsl/conda/manifest-rules  project. All jobs are
11
12
# assumed to be executed in an environment in which
# conda is available.
13
14
default:
  before_script:
15
16
17
18
    - |
      mkdir ./conda-dir;
      export CONDA_ENVS_DIRS=$(pwd)/conda-dir/;
      export CONDA_PKGS_DIRS=$(pwd)/conda-dir/.pkgs;
19
      CONDABINDIR=$(dirname $(which conda))/../bin/;
20
      conda create -c conda-forge -p ./env python=3.9;
21
      if [ -f "$CONDABINDIR"/activate ]; then source $CONDABINDIR/activate ./env;
Paul McCarthy's avatar
Paul McCarthy committed
22
      else                                    conda               activate ./env; fi;
23
24
      python -m pip install git+https://git.fmrib.ox.ac.uk/fsl/fsl-ci-rules.git;
      python -m pip install git+https://git.fmrib.ox.ac.uk/fsl/conda/manifest-rules.git;
25
26


Paul McCarthy's avatar
Paul McCarthy committed
27
# Don't run pipeline on MRs
28
29
workflow:
  rules:
Paul McCarthy's avatar
Paul McCarthy committed
30
31
32
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: never
    - when: always
33
34
35


variables:
36
37
38
39
40
41
42
43
44
45
46
47

  # Manifest and environment files can only be published
  # from tags, or from these branches. Attempts to run a
  # publish job from a different branch will fail.
  #
  # Additionally, the test stages will only perform a
  # full test on these branches - a fast test is
  # performed on other branches. See these files for more
  # details:
  #
  #   - fsl/conda/manifest-rules:manifest_rules/test_environment.py
  #   - fsl/conda/manifest-rules:manifest_rules/deploy_files.py
48
  PUBLISH_FROM_BRANCHES: "master external"
49

50
51
52
53
54
55
56
57
58
59
  # The following variables must be added
  # as GitLab CI / CD variables

  # Username/password to authenticate
  # to the internal FSL conda channel
  FSLCONDA_USERNAME: ""
  FSLCONDA_PASSWORD: ""
  # GitLab API access token
  FSL_CI_API_TOKEN:  ""

60
61
62
63
64
65

stages:
  # Generate FSL conda environment specification
  # files for all platforms and CUDA versions
  - generate-environment-files

66
67
68
69
70
71
72
  # Do a test installation for all environment files,
  # and calculate the number of expected output lines
  # so that the fslinstaller script can report
  # progress to the user.  A full test is performed on
  # PUBLISH_FROM_BRANCHES branches, otherwise a fast
  # test is performed - see the test_environmemt.py
  # script.
73
74
75
76
77
78
79
  - test-environments

  # Generate release or development manifest.json
  # files
  - generate-manifest-files

  # Run the fslinstaller.py script against
80
81
82
  # the generated manifest files. This stage
  # is only run on PUBLISH_FROM_BRANCHES
  # branches - see the test_manifest.py script.
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
  - test-manifests

  # Publish the manifest and/or environment files.
  # All jobs in this stage must be manually triggered.
  - publish


# Generate FSL conda environment.yml files, for
# all supported platforms and CUDA versions,
# from fsl-release.yml. Refer to the
# manifest_rules//generate_environment_files.py
# script for details.
generate-environment-files:
  stage: generate-environment-files
  image: continuumio/miniconda3
  tags:
    - fsl-ci
    - docker
  script:
    - generate_environment_files fsl-release.yml ./environments
  artifacts:
Paul McCarthy's avatar
Paul McCarthy committed
104
    expire_in: 24 hrs
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
    paths:
      - environments


# Do a test install of all environment files on
# all platforms.  Information about the
# installation of each environment file is saved
# to test_install/<environment_file>.install.txt.
# The macos-64 test job is run on a macOS shell
# runner - it is assumed that that conda is
# installed in the environment.

# This is a template. Concrete test jobs (below)
# specify a SUFFIX variable, denoting the
# environment file to test.
.test-environments:
  stage: test-environments
  dependencies:
    - generate-environment-files
  # tags and image is overridden by the macOS job
  tags:
    - fsl-ci
    - docker
  script:
    - mkdir ./test_environments
    - test_environment ./fsl-release.yml ./environments/*${SUFFIX}.yml 2>&1 | tee test_environments/${SUFFIX}.install.txt
  artifacts:
Paul McCarthy's avatar
Paul McCarthy committed
132
    expire_in: 24 hrs
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
    when: always
    paths:
      - test_environments

test-environment:macos-64:
  extends: .test-environments
  tags:
    - fsl-ci
    - macOS
  variables:
    SUFFIX: "macos-64"

test-environment:linux-64:
  extends: .test-environments
  image:   fsldevelopment/fsl-linux-64
  variables:
    SUFFIX: "linux-64"


test-environment:linux-64_cudaX.Y:
  extends: .test-environments
  image:   fsldevelopment/fsl-linux-64
  parallel:
    matrix:
Paul McCarthy's avatar
Paul McCarthy committed
157
      - CUDA: ["9.2", "10.2", "11.0", "11.1", "11.3"]
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
  variables:
    SUFFIX: "linux-64_cuda${CUDA}"


# Generate release manifest files. Both release
# and development manifest files are generated
# on every commit, but as the deploy jobs must
# be manually invoked, the user can choose which
# one they want to publish.

.generate-manifest-file:
  stage: generate-manifest-files
  image: continuumio/miniconda3
  tags:
    - fsl-ci
    - docker
  dependencies:
    - generate-environment-files
    - test-environment:linux-64
    - test-environment:macos-64
    - test-environment:linux-64_cudaX.Y
  script:
    - mkdir -p ./manifest
    - generate_manifest_file ./fsl-release.yml "$OFFICIAL" ./manifest ./environments ./test_environments
  artifacts:
Paul McCarthy's avatar
Paul McCarthy committed
183
    expire_in: 24 hrs
184
185
186
187
188
189
190
191
192
193
    paths:
      - manifest

generate-release-manifest-file:
  extends: .generate-manifest-file
  variables:
    OFFICIAL: "true"

generate-development-manifest-file:
  extends: .generate-manifest-file
194
195
  rules:
    - if: '$CI_COMMIT_TAG == null'
196
197
198
199
200
201
  variables:
    OFFICIAL: "false"


# Run the latest fslinstaller.py script
# on the generated manifest and environment
202
# files. Only the release manifest is tested.
203
204
205
206
207
208
#
# This is a template. Each concrete job
# specifies a variable SUFFIX which denotes
# the environment file to be installed.
.test-manifest:
  stage: test-manifests
209
210
  rules:
    - if: '$CI_COMMIT_TAG != null'
211
212
213
214
215
216
  script:
    # there should only be one manifest file, so
    # manifest/*.json will resolve to a single
    # file. Concrete jobs should define SUFFIX
    # so that environments/*${SUFFIX}.yml should
    # also resolve to one file.
217
    - test_manifest manifest/*.json environments/
218
219


220
test-manifest:linux-64:
221
222
223
224
225
226
227
228
229
230
231
232
  stage:   test-manifests
  extends: .test-manifest
  image:   fsldevelopment/fsl-linux-64
  variables:
    SUFFIX: "linux-64"
  tags:
    - fsl-ci
    - docker
  dependencies:
    - generate-environment-files
    - generate-release-manifest-file

233
test-manifest:linux-64_cudaX.Y:
234
235
236
237
238
  stage:   test-manifests
  extends: .test-manifest
  image:   fsldevelopment/fsl-linux-64
  parallel:
    matrix:
Paul McCarthy's avatar
Paul McCarthy committed
239
      - CUDA: ["9.2", "10.2", "11.0", "11.1", "11.3"]
240
241
242
243
244
245
246
247
248
  variables:
    SUFFIX: "linux-64_cuda$CUDA"
  tags:
    - fsl-ci
    - docker
  dependencies:
    - generate-environment-files
    - generate-release-manifest-file

249
test-manifest:macos-64:
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
  stage:   test-manifests
  extends: .test-manifest
  variables:
    SUFFIX: "macos-64"
  tags:
    - fsl-ci
    - macOS
  dependencies:
    - generate-environment-files
    - generate-release-manifest-file


# Deploy/publish environment and manifest files.
# All deploy jobs must be manually invoked.
# Deployment destination is assumed to be
# available as a mount point at /releases/

.deploy-job-template:
  stage: publish
  image: continuumio/miniconda3
  tags:
    - fslconda-channel-host
  when: manual

deploy-environment-files:
  extends: .deploy-job-template
  dependencies:
    - generate-environment-files
  script:
    - deploy_files /releases/ ./environments/*.yml

deploy-development-manifest:
  extends: .deploy-job-template
  dependencies:
    - generate-development-manifest-file
  script:
    - deploy_files /releases/ ./manifest/*.json

deploy-release-manifest:
  extends: .deploy-job-template
  dependencies:
    - generate-release-manifest-file
  script:
    - deploy_files /releases/ ./manifest/*.json