.gitlab-ci.yml 10.5 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
###########################################################################
# This file defines the build process for fslpy, as hosted at:
#
#    https://git.fmrib.ox.ac.uk/fsl/fslpy
#
# The build pipeline comprises four stages:
#
#    1. test:   Unit tests
#
#    2. doc:    Building API documentation
#
#    3. build:  Building source distributions and wheels
#
#    4. deploy: Uploading the build outputs to pypi, and the documentation
#               to a hosting server.
#
# The test stage is executed on all branches of upstream and fork
# repositories.
#
# The doc stage, and the deploy-doc job, is executed on all branches of the
# upstream repository.
#
# The build stage, and the remaining jobs in the  deploy stage, are only
# executed on the upstream repository, and only for release tags.
#
# The deploy stages are manually instantiated.
###########################################################################


stages:
 - test
 - doc
 - build
 - deploy


37
#################################################################################
38
39
40
41
42
43
44
45
46
47
# A number of variables must be set for the jobs to work. The following
# variables are implicitly defined in any gitlab CI job:
#
#   - CI_PROJECT_PATH    - gitlab namespace/project
#   - CI_COMMIT_REF_NAME - branch name, provided by gitlab
#   - CI_COMMIT_TAG      - present if build is running on a tag
#
# These variables must be explicitly set as "secret" variables:
#
#   - SSH_PRIVATE_KEY_GIT            - private key for git login to remote host
Paul McCarthy's avatar
Paul McCarthy committed
48
49
50
51
#                                      (UPSTREAM_URL)
#
#   - SSH_PRIVATE_KEY_FSL_DOWNLOAD   - private key for downloading some FSL
#                                      files from a remote server (FSL_HOST)
52
53
#
#   - SSH_PRIVATE_KEY_DOC_DEPLOY     - private key for rsyncing documentation
Paul McCarthy's avatar
Paul McCarthy committed
54
#                                      to remote host (DOC_HOST)
55
56
57
#
#   - SSH_SERVER_HOSTKEYS            - List of trusted SSH hosts
#
58
59
60
61
62
63
64
65
#   - DOC_HOST:                      - Username@host to upload documentation to
#                                      (e.g. "paulmc@jalapeno.fmrib.ox.ac.uk")
#
#   - FSL_HOST:                      - Username@host to download FSL data from
#                                      (e.g. "paulmc@jalapeno.fmrib.ox.ac.uk")
#
#   - TWINE_USERNAME:                - Username to use when uploading to pypi
#
66
#   - TWINE_PASSWORD:                - Password to use when uploading to pypi
67
68
#
#   - TWINE_REPOSITORY_URL:          - Pypi repository to upload to
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
###############################################################################


variables:
  UPSTREAM_PROJECT:     "fsl/fslpy"
  UPSTREAM_URL:         "git@git.fmrib.ox.ac.uk"


####################################
# These anchors are used to restrict
# when and where jobs are executed.
####################################


.only_upstream: &only_upstream
  only:
    - branches@fsl/fslpy


.only_master: &only_master
  only:
    - master@fsl/fslpy


.only_releases: &only_releases
  only:
    - tags@fsl/fslpy


.except_releases: &except_releases
  except:
    - tags


##########################################################
# The setup_ssh anchor contains a before_script section
# which does the following:
#
#  - Sets up key-based SSH login, and
#    installs the private keys, so
#    we can connect to servers.
#
#  - Configures git, and adds the
#    upstream repo as a remote
#
# (see https://docs.gitlab.com/ce/ci/ssh_keys/README.html)
#
# NOTE: It is assumed that non-docker
#       executors are already configured
#       (or don't need any configuration).
##########################################################


.setup_ssh: &setup_ssh
  before_script:

    - if [[ -f /.dockerenv ]]; then

        apt-get update -y                 || yum -y check-update           || true;
        apt-get install -y openssh-client || yum install -y openssh-client || true;
        apt-get install -y rsync          || yum install -y rsync          || true;

        eval $(ssh-agent -s);
        mkdir -p $HOME/.ssh;

Paul McCarthy's avatar
Paul McCarthy committed
134
135
        echo "$SSH_PRIVATE_KEY_GIT"          > $HOME/.ssh/id_git;
        echo "$SSH_PRIVATE_KEY_FSL_DOWNLOAD" > $HOME/.ssh/id_fsl_download;
136
137
138
139

        if [[ "$CI_PROJECT_PATH" == "$UPSTREAM_PROJECT" ]]; then
          echo "$SSH_PRIVATE_KEY_DOC_DEPLOY" > $HOME/.ssh/id_doc_deploy;
        fi;
140
141
142
143

        chmod go-rwx $HOME/.ssh/id_*;

        ssh-add $HOME/.ssh/id_git;
Paul McCarthy's avatar
Paul McCarthy committed
144
        ssh-add $HOME/.ssh/id_fsl_download;
145
146
147
148

        if [[ "$CI_PROJECT_PATH" == "$UPSTREAM_PROJECT" ]]; then
          ssh-add $HOME/.ssh/id_doc_deploy;
        fi
149
150
151
152
153

        echo "$SSH_SERVER_HOSTKEYS" > $HOME/.ssh/known_hosts;

        touch $HOME/.ssh/config;

Paul McCarthy's avatar
Paul McCarthy committed
154
155
156
        echo "Host ${UPSTREAM_URL##*@}"                    >> $HOME/.ssh/config;
        echo "    User ${UPSTREAM_URL%@*}"                 >> $HOME/.ssh/config;
        echo "    IdentityFile $HOME/.ssh/id_git"          >> $HOME/.ssh/config;
157

Paul McCarthy's avatar
Paul McCarthy committed
158
159
160
161
        echo "Host docdeploy"                              >> $HOME/.ssh/config;
        echo "    HostName ${DOC_HOST##*@}"                >> $HOME/.ssh/config;
        echo "    User ${DOC_HOST%@*}"                     >> $HOME/.ssh/config;
        echo "    IdentityFile $HOME/.ssh/id_doc_deploy"   >> $HOME/.ssh/config;
162

Paul McCarthy's avatar
Paul McCarthy committed
163
164
165
166
        echo "Host fsldownload"                            >> $HOME/.ssh/config;
        echo "    HostName ${FSL_HOST##*@}"                >> $HOME/.ssh/config;
        echo "    User ${FSL_HOST%@*}"                     >> $HOME/.ssh/config;
        echo "    IdentityFile $HOME/.ssh/id_fsl_download" >> $HOME/.ssh/config;
Paul McCarthy's avatar
Paul McCarthy committed
167

Paul McCarthy's avatar
Paul McCarthy committed
168
169
        echo "Host *"                                      >> $HOME/.ssh/config;
        echo "    IdentitiesOnly yes"                      >> $HOME/.ssh/config;
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191

        git config --global user.name  "Gitlab CI";
        git config --global user.email "gitlabci@localhost";

        if [[ `git remote -v` == *"upstream"* ]]; then
            git remote remove upstream;
        fi;
        git remote add upstream "$UPSTREAM_URL:$UPSTREAM_PROJECT";
      fi


###################################################
# The patch_version anchor contains a before_script
# section which is run on release builds, and makes
# sure that the version in the code is up to date
# (i.e. equal to the tag name).
###################################################


.patch_version: &patch_version
  before_script:

192
    - if [[ "x$CI_COMMIT_TAG" != "x" ]]; then
193
        echo "Release detected - patching version - $CI_COMMIT_REF_NAME";
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
        python -c "import fsl.version as v; v.patchVersion('fsl/version.py', '$CI_COMMIT_REF_NAME')";
      fi


############
# Test stage
############


.test: &test_template
  stage: test
  <<: *setup_ssh

  # Releases are just tags on a release
  # branch, so we don't need to test them.
  <<: *except_releases

  tags:
    - docker

Paul McCarthy's avatar
Paul McCarthy committed
214
  script:
215
216
217
218
219
220
221
222
223
224
225

    # If running on a fork repository, we merge in the
    # upstream/master branch. This is done so that merge
    # requests from fork to the parent repository will
    # have unit tests run on the merged code, something
    # which gitlab CE does not currently do for us.
    - if [[ "$CI_PROJECT_PATH" != "$UPSTREAM_PROJECT" ]]; then
        git fetch upstream;
        git merge --no-commit --no-ff upstream/master;
      fi;

Paul McCarthy's avatar
Paul McCarthy committed
226
227
228
229
    # I am currently assuming that we are
    # running a debian 8/jessie container
    # (the python:2.7 and 3.6 images are
    # based on this).
Paul McCarthy's avatar
Paul McCarthy committed
230
231
232
    # We need to install xvfb, and all of
    # the wxpython dependencies.
    - apt-get update  -y
233
234
    - apt-get install -y xvfb libgtk-3-0
    - apt-get install -y libnotify4 freeglut3 libsdl1.2debian
Paul McCarthy's avatar
Paul McCarthy committed
235
236

    # Linux builds for wxPython are currently not
Paul McCarthy's avatar
Paul McCarthy committed
237
    # on pypi, but are available at this url.
238
239
    - pip install -f https://wxpython.org/Phoenix/release-extras/linux/gtk3/debian-8/ wxpython

Paul McCarthy's avatar
Paul McCarthy committed
240
241
242
    # All other deps can be installed as
    # normal. scipy is required by nibabel,
    # but not listed in its requirements.
243
244
245
246
    # We install test dependencies through
    # pip, because if we let setuptools do
    # it, it will build/install everything
    # from source, rather than using wheels.
247
    - pip install -r requirements.txt
Paul McCarthy's avatar
Paul McCarthy committed
248
    - pip install scipy
249
250
    - pip install sphinx sphinx-rtd-theme
    - pip install pytest pytest-cov pytest-html pytest-runner mock coverage
Paul McCarthy's avatar
Paul McCarthy committed
251
252
253
254

    # We need the FSL atlases for the atlas
    # tests, and need $FSLDIR to be defined
    - export FSLDIR=/fsl/
Paul McCarthy's avatar
Paul McCarthy committed
255
    - mkdir -p $FSLDIR/data/
Paul McCarthy's avatar
Paul McCarthy committed
256
257
    - rsync -rv "fsldownload:data/atlases/" "$FSLDIR/data/atlases/"

258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
    # Finally, run the damned tests.

    # We run some tests under xvfb-run
    # because they invoke wx. Sleep in
    # between, otherwise xvfb gets upset.
    - xvfb-run python setup.py test --addopts="tests/test_async.py"
    - sleep 5
    - xvfb-run python setup.py test --addopts="tests/test_platform.py"

    # We run the immv/imcpy tests as the nobody
    # user because some tests expect permission
    # denied errors when looking at files, and
    # root never gets denied. Make everything in
    # this directory writable by anybody (which,
    # unintuitively, includes nobody)
    - chmod -R a+w `pwd`
    - su -s /bin/bash -c "python setup.py test --addopts='tests/test_immv_imcp.py'" nobody

    # All other tests can be run as normal
    - python setup.py test --addopts="--ignore=tests/test_async.py --ignore=tests/test_platform.py --ignore=tests/test_immv_imcp.py"
278
    - python -m coverage report
279

280
281
282
283
284
285

test:2.7:
  <<: *test_template
  image: python:2.7


Paul McCarthy's avatar
Gaargh    
Paul McCarthy committed
286
test:3.4:
287
  <<: *test_template
Paul McCarthy's avatar
Gaargh    
Paul McCarthy committed
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
  image: python:3.4


# a wxphoenix/3.5 build
# is not yet available
# test:3.5:
#   <<: *test_template
#   image: python:3.5


# a wxphoenix/3.6 build
# is not yet available
# test:3.6:
#   <<: *test_template
#   image: python:3.6
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318


###########
# Doc stage
###########

build-doc:
  <<: *only_upstream
  <<: *patch_version

  tags:
   - docker

  stage: doc
  image: python:3.5

Paul McCarthy's avatar
Paul McCarthy committed
319
  script:
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
    - python setup.py doc
    - mv doc/html doc/"$CI_COMMIT_REF_NAME"
  artifacts:
    paths:
      - doc/$CI_COMMIT_REF_NAME


#############
# Build stage
#############


build-dist:
  <<: *only_releases
  <<: *patch_version

  stage: build
  image: python:3.5
338
339
340
  
  tags:
    - docker
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361

  script:
   - pip install wheel
   - python setup.py sdist
   - python setup.py bdist_wheel

  artifacts:
    paths:
      - dist/*


##############
# Deploy stage
##############


deploy-doc:
  <<: *only_upstream
  <<: *setup_ssh
  stage: deploy
  when:  manual
362
  image: python:3.5
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388

  tags:
    - docker

  dependencies:
    - build-doc

  script:
    - rsync -rv doc/"$CI_COMMIT_REF_NAME" "docdeploy:"


deploy-pypi:
  <<: *only_releases
  <<: *setup_ssh
  stage: deploy
  when:  manual
  image: python:3.5

  tags:
    - docker

  dependencies:
    - build-dist

  script:
    - pip install setuptools wheel twine
389
    - twine upload dist/*