.gitlab-ci.yml 10.3 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
###########################################################################
# This file defines the build process for fslpy, as hosted at:
#
#    https://git.fmrib.ox.ac.uk/fsl/fslpy
#
# The build pipeline comprises four stages:
#
#    1. test:   Unit tests
#
#    2. doc:    Building API documentation
#
#    3. build:  Building source distributions and wheels
#
#    4. deploy: Uploading the build outputs to pypi, and the documentation
#               to a hosting server.
#
# The test stage is executed on all branches of upstream and fork
# repositories.
#
# The doc stage, and the deploy-doc job, is executed on all branches of the
# upstream repository.
#
# The build stage, and the remaining jobs in the  deploy stage, are only
# executed on the upstream repository, and only for release tags.
#
# The deploy stages are manually instantiated.
###########################################################################


stages:
 - test
 - doc
 - build
 - deploy


###############################################################################
# A number of variables must be set for the jobs to work. The following
# variables are implicitly defined in any gitlab CI job:
#
#   - CI_PROJECT_PATH    - gitlab namespace/project
#   - CI_COMMIT_REF_NAME - branch name, provided by gitlab
#   - CI_COMMIT_TAG      - present if build is running on a tag
#
# These variables must be explicitly set as "secret" variables:
#
#   - SSH_PRIVATE_KEY_GIT            - private key for git login to remote host
Paul McCarthy's avatar
Paul McCarthy committed
48
49
50
51
#                                      (UPSTREAM_URL)
#
#   - SSH_PRIVATE_KEY_FSL_DOWNLOAD   - private key for downloading some FSL
#                                      files from a remote server (FSL_HOST)
52
53
#
#   - SSH_PRIVATE_KEY_DOC_DEPLOY     - private key for rsyncing documentation
Paul McCarthy's avatar
Paul McCarthy committed
54
#                                      to remote host (DOC_HOST)
55
56
57
58
59
60
61
62
63
64
65
#
#   - SSH_SERVER_HOSTKEYS            - List of trusted SSH hosts
#
#   - TWINE_PASSWORD:                - Password to use when uploading to pypi
###############################################################################


variables:
  UPSTREAM_PROJECT:     "fsl/fslpy"
  UPSTREAM_URL:         "git@git.fmrib.ox.ac.uk"
  DOC_HOST:             "paulmc@jalapeno.fmrib.ox.ac.uk"
Paul McCarthy's avatar
Paul McCarthy committed
66
  FSL_HOST:             "paulmc@jalapeno.fmrib.ox.ac.uk"
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
  TWINE_USERNAME:       "pauldmccarthy"
  TWINE_REPOSITORY_URL: "https://testpypi.python.org/pypi"


####################################
# These anchors are used to restrict
# when and where jobs are executed.
####################################


.only_upstream: &only_upstream
  only:
    - branches@fsl/fslpy


.only_master: &only_master
  only:
    - master@fsl/fslpy


.only_releases: &only_releases
  only:
    - tags@fsl/fslpy


.except_releases: &except_releases
  except:
    - tags


##########################################################
# The setup_ssh anchor contains a before_script section
# which does the following:
#
#  - Sets up key-based SSH login, and
#    installs the private keys, so
#    we can connect to servers.
#
#  - Configures git, and adds the
#    upstream repo as a remote
#
# (see https://docs.gitlab.com/ce/ci/ssh_keys/README.html)
#
# NOTE: It is assumed that non-docker
#       executors are already configured
#       (or don't need any configuration).
##########################################################


.setup_ssh: &setup_ssh
  before_script:

    - if [[ -f /.dockerenv ]]; then

        apt-get update -y                 || yum -y check-update           || true;
        apt-get install -y openssh-client || yum install -y openssh-client || true;
        apt-get install -y rsync          || yum install -y rsync          || true;

        eval $(ssh-agent -s);
        mkdir -p $HOME/.ssh;

Paul McCarthy's avatar
Paul McCarthy committed
128
129
        echo "$SSH_PRIVATE_KEY_GIT"          > $HOME/.ssh/id_git;
        echo "$SSH_PRIVATE_KEY_FSL_DOWNLOAD" > $HOME/.ssh/id_fsl_download;
130
131
132
133

        if [[ "$CI_PROJECT_PATH" == "$UPSTREAM_PROJECT" ]]; then
          echo "$SSH_PRIVATE_KEY_DOC_DEPLOY" > $HOME/.ssh/id_doc_deploy;
        fi;
134
135
136
137

        chmod go-rwx $HOME/.ssh/id_*;

        ssh-add $HOME/.ssh/id_git;
Paul McCarthy's avatar
Paul McCarthy committed
138
        ssh-add $HOME/.ssh/id_fsl_download;
139
140
141
142

        if [[ "$CI_PROJECT_PATH" == "$UPSTREAM_PROJECT" ]]; then
          ssh-add $HOME/.ssh/id_doc_deploy;
        fi
143
144
145
146
147

        echo "$SSH_SERVER_HOSTKEYS" > $HOME/.ssh/known_hosts;

        touch $HOME/.ssh/config;

Paul McCarthy's avatar
Paul McCarthy committed
148
149
150
        echo "Host ${UPSTREAM_URL##*@}"                    >> $HOME/.ssh/config;
        echo "    User ${UPSTREAM_URL%@*}"                 >> $HOME/.ssh/config;
        echo "    IdentityFile $HOME/.ssh/id_git"          >> $HOME/.ssh/config;
151

Paul McCarthy's avatar
Paul McCarthy committed
152
153
154
155
        echo "Host docdeploy"                              >> $HOME/.ssh/config;
        echo "    HostName ${DOC_HOST##*@}"                >> $HOME/.ssh/config;
        echo "    User ${DOC_HOST%@*}"                     >> $HOME/.ssh/config;
        echo "    IdentityFile $HOME/.ssh/id_doc_deploy"   >> $HOME/.ssh/config;
156

Paul McCarthy's avatar
Paul McCarthy committed
157
158
159
160
        echo "Host fsldownload"                            >> $HOME/.ssh/config;
        echo "    HostName ${FSL_HOST##*@}"                >> $HOME/.ssh/config;
        echo "    User ${FSL_HOST%@*}"                     >> $HOME/.ssh/config;
        echo "    IdentityFile $HOME/.ssh/id_fsl_download" >> $HOME/.ssh/config;
Paul McCarthy's avatar
Paul McCarthy committed
161

Paul McCarthy's avatar
Paul McCarthy committed
162
163
        echo "Host *"                                      >> $HOME/.ssh/config;
        echo "    IdentitiesOnly yes"                      >> $HOME/.ssh/config;
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185

        git config --global user.name  "Gitlab CI";
        git config --global user.email "gitlabci@localhost";

        if [[ `git remote -v` == *"upstream"* ]]; then
            git remote remove upstream;
        fi;
        git remote add upstream "$UPSTREAM_URL:$UPSTREAM_PROJECT";
      fi


###################################################
# The patch_version anchor contains a before_script
# section which is run on release builds, and makes
# sure that the version in the code is up to date
# (i.e. equal to the tag name).
###################################################


.patch_version: &patch_version
  before_script:

186
    - if [[ "x$CI_COMMIT_TAG" != "x" ]]; then
187
        echo "Release detected - patching version - $CI_COMMIT_REF_NAME";
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
        python -c "import fsl.version as v; v.patchVersion('fsl/version.py', '$CI_COMMIT_REF_NAME')";
      fi


############
# Test stage
############


.test: &test_template
  stage: test
  <<: *setup_ssh

  # Releases are just tags on a release
  # branch, so we don't need to test them.
  <<: *except_releases

  tags:
    - docker

Paul McCarthy's avatar
Paul McCarthy committed
208
  script:
209
210
211
212
213
214
215
216
217
218
219

    # If running on a fork repository, we merge in the
    # upstream/master branch. This is done so that merge
    # requests from fork to the parent repository will
    # have unit tests run on the merged code, something
    # which gitlab CE does not currently do for us.
    - if [[ "$CI_PROJECT_PATH" != "$UPSTREAM_PROJECT" ]]; then
        git fetch upstream;
        git merge --no-commit --no-ff upstream/master;
      fi;

Paul McCarthy's avatar
Paul McCarthy committed
220
221
222
223
    # I am currently assuming that we are
    # running a debian 8/jessie container
    # (the python:2.7 and 3.6 images are
    # based on this).
Paul McCarthy's avatar
Paul McCarthy committed
224
225
226
    # We need to install xvfb, and all of
    # the wxpython dependencies.
    - apt-get update  -y
227
228
    - apt-get install -y xvfb libgtk-3-0
    - apt-get install -y libnotify4 freeglut3 libsdl1.2debian
Paul McCarthy's avatar
Paul McCarthy committed
229
230

    # Linux builds for wxPython are currently not
Paul McCarthy's avatar
Paul McCarthy committed
231
    # on pypi, but are available at this url.
232
233
    - pip install -f https://wxpython.org/Phoenix/release-extras/linux/gtk3/debian-8/ wxpython

Paul McCarthy's avatar
Paul McCarthy committed
234
235
236
    # All other deps can be installed as
    # normal. scipy is required by nibabel,
    # but not listed in its requirements.
237
238
239
240
    # We install test dependencies through
    # pip, because if we let setuptools do
    # it, it will build/install everything
    # from source, rather than using wheels.
241
    - pip install -r requirements.txt
Paul McCarthy's avatar
Paul McCarthy committed
242
    - pip install scipy
243
244
    - pip install sphinx sphinx-rtd-theme
    - pip install pytest pytest-cov pytest-html pytest-runner mock coverage
Paul McCarthy's avatar
Paul McCarthy committed
245
246
247
248

    # We need the FSL atlases for the atlas
    # tests, and need $FSLDIR to be defined
    - export FSLDIR=/fsl/
Paul McCarthy's avatar
Paul McCarthy committed
249
    - mkdir -p $FSLDIR/data/
Paul McCarthy's avatar
Paul McCarthy committed
250
251
    - rsync -rv "fsldownload:data/atlases/" "$FSLDIR/data/atlases/"

252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
    # Finally, run the damned tests.

    # We run some tests under xvfb-run
    # because they invoke wx. Sleep in
    # between, otherwise xvfb gets upset.
    - xvfb-run python setup.py test --addopts="tests/test_async.py"
    - sleep 5
    - xvfb-run python setup.py test --addopts="tests/test_platform.py"

    # We run the immv/imcpy tests as the nobody
    # user because some tests expect permission
    # denied errors when looking at files, and
    # root never gets denied. Make everything in
    # this directory writable by anybody (which,
    # unintuitively, includes nobody)
    - chmod -R a+w `pwd`
    - su -s /bin/bash -c "python setup.py test --addopts='tests/test_immv_imcp.py'" nobody

    # All other tests can be run as normal
    - python setup.py test --addopts="--ignore=tests/test_async.py --ignore=tests/test_platform.py --ignore=tests/test_immv_imcp.py"
272
    - python -m coverage report
273

274
275
276
277
278
279

test:2.7:
  <<: *test_template
  image: python:2.7


Paul McCarthy's avatar
Gaargh    
Paul McCarthy committed
280
test:3.4:
281
  <<: *test_template
Paul McCarthy's avatar
Gaargh    
Paul McCarthy committed
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
  image: python:3.4


# a wxphoenix/3.5 build
# is not yet available
# test:3.5:
#   <<: *test_template
#   image: python:3.5


# a wxphoenix/3.6 build
# is not yet available
# test:3.6:
#   <<: *test_template
#   image: python:3.6
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312


###########
# Doc stage
###########

build-doc:
  <<: *only_upstream
  <<: *patch_version

  tags:
   - docker

  stage: doc
  image: python:3.5

Paul McCarthy's avatar
Paul McCarthy committed
313
  script:
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
    - python setup.py doc
    - mv doc/html doc/"$CI_COMMIT_REF_NAME"
  artifacts:
    paths:
      - doc/$CI_COMMIT_REF_NAME


#############
# Build stage
#############


build-dist:
  <<: *only_releases
  <<: *patch_version

  stage: build
  image: python:3.5

  script:
   - pip install wheel
   - python setup.py sdist
   - python setup.py bdist_wheel

  artifacts:
    paths:
      - dist/*


##############
# Deploy stage
##############


deploy-doc:
  <<: *only_upstream
  <<: *setup_ssh
  stage: deploy
  when:  manual
353
  image: python:3.5
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379

  tags:
    - docker

  dependencies:
    - build-doc

  script:
    - rsync -rv doc/"$CI_COMMIT_REF_NAME" "docdeploy:"


deploy-pypi:
  <<: *only_releases
  <<: *setup_ssh
  stage: deploy
  when:  manual
  image: python:3.5

  tags:
    - docker

  dependencies:
    - build-dist

  script:
    - pip install setuptools wheel twine
380
    - twine upload dist/*