fsl-ci-job-template.yml 4.66 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
###############################################################################
# This file defines a standard job template that may be extended by concrete
# job definitions. It defines standard before_script and after_script rules
# which are common to mots jobs executed via the fsl-ci-rules. It is included
# by the main fsl-ci-rules/.gitlab-ci.yml configuration file.
#
# Other FSL CI jobs may use the before_script and after_script by extending the
# .fsl-ci-rules-job template.
###############################################################################


.fsl-ci-rules-job:

  # Initial configuration for all CI jobs
  before_script:
    # CI jobs are either run within a Docker container,
17
18
19
20
21
22
    # which is nice, or on a physical/virtual machine
    # (macOS builds and deploy jobs), which is not nice,
    # because multiple jobs may be running concurrently,
    # which means that it is difficult to configure git
    # and conda. We do things differently depending on
    # which environment we are in.
23
24
25
26
27
28
    #
    # (Things would be *so* much easier if both git and
    # conda made it easier to set custom configuration
    # file paths, but they don't.)
    - if [ -f /.dockerenv ]; then DOCKER="1"; else DOCKER="0"; fi

29
    # Create a unique working dir for this job -
30
31
32
33
34
35
    # we do this to prevent conflicts in conda
    # package caches/environment dirs between
    # jobs which are running concurrently on
    # the same machine (e.g. macOS builds).
    # We write its location to a file, so the
    # after_script can delete it
36
37
38
39
40
    - if [ -z "$TMPDIR" ]; then
        FSL_CI_CONFIG_DIR=/tmp/fsl_ci_config_${CI_JOB_ID};
      else
        FSL_CI_CONFIG_DIR=$TMPDIR/fsl_ci_config_${CI_JOB_ID};
      fi
41
42
43
44
45
46
    - mkdir ${FSL_CI_CONFIG_DIR}
    - echo  ${FSL_CI_CONFIG_DIR} > ./.fsl-ci-config-dir

    # Allow cloning and pushing to other repositories
    # from gitlab by using the FSL_CI_API_TOKEN. Also
    # always use https://, and not git@ URLs.
47
48
49
    # On shared machines (e.g. macOS), we have to
    # set this in the global (user) git configuration,
    # so we assume that this is ok.
50
51
52
53
54
55
56
57
58
59
    #
    # We fall back to the CI_JOB_TOKEN if FSL_CI_API_TOKEN
    # is not defined - this gives us read-only (clone)
    # access, although this mightchange in a future
    # version of gitlab.
    - if [ ! -z "$FSL_CI_API_TOKEN" ]; then
        giturl=https://gitlab-ci-token:${FSL_CI_API_TOKEN}@git.fmrib.ox.ac.uk;
      else
        giturl=https://gitlab-ci-token:${CI_JOB_TOKEN}@git.fmrib.ox.ac.uk;
      fi
60
61
62
63
64
    - |
      GIT_CREDENTIALS=${FSL_CI_CONFIG_DIR}/.git-credentials;
      echo ${giturl} > ${GIT_CREDENTIALS};
      git config --global credential.helper "store --file ${GIT_CREDENTIALS}";
      git config --global "url.${giturl}/.insteadOf" "git@git.fmrib.ox.ac.uk:";
65
      git config --global "url.${giturl}.insteadOf"  "https://git.fmrib.ox.ac.uk";
66
67
68
69
70
71
72
73

    # Isolate conda configuration to unique
    # config dir
    - export CONDA_CHANNEL_PRIORITY=strict
    - export CONDA_AUTO_UPDATE_CONDA=false
    - export CONDA_ENVS_DIRS=${FSL_CI_CONFIG_DIR}/conda
    - export CONDA_PKGS_DIRS=${FSL_CI_CONFIG_DIR}/conda/.pkgs

74
75
    # If running on docker, install fsl-ci-rules
    # directly into the miniconda base environment.
76
    # Otherwise create a conda env in the job work
77
78
79
    # dir, and install fsl-ci-rules into it. If
    # conda is not on $PATH, assume it is in
    # ${HOME}/miniconda3/
80
81
82
83
84
85
    #
    # If the <conda>/bin/activate script is not
    # present for some reason, we assume that
    # conda has been configured in the shell.
    # This is the case on the FSL conda deploy
    # runner.
86
    - if [ "$DOCKER" == "0" ]; then
87
88
89
        CONDABIN=$(which conda || echo "${HOME}/miniconda3/bin/conda");
        CONDADIR=$(dirname ${CONDABIN})/../bin/;
        ${CONDABIN} create -y -p ${FSL_CI_CONFIG_DIR}/env -c conda-forge python=3.9 conda-build;
90
        if [ -f "${CONDADIR}"/activate ]; then source ${CONDADIR}/activate ${FSL_CI_CONFIG_DIR}/env;
91
        else                                   conda activate              ${FSL_CI_CONFIG_DIR}/env; fi;
92
      fi
93
    - if [ "$CI_PROJECT_PATH" == "fsl/conda/fsl-ci-rules" ]; then
94
95
        python -m pip install .;
      else
96
97
        for ((i=1; i<6; i++)); do
          echo "Installing CI rules from git (attempt $i / 5)...";
98
          (python -m pip install --no-cache git+${CI_RULES_REPOSITORY}@${CI_RULES_BRANCH} && touch success) || true;
99
          if [ -f success ]; then break; fi;
100
          sleep 5;
101
102
        done;
        if [ ! -f success ]; then exit 1; fi;
103
      fi
104
105
106
107
108
109

  # Post-job - remove the custom config directory
  # that was created in before_script.
  after_script:
    - FSL_CI_CONFIG_DIR=$(cat ./.fsl-ci-config-dir)
    - rm -r ${FSL_CI_CONFIG_DIR}
110
    - rm ./.fsl-ci-config-dir