From 231fc84d4afa1da02169b20fbf2dd23988ef66b9 Mon Sep 17 00:00:00 2001
From: Paul McCarthy <pauldmccarthy@gmail.com>
Date: Wed, 8 Nov 2017 12:14:08 +0000
Subject: [PATCH] Moved all CI stuff out of gitlab yml because it is much
cleaner. Added conda skeleton stuff for building conda packages. Still some
work to do
---
.ci/build_conda_dist.sh | 46 ++++++++++
.ci/build_doc.sh | 4 +
.ci/build_pypi_dist.sh | 5 ++
.ci/deploy_doc.sh | 3 +
.ci/deploy_pypi.sh | 4 +
.ci/patch_version.sh | 6 ++
.ci/setup_ssh.sh | 72 ++++++++++++++++
.ci/test_template.sh | 67 +++++++++++++++
.conda/bld.bat | 2 +
.conda/build.sh | 1 +
.conda/meta.yaml | 21 +++++
.gitlab-ci.yml | 184 +++++++---------------------------------
12 files changed, 261 insertions(+), 154 deletions(-)
create mode 100644 .ci/build_conda_dist.sh
create mode 100644 .ci/build_doc.sh
create mode 100644 .ci/build_pypi_dist.sh
create mode 100644 .ci/deploy_doc.sh
create mode 100644 .ci/deploy_pypi.sh
create mode 100644 .ci/patch_version.sh
create mode 100644 .ci/setup_ssh.sh
create mode 100644 .ci/test_template.sh
create mode 100644 .conda/bld.bat
create mode 100644 .conda/build.sh
create mode 100644 .conda/meta.yaml
diff --git a/.ci/build_conda_dist.sh b/.ci/build_conda_dist.sh
new file mode 100644
index 000000000..1e1fd35bc
--- /dev/null
+++ b/.ci/build_conda_dist.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+conda update conda
+conda install setuptools conda-build
+cd .conda
+
+# get version and name
+version=`python ../setup.py -V`
+name=`python ../setup.py --name`
+
+# get requirements, and make
+# them conda compatible...
+
+# strip all spaces
+reqs=`cat ../requirements.txt | sed -e 's/ //g'`
+
+# add a space after package name -
+# package names must match a-zA-Z0-9_
+reqs=`echo "$reqs" | sed -e "s/^[a-zA-Z0-9_][a-zA-Z0-9_]*/& /g"`
+
+# remove ==, replace it with a space
+reqs=`echo "$reqs" | sed -e "s/==/ /g"`
+
+# wrap each dep in quotes
+reqs=`echo "$reqs" | sed -e "s/^.*$/'&'/g"`
+
+# add a comma at the end
+reqs=`echo "$reqs" | sed -e "s/$/,/g"`
+
+# remove newlines
+reqs=`echo $reqs`
+
+echo "version: $version"
+echo "name: $name"
+echo "reqs: $reqs"
+
+echo "{% set name = 'name' %}" > vars.txt
+echo "{% set version = '$version' %}" >> vars.txt
+echo "{% set requirements = [$reqs] %}" >> vars.txt
+cat vars.txt meta.yaml > tempfile
+mv tempfile meta.yaml
+rm vars.txt
+
+conda build fslpy
+
+cd ..
diff --git a/.ci/build_doc.sh b/.ci/build_doc.sh
new file mode 100644
index 000000000..b3d1f20db
--- /dev/null
+++ b/.ci/build_doc.sh
@@ -0,0 +1,4 @@
+#!/usr/bin/env /bash
+
+python setup.py doc
+mv doc/html doc/"$CI_COMMIT_REF_NAME"
diff --git a/.ci/build_pypi_dist.sh b/.ci/build_pypi_dist.sh
new file mode 100644
index 000000000..24d24cb80
--- /dev/null
+++ b/.ci/build_pypi_dist.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+pip install wheel
+python setup.py sdist
+python setup.py bdist_wheel
diff --git a/.ci/deploy_doc.sh b/.ci/deploy_doc.sh
new file mode 100644
index 000000000..652eefd73
--- /dev/null
+++ b/.ci/deploy_doc.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+rsync -rv doc/"$CI_COMMIT_REF_NAME" "docdeploy:"
diff --git a/.ci/deploy_pypi.sh b/.ci/deploy_pypi.sh
new file mode 100644
index 000000000..23746faed
--- /dev/null
+++ b/.ci/deploy_pypi.sh
@@ -0,0 +1,4 @@
+#!/usr/bin/env BASH
+
+pip install setuptools wheel twine
+twine upload dist/*
diff --git a/.ci/patch_version.sh b/.ci/patch_version.sh
new file mode 100644
index 000000000..90f2baa9a
--- /dev/null
+++ b/.ci/patch_version.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+if [[ "x$CI_COMMIT_TAG" != "x" ]]; then
+ echo "Release detected - patching version - $CI_COMMIT_REF_NAME";
+ python -c "import fsl.version as v; v.patchVersion('fsl/version.py', '$CI_COMMIT_REF_NAME')";
+fi
diff --git a/.ci/setup_ssh.sh b/.ci/setup_ssh.sh
new file mode 100644
index 000000000..2ef89d1cc
--- /dev/null
+++ b/.ci/setup_ssh.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env /bash
+
+##########################################################
+# The setup_ssh script does the following:
+#
+# - Sets up key-based SSH login, and
+# installs the private keys, so
+# we can connect to servers.
+#
+# - Configures git, and adds the
+# upstream repo as a remote
+#
+# (see https://docs.gitlab.com/ce/ci/ssh_keys/README.html)
+#
+# NOTE: It is assumed that non-docker
+# executors are already configured
+# (or don't need any configuration).
+##########################################################
+
+if [[ -f /.dockerenv ]]; then
+
+ apt-get update -y || yum -y check-update || true;
+ apt-get install -y openssh-client rsync git || yum install -y openssh-client rsync git || true;
+
+ eval $(ssh-agent -s);
+ mkdir -p $HOME/.ssh;
+
+ echo "$SSH_PRIVATE_KEY_GIT" > $HOME/.ssh/id_git;
+ echo "$SSH_PRIVATE_KEY_FSL_DOWNLOAD" > $HOME/.ssh/id_fsl_download;
+
+ if [[ "$CI_PROJECT_PATH" == "$UPSTREAM_PROJECT" ]]; then
+ echo "$SSH_PRIVATE_KEY_DOC_DEPLOY" > $HOME/.ssh/id_doc_deploy;
+ fi;
+
+ chmod go-rwx $HOME/.ssh/id_*;
+
+ ssh-add $HOME/.ssh/id_git;
+ ssh-add $HOME/.ssh/id_fsl_download;
+
+ if [[ "$CI_PROJECT_PATH" == "$UPSTREAM_PROJECT" ]]; then
+ ssh-add $HOME/.ssh/id_doc_deploy;
+ fi
+
+ echo "$SSH_SERVER_HOSTKEYS" > $HOME/.ssh/known_hosts;
+
+ touch $HOME/.ssh/config;
+
+ echo "Host ${UPSTREAM_URL##*@}" >> $HOME/.ssh/config;
+ echo " User ${UPSTREAM_URL%@*}" >> $HOME/.ssh/config;
+ echo " IdentityFile $HOME/.ssh/id_git" >> $HOME/.ssh/config;
+
+ echo "Host docdeploy" >> $HOME/.ssh/config;
+ echo " HostName ${DOC_HOST##*@}" >> $HOME/.ssh/config;
+ echo " User ${DOC_HOST%@*}" >> $HOME/.ssh/config;
+ echo " IdentityFile $HOME/.ssh/id_doc_deploy" >> $HOME/.ssh/config;
+
+ echo "Host fsldownload" >> $HOME/.ssh/config;
+ echo " HostName ${FSL_HOST##*@}" >> $HOME/.ssh/config;
+ echo " User ${FSL_HOST%@*}" >> $HOME/.ssh/config;
+ echo " IdentityFile $HOME/.ssh/id_fsl_download" >> $HOME/.ssh/config;
+
+ echo "Host *" >> $HOME/.ssh/config;
+ echo " IdentitiesOnly yes" >> $HOME/.ssh/config;
+
+ git config --global user.name "Gitlab CI";
+ git config --global user.email "gitlabci@localhost";
+
+ if [[ `git remote -v` == *"upstream"* ]]; then
+ git remote remove upstream;
+ fi;
+ git remote add upstream "$UPSTREAM_URL:$UPSTREAM_PROJECT";
+fi
diff --git a/.ci/test_template.sh b/.ci/test_template.sh
new file mode 100644
index 000000000..87891ec98
--- /dev/null
+++ b/.ci/test_template.sh
@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+
+# Install $PY_VERSION, xvfb, and all
+# of the wxpython dependencies.
+apt-get update -y || true
+apt-get install -y software-properties-common python-software-properties xvfb libgtk2.0-0 libnotify4 freeglut3 libsdl1.2debian
+add-apt-repository -y ppa:deadsnakes/ppa
+apt-get update -y || true
+apt-get install -y $PY_VERSION "$PY_VERSION"-dev $PY_PACKAGES
+$PY_VENV test.venv
+source test.venv/bin/activate
+pip install --upgrade pip setuptools
+
+# If running on a fork repository, we merge in the
+# upstream/master branch. This is done so that merge
+# requests from fork to the parent repository will
+# have unit tests run on the merged code, something
+# which gitlab CE does not currently do for us.
+if [[ "$CI_PROJECT_PATH" != "$UPSTREAM_PROJECT" ]]; then
+ git fetch upstream;
+ git merge --no-commit --no-ff upstream/master;
+fi;
+
+$INSTALL_WX
+
+# All other deps can be installed as normal.
+# We install test dependenciesd through pip,
+# because if we let setuptools do it, it
+# will build/install everything from source,
+# rather than using wheels.
+pip install -r requirements.txt
+pip install sphinx sphinx-rtd-theme
+pip install pytest pytest-cov pytest-html pytest-runner mock coverage
+
+# style stage
+if [ "$TEST_STYLE"x != "x" ]; then pip install pylint flake8; fi;
+if [ "$TEST_STYLE"x != "x" ]; then flake8 fsl || true; fi;
+if [ "$TEST_STYLE"x != "x" ]; then pylint --output-format=colorized fsl || true; fi;
+if [ "$TEST_STYLE"x != "x" ]; then exit 0; fi
+
+# We need the FSL atlases for the atlas
+# tests, and need $FSLDIR to be defined
+export FSLDIR=/fsl/
+mkdir -p $FSLDIR/data/
+rsync -rv "fsldownload:data/atlases/" "$FSLDIR/data/atlases/"
+
+# Finally, run the damned tests.
+
+# We run some tests under xvfb-run
+# because they invoke wx. Sleep in
+# between, otherwise xvfb gets upset.
+xvfb-run python setup.py test --addopts="$TEST_OPTS tests/test_idle.py"
+sleep 5
+xvfb-run python setup.py test --addopts="$TEST_OPTS tests/test_platform.py"
+
+# We run the immv/imcpy tests as the nobody
+# user because some tests expect permission
+# denied errors when looking at files, and
+# root never gets denied. Make everything in
+# this directory writable by anybody (which,
+# unintuitively, includes nobody)
+chmod -R a+w `pwd`
+su -s /bin/bash -c 'source test.venv/bin/activate && python setup.py test --addopts="$TEST_OPTS tests/test_immv_imcp.py"' nobody
+
+# All other tests can be run as normal
+python setup.py test --addopts="$TEST_OPTS --ignore=tests/test_idle.py --ignore=tests/test_platform.py --ignore=tests/test_immv_imcp.py"
+python -m coverage report
diff --git a/.conda/bld.bat b/.conda/bld.bat
new file mode 100644
index 000000000..c40a9bbef
--- /dev/null
+++ b/.conda/bld.bat
@@ -0,0 +1,2 @@
+"%PYTHON%" setup.py install
+if errorlevel 1 exit 1
diff --git a/.conda/build.sh b/.conda/build.sh
new file mode 100644
index 000000000..a40f1097a
--- /dev/null
+++ b/.conda/build.sh
@@ -0,0 +1 @@
+$PYTHON setup.py install # Python command to install the script.
diff --git a/.conda/meta.yaml b/.conda/meta.yaml
new file mode 100644
index 000000000..af27c70ef
--- /dev/null
+++ b/.conda/meta.yaml
@@ -0,0 +1,21 @@
+package:
+ name: '{{ name }}'
+ version: '{{ version }}'
+
+source:
+ path:
+ ../
+
+requirements:
+ build:
+ - python
+ - setuptools
+ {% for r in requirements %}
+ - {{ r }}
+ {% endfor %}
+
+ run:
+ - python
+ {% for r in requirements %}
+ - {{ r }}
+ {% endfor %}
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4e96f2137..e5893aa75 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -110,81 +110,9 @@ variables:
- tags
-##########################################################
-# The setup_ssh anchor contains a before_script section
-# which does the following:
-#
-# - Sets up key-based SSH login, and
-# installs the private keys, so
-# we can connect to servers.
-#
-# - Configures git, and adds the
-# upstream repo as a remote
-#
-# (see https://docs.gitlab.com/ce/ci/ssh_keys/README.html)
-#
-# NOTE: It is assumed that non-docker
-# executors are already configured
-# (or don't need any configuration).
-##########################################################
-
-
.setup_ssh: &setup_ssh
before_script:
-
- - if [[ -f /.dockerenv ]]; then
-
- apt-get update -y || yum -y check-update || true;
- apt-get install -y openssh-client rsync git || yum install -y openssh-client rsync git || true;
-
- eval $(ssh-agent -s);
- mkdir -p $HOME/.ssh;
-
- echo "$SSH_PRIVATE_KEY_GIT" > $HOME/.ssh/id_git;
- echo "$SSH_PRIVATE_KEY_FSL_DOWNLOAD" > $HOME/.ssh/id_fsl_download;
-
- if [[ "$CI_PROJECT_PATH" == "$UPSTREAM_PROJECT" ]]; then
- echo "$SSH_PRIVATE_KEY_DOC_DEPLOY" > $HOME/.ssh/id_doc_deploy;
- fi;
-
- chmod go-rwx $HOME/.ssh/id_*;
-
- ssh-add $HOME/.ssh/id_git;
- ssh-add $HOME/.ssh/id_fsl_download;
-
- if [[ "$CI_PROJECT_PATH" == "$UPSTREAM_PROJECT" ]]; then
- ssh-add $HOME/.ssh/id_doc_deploy;
- fi
-
- echo "$SSH_SERVER_HOSTKEYS" > $HOME/.ssh/known_hosts;
-
- touch $HOME/.ssh/config;
-
- echo "Host ${UPSTREAM_URL##*@}" >> $HOME/.ssh/config;
- echo " User ${UPSTREAM_URL%@*}" >> $HOME/.ssh/config;
- echo " IdentityFile $HOME/.ssh/id_git" >> $HOME/.ssh/config;
-
- echo "Host docdeploy" >> $HOME/.ssh/config;
- echo " HostName ${DOC_HOST##*@}" >> $HOME/.ssh/config;
- echo " User ${DOC_HOST%@*}" >> $HOME/.ssh/config;
- echo " IdentityFile $HOME/.ssh/id_doc_deploy" >> $HOME/.ssh/config;
-
- echo "Host fsldownload" >> $HOME/.ssh/config;
- echo " HostName ${FSL_HOST##*@}" >> $HOME/.ssh/config;
- echo " User ${FSL_HOST%@*}" >> $HOME/.ssh/config;
- echo " IdentityFile $HOME/.ssh/id_fsl_download" >> $HOME/.ssh/config;
-
- echo "Host *" >> $HOME/.ssh/config;
- echo " IdentitiesOnly yes" >> $HOME/.ssh/config;
-
- git config --global user.name "Gitlab CI";
- git config --global user.email "gitlabci@localhost";
-
- if [[ `git remote -v` == *"upstream"* ]]; then
- git remote remove upstream;
- fi;
- git remote add upstream "$UPSTREAM_URL:$UPSTREAM_PROJECT";
- fi
+ - bash ./.ci/setup_ssh.sh
###################################################
@@ -197,11 +125,7 @@ variables:
.patch_version: &patch_version
before_script:
-
- - if [[ "x$CI_COMMIT_TAG" != "x" ]]; then
- echo "Release detected - patching version - $CI_COMMIT_REF_NAME";
- python -c "import fsl.version as v; v.patchVersion('fsl/version.py', '$CI_COMMIT_REF_NAME')";
- fi
+ - bash ./.ci/patch_version.sh
############
@@ -220,72 +144,7 @@ variables:
- docker
script:
-
- # Install $PY_VERSION, xvfb, and all
- # of the wxpython dependencies.
- - apt-get update -y || true
- - apt-get install -y software-properties-common python-software-properties xvfb libgtk2.0-0 libnotify4 freeglut3 libsdl1.2debian
- - add-apt-repository -y ppa:deadsnakes/ppa
- - apt-get update -y || true
- - apt-get install -y $PY_VERSION "$PY_VERSION"-dev $PY_PACKAGES
- - $PY_VENV test.venv
- - source test.venv/bin/activate
- - pip install --upgrade pip setuptools
-
- # If running on a fork repository, we merge in the
- # upstream/master branch. This is done so that merge
- # requests from fork to the parent repository will
- # have unit tests run on the merged code, something
- # which gitlab CE does not currently do for us.
- - if [[ "$CI_PROJECT_PATH" != "$UPSTREAM_PROJECT" ]]; then
- git fetch upstream;
- git merge --no-commit --no-ff upstream/master;
- fi;
-
- - $INSTALL_WX
-
- # All other deps can be installed as normal.
- # We install test dependenciesd through pip,
- # because if we let setuptools do it, it
- # will build/install everything from source,
- # rather than using wheels.
- - pip install -r requirements.txt
- - pip install sphinx sphinx-rtd-theme
- - pip install pytest pytest-cov pytest-html pytest-runner mock coverage
-
- # style stage
- - if [ "$TEST_STYLE"x != "x" ]; then pip install pylint flake8; fi;
- - if [ "$TEST_STYLE"x != "x" ]; then flake8 fsl || true; fi;
- - if [ "$TEST_STYLE"x != "x" ]; then pylint --output-format=colorized fsl || true; fi;
- - if [ "$TEST_STYLE"x != "x" ]; then exit 0; fi
-
- # We need the FSL atlases for the atlas
- # tests, and need $FSLDIR to be defined
- - export FSLDIR=/fsl/
- - mkdir -p $FSLDIR/data/
- - rsync -rv "fsldownload:data/atlases/" "$FSLDIR/data/atlases/"
-
- # Finally, run the damned tests.
-
- # We run some tests under xvfb-run
- # because they invoke wx. Sleep in
- # between, otherwise xvfb gets upset.
- - xvfb-run python setup.py test --addopts="$TEST_OPTS tests/test_idle.py"
- - sleep 5
- - xvfb-run python setup.py test --addopts="$TEST_OPTS tests/test_platform.py"
-
- # We run the immv/imcpy tests as the nobody
- # user because some tests expect permission
- # denied errors when looking at files, and
- # root never gets denied. Make everything in
- # this directory writable by anybody (which,
- # unintuitively, includes nobody)
- - chmod -R a+w `pwd`
- - su -s /bin/bash -c 'source test.venv/bin/activate && python setup.py test --addopts="$TEST_OPTS tests/test_immv_imcp.py"' nobody
-
- # All other tests can be run as normal
- - python setup.py test --addopts="$TEST_OPTS --ignore=tests/test_idle.py --ignore=tests/test_platform.py --ignore=tests/test_immv_imcp.py"
- - python -m coverage report
+ - bash ./.ci/test_template.sh
test:2.7:
@@ -366,8 +225,8 @@ build-doc:
image: python:3.5
script:
- - python setup.py doc
- - mv doc/html doc/"$CI_COMMIT_REF_NAME"
+ - bash ./.ci/build_doc.sh
+
artifacts:
expire_in: 1 day
paths:
@@ -379,7 +238,7 @@ build-doc:
#############
-build-dist:
+build-pypi-dist:
<<: *only_releases
<<: *patch_version
@@ -390,9 +249,27 @@ build-dist:
- docker
script:
- - pip install wheel
- - python setup.py sdist
- - python setup.py bdist_wheel
+ - bash ./.ci/build_pypi_dist.sh
+
+ artifacts:
+ expire_in: 1 day
+ paths:
+ - dist/*
+
+
+build-conda-dist:
+ <<: *only_releases
+ <<: *patch_version
+
+ stage: build
+ image: continuumio/minconda
+
+ tags:
+ - docker
+
+ script:
+
+ - bash ./.ci/build_conda_dist.sh
artifacts:
expire_in: 1 day
@@ -419,7 +296,7 @@ deploy-doc:
- build-doc
script:
- - rsync -rv doc/"$CI_COMMIT_REF_NAME" "docdeploy:"
+ - bash ./.ci/deploy_doc.sh
deploy-pypi:
@@ -433,8 +310,7 @@ deploy-pypi:
- docker
dependencies:
- - build-dist
+ - build-pypi-dist
script:
- - pip install setuptools wheel twine
- - twine upload dist/*
+ - bash ./.ci/deploy_pypi.sh
--
GitLab