From 1e95b86d9a7f61c89196e47a7c853c616aa4ce17 Mon Sep 17 00:00:00 2001
From: Paul McCarthy <pauldmccarthy@gmail.com>
Date: Wed, 8 Nov 2017 18:43:21 +0000
Subject: [PATCH] CI for conda. Only building 3.6 because conda is a complete
pain in the arse. Tweaks to other CI scripts
---
.ci/build_conda_dist.sh | 41 +++++++++++-----------------
.ci/build_doc.sh | 2 ++
.ci/build_pypi_dist.sh | 2 ++
.ci/deploy_conda.sh | 2 ++
.ci/deploy_doc.sh | 2 ++
.ci/deploy_pypi.sh | 2 ++
.ci/patch_version.sh | 2 ++
.ci/setup_ssh.sh | 11 +++++++-
.ci/test_template.sh | 2 ++
.conda/conda_build_config.yaml | 2 ++
.conda/meta.yaml | 2 +-
.gitlab-ci.yml | 49 +++++++++++++++++++++++++++-------
12 files changed, 82 insertions(+), 37 deletions(-)
create mode 100644 .conda/conda_build_config.yaml
diff --git a/.ci/build_conda_dist.sh b/.ci/build_conda_dist.sh
index 07d4f87be..0ef6e7a81 100644
--- a/.ci/build_conda_dist.sh
+++ b/.ci/build_conda_dist.sh
@@ -1,37 +1,28 @@
#!/usr/bin/env bash
-conda update conda
-conda install setuptools conda-build
+set -e
-mkdir -p dist
-
-cd .conda
+name=$1
+version=$2
-# get version and name. We call
-# setup.py beforehand because it
-# will install a bunch of deps,
-# and output a bunch of stuff.
-python ../setup.py -V &> /dev/null
-version=`python ../setup.py -V`
-name=`python ../setup.py --name`
+# add any extra channels that are needed
+for channel in $CONDA_CHANNELS; do
+ conda config --append channels $channel
+done
-# invoking setup.py causes it to
-# install deps, which conda will
-# incklude in thye build
-rm -rf .eggs
-
-# insert name/version into meta.yaml
+# insert project name/version into meta.yaml
echo "{% set name = '$name' %}" > vars.txt
echo "{% set version = '$version' %}" >> vars.txt
-
-cat vars.txt meta.yaml > tempfile
-mv tempfile meta.yaml
+cat vars.txt .conda/meta.yaml > tempfile
+mv tempfile .conda/meta.yaml
rm vars.txt
-# do the build
-conda build --output-folder=../dist .
+mkdir -p dist
+
+conda update conda setuptools conda-build
+
+conda build --output-folder=dist .conda
# tar it up
-cd ../dist
+cd dist
tar czf "$name"-"$version"-conda.tar.gz *
-cd ..
diff --git a/.ci/build_doc.sh b/.ci/build_doc.sh
index b3d1f20db..1867d67d0 100644
--- a/.ci/build_doc.sh
+++ b/.ci/build_doc.sh
@@ -1,4 +1,6 @@
#!/usr/bin/env /bash
+set -e
+
python setup.py doc
mv doc/html doc/"$CI_COMMIT_REF_NAME"
diff --git a/.ci/build_pypi_dist.sh b/.ci/build_pypi_dist.sh
index 24d24cb80..3a840cb2d 100644
--- a/.ci/build_pypi_dist.sh
+++ b/.ci/build_pypi_dist.sh
@@ -1,5 +1,7 @@
#!/usr/bin/env bash
+set -e
+
pip install wheel
python setup.py sdist
python setup.py bdist_wheel
diff --git a/.ci/deploy_conda.sh b/.ci/deploy_conda.sh
index ad8eb3052..802227622 100644
--- a/.ci/deploy_conda.sh
+++ b/.ci/deploy_conda.sh
@@ -1,3 +1,5 @@
#!/usr/bin/env bash
+set -e
+
rsync -rv dist/*conda.tar.gz "condadeploy:"
diff --git a/.ci/deploy_doc.sh b/.ci/deploy_doc.sh
index 652eefd73..a8fdc6559 100644
--- a/.ci/deploy_doc.sh
+++ b/.ci/deploy_doc.sh
@@ -1,3 +1,5 @@
#!/usr/bin/env bash
+set -e
+
rsync -rv doc/"$CI_COMMIT_REF_NAME" "docdeploy:"
diff --git a/.ci/deploy_pypi.sh b/.ci/deploy_pypi.sh
index 23746faed..a49f0f024 100644
--- a/.ci/deploy_pypi.sh
+++ b/.ci/deploy_pypi.sh
@@ -1,4 +1,6 @@
#!/usr/bin/env BASH
+set -e
+
pip install setuptools wheel twine
twine upload dist/*
diff --git a/.ci/patch_version.sh b/.ci/patch_version.sh
index 90f2baa9a..482e4fc4a 100644
--- a/.ci/patch_version.sh
+++ b/.ci/patch_version.sh
@@ -1,5 +1,7 @@
#!/usr/bin/env bash
+set -e
+
if [[ "x$CI_COMMIT_TAG" != "x" ]]; then
echo "Release detected - patching version - $CI_COMMIT_REF_NAME";
python -c "import fsl.version as v; v.patchVersion('fsl/version.py', '$CI_COMMIT_REF_NAME')";
diff --git a/.ci/setup_ssh.sh b/.ci/setup_ssh.sh
index 2ef89d1cc..17795f309 100644
--- a/.ci/setup_ssh.sh
+++ b/.ci/setup_ssh.sh
@@ -1,5 +1,7 @@
#!/usr/bin/env /bash
+set -e
+
##########################################################
# The setup_ssh script does the following:
#
@@ -29,7 +31,8 @@ if [[ -f /.dockerenv ]]; then
echo "$SSH_PRIVATE_KEY_FSL_DOWNLOAD" > $HOME/.ssh/id_fsl_download;
if [[ "$CI_PROJECT_PATH" == "$UPSTREAM_PROJECT" ]]; then
- echo "$SSH_PRIVATE_KEY_DOC_DEPLOY" > $HOME/.ssh/id_doc_deploy;
+ echo "$SSH_PRIVATE_KEY_DOC_DEPLOY" > $HOME/.ssh/id_doc_deploy;
+ echo "$SSH_PRIVATE_KEY_CONDA_DEPLOY" > $HOME/.ssh/id_conda_deploy;
fi;
chmod go-rwx $HOME/.ssh/id_*;
@@ -39,6 +42,7 @@ if [[ -f /.dockerenv ]]; then
if [[ "$CI_PROJECT_PATH" == "$UPSTREAM_PROJECT" ]]; then
ssh-add $HOME/.ssh/id_doc_deploy;
+ ssh-add $HOME/.ssh/id_conda_deploy;
fi
echo "$SSH_SERVER_HOSTKEYS" > $HOME/.ssh/known_hosts;
@@ -54,6 +58,11 @@ if [[ -f /.dockerenv ]]; then
echo " User ${DOC_HOST%@*}" >> $HOME/.ssh/config;
echo " IdentityFile $HOME/.ssh/id_doc_deploy" >> $HOME/.ssh/config;
+ echo "Host condadeploy" >> $HOME/.ssh/config;
+ echo " HostName ${CONDA_HOST##*@}" >> $HOME/.ssh/config;
+ echo " User ${CONDA_HOST%@*}" >> $HOME/.ssh/config;
+ echo " IdentityFile $HOME/.ssh/id_conda_deploy" >> $HOME/.ssh/config;
+
echo "Host fsldownload" >> $HOME/.ssh/config;
echo " HostName ${FSL_HOST##*@}" >> $HOME/.ssh/config;
echo " User ${FSL_HOST%@*}" >> $HOME/.ssh/config;
diff --git a/.ci/test_template.sh b/.ci/test_template.sh
index 87891ec98..3f6f82d1f 100644
--- a/.ci/test_template.sh
+++ b/.ci/test_template.sh
@@ -1,5 +1,7 @@
#!/usr/bin/env bash
+set -e
+
# Install $PY_VERSION, xvfb, and all
# of the wxpython dependencies.
apt-get update -y || true
diff --git a/.conda/conda_build_config.yaml b/.conda/conda_build_config.yaml
new file mode 100644
index 000000000..2d1bcb889
--- /dev/null
+++ b/.conda/conda_build_config.yaml
@@ -0,0 +1,2 @@
+python:
+ - 3.6
\ No newline at end of file
diff --git a/.conda/meta.yaml b/.conda/meta.yaml
index ccb3cc475..4587cc4fb 100644
--- a/.conda/meta.yaml
+++ b/.conda/meta.yaml
@@ -13,7 +13,7 @@ source:
requirements:
build:
- - python
+ - python {{ python }}
- setuptools
- six 1.*
- deprecation 1.*
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e5893aa75..13bf9dbd2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,4 +1,4 @@
-###########################################################################
+#############################################################################
# This file defines the build process for fslpy, as hosted at:
#
# https://git.fmrib.ox.ac.uk/fsl/fslpy
@@ -11,10 +11,10 @@
#
# 3. doc: Building API documentation
#
-# 4. build: Building source distributions and wheels
+# 4. build: Building source, wheel and conda distributions
#
-# 5. deploy: Uploading the build outputs to pypi, and the documentation
-# to a hosting server.
+# 5. deploy: Uploading the build outputs to pypi/hosting servers, and the
+# documentation to a hosting server.
#
# The test and style stages are executed on all branches of upstream and fork
# repositories.
@@ -22,11 +22,14 @@
# The doc stage, and the deploy-doc job, is executed on all branches of the
# upstream repository.
#
-# The build stage, and the remaining jobs in the deploy stage, are only
+# The build stage, and the remaining jobs in the deploy stage, are only
# executed on the upstream repository, and only for release tags.
#
# The deploy stages are manually instantiated.
-###########################################################################
+#
+# Most of the logic for each job is defined in shell scripts in the .ci
+# sub-directory.
+#############################################################################
stages:
@@ -56,6 +59,9 @@ stages:
# - SSH_PRIVATE_KEY_DOC_DEPLOY - private key for rsyncing documentation
# to remote host (DOC_HOST)
#
+# - SSH_PRIVATE_KEY_CONDA_DEPLOY - private key for rsyncing conda builds
+# to remote host (CONDA_HOST)
+#
# - SSH_SERVER_HOSTKEYS - List of trusted SSH hosts
#
# - DOC_HOST: - Username@host to upload documentation to
@@ -64,6 +70,12 @@ stages:
# - FSL_HOST: - Username@host to download FSL data from
# (e.g. "paulmc@jalapeno.fmrib.ox.ac.uk")
#
+# - CONDA_HOST: - Username@host to upload conda build to
+# (e.g. "paulmc@jalapeno.fmrib.ox.ac.uk")
+#
+# - CONDA_CHANNELS - List of additional conda channels to
+# use for conda build.
+#
# - TWINE_USERNAME: - Username to use when uploading to pypi
#
# - TWINE_PASSWORD: - Password to use when uploading to pypi
@@ -262,19 +274,18 @@ build-conda-dist:
<<: *patch_version
stage: build
- image: continuumio/minconda
+ image: continuumio/miniconda3
tags:
- docker
script:
-
- - bash ./.ci/build_conda_dist.sh
+ - bash ./.ci/build_conda_dist.sh fslpy "$CI_COMMIT_REF_NAME"
artifacts:
expire_in: 1 day
paths:
- - dist/*
+ - dist/*conda.tar.gz
##############
@@ -314,3 +325,21 @@ deploy-pypi:
script:
- bash ./.ci/deploy_pypi.sh
+
+
+
+deploy-conda:
+ <<: *only_releases
+ <<: *setup_ssh
+ stage: deploy
+ when: manual
+ image: python:3.5
+
+ tags:
+ - docker
+
+ dependencies:
+ - build-conda-dist
+
+ script:
+ - bash ./.ci/deploy_conda.sh
--
GitLab