Commit 231fc84d authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

Moved all CI stuff out of gitlab yml because it is much cleaner. Added conda...

Moved all CI stuff out of gitlab yml because it is much cleaner. Added conda skeleton stuff for building conda packages. Still some work to do
parent 3c0a0d7a
#!/usr/bin/env bash
conda update conda
conda install setuptools conda-build
cd .conda
# get version and name
version=`python ../setup.py -V`
name=`python ../setup.py --name`
# get requirements, and make
# them conda compatible...
# strip all spaces
reqs=`cat ../requirements.txt | sed -e 's/ //g'`
# add a space after package name -
# package names must match a-zA-Z0-9_
reqs=`echo "$reqs" | sed -e "s/^[a-zA-Z0-9_][a-zA-Z0-9_]*/& /g"`
# remove ==, replace it with a space
reqs=`echo "$reqs" | sed -e "s/==/ /g"`
# wrap each dep in quotes
reqs=`echo "$reqs" | sed -e "s/^.*$/'&'/g"`
# add a comma at the end
reqs=`echo "$reqs" | sed -e "s/$/,/g"`
# remove newlines
reqs=`echo $reqs`
echo "version: $version"
echo "name: $name"
echo "reqs: $reqs"
echo "{% set name = 'name' %}" > vars.txt
echo "{% set version = '$version' %}" >> vars.txt
echo "{% set requirements = [$reqs] %}" >> vars.txt
cat vars.txt meta.yaml > tempfile
mv tempfile meta.yaml
rm vars.txt
conda build fslpy
cd ..
#!/usr/bin/env /bash
python setup.py doc
mv doc/html doc/"$CI_COMMIT_REF_NAME"
#!/usr/bin/env bash
pip install wheel
python setup.py sdist
python setup.py bdist_wheel
#!/usr/bin/env bash
rsync -rv doc/"$CI_COMMIT_REF_NAME" "docdeploy:"
#!/usr/bin/env BASH
pip install setuptools wheel twine
twine upload dist/*
#!/usr/bin/env bash
if [[ "x$CI_COMMIT_TAG" != "x" ]]; then
echo "Release detected - patching version - $CI_COMMIT_REF_NAME";
python -c "import fsl.version as v; v.patchVersion('fsl/version.py', '$CI_COMMIT_REF_NAME')";
fi
#!/usr/bin/env /bash
##########################################################
# The setup_ssh script does the following:
#
# - Sets up key-based SSH login, and
# installs the private keys, so
# we can connect to servers.
#
# - Configures git, and adds the
# upstream repo as a remote
#
# (see https://docs.gitlab.com/ce/ci/ssh_keys/README.html)
#
# NOTE: It is assumed that non-docker
# executors are already configured
# (or don't need any configuration).
##########################################################
if [[ -f /.dockerenv ]]; then
apt-get update -y || yum -y check-update || true;
apt-get install -y openssh-client rsync git || yum install -y openssh-client rsync git || true;
eval $(ssh-agent -s);
mkdir -p $HOME/.ssh;
echo "$SSH_PRIVATE_KEY_GIT" > $HOME/.ssh/id_git;
echo "$SSH_PRIVATE_KEY_FSL_DOWNLOAD" > $HOME/.ssh/id_fsl_download;
if [[ "$CI_PROJECT_PATH" == "$UPSTREAM_PROJECT" ]]; then
echo "$SSH_PRIVATE_KEY_DOC_DEPLOY" > $HOME/.ssh/id_doc_deploy;
fi;
chmod go-rwx $HOME/.ssh/id_*;
ssh-add $HOME/.ssh/id_git;
ssh-add $HOME/.ssh/id_fsl_download;
if [[ "$CI_PROJECT_PATH" == "$UPSTREAM_PROJECT" ]]; then
ssh-add $HOME/.ssh/id_doc_deploy;
fi
echo "$SSH_SERVER_HOSTKEYS" > $HOME/.ssh/known_hosts;
touch $HOME/.ssh/config;
echo "Host ${UPSTREAM_URL##*@}" >> $HOME/.ssh/config;
echo " User ${UPSTREAM_URL%@*}" >> $HOME/.ssh/config;
echo " IdentityFile $HOME/.ssh/id_git" >> $HOME/.ssh/config;
echo "Host docdeploy" >> $HOME/.ssh/config;
echo " HostName ${DOC_HOST##*@}" >> $HOME/.ssh/config;
echo " User ${DOC_HOST%@*}" >> $HOME/.ssh/config;
echo " IdentityFile $HOME/.ssh/id_doc_deploy" >> $HOME/.ssh/config;
echo "Host fsldownload" >> $HOME/.ssh/config;
echo " HostName ${FSL_HOST##*@}" >> $HOME/.ssh/config;
echo " User ${FSL_HOST%@*}" >> $HOME/.ssh/config;
echo " IdentityFile $HOME/.ssh/id_fsl_download" >> $HOME/.ssh/config;
echo "Host *" >> $HOME/.ssh/config;
echo " IdentitiesOnly yes" >> $HOME/.ssh/config;
git config --global user.name "Gitlab CI";
git config --global user.email "gitlabci@localhost";
if [[ `git remote -v` == *"upstream"* ]]; then
git remote remove upstream;
fi;
git remote add upstream "$UPSTREAM_URL:$UPSTREAM_PROJECT";
fi
#!/usr/bin/env bash
# Install $PY_VERSION, xvfb, and all
# of the wxpython dependencies.
apt-get update -y || true
apt-get install -y software-properties-common python-software-properties xvfb libgtk2.0-0 libnotify4 freeglut3 libsdl1.2debian
add-apt-repository -y ppa:deadsnakes/ppa
apt-get update -y || true
apt-get install -y $PY_VERSION "$PY_VERSION"-dev $PY_PACKAGES
$PY_VENV test.venv
source test.venv/bin/activate
pip install --upgrade pip setuptools
# If running on a fork repository, we merge in the
# upstream/master branch. This is done so that merge
# requests from fork to the parent repository will
# have unit tests run on the merged code, something
# which gitlab CE does not currently do for us.
if [[ "$CI_PROJECT_PATH" != "$UPSTREAM_PROJECT" ]]; then
git fetch upstream;
git merge --no-commit --no-ff upstream/master;
fi;
$INSTALL_WX
# All other deps can be installed as normal.
# We install test dependenciesd through pip,
# because if we let setuptools do it, it
# will build/install everything from source,
# rather than using wheels.
pip install -r requirements.txt
pip install sphinx sphinx-rtd-theme
pip install pytest pytest-cov pytest-html pytest-runner mock coverage
# style stage
if [ "$TEST_STYLE"x != "x" ]; then pip install pylint flake8; fi;
if [ "$TEST_STYLE"x != "x" ]; then flake8 fsl || true; fi;
if [ "$TEST_STYLE"x != "x" ]; then pylint --output-format=colorized fsl || true; fi;
if [ "$TEST_STYLE"x != "x" ]; then exit 0; fi
# We need the FSL atlases for the atlas
# tests, and need $FSLDIR to be defined
export FSLDIR=/fsl/
mkdir -p $FSLDIR/data/
rsync -rv "fsldownload:data/atlases/" "$FSLDIR/data/atlases/"
# Finally, run the damned tests.
# We run some tests under xvfb-run
# because they invoke wx. Sleep in
# between, otherwise xvfb gets upset.
xvfb-run python setup.py test --addopts="$TEST_OPTS tests/test_idle.py"
sleep 5
xvfb-run python setup.py test --addopts="$TEST_OPTS tests/test_platform.py"
# We run the immv/imcpy tests as the nobody
# user because some tests expect permission
# denied errors when looking at files, and
# root never gets denied. Make everything in
# this directory writable by anybody (which,
# unintuitively, includes nobody)
chmod -R a+w `pwd`
su -s /bin/bash -c 'source test.venv/bin/activate && python setup.py test --addopts="$TEST_OPTS tests/test_immv_imcp.py"' nobody
# All other tests can be run as normal
python setup.py test --addopts="$TEST_OPTS --ignore=tests/test_idle.py --ignore=tests/test_platform.py --ignore=tests/test_immv_imcp.py"
python -m coverage report
"%PYTHON%" setup.py install
if errorlevel 1 exit 1
$PYTHON setup.py install # Python command to install the script.
package:
name: '{{ name }}'
version: '{{ version }}'
source:
path:
../
requirements:
build:
- python
- setuptools
{% for r in requirements %}
- {{ r }}
{% endfor %}
run:
- python
{% for r in requirements %}
- {{ r }}
{% endfor %}
......@@ -110,81 +110,9 @@ variables:
- tags
##########################################################
# The setup_ssh anchor contains a before_script section
# which does the following:
#
# - Sets up key-based SSH login, and
# installs the private keys, so
# we can connect to servers.
#
# - Configures git, and adds the
# upstream repo as a remote
#
# (see https://docs.gitlab.com/ce/ci/ssh_keys/README.html)
#
# NOTE: It is assumed that non-docker
# executors are already configured
# (or don't need any configuration).
##########################################################
.setup_ssh: &setup_ssh
before_script:
- if [[ -f /.dockerenv ]]; then
apt-get update -y || yum -y check-update || true;
apt-get install -y openssh-client rsync git || yum install -y openssh-client rsync git || true;
eval $(ssh-agent -s);
mkdir -p $HOME/.ssh;
echo "$SSH_PRIVATE_KEY_GIT" > $HOME/.ssh/id_git;
echo "$SSH_PRIVATE_KEY_FSL_DOWNLOAD" > $HOME/.ssh/id_fsl_download;
if [[ "$CI_PROJECT_PATH" == "$UPSTREAM_PROJECT" ]]; then
echo "$SSH_PRIVATE_KEY_DOC_DEPLOY" > $HOME/.ssh/id_doc_deploy;
fi;
chmod go-rwx $HOME/.ssh/id_*;
ssh-add $HOME/.ssh/id_git;
ssh-add $HOME/.ssh/id_fsl_download;
if [[ "$CI_PROJECT_PATH" == "$UPSTREAM_PROJECT" ]]; then
ssh-add $HOME/.ssh/id_doc_deploy;
fi
echo "$SSH_SERVER_HOSTKEYS" > $HOME/.ssh/known_hosts;
touch $HOME/.ssh/config;
echo "Host ${UPSTREAM_URL##*@}" >> $HOME/.ssh/config;
echo " User ${UPSTREAM_URL%@*}" >> $HOME/.ssh/config;
echo " IdentityFile $HOME/.ssh/id_git" >> $HOME/.ssh/config;
echo "Host docdeploy" >> $HOME/.ssh/config;
echo " HostName ${DOC_HOST##*@}" >> $HOME/.ssh/config;
echo " User ${DOC_HOST%@*}" >> $HOME/.ssh/config;
echo " IdentityFile $HOME/.ssh/id_doc_deploy" >> $HOME/.ssh/config;
echo "Host fsldownload" >> $HOME/.ssh/config;
echo " HostName ${FSL_HOST##*@}" >> $HOME/.ssh/config;
echo " User ${FSL_HOST%@*}" >> $HOME/.ssh/config;
echo " IdentityFile $HOME/.ssh/id_fsl_download" >> $HOME/.ssh/config;
echo "Host *" >> $HOME/.ssh/config;
echo " IdentitiesOnly yes" >> $HOME/.ssh/config;
git config --global user.name "Gitlab CI";
git config --global user.email "gitlabci@localhost";
if [[ `git remote -v` == *"upstream"* ]]; then
git remote remove upstream;
fi;
git remote add upstream "$UPSTREAM_URL:$UPSTREAM_PROJECT";
fi
- bash ./.ci/setup_ssh.sh
###################################################
......@@ -197,11 +125,7 @@ variables:
.patch_version: &patch_version
before_script:
- if [[ "x$CI_COMMIT_TAG" != "x" ]]; then
echo "Release detected - patching version - $CI_COMMIT_REF_NAME";
python -c "import fsl.version as v; v.patchVersion('fsl/version.py', '$CI_COMMIT_REF_NAME')";
fi
- bash ./.ci/patch_version.sh
############
......@@ -220,72 +144,7 @@ variables:
- docker
script:
# Install $PY_VERSION, xvfb, and all
# of the wxpython dependencies.
- apt-get update -y || true
- apt-get install -y software-properties-common python-software-properties xvfb libgtk2.0-0 libnotify4 freeglut3 libsdl1.2debian
- add-apt-repository -y ppa:deadsnakes/ppa
- apt-get update -y || true
- apt-get install -y $PY_VERSION "$PY_VERSION"-dev $PY_PACKAGES
- $PY_VENV test.venv
- source test.venv/bin/activate
- pip install --upgrade pip setuptools
# If running on a fork repository, we merge in the
# upstream/master branch. This is done so that merge
# requests from fork to the parent repository will
# have unit tests run on the merged code, something
# which gitlab CE does not currently do for us.
- if [[ "$CI_PROJECT_PATH" != "$UPSTREAM_PROJECT" ]]; then
git fetch upstream;
git merge --no-commit --no-ff upstream/master;
fi;
- $INSTALL_WX
# All other deps can be installed as normal.
# We install test dependenciesd through pip,
# because if we let setuptools do it, it
# will build/install everything from source,
# rather than using wheels.
- pip install -r requirements.txt
- pip install sphinx sphinx-rtd-theme
- pip install pytest pytest-cov pytest-html pytest-runner mock coverage
# style stage
- if [ "$TEST_STYLE"x != "x" ]; then pip install pylint flake8; fi;
- if [ "$TEST_STYLE"x != "x" ]; then flake8 fsl || true; fi;
- if [ "$TEST_STYLE"x != "x" ]; then pylint --output-format=colorized fsl || true; fi;
- if [ "$TEST_STYLE"x != "x" ]; then exit 0; fi
# We need the FSL atlases for the atlas
# tests, and need $FSLDIR to be defined
- export FSLDIR=/fsl/
- mkdir -p $FSLDIR/data/
- rsync -rv "fsldownload:data/atlases/" "$FSLDIR/data/atlases/"
# Finally, run the damned tests.
# We run some tests under xvfb-run
# because they invoke wx. Sleep in
# between, otherwise xvfb gets upset.
- xvfb-run python setup.py test --addopts="$TEST_OPTS tests/test_idle.py"
- sleep 5
- xvfb-run python setup.py test --addopts="$TEST_OPTS tests/test_platform.py"
# We run the immv/imcpy tests as the nobody
# user because some tests expect permission
# denied errors when looking at files, and
# root never gets denied. Make everything in
# this directory writable by anybody (which,
# unintuitively, includes nobody)
- chmod -R a+w `pwd`
- su -s /bin/bash -c 'source test.venv/bin/activate && python setup.py test --addopts="$TEST_OPTS tests/test_immv_imcp.py"' nobody
# All other tests can be run as normal
- python setup.py test --addopts="$TEST_OPTS --ignore=tests/test_idle.py --ignore=tests/test_platform.py --ignore=tests/test_immv_imcp.py"
- python -m coverage report
- bash ./.ci/test_template.sh
test:2.7:
......@@ -366,8 +225,8 @@ build-doc:
image: python:3.5
script:
- python setup.py doc
- mv doc/html doc/"$CI_COMMIT_REF_NAME"
- bash ./.ci/build_doc.sh
artifacts:
expire_in: 1 day
paths:
......@@ -379,7 +238,7 @@ build-doc:
#############
build-dist:
build-pypi-dist:
<<: *only_releases
<<: *patch_version
......@@ -390,9 +249,27 @@ build-dist:
- docker
script:
- pip install wheel
- python setup.py sdist
- python setup.py bdist_wheel
- bash ./.ci/build_pypi_dist.sh
artifacts:
expire_in: 1 day
paths:
- dist/*
build-conda-dist:
<<: *only_releases
<<: *patch_version
stage: build
image: continuumio/minconda
tags:
- docker
script:
- bash ./.ci/build_conda_dist.sh
artifacts:
expire_in: 1 day
......@@ -419,7 +296,7 @@ deploy-doc:
- build-doc
script:
- rsync -rv doc/"$CI_COMMIT_REF_NAME" "docdeploy:"
- bash ./.ci/deploy_doc.sh
deploy-pypi:
......@@ -433,8 +310,7 @@ deploy-pypi:
- docker
dependencies:
- build-dist
- build-pypi-dist
script:
- pip install setuptools wheel twine
- twine upload dist/*
- bash ./.ci/deploy_pypi.sh
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment