Commit 375afae7 authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

Merge branch 'master' into 'master'

Full build pipeline

See merge request !21
parents c74de29c f8943bb0
Pipeline #682 passed with stages
in 47 minutes and 27 seconds
test:2.7: ###########################################################################
image: fsleyes-py27 # This file defines the build process for fslpy, as hosted at:
#
# https://git.fmrib.ox.ac.uk/fsl/fslpy
#
# The build pipeline comprises four stages:
#
# 1. test: Unit tests
#
# 2. doc: Building API documentation
#
# 3. build: Building source distributions and wheels
#
# 4. deploy: Uploading the build outputs to pypi, and the documentation
# to a hosting server.
#
# The test stage is executed on all branches of upstream and fork
# repositories.
#
# The doc stage, and the deploy-doc job, is executed on all branches of the
# upstream repository.
#
# The build stage, and the remaining jobs in the deploy stage, are only
# executed on the upstream repository, and only for release tags.
#
# The deploy stages are manually instantiated.
###########################################################################
stages:
- test
- doc
- build
- deploy
###############################################################################
# A number of variables must be set for the jobs to work. The following
# variables are implicitly defined in any gitlab CI job:
#
# - CI_PROJECT_PATH - gitlab namespace/project
# - CI_COMMIT_REF_NAME - branch name, provided by gitlab
# - CI_COMMIT_TAG - present if build is running on a tag
#
# These variables must be explicitly set as "secret" variables:
#
# - SSH_PRIVATE_KEY_GIT - private key for git login to remote host
# (UPSTREAM_URL)
#
# - SSH_PRIVATE_KEY_FSL_DOWNLOAD - private key for downloading some FSL
# files from a remote server (FSL_HOST)
#
# - SSH_PRIVATE_KEY_DOC_DEPLOY - private key for rsyncing documentation
# to remote host (DOC_HOST)
#
# - SSH_SERVER_HOSTKEYS - List of trusted SSH hosts
#
# - TWINE_PASSWORD: - Password to use when uploading to pypi
###############################################################################
variables:
UPSTREAM_PROJECT: "fsl/fslpy"
UPSTREAM_URL: "git@git.fmrib.ox.ac.uk"
DOC_HOST: "paulmc@jalapeno.fmrib.ox.ac.uk"
FSL_HOST: "paulmc@jalapeno.fmrib.ox.ac.uk"
TWINE_USERNAME: "pauldmccarthy"
TWINE_REPOSITORY_URL: "https://testpypi.python.org/pypi"
####################################
# These anchors are used to restrict
# when and where jobs are executed.
####################################
.only_upstream: &only_upstream
only:
- branches@fsl/fslpy
.only_master: &only_master
only:
- master@fsl/fslpy
.only_releases: &only_releases
only:
- tags@fsl/fslpy
.except_releases: &except_releases
except:
- tags
##########################################################
# The setup_ssh anchor contains a before_script section
# which does the following:
#
# - Sets up key-based SSH login, and
# installs the private keys, so
# we can connect to servers.
#
# - Configures git, and adds the
# upstream repo as a remote
#
# (see https://docs.gitlab.com/ce/ci/ssh_keys/README.html)
#
# NOTE: It is assumed that non-docker
# executors are already configured
# (or don't need any configuration).
##########################################################
.setup_ssh: &setup_ssh
before_script:
- if [[ -f /.dockerenv ]]; then
apt-get update -y || yum -y check-update || true;
apt-get install -y openssh-client || yum install -y openssh-client || true;
apt-get install -y rsync || yum install -y rsync || true;
eval $(ssh-agent -s);
mkdir -p $HOME/.ssh;
echo "$SSH_PRIVATE_KEY_GIT" > $HOME/.ssh/id_git;
echo "$SSH_PRIVATE_KEY_FSL_DOWNLOAD" > $HOME/.ssh/id_fsl_download;
if [[ "$CI_PROJECT_PATH" == "$UPSTREAM_PROJECT" ]]; then
echo "$SSH_PRIVATE_KEY_DOC_DEPLOY" > $HOME/.ssh/id_doc_deploy;
fi;
chmod go-rwx $HOME/.ssh/id_*;
ssh-add $HOME/.ssh/id_git;
ssh-add $HOME/.ssh/id_fsl_download;
if [[ "$CI_PROJECT_PATH" == "$UPSTREAM_PROJECT" ]]; then
ssh-add $HOME/.ssh/id_doc_deploy;
fi
echo "$SSH_SERVER_HOSTKEYS" > $HOME/.ssh/known_hosts;
touch $HOME/.ssh/config;
echo "Host ${UPSTREAM_URL##*@}" >> $HOME/.ssh/config;
echo " User ${UPSTREAM_URL%@*}" >> $HOME/.ssh/config;
echo " IdentityFile $HOME/.ssh/id_git" >> $HOME/.ssh/config;
echo "Host docdeploy" >> $HOME/.ssh/config;
echo " HostName ${DOC_HOST##*@}" >> $HOME/.ssh/config;
echo " User ${DOC_HOST%@*}" >> $HOME/.ssh/config;
echo " IdentityFile $HOME/.ssh/id_doc_deploy" >> $HOME/.ssh/config;
echo "Host fsldownload" >> $HOME/.ssh/config;
echo " HostName ${FSL_HOST##*@}" >> $HOME/.ssh/config;
echo " User ${FSL_HOST%@*}" >> $HOME/.ssh/config;
echo " IdentityFile $HOME/.ssh/id_fsl_download" >> $HOME/.ssh/config;
echo "Host *" >> $HOME/.ssh/config;
echo " IdentitiesOnly yes" >> $HOME/.ssh/config;
git config --global user.name "Gitlab CI";
git config --global user.email "gitlabci@localhost";
if [[ `git remote -v` == *"upstream"* ]]; then
git remote remove upstream;
fi;
git remote add upstream "$UPSTREAM_URL:$UPSTREAM_PROJECT";
fi
###################################################
# The patch_version anchor contains a before_script
# section which is run on release builds, and makes
# sure that the version in the code is up to date
# (i.e. equal to the tag name).
###################################################
.patch_version: &patch_version
before_script:
- if [[ "x$CI_COMMIT_TAG" != "x" ]]; then
echo "Release detected - patching version - $CI_COMMIT_REF_NAME";
python -c "import fsl.version as v; v.patchVersion('fsl/version.py', '$CI_COMMIT_REF_NAME')";
fi
############
# Test stage
############
.test: &test_template
stage: test
<<: *setup_ssh
# Releases are just tags on a release
# branch, so we don't need to test them.
<<: *except_releases
tags:
- docker
script: script:
- cat requirements.txt | xargs -n 1 pip install
# If running on a fork repository, we merge in the
# upstream/master branch. This is done so that merge
# requests from fork to the parent repository will
# have unit tests run on the merged code, something
# which gitlab CE does not currently do for us.
- if [[ "$CI_PROJECT_PATH" != "$UPSTREAM_PROJECT" ]]; then
git fetch upstream;
git merge --no-commit --no-ff upstream/master;
fi;
# I am currently assuming that we are
# running a debian 8/jessie container
# (the python:2.7 and 3.6 images are
# based on this).
# We need to install xvfb, and all of
# the wxpython dependencies.
- apt-get update -y
- apt-get install -y xvfb libgtk-3-0
- apt-get install -y libnotify4 freeglut3 libsdl1.2debian
# Linux builds for wxPython are currently not
# on pypi, but are available at this url.
- pip install -f https://wxpython.org/Phoenix/release-extras/linux/gtk3/debian-8/ wxpython
# All other deps can be installed as
# normal. scipy is required by nibabel,
# but not listed in its requirements.
# We install test dependencies through
# pip, because if we let setuptools do
# it, it will build/install everything
# from source, rather than using wheels.
- pip install -r requirements.txt
- pip install scipy - pip install scipy
- pip install coverage - pip install sphinx sphinx-rtd-theme
- su -s /bin/bash -c "xvfb-run python setup.py test" nobody - pip install pytest pytest-cov pytest-html pytest-runner mock coverage
- coverage report -m
# We need the FSL atlases for the atlas
# tests, and need $FSLDIR to be defined
- export FSLDIR=/fsl/
- mkdir -p $FSLDIR/data/
- rsync -rv "fsldownload:data/atlases/" "$FSLDIR/data/atlases/"
# Finally, run the damned tests.
# We run some tests under xvfb-run
# because they invoke wx. Sleep in
# between, otherwise xvfb gets upset.
- xvfb-run python setup.py test --addopts="tests/test_async.py"
- sleep 5
- xvfb-run python setup.py test --addopts="tests/test_platform.py"
# We run the immv/imcpy tests as the nobody
# user because some tests expect permission
# denied errors when looking at files, and
# root never gets denied. Make everything in
# this directory writable by anybody (which,
# unintuitively, includes nobody)
- chmod -R a+w `pwd`
- su -s /bin/bash -c "python setup.py test --addopts='tests/test_immv_imcp.py'" nobody
# All other tests can be run as normal
- python setup.py test --addopts="--ignore=tests/test_async.py --ignore=tests/test_platform.py --ignore=tests/test_immv_imcp.py"
- python -m coverage report
test:2.7:
<<: *test_template
image: python:2.7
test:3.4:
<<: *test_template
image: python:3.4
# a wxphoenix/3.5 build
# is not yet available
# test:3.5:
# <<: *test_template
# image: python:3.5
# a wxphoenix/3.6 build
# is not yet available
# test:3.6:
# <<: *test_template
# image: python:3.6
###########
# Doc stage
###########
build-doc:
<<: *only_upstream
<<: *patch_version
tags:
- docker
stage: doc
image: python:3.5
test:3.6:
image: fsleyes-py36
script: script:
- cat requirements.txt | xargs -n 1 pip install - python setup.py doc
- pip install scipy - mv doc/html doc/"$CI_COMMIT_REF_NAME"
- pip install coverage artifacts:
- su -s /bin/bash -c "xvfb-run python setup.py test" nobody paths:
- coverage report -m - doc/$CI_COMMIT_REF_NAME
#############
# Build stage
#############
build-dist:
<<: *only_releases
<<: *patch_version
stage: build
image: python:3.5
script:
- pip install wheel
- python setup.py sdist
- python setup.py bdist_wheel
artifacts:
paths:
- dist/*
##############
# Deploy stage
##############
deploy-doc:
<<: *only_upstream
<<: *setup_ssh
stage: deploy
when: manual
image: python:3.5
tags:
- docker
dependencies:
- build-doc
script:
- rsync -rv doc/"$CI_COMMIT_REF_NAME" "docdeploy:"
deploy-pypi:
<<: *only_releases
<<: *setup_ssh
stage: deploy
when: manual
image: python:3.5
tags:
- docker
dependencies:
- build-dist
script:
- pip install setuptools wheel twine
- twine upload dist/*
include LICENSE include LICENSE
include COPYRIGHT include COPYRIGHT
include README.md
include requirements.txt include requirements.txt
include pytest.ini include pytest.ini
recursive-include doc * recursive-include doc *
......
fslpy
=====
The `fslpy` project is a [FSL](http://fsl.fmrib.ox.ac.uk/fsl/fslwiki/)
programming library written in Python. It is used by
[FSLeyes](https://git.fmrib.ox.ac.uk/paulmc/fsleyes/).
Dependencies
------------
All of the dependencies of `fslpy` are listed in the
[requirements.txt](requirements.txt) file. Some `fslpy` modules require
[wxPython](http://www.wxpython.org) 3.0.2.0.
Documentation
-------------
`fslpy` is documented using [sphinx](http://http://sphinx-doc.org/). You can
build the API documentation by running:
python setup.py doc
The HTML documentation will be generated and saved in the `doc/html/` directory.
If you are interested in contributing to `fslpy`, check out the [contributing
guide](doc/contributing.rst).
Tests
-----
Run the test suite via:
python setup.py test
A test report will be generated at `report.html`, and a code coverage report
will be generated in `htmlcov/`.
fslpy
=====
.. image:: https://git.fmrib.ox.ac.uk/fsl/fslpy/badges/master/build.svg
:target: https://git.fmrib.ox.ac.uk/fsl/fslpy/commits/master/
.. image:: https://git.fmrib.ox.ac.uk/fsl/fslpy/badges/master/coverage.svg
:target: https://git.fmrib.ox.ac.uk/fsl/fslpy/commits/master/
The ``fslpy`` project is a `FSL <http://fsl.fmrib.ox.ac.uk/fsl/fslwiki/>`_
programming library written in Python. It is used by `FSLeyes
<https://git.fmrib.ox.ac.uk/paulmc/fsleyes/>`_.
Dependencies
------------
All of the dependencies of ``fslpy`` are listed in the `requirements.txt
<requirements.txt>`_ file. Some ``fslpy`` modules require `wxPython
<http://www.wxpython.org>`_ 3.0.2.0 or higher.
Documentation
-------------
``fslpy`` is documented using `sphinx <http://http://sphinx-doc.org/>`_. You
can build the API documentation by running::
python setup.py doc
The HTML documentation will be generated and saved in the ``doc/html/``
directory.
If you are interested in contributing to ``fslpy``, check out the
`contributing guide <doc/contributing.rst>`_.
Tests
-----
Run the test suite via::
python setup.py test
A test report will be generated at ``report.html``, and a code coverage report
will be generated in ``htmlcov/``.
...@@ -45,6 +45,12 @@ numbers:: ...@@ -45,6 +45,12 @@ numbers::
backwards-incompatible changes. backwards-incompatible changes.
The version number in the ``master`` branch should be of the form
``major.minor.patch.dev``, to indicate that any releases made from this branch
are development releases (although development releases are not part of the
release model).
Releases Releases
-------- --------
...@@ -72,12 +78,10 @@ Testing ...@@ -72,12 +78,10 @@ Testing
Unit and integration tests are currently run with ``py.test`` and Unit and integration tests are currently run with ``py.test`` and
``coverage``. We don't have CI configured yet, so tests have to be run ``coverage``.
manually.
- Aim for 100% code coverage. - Aim for 100% code coverage.
- Tests must pass on both python 2.7 and 3.5 - Tests must pass on python 2.7, 3.4, 3.5, and 3.6
- Tests must pass on both wxPython 3.0.2.0 and 4.0.0
Coding conventions Coding conventions
......
...@@ -55,14 +55,14 @@ def memoize(func): ...@@ -55,14 +55,14 @@ def memoize(func):
try: try:
result = cache[key] result = cache[key]
log.debug('Retrieved from cache[{}]: {}'.format(key, result)) log.debug(u'Retrieved from cache[{}]: {}'.format(key, result))
except KeyError: except KeyError:
result = func(*a, **kwa) result = func(*a, **kwa)
cache[key] = result cache[key] = result
log.debug('Adding to cache[{}]: {}'.format(key, result)) log.debug(u'Adding to cache[{}]: {}'.format(key, result))
return result return result
return wrapper return wrapper
...@@ -82,8 +82,15 @@ def memoizeMD5(func): ...@@ -82,8 +82,15 @@ def memoizeMD5(func):
hashobj = hashlib.md5() hashobj = hashlib.md5()
# Convert each arg to a string
# representation, then encode
# it into a sequence of (utf-8
# compatible) bytes , and take
# the hash of those bytes.
for arg in args: for arg in args:
arg = six.u(str(arg)).encode('utf-8') if not isinstance(arg, six.string_types):
arg = str(arg)
arg = arg.encode('utf-8')
hashobj.update(arg) hashobj.update(arg)
digest = hashobj.hexdigest() digest = hashobj.hexdigest()
...@@ -94,7 +101,7 @@ def memoizeMD5(func): ...@@ -94,7 +101,7 @@ def memoizeMD5(func):
result = func(*args, **kwargs) result = func(*args, **kwargs)
log.debug('Adding to MD5 cache[{}]: {}'.format( log.debug(u'Adding to MD5 cache[{}]: {}'.format(
digest, result)) digest, result))
cache[digest] = result cache[digest] = result
......
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
# Author: Paul McCarthy <pauldmccarthy@gmail.com> # Author: Paul McCarthy <pauldmccarthy@gmail.com>
# #
"""The primary purpose of this module is as a container for the ``fslpy`` """The primary purpose of this module is as a container for the ``fslpy``
version number. A couple of conveniense functions for comparing version version number. A handful of convenience functions for managing version
numbers are also defined here. numbers are also defined here.
.. autosummary:: .. autosummary::
...@@ -14,10 +14,13 @@ numbers are also defined here. ...@@ -14,10 +14,13 @@ numbers are also defined here.
__version__ __version__
parseVersionString parseVersionString
compareVersions compareVersions
patchVersion
The ``fslpy`` version number consists of three numbers, separated by a period, The ``fslpy`` version number consists of three numbers, separated by a period,
which roughly obeys the Semantic Versioning conventions (http://semver.org/): roughly obeys the Semantic Versioning conventions (http://semver.org/), and
is compatible with PEP 440 (https://www.python.org/dev/peps/pep-0440/):
1. The major release number. This gets updated for major/external releases. 1. The major release number. This gets updated for major/external releases.
...@@ -26,13 +29,19 @@ which roughly obeys the Semantic Versioning conventions (http://semver.org/): ...@@ -26,13 +29,19 @@ which roughly obeys the Semantic Versioning conventions (http://semver.org/):
3. The point release number. This gets updated for minor/internal releases, 3. The point release number. This gets updated for minor/internal releases,
which primarily involve bug-fixes and minor changes. which primarily involve bug-fixes and minor changes.
The sole exception to the above convention are evelopment versions, which end
in ``'.dev'``.
""" """
import string import os.path as op
import re
import string
__version__ = 'dev' __version__ = '1.0.2.dev'