build_conda_package.py

#!/usr/bin/env python
#
# Build a conda package from a FSL recipe repository.
#
# Author: Paul McCarthy <pauldmccarthy@gmail.com>
#
# (See rules/fsl-ci-build-rules.yml)


import                 os
import os.path      as op
import urllib.parse as urlparse

from fsl_ci            import (sprun,
                               indir,
                               fprint,
                               add_credentials)
from fsl_ci.versioning import  generate_staging_version
from fsl_ci.platform   import  get_platform_shortcut_if_not_applicable
from fsl_ci.conda      import  load_meta_yaml
from fsl_ci.gitlab     import  get_revision_hash
from fsl_ci.recipe     import (get_recipe_variable,
                               patch_recipe_version)


def build_recipe(recipe_dir, repo, ref, output_dir, *channels):
    """Build the conda recipe in the given directory. """

    env = dict(os.environ)

    if repo in ('', None): env.pop('FSLCONDA_REPOSITORY', None)
    else:                  env['FSLCONDA_REPOSITORY'] = repo
    if ref  in ('', None): env.pop('FSLCONDA_REVISION', None)
    else:                  env['FSLCONDA_REVISION'] = ref

    cmd = 'conda build -c conda-forge -c defaults'
    for chan in channels:
        cmd = f'{cmd} -c {chan}'

    sprun(f'{cmd} --output-folder={output_dir} {recipe_dir}', env=env)


def build_report(output_dir):
    """Lists all files produced by build_recipe."""
    for dirpath, _, filenames in os.walk(output_dir):
        for filename in filenames:
            filepath = op.join(dirpath, filename)
            size     = op.getsize(filepath) / 1048576
            fprint(f'{size:8.2f}MB {filepath}')


def patch_recipe(recipe_dir, project_ref, server, token):
    """Patches the package version number in the recipe metadata.

    For staging builds, a development suffix is added to the most recent
    stable project version number.

    :arg recipe_dir:  Directory to find the recipe meta.yaml
    :arg project_ref: Git reference of project being built
    :arg server:      GitLab server URL
    :arg token:       GitLab API access token
    """

    with open(op.join(recipe_dir, 'meta.yaml')) as f:
        meta = f.read()

    # Get project repository and version string
    # for most recent production release
    repo   = get_recipe_variable(meta, 'repository')
    ver    = get_recipe_variable(meta, 'version')
    gitref = None

    # if this is a FSL project, retrieve the git
    # revision and bake it into the version string
    # Staging packages are generally not built for
    # externally hosted projects, but the option
    # is there if we need it. See the
    # generate_staging_version function.
    if repo is not None and server in repo:
        project_path = urlparse.urlparse(repo).path.replace('.git', '')[1:]
        gitref = get_revision_hash(project_path, server, token, project_ref)

    ver  = generate_staging_version(ver, gitref)
    meta = patch_recipe_version(meta, ver)

    with open(op.join(recipe_dir, 'meta.yaml'), 'wt') as f:
        f.write(meta)


def main():
    """Build a conda package from a FSL recipe repository.

    This script assumes that the following environment variables
    are set, in addition to the Gitlab CI predefined environment variables:

     - FSLCONDA_REVISION: Name of the git ref (e.g. tag, branch) to build the
       recipe from. If empty or unset, ref specified in the recipe meta.yaml
       file is used.

     - FSLCONDA_REPOSITORY: URL of the git repository to build the recipe
       from. If empty or unset, the repo specified in the recipe meta.yaml
       file is used.

     - STAGING: If equal to "true", the package is marked as a staging/
       development package.

     - FSLCONDA_INTERNAL: Marks this package as internal - if set, the
       FSLCONDA_INTENRAL_CHANNEL_URL will be added to the list of conda
       channels to source from, when building the package.

     - FSL_CI_API_TOKEN: GitLab API access token
    """

    project_ref    = os.environ.get('FSLCONDA_REVISION',   '')
    project_repo   = os.environ.get('FSLCONDA_REPOSITORY', '')
    recipe_url     = os.environ['CI_PROJECT_URL']
    recipe_ref     = os.environ['CI_COMMIT_REF_NAME']
    job_name       = os.environ['CI_JOB_NAME']
    server_url     = os.environ['CI_SERVER_URL']
    token          = os.environ['FSL_CI_API_TOKEN']
    staging        = os.environ['STAGING'].lower() == 'true'
    internal       = 'FSLCONDA_INTERNAL' in os.environ
    username       = os.environ.get('FSLCONDA_INTERNAL_CHANNEL_USERNAME', None)
    password       = os.environ.get('FSLCONDA_INTERNAL_CHANNEL_PASSWORD', None)
    skip_platforms = os.environ.get('FSLCONDA_SKIP_PLATFORM', '')
    skip_platforms = skip_platforms.split()

    # get the package name from the un-rendered metayaml text
    with open('meta.yaml', 'rt') as f:
        meta = f.read()
    package_name = get_recipe_variable(meta, 'name')

    # then load the rendered meta.yaml for use
    # by the get_platform_shortcut.. function
    meta     = load_meta_yaml('meta.yaml')
    platform = get_platform_shortcut_if_not_applicable(
        meta, package_name, job_name, skip_platforms)

    # We direct builds for each platform type into a
    # separate sub-directory, because the outputs from
    # all builds will be forwarded to the deploy-package
    # job, and we want to avoid file collisions.
    #
    # conda has some seriously fucked up behaviour -
    # if you try to direct the build into a directory
    # called "noarch", it thinks that you are directing
    # it to a channel subdirectory, and explodes. So
    # we output to _platform_, rather thah platform.
    output_dir = op.join(os.getcwd(), 'conda_build', f'_{platform}_')

    pubchan = os.environ['FSLCONDA_PUBLIC_CHANNEL_URL']
    intchan = os.environ['FSLCONDA_INTERNAL_CHANNEL_URL']
    if username is not None:
        intchan = add_credentials(intchan, username, password)
    if internal: channel_urls = [intchan, pubchan]
    else:        channel_urls = [pubchan]

    os.makedirs(output_dir)

    fprint('************************************')
    fprint(f'Building conda recipe for:                 {package_name}')
    fprint(f'Recipe URL:                                {recipe_url}')
    fprint(f'Recipe revision:                           {recipe_ref}')
    fprint( 'Project repository (empty means to ')
    fprint(f'  build from repo specified in meta.yaml): {project_repo}')
    fprint( 'Project revision (empty means to ')
    fprint(f'  build release specified in meta.yaml):   {project_ref}')
    fprint(f'FSL conda channel URLs:                    {channel_urls}')
    fprint('************************************')

    if project_repo == '': project_repo = None
    if project_ref  == '': project_ref  = None

    # patch the recipe so it has a staging version number
    if staging and project_ref is not None:
        patch_recipe('.', project_ref, server_url, token)

    build_recipe('.',
                 project_repo,
                 project_ref,
                 output_dir,
                 *channel_urls)
    with indir(output_dir):
        build_report('.')


if __name__ == '__main__':
    main()