#!/usr/bin/env python # # Build a conda package from a FSL recipe repository. # # Author: Paul McCarthy # # (See rules/fsl-ci-build-rules.yml) import os import os.path as op import urllib.parse as urlparse from fsl_ci import (sprun, indir, fprint, add_credentials) from fsl_ci.versioning import generate_staging_version from fsl_ci.platform import get_platform_shortcut_if_not_applicable from fsl_ci.conda import load_meta_yaml from fsl_ci.gitlab import get_revision_hash from fsl_ci.recipe import (get_recipe_variable, patch_recipe_version) def build_recipe(recipe_dir, repo, ref, output_dir, *channels): """Build the conda recipe in the given directory. """ env = dict(os.environ) if repo in ('', None): env.pop('FSLCONDA_REPOSITORY', None) else: env['FSLCONDA_REPOSITORY'] = repo if ref in ('', None): env.pop('FSLCONDA_REVISION', None) else: env['FSLCONDA_REVISION'] = ref cmd = 'conda build -c conda-forge -c defaults' for chan in channels: cmd = f'{cmd} -c {chan}' sprun(f'{cmd} --output-folder={output_dir} {recipe_dir}', env=env) def build_report(output_dir): """Lists all files produced by build_recipe.""" for dirpath, _, filenames in os.walk(output_dir): for filename in filenames: filepath = op.join(dirpath, filename) size = op.getsize(filepath) / 1048576 fprint(f'{size:8.2f}MB {filepath}') def patch_recipe(recipe_dir, project_ref, server, token): """Patches the package version number in the recipe metadata. For staging builds, a development suffix is added to the most recent stable project version number. :arg recipe_dir: Directory to find the recipe meta.yaml :arg project_ref: Git reference of project being built :arg server: GitLab server URL :arg token: GitLab API access token """ with open(op.join(recipe_dir, 'meta.yaml')) as f: meta = f.read() # Get project repository and version string # for most recent production release repo = get_recipe_variable(meta, 'repository') ver = get_recipe_variable(meta, 'version') gitref = None # if this is a FSL project, retrieve the git # revision and bake it into the version string # Staging packages are generally not built for # externally hosted projects, but the option # is there if we need it. See the # generate_staging_version function. if repo is not None and server in repo: project_path = urlparse.urlparse(repo).path.replace('.git', '')[1:] gitref = get_revision_hash(project_path, server, token, project_ref) ver = generate_staging_version(ver, gitref) meta = patch_recipe_version(meta, ver) with open(op.join(recipe_dir, 'meta.yaml'), 'wt') as f: f.write(meta) def main(): """Build a conda package from a FSL recipe repository. This script assumes that the following environment variables are set, in addition to the Gitlab CI predefined environment variables: - FSLCONDA_REVISION: Name of the git ref (e.g. tag, branch) to build the recipe from. If empty or unset, ref specified in the recipe meta.yaml file is used. - FSLCONDA_REPOSITORY: URL of the git repository to build the recipe from. If empty or unset, the repo specified in the recipe meta.yaml file is used. - STAGING: If equal to "true", the package is marked as a staging/ development package. - FSLCONDA_INTERNAL: Marks this package as internal - if set, the FSLCONDA_INTENRAL_CHANNEL_URL will be added to the list of conda channels to source from, when building the package. - FSL_CI_API_TOKEN: GitLab API access token """ project_ref = os.environ.get('FSLCONDA_REVISION', '') project_repo = os.environ.get('FSLCONDA_REPOSITORY', '') recipe_url = os.environ['CI_PROJECT_URL'] recipe_ref = os.environ['CI_COMMIT_REF_NAME'] job_name = os.environ['CI_JOB_NAME'] server_url = os.environ['CI_SERVER_URL'] token = os.environ['FSL_CI_API_TOKEN'] staging = os.environ['STAGING'].lower() == 'true' internal = 'FSLCONDA_INTERNAL' in os.environ username = os.environ.get('FSLCONDA_INTERNAL_CHANNEL_USERNAME', None) password = os.environ.get('FSLCONDA_INTERNAL_CHANNEL_PASSWORD', None) skip_platforms = os.environ.get('FSLCONDA_SKIP_PLATFORM', '') skip_platforms = skip_platforms.split() # get the package name from the un-rendered metayaml text with open('meta.yaml', 'rt') as f: meta = f.read() package_name = get_recipe_variable(meta, 'name') # then load the rendered meta.yaml for use # by the get_platform_shortcut.. function meta = load_meta_yaml('meta.yaml') platform = get_platform_shortcut_if_not_applicable( meta, package_name, job_name, skip_platforms) # We direct builds for each platform type into a # separate sub-directory, because the outputs from # all builds will be forwarded to the deploy-package # job, and we want to avoid file collisions. # # conda has some seriously fucked up behaviour - # if you try to direct the build into a directory # called "noarch", it thinks that you are directing # it to a channel subdirectory, and explodes. So # we output to _platform_, rather thah platform. output_dir = op.join(os.getcwd(), 'conda_build', f'_{platform}_') pubchan = os.environ['FSLCONDA_PUBLIC_CHANNEL_URL'] intchan = os.environ['FSLCONDA_INTERNAL_CHANNEL_URL'] if username is not None: intchan = add_credentials(intchan, username, password) if internal: channel_urls = [intchan, pubchan] else: channel_urls = [pubchan] os.makedirs(output_dir) fprint('************************************') fprint(f'Building conda recipe for: {package_name}') fprint(f'Recipe URL: {recipe_url}') fprint(f'Recipe revision: {recipe_ref}') fprint( 'Project repository (empty means to ') fprint(f' build from repo specified in meta.yaml): {project_repo}') fprint( 'Project revision (empty means to ') fprint(f' build release specified in meta.yaml): {project_ref}') fprint(f'FSL conda channel URLs: {channel_urls}') fprint('************************************') if project_repo == '': project_repo = None if project_ref == '': project_ref = None # patch the recipe so it has a staging version number if staging and project_ref is not None: patch_recipe('.', project_ref, server_url, token) build_recipe('.', project_repo, project_ref, output_dir, *channel_urls) with indir(output_dir): build_report('.') if __name__ == '__main__': main()