Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
FSL
conda
manifest-rules
Commits
44353549
Commit
44353549
authored
Jul 29, 2021
by
Paul McCarthy
🚵
Browse files
MNT: initial commit
parents
Changes
10
Hide whitespace changes
Inline
Side-by-side
.gitlab-ci.yml
0 → 100644
View file @
44353549
stages
:
-
check-version
# Only run on merge requests
workflow
:
rules
:
-
if
:
'
$CI_MERGE_REQUEST_ID
!=
null'
check-version
:
stage
:
check-version
image
:
python:3.9
tags
:
-
fsl-ci
-
docker
script
:
-
thisver=$( cat manifest_rules/__init__.py | grep __version__ | cut -d " " -f 3 | tr -d "'")
-
masterver=$(git show master:manifest_rules/__init__.py | grep __version__ | cut -d " " -f 3 | tr -d "'")
-
|
if [ "$thisver" = "$masterver" ]; then
echo "Version has not been updated!"
echo "Version on master branch: $masterver"
echo "Version in this MR: $thisver"
echo "The version number must be updated before this MR can be merged."
exit 1
else
echo "Version on master branch: $masterver"
echo "Version in this MR: $thisver"
fi
README.md
0 → 100644
View file @
44353549
# FSL release manifest rules
This repository contains scripts which are used in the fsl/conda/manifest>
repository to generate, test, and deploy FSL release manifest and environment
files. Refer to the fsl/conda/manifest> repository for more details.
These scripts are hosted separately from the
`fsl/conda/manifest`
project so
that they can be updated independently of FSL releases.
Whenever changes to these scripts are made, the version number in
`manifest_rules/__init__.py`
must be incremented.
manifest_rules/__init__.py
0 → 100644
View file @
44353549
#!/usr/bin/env python
__version__
=
'0.1.0'
manifest_rules/deploy_files.py
0 → 100644
View file @
44353549
#!/usr/bin/env python
import
os
import
sys
import
shutil
def
main
():
dest
=
sys
.
argv
[
1
]
files
=
sys
.
argv
[
2
:]
publish_from_branches
=
os
.
environ
.
get
(
'PUBLISH_FROM_BRANCHES'
,
''
)
branch
=
os
.
environ
.
get
(
'CI_COMMIT_BRANCH'
,
None
)
tag
=
os
.
environ
.
get
(
'CI_COMMIT_TAG'
,
None
)
allow_publish
=
((
tag
is
not
None
)
or
((
branch
is
not
None
)
and
(
branch
in
publish_from_branches
)))
if
not
allow_publish
:
print
(
'Publishing is only allowed from tags or '
f
'these branches:
{
publish_from_branches
}
'
)
sys
.
exit
(
1
)
if
tag
is
not
None
:
print
(
f
'Publishing files from tag
{
tag
}
'
)
else
:
print
(
f
'Publishing files from branch
{
branch
}
'
)
for
src
in
files
:
print
(
f
'Copying
{
src
}
to
{
dest
}
'
)
shutil
.
copy
(
src
,
dest
)
if
__name__
==
'__main__'
:
main
()
manifest_rules/generate_environment_files.py
0 → 100644
View file @
44353549
#!/usr/bin/env python
#
# Generate FSL conda environment files from fsl-release.yml
import
itertools
as
it
import
os.path
as
op
import
os
import
re
import
sys
from
manifest_rules.utils
import
(
sprun
,
load_release_info
,
generate_environment_file_name
,
generate_development_version_identifier
)
def
filter_packages
(
release_info
,
platform
,
cudaver
):
"""Return a list of packages that should be included in the environment
file for the platform and CUDA version.
"""
packages
=
release_info
[
'packages'
]
cudapat
=
r
'cuda-[\d]+\.[\d]+'
cudapkgs
=
[
p
for
p
in
packages
if
re
.
search
(
cudapat
,
p
)
is
not
None
]
otherpkgs
=
[
p
for
p
in
packages
if
p
not
in
cudapkgs
]
if
cudaver
is
None
:
return
otherpkgs
else
:
include
=
f
'cuda-
{
cudaver
}
'
cudapkgs
=
[
p
for
p
in
cudapkgs
if
include
in
p
]
return
otherpkgs
+
cudapkgs
def
generate_environment
(
release_info
,
version
,
platform
,
cudaver
,
outfile
):
"""Genereate an environment file for the platform and CUDA version.
"""
channels
=
list
(
release_info
[
'channels'
])
packages
=
filter_packages
(
release_info
,
platform
,
cudaver
)
# Dev release
if
version
is
None
:
channels
=
[
release_info
[
'internal_channel'
]]
+
channels
with
open
(
outfile
,
'wt'
)
as
f
:
f
.
write
(
'name: FSL
\n
'
)
f
.
write
(
'channels:
\n
'
)
for
channel
in
channels
:
f
.
write
(
f
' -
{
channel
}
\n
'
)
f
.
write
(
'dependencies:
\n
'
)
for
package
in
packages
:
f
.
write
(
f
' -
{
package
}
\n
'
)
def
generate_variants
(
release_info
):
"""Generate a list of (platform, cudaver) pairs for which an environment
file should be generated. For non-CUDA environments, cudaver will be None.
"""
cudas
=
[
None
]
+
list
(
release_info
[
'cuda'
])
platforms
=
list
(
release_info
[
'miniconda'
].
keys
())
variants
=
list
(
it
.
product
(
platforms
,
cudas
))
# CUDA builds are only supported
# on linux-64 at this time
variants
=
[(
plat
,
cuda
)
for
plat
,
cuda
in
variants
if
not
((
'linux-64'
not
in
plat
)
and
cuda
is
not
None
)]
return
variants
def
main
():
# Save generated environment files here
release_file
=
op
.
abspath
(
sys
.
argv
[
1
])
outdir
=
op
.
abspath
(
sys
.
argv
[
2
])
# Tags on fsl/conda/manifest denote a
# public FSL release. All other
# commits denote an internal release.
version
=
os
.
environ
.
get
(
'CI_COMMIT_TAG'
,
None
)
release_info
=
load_release_info
(
release_file
)
# Generate environment files
# for all platforms+CUDA versions
variants
=
generate_variants
(
release_info
)
if
not
op
.
exists
(
outdir
):
os
.
mkdir
(
outdir
)
generated
=
[]
for
platform
,
cuda
in
variants
:
filename
=
generate_environment_file_name
(
version
,
platform
,
cuda
)
filename
=
op
.
join
(
outdir
,
filename
)
generate_environment
(
release_info
,
version
,
platform
,
cuda
,
filename
)
generated
.
append
((
platform
,
cuda
,
op
.
basename
(
filename
)))
for
platform
,
cuda
,
filename
in
generated
:
print
(
f
'Platform
{
platform
}
[CUDA
{
cuda
}
]:
{
filename
}
'
)
if
__name__
==
'__main__'
:
sys
.
exit
(
main
())
manifest_rules/generate_manifest_file.py
0 → 100644
View file @
44353549
#!/usr/bin/env python
#
# Generate FSL release manifest file from fsl-release.yml
import
os.path
as
op
import
os
import
sys
import
glob
import
json
import
urllib.parse
as
urlparse
from
collections
import
defaultdict
from
manifest_rules.utils
import
(
load_release_info
,
tempdir
,
download_file
,
sha256
,
parse_environment_file_name
,
generate_development_version_identifier
)
def
load_test_install_info
(
install_dir
):
"""Returns information (number of output lines produced) about
the test installations. Returns them in a dict.
"""
inst_info
=
defaultdict
(
dict
)
inst_files
=
glob
.
glob
(
op
.
join
(
install_dir
,
'*.install.txt'
))
for
fname
in
inst_files
:
with
open
(
fname
,
'rt'
)
as
f
:
lines
=
f
.
readlines
()
info
=
lines
[
-
1
].
strip
()
envfile
,
inst_lines
,
env_lines
=
info
.
split
()
platform
=
parse_environment_file_name
(
envfile
)[
1
]
inst_lines
=
int
(
inst_lines
)
env_lines
=
int
(
env_lines
)
prev_inst_lines
=
inst_info
[
'miniconda'
].
get
(
'platform'
,
0
)
inst_lines
=
max
((
inst_lines
,
prev_inst_lines
))
inst_info
[
'miniconda'
][
platform
]
=
inst_lines
inst_info
[
envfile
]
=
env_lines
return
inst_info
def
load_previous_manifest
(
release_info
):
"""Downloads the previous official FSL manifest, returning it
as a dict. Returns None if the manifest cannot be downloaded.
"""
url
=
urlparse
.
urljoin
(
release_info
[
'release_url'
],
'manifest.json'
)
with
tempdir
():
try
:
download_file
(
url
,
'manifest.json'
)
except
Exception
:
return
None
with
open
(
'manifest.json'
,
'rt'
)
as
f
:
lines
=
f
.
readlines
()
# Drop comments
lines
=
[
l
for
l
in
lines
if
not
l
.
lstrip
().
startswith
(
'//'
)]
manifest
=
json
.
loads
(
'
\n
'
.
join
(
lines
))
# Filter out any releases/builds for which
# the environment file is not available
versions
=
dict
(
manifest
.
get
(
'versions'
,
{}))
for
version
,
builds
in
versions
.
items
():
for
i
,
build
in
enumerate
(
list
(
builds
)):
try
:
with
tempdir
():
download_file
(
build
[
'url'
],
'env.yml'
)
except
Exception
:
print
(
f
'WARNING: Build environment file '
f
'is not available:
{
build
}
'
)
builds
.
pop
(
i
)
if
len
(
builds
)
==
0
:
versions
.
pop
(
version
)
else
:
versions
[
version
]
=
builds
latest
=
versions
.
get
(
'latest'
,
None
)
if
(
latest
is
not
None
)
and
(
latest
not
in
versions
):
print
(
f
'WARNING: Build for latest FSL '
f
'version
{
latest
}
is not available'
)
versions
.
pop
(
'latest'
)
manifest
[
'versions'
]
=
versions
return
manifest
def
generate_installer_section
(
release_info
):
"""Generates the "installer" manifest section, returning a dict. """
version
=
release_info
[
'installer'
]
url
=
urlparse
.
urljoin
(
release_info
[
'release_url'
],
'fslinstaller.py'
)
try
:
with
tempdir
():
download_file
(
url
,
'fslinstaller.py'
)
checksum
=
sha256
(
'fslinstaller.py'
)
except
Exception
:
print
(
f
'WARNING: fslinstaller.py script not available at
{
url
}
!'
)
checksum
=
''
return
{
'version'
:
version
,
'url'
:
url
,
'sha256'
:
checksum
}
def
generate_miniconda_section
(
release_info
,
install_info
):
"""Generates the "miniconda" manifest section, returning a dict. """
section
=
{}
outputs
=
install_info
.
get
(
'miniconda'
,
{})
for
platform
,
url
in
release_info
[
'miniconda'
].
items
():
output
=
outputs
.
get
(
platform
,
None
)
with
tempdir
():
download_file
(
url
,
'miniconda.sh'
)
checksum
=
sha256
(
'miniconda.sh'
)
section
[
platform
]
=
{
'url'
:
url
,
'sha256'
:
checksum
,
}
if
output
is
not
None
:
section
[
platform
][
'output'
]
=
str
(
output
)
return
section
def
generate_version_section
(
version
,
include_current_release
,
include_past_releases
,
envdir
,
release_info
,
install_info
):
"""Generates the "versions" manifest section, returning a dict.
Does not add the "latest" entry.
"""
section
=
{}
if
include_past_releases
:
prev_manifest
=
load_previous_manifest
(
release_info
)
if
prev_manifest
is
not
None
:
section
=
prev_manifest
[
'versions'
]
if
not
include_current_release
:
return
section
versions
=
[]
section
=
{
version
:
versions
}
for
envfile
in
glob
.
glob
(
op
.
join
(
envdir
,
'*.yml'
)):
checksum
=
sha256
(
envfile
)
envfile
=
op
.
basename
(
envfile
)
output
=
install_info
.
get
(
envfile
,
None
)
url
=
urlparse
.
urljoin
(
release_info
[
'release_url'
],
envfile
)
version
,
platform
,
cuda
=
parse_environment_file_name
(
envfile
)
build
=
{
'platform'
:
platform
,
'environment'
:
url
,
'sha256'
:
checksum
}
if
cuda
is
not
None
:
build
[
'cuda'
]
=
cuda
versions
.
append
(
build
)
if
output
is
not
None
:
versions
[
-
1
][
'output'
]
=
{
'install'
:
str
(
output
)}
return
section
def
main
():
# This script is run in the following scenarios:
# - New public FSL release - generate a new official
# manifest, describing the new release, and all past
# public releases
# - New internal/development release - generate a
# development manifest describing just the development
# release
# - Update to official manifest - re-generate the
# official manifest, describing all past public
# releases
# We use the "official" and "add_current_release"
# variables to figure out the scenario we are in
release_file
=
op
.
abspath
(
sys
.
argv
[
1
])
# See .gitlab-ci.yml - we are passed "true" or "false"
# to determine whether we are to generate an official
# or development manifest
official
=
sys
.
argv
[
2
]
==
'true'
# Directory to save the generated manifest file
outdir
=
op
.
abspath
(
sys
.
argv
[
3
])
# Directory containing the environment files
# to be added to/described by the manifest
envdir
=
op
.
abspath
(
sys
.
argv
[
4
])
# Directory containing outputs of the test
# installation
installdir
=
op
.
abspath
(
sys
.
argv
[
5
])
# FSL version identifier
version
=
os
.
environ
.
get
(
'CI_COMMIT_TAG'
,
None
)
# Add information about the current release
# (tag or commit) to the generated manifest?
# We do this for new official releases, or
# for development releases
add_current_release
=
(
version
is
not
None
)
or
(
not
official
)
# Generate dev version identnfier
if
version
is
None
:
version
=
generate_development_version_identifier
()
# Load fsl-release.yml, and load
# outputs of test install stage
release_info
=
load_release_info
(
release_file
)
install_info
=
load_test_install_info
(
installdir
)
# Generate the manifest
manifest
=
{
'installer'
:
generate_installer_section
(
release_info
),
'miniconda'
:
generate_miniconda_section
(
release_info
,
install_info
),
'versions'
:
generate_version_section
(
version
,
add_current_release
,
official
,
envdir
,
release_info
,
install_info
),
}
# Set the "latest" field for for
# new public or developmenmt releases
if
add_current_release
:
manifest
[
'versions'
][
'latest'
]
=
version
if
official
:
fname
=
'manifest.json'
else
:
fname
=
f
'manifest-
{
version
}
.json'
manifest
=
json
.
dumps
(
manifest
,
indent
=
4
,
sort_keys
=
True
)
with
open
(
op
.
join
(
outdir
,
fname
),
'wt'
)
as
f
:
f
.
write
(
manifest
)
print
(
f
'Generated manifest file
{
fname
}
:'
)
print
(
manifest
)
if
__name__
==
'__main__'
:
sys
.
exit
(
main
())
manifest_rules/test_environment.py
0 → 100644
View file @
44353549
#!/usr/bin/env python
#
# Performs a test installation for a FSL conda environment file. Counts
# the number of output lines produced when installing miniconda, and
# when installing the FSL environment.
#
import
os.path
as
op
import
os
import
sys
import
shutil
import
yaml
from
manifest_rules.utils
import
(
sprun
,
tempdir
,
download_file
,
load_release_info
,
parse_environment_file_name
)
def
preprocess_environment
(
envfile
,
condadir
,
regen_envfile
=
False
):
with
open
(
envfile
,
'rt'
)
as
f
:
env
=
f
.
read
()
env
=
yaml
.
load
(
env
,
Loader
=
yaml
.
Loader
)
packages
=
env
[
'dependencies'
]
channels
=
env
[
'channels'
]
for
i
,
pkg
in
enumerate
(
packages
):
pkg
,
ver
=
pkg
.
split
()
packages
[
i
]
=
f
'
{
pkg
}
=
{
ver
}
'
with
open
(
op
.
join
(
condadir
,
'condarc'
),
'wt'
)
as
f
:
f
.
write
(
'channels:
\n
'
)
for
c
in
channels
:
f
.
write
(
f
' -
{
c
}
\n
'
)
if
regen_envfile
:
with
open
(
envfile
,
'wt'
)
as
f
:
f
.
write
(
'channels:
\n
'
)
for
c
in
channels
:
f
.
write
(
f
' -
{
c
}
\n
'
)
f
.
write
(
'dependencies:
\n
'
)
for
p
in
packages
:
f
.
write
(
f
' -
{
p
}
\n
'
)
return
packages
def
fast_test
(
envfile
,
release_info
):
install_out
=
sprun
(
'conda create -y -p fsl'
)
packages
=
preprocess_environment
(
envfile
,
'fsl'
)
env
=
os
.
environ
.
copy
()
env
[
'CONDARC'
]
=
op
.
abspath
(
op
.
join
(
'fsl'
,
'condarc'
))
cmd
=
'conda install -p fsl -y --dry-run '
+
' '
.
join
(
packages
)
env_out
=
sprun
(
cmd
,
env
=
env
)
return
install_out
,
env_out
def
full_test
(
envfile
,
release_info
):
platform
=
parse_environment_file_name
(
envfile
)[
1
]
miniconda_url
=
release_info
[
'miniconda'
][
platform
]
download_file
(
miniconda_url
,
'miniconda.sh'
)
install_out
=
sprun
(
'sh ./miniconda.sh -b -p ./fsl'
)
preprocess_environment
(
envfile
,
'fsl'
,
True
)
env
=
os
.
environ
.
copy
()
env
[
'CONDARC'
]
=
op
.
abspath
(
op
.
join
(
'fsl'
,
'condarc'
))
env_out
=
sprun
(
f
'./fsl/bin/conda env update -n base -f
{
envfile
}
'
,
env
=
env
)
return
install_out
,
env_out
def
main
():
release_file
=
op
.
abspath
(
sys
.
argv
[
1
])
envfile
=
op
.
abspath
(
sys
.
argv
[
2
])
release_info
=
load_release_info
(
release_file
)
publish_from_branches
=
os
.
environ
.
get
(
'PUBLISH_FROM_BRANCHES'
,
''
)
branch_name
=
os
.
environ
.
get
(
'CI_COMMIT_BRANCH'
,
None
)
is_tag
=
'CI_COMMIT_TAG'
in
os
.
environ
run_full_test
=
is_tag
or
((
branch_name
is
not
None
)
and
(
branch_name
in
publish_from_branches
))
username
=
os
.
environ
.
get
(
'FSLCONDA_USERNAME'
,
'username'
)
password
=
os
.
environ
.
get
(
'FSLCONDA_PASSWORD'
,
'password'
)
with
tempdir
():
shutil
.
copy
(
envfile
,
'.environment.yml'
)
with
open
(
'.environment.yml'
,
'rt'
)
as
inf
,
\
open
(
envfile
,
'wt'
)
as
outf
:
contents
=
inf
.
read
()
contents
=
contents
.
replace
(
'${FSLCONDA_USERNAME}'
,
username
)
contents
=
contents
.
replace
(
'${FSLCONDA_PASSWORD}'
,
password
)
outf
.
write
(
contents
)
if
run_full_test
:
install_out
,
env_out
=
full_test
(
envfile
,
release_info
)
else
:
install_out
,
env_out
=
fast_test
(
envfile
,
release_info
)
print
(
install_out
)
print
(
env_out
)
# TODO carriage returns in output are being
# interpreted as newlines. Maybe need to
# change sprun to capture output as binary
install_lines
=
len
(
install_out
.
split
(
'
\n
'
))
env_lines
=
len
(
env_out
.
split
(
'
\n
'
))
envfile
=
op
.
basename
(
envfile
)
print
(
f
'
{
envfile
}
{
install_lines
}
{
env_lines
}
'
)
if
__name__
==
'__main__'
:
sys
.
exit
(
main
())
manifest_rules/test_manifest.py
0 → 100644
View file @
44353549
#!/usr/bin/env python
#
# test_manifest.py - Run the fslinstaller script on a generated manifest file.
import
os.path
as
op
import
os
import
sys
import
json
from
manifest_rules.utils
import
sprun
,
tempdir
,
download_file
def
main
():
manifest_file
=
op
.
abspath
(
sys
.
argv
[
1
])
environment_file
=
op
.
abspath
(
sys
.
argv
[
2
])
# credentials for logging into
# internal FSL conda channel
username
=
os
.
environ
[
'FSLCONDA_USERNAME'
]
password
=
os
.
environ
[
'FSLCONDA_PASSWORD'
]