Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
FSL
conda
fsl-ci-rules
Commits
a685dab1
Commit
a685dab1
authored
Oct 04, 2021
by
Paul McCarthy
🚵
Browse files
Merge branch 'mnt/auth' into 'master'
Mnt/auth See merge request fsl/fsl-ci-rules!94
parents
72a01f51
c2ac5316
Changes
4
Hide whitespace changes
Inline
Side-by-side
fsl_ci/__init__.py
View file @
a685dab1
...
@@ -6,20 +6,21 @@
...
@@ -6,20 +6,21 @@
#
#
import
os.path
as
op
import
os.path
as
op
import
os
import
os
import
sys
import
sys
import
errno
import
errno
import
shlex
import
shlex
import
time
import
time
import
tempfile
import
tempfile
import
contextlib
as
ctxlib
import
urllib.parse
as
urlparse
import
subprocess
as
sp
import
contextlib
as
ctxlib
import
subprocess
as
sp
import
yaml
import
yaml
__version__
=
'0.1
6.9
'
__version__
=
'0.1
7.0
'
"""Current version of the fsl-ci-rules."""
"""Current version of the fsl-ci-rules."""
...
@@ -133,6 +134,14 @@ def dumpyaml(o):
...
@@ -133,6 +134,14 @@ def dumpyaml(o):
return
yaml
.
dump
(
o
,
sort_keys
=
False
)
return
yaml
.
dump
(
o
,
sort_keys
=
False
)
def
add_credentials
(
url
,
username
,
password
):
"""Returns a URL with the username/password added."""
username
=
urlparse
.
quote_plus
(
username
)
password
=
urlparse
.
quote_plus
(
password
)
scheme
,
path
=
url
.
split
(
'://'
)
return
f
'
{
scheme
}
://
{
username
}
:
{
password
}
@
{
path
}
'
class
CaptureStdout
:
class
CaptureStdout
:
"""Context manager which captures stdout and stderr. """
"""Context manager which captures stdout and stderr. """
...
...
fsl_ci/scripts/build_conda_package.py
View file @
a685dab1
...
@@ -13,7 +13,8 @@ import urllib.parse as urlparse
...
@@ -13,7 +13,8 @@ import urllib.parse as urlparse
from
fsl_ci
import
(
sprun
,
from
fsl_ci
import
(
sprun
,
indir
,
indir
,
fprint
)
fprint
,
add_credentials
)
from
fsl_ci.versioning
import
generate_staging_version
from
fsl_ci.versioning
import
generate_staging_version
from
fsl_ci.platform
import
get_platform_shortcut_if_not_applicable
from
fsl_ci.platform
import
get_platform_shortcut_if_not_applicable
from
fsl_ci.conda
import
load_meta_yaml
from
fsl_ci.conda
import
load_meta_yaml
...
@@ -119,6 +120,8 @@ def main():
...
@@ -119,6 +120,8 @@ def main():
token
=
os
.
environ
[
'FSL_CI_API_TOKEN'
]
token
=
os
.
environ
[
'FSL_CI_API_TOKEN'
]
staging
=
os
.
environ
[
'STAGING'
].
lower
()
==
'true'
staging
=
os
.
environ
[
'STAGING'
].
lower
()
==
'true'
internal
=
'FSLCONDA_INTERNAL'
in
os
.
environ
internal
=
'FSLCONDA_INTERNAL'
in
os
.
environ
username
=
os
.
environ
.
get
(
'FSLCONDA_INTERNAL_CHANNEL_USERNAME'
,
None
)
password
=
os
.
environ
.
get
(
'FSLCONDA_INTERNAL_CHANNEL_PASSWORD'
,
None
)
skip_platforms
=
os
.
environ
.
get
(
'FSLCONDA_SKIP_PLATFORM'
,
''
)
skip_platforms
=
os
.
environ
.
get
(
'FSLCONDA_SKIP_PLATFORM'
,
''
)
skip_platforms
=
skip_platforms
.
split
()
skip_platforms
=
skip_platforms
.
split
()
...
@@ -145,11 +148,12 @@ def main():
...
@@ -145,11 +148,12 @@ def main():
# we output to _platform_, rather thah platform.
# we output to _platform_, rather thah platform.
output_dir
=
op
.
join
(
os
.
getcwd
(),
'conda_build'
,
f
'_
{
platform
}
_'
)
output_dir
=
op
.
join
(
os
.
getcwd
(),
'conda_build'
,
f
'_
{
platform
}
_'
)
if
internal
:
pubchan
=
os
.
environ
[
'FSLCONDA_PUBLIC_CHANNEL_URL'
]
channel_urls
=
[
os
.
environ
[
'FSLCONDA_INTERNAL_CHANNEL_URL'
],
intchan
=
os
.
environ
[
'FSLCONDA_INTERNAL_CHANNEL_URL'
]
os
.
environ
[
'FSLCONDA_PUBLIC_CHANNEL_URL'
]]
if
username
is
not
None
:
else
:
intchan
=
add_credentials
(
intchan
,
username
,
password
)
channel_urls
=
[
os
.
environ
[
'FSLCONDA_PUBLIC_CHANNEL_URL'
]]
if
internal
:
channel_urls
=
[
intchan
,
pubchan
]
else
:
channel_urls
=
[
pubchan
]
os
.
makedirs
(
output_dir
)
os
.
makedirs
(
output_dir
)
...
...
fsl_ci/scripts/run_unit_tests.py
View file @
a685dab1
...
@@ -12,7 +12,7 @@ import os
...
@@ -12,7 +12,7 @@ import os
import
sys
import
sys
import
glob
import
glob
from
fsl_ci
import
sprun
from
fsl_ci
import
sprun
,
add_credentials
from
fsl_ci.platform
import
get_platform_shortcut_if_not_applicable
from
fsl_ci.platform
import
get_platform_shortcut_if_not_applicable
from
fsl_ci.conda
import
(
load_meta_yaml
,
from
fsl_ci.conda
import
(
load_meta_yaml
,
get_project_repository_and_revision
)
get_project_repository_and_revision
)
...
@@ -55,6 +55,11 @@ def create_test_env(env_dir, build_dir):
...
@@ -55,6 +55,11 @@ def create_test_env(env_dir, build_dir):
internal
=
'FSLCONDA_INTERNAL'
in
os
.
environ
internal
=
'FSLCONDA_INTERNAL'
in
os
.
environ
intchannel
=
os
.
environ
[
'FSLCONDA_INTERNAL_CHANNEL_URL'
]
intchannel
=
os
.
environ
[
'FSLCONDA_INTERNAL_CHANNEL_URL'
]
pubchannel
=
os
.
environ
[
'FSLCONDA_PUBLIC_CHANNEL_URL'
]
pubchannel
=
os
.
environ
[
'FSLCONDA_PUBLIC_CHANNEL_URL'
]
username
=
os
.
environ
.
get
(
'FSLCONDA_INTERNAL_CHANNEL_USERNAME'
,
None
)
password
=
os
.
environ
.
get
(
'FSLCONDA_INTERNAL_CHANNEL_PASSWORD'
,
None
)
if
username
is
not
None
:
intchannel
=
add_credentials
(
intchannel
,
username
,
password
)
if
internal
:
pkgchannels
=
[
intchannel
,
pubchannel
]
if
internal
:
pkgchannels
=
[
intchannel
,
pubchannel
]
else
:
pkgchannels
=
[
pubchannel
]
else
:
pkgchannels
=
[
pubchannel
]
...
...
fsl_ci/utils/trigger_build.py
View file @
a685dab1
...
@@ -7,10 +7,12 @@
...
@@ -7,10 +7,12 @@
#
#
import
sys
import
os.path
as
op
import
os.path
as
op
import
functools
as
ft
import
functools
as
ft
import
textwrap
as
tw
import
textwrap
as
tw
import
argparse
import
argparse
import
datetime
import
multiprocessing.dummy
as
mp
import
multiprocessing.dummy
as
mp
from
fsl_ci.recipe
import
get_recipe_variable
from
fsl_ci.recipe
import
get_recipe_variable
...
@@ -32,6 +34,11 @@ SERVER_URL = 'https://git.fmrib.ox.ac.uk'
...
@@ -32,6 +34,11 @@ SERVER_URL = 'https://git.fmrib.ox.ac.uk'
"""Default gitlab instance URL, if not specified on the command.line."""
"""Default gitlab instance URL, if not specified on the command.line."""
def
now
():
"""Returns the current time as a string. """
return
datetime
.
datetime
.
now
().
strftime
(
'%H:%M:%S'
)
def
get_revision
(
recipe_path
,
server
,
token
):
def
get_revision
(
recipe_path
,
server
,
token
):
"""Return the value of the FSLCONDA_REVISION variable on the given
"""Return the value of the FSLCONDA_REVISION variable on the given
conda recipe repository, or None if it is not set.
conda recipe repository, or None if it is not set.
...
@@ -85,7 +92,7 @@ def trigger_build(project, server, token, production):
...
@@ -85,7 +92,7 @@ def trigger_build(project, server, token, production):
pid
=
pipeline
[
'id'
]
pid
=
pipeline
[
'id'
]
print
(
f
'Pipeline triggered on
{
project
}
(
{
channel
}
build) '
print
(
f
'
{
now
()
}
Pipeline triggered on
{
project
}
(
{
channel
}
build) '
f
'- see
{
pipeline
[
"web_url"
]
}
'
)
f
'- see
{
pipeline
[
"web_url"
]
}
'
)
try
:
try
:
...
@@ -93,7 +100,7 @@ def trigger_build(project, server, token, production):
...
@@ -93,7 +100,7 @@ def trigger_build(project, server, token, production):
except
Exception
:
except
Exception
:
return
None
return
None
print
(
f
'Build pipeline for
{
project
}
has finished:
{
status
}
'
)
print
(
f
'
{
now
()
}
Build pipeline for
{
project
}
has finished:
{
status
}
'
)
if
status
!=
'manual'
:
if
status
!=
'manual'
:
return
None
return
None
...
@@ -106,6 +113,11 @@ def trigger_deploy(project, pid, server, token, production):
...
@@ -106,6 +113,11 @@ def trigger_deploy(project, pid, server, token, production):
and waits for it to complete.
and waits for it to complete.
"""
"""
# trigger_build returns None
# if the build failed
if
pid
is
None
:
return
False
# deployment to staging/production gets set at
# deployment to staging/production gets set at
# build time, so we don't need to pass the STAGING
# build time, so we don't need to pass the STAGING
# variable here, like we do in trigger_build
# variable here, like we do in trigger_build
...
@@ -116,7 +128,7 @@ def trigger_deploy(project, pid, server, token, production):
...
@@ -116,7 +128,7 @@ def trigger_deploy(project, pid, server, token, production):
meta
=
load_meta_yaml
(
meta
)
meta
=
load_meta_yaml
(
meta
)
platforms
=
get_platform_ids
(
meta
)
platforms
=
get_platform_ids
(
meta
)
print
(
f
'Triggering deploy-
{
platforms
}
-conda-package jobs '
print
(
f
'
{
now
()
}
Triggering deploy-
{
platforms
}
-conda-package jobs '
f
'on
{
project
}
(deploying to
{
channel
}
channel)'
)
f
'on
{
project
}
(deploying to
{
channel
}
channel)'
)
try
:
try
:
...
@@ -128,9 +140,11 @@ def trigger_deploy(project, pid, server, token, production):
...
@@ -128,9 +140,11 @@ def trigger_deploy(project, pid, server, token, production):
trigger_job
(
project
,
j
[
'id'
],
server
,
token
)
trigger_job
(
project
,
j
[
'id'
],
server
,
token
)
status
=
wait_on_pipeline
(
project
,
pid
,
server
,
token
)
status
=
wait_on_pipeline
(
project
,
pid
,
server
,
token
)
print
(
f
'Deploy job
{
project
}
has finished:
{
status
}
'
)
print
(
f
'
{
now
()
}
Deploy job
{
project
}
has finished:
{
status
}
'
)
return
True
except
Exception
as
e
:
except
Exception
as
e
:
print
(
f
'Error triggering deploy job on
{
project
}
:
{
e
}
'
)
print
(
f
'Error triggering deploy job on
{
project
}
:
{
e
}
'
)
return
False
def
parseArgs
(
argv
=
None
):
def
parseArgs
(
argv
=
None
):
...
@@ -183,19 +197,22 @@ def main(argv=None):
...
@@ -183,19 +197,22 @@ def main(argv=None):
server
=
args
.
server
,
server
=
args
.
server
,
token
=
args
.
token
,
token
=
args
.
token
,
production
=
args
.
production
)
production
=
args
.
production
)
result
=
True
if
args
.
sequential
:
if
args
.
sequential
:
for
project
in
projects
:
for
project
in
projects
:
pid
=
build
(
project
)
pid
=
build
(
project
)
deploy
(
project
,
pid
)
if
not
deploy
(
project
,
pid
):
result
=
False
else
:
else
:
pool
=
mp
.
Pool
(
len
(
projects
))
pool
=
mp
.
Pool
(
len
(
projects
))
pids
=
pool
.
map
(
build
,
projects
)
pids
=
pool
.
map
(
build
,
projects
)
pool
.
starmap
(
deploy
,
zip
(
projects
,
pids
))
result
=
all
(
pool
.
starmap
(
deploy
,
zip
(
projects
,
pids
))
)
pool
.
close
()
pool
.
close
()
pool
.
join
()
pool
.
join
()
return
0
if
result
else
1
if
__name__
==
'__main__'
:
if
__name__
==
'__main__'
:
main
()
sys
.
exit
(
main
()
)
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment