Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
FSL
fslpy
Commits
aa198fc7
Commit
aa198fc7
authored
Jul 09, 2020
by
Paul McCarthy
🚵
Browse files
Merge branch 'rel/3.2.2' into 'v3.2'
Rel/3.2.2 See merge request fsl/fslpy!244
parents
33e55210
2e1d49b5
Pipeline
#5551
passed with stages
in 1 minute and 52 seconds
Changes
4
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
CHANGELOG.rst
View file @
aa198fc7
...
...
@@ -2,6 +2,17 @@ This document contains the ``fslpy`` release history in reverse chronological
order.
3.2.2 (Thursday 9th July 2020)
------------------------------
Changed
^^^^^^^
* The :func:`.fslsub.func_to_cmd` function allows more fine-grained control
over whether the script file is removed after the job has finished running.
3.2.1 (Tuesday 23rd June 2020)
------------------------------
...
...
fsl/utils/fslsub.py
View file @
aa198fc7
...
...
@@ -435,14 +435,13 @@ def hold(job_ids, hold_filename=None):
os
.
remove
(
hold_filename
)
_external_job
=
"""#!{}
_external_job
=
(
"""#!{}
# This is a temporary file designed to run the python function {},
# so that it can be submitted to the cluster
import pickle
from six import BytesIO
from importlib import import_module
{}
pickle_bytes = BytesIO({})
name_type, name, func_name, args, kwargs = pickle.load(pickle_bytes)
...
...
@@ -456,15 +455,13 @@ elif name_type == 'script':
func = local_execute[func_name]
else:
raise ValueError('Unknown name_type: %r' % name_type)
{}
res = func(*args, **kwargs)
if res is not None:
with open(__file__ + '_out.pickle', 'w') as f:
pickle.dump(f, res)
"""
"""
)
def
func_to_cmd
(
func
,
args
,
kwargs
,
tmp_dir
=
None
,
clean
=
False
):
def
func_to_cmd
(
func
,
args
=
None
,
kwargs
=
None
,
tmp_dir
=
None
,
clean
=
"never"
,
verbose
=
False
):
"""Defines the command needed to run the function from the command line
WARNING: if submitting a function defined in the __main__ script,
...
...
@@ -475,9 +472,21 @@ def func_to_cmd(func, args, kwargs, tmp_dir=None, clean=False):
:arg args: positional arguments
:arg kwargs: keyword arguments
:arg tmp_dir: directory where to store the temporary file
:arg clean: if True removes the submitted script after running it
:arg clean: Whether the script should be removed after running. There are three options:
- "never" (default): Script is kept
- "on_success": only remove if script successfully finished (i.e., no error is raised)
- "always": always remove the script, even if it raises an error
:arg verbose: If set to True, the script will print its own filename before running
:return: string which will run the function
"""
if
clean
not
in
(
'never'
,
'always'
,
'on_success'
):
raise
ValueError
(
f
"Clean should be one of 'never', 'always', or 'on_success', not
{
clean
}
"
)
if
args
is
None
:
args
=
()
if
kwargs
is
None
:
kwargs
=
{}
pickle_bytes
=
BytesIO
()
if
func
.
__module__
==
'__main__'
:
pickle
.
dump
((
'script'
,
importlib
.
import_module
(
'__main__'
).
__file__
,
func
.
__name__
,
...
...
@@ -485,15 +494,29 @@ def func_to_cmd(func, args, kwargs, tmp_dir=None, clean=False):
else
:
pickle
.
dump
((
'module'
,
func
.
__module__
,
func
.
__name__
,
args
,
kwargs
),
pickle_bytes
)
python_cmd
=
_external_job
.
format
(
sys
.
executable
,
func
.
__name__
,
pickle_bytes
.
getvalue
())
_
,
filename
=
tempfile
.
mkstemp
(
prefix
=
func
.
__name__
+
'_'
,
suffix
=
'.py'
,
dir
=
tmp_dir
)
verbose_script
=
f
'
\n
print("running
{
filename
}
")
\n
'
if
verbose
else
''
if
clean
==
'never'
:
run_script
=
"res = func(*args, **kwargs)"
elif
clean
==
'always'
:
run_script
=
f
"""try:
res = func(*args, **kwargs)
finally:
import os; os.remove("
{
filename
}
")"""
elif
clean
==
'on_success'
:
run_script
=
f
"""res = func(*args, **kwargs)
import os; os.remove("
{
filename
}
")"""
python_cmd
=
_external_job
.
format
(
sys
.
executable
,
func
.
__name__
,
verbose_script
,
pickle_bytes
.
getvalue
(),
run_script
)
with
open
(
filename
,
'w'
)
as
python_file
:
python_file
.
write
(
python_cmd
)
return
sys
.
executable
+
" "
+
filename
+
(
'; rm '
+
filename
if
clean
else
''
)
return
sys
.
executable
+
" "
+
filename
fsl/version.py
View file @
aa198fc7
...
...
@@ -47,7 +47,7 @@ import re
import
string
__version__
=
'3.2.
1
'
__version__
=
'3.2.
2
'
"""Current version number, as a string. """
...
...
tests/test_fslsub.py
View file @
aa198fc7
...
...
@@ -16,7 +16,7 @@ import argparse
import
pytest
import
fsl
from
fsl.utils
import
fslsub
from
fsl.utils
import
fslsub
,
run
from
fsl.utils.tempdir
import
tempdir
from
.
import
mockFSLDIR
...
...
@@ -256,3 +256,44 @@ def test_info():
with
pytest
.
raises
(
ValueError
):
fslsub
.
_parse_qstat
(
valid_job_ids
[
0
],
example_qstat_reply
)
def
_good_func
():
print
(
'hello'
)
def
_bad_func
():
1
/
0
def
test_func_to_cmd
():
cwd
=
os
.
getcwd
()
with
tempdir
():
for
tmp_dir
in
(
None
,
'.'
):
for
clean
in
(
'never'
,
'on_success'
,
'always'
):
for
verbose
in
(
False
,
True
):
cmd
=
fslsub
.
func_to_cmd
(
_good_func
,
clean
=
clean
,
tmp_dir
=
tmp_dir
,
verbose
=
verbose
)
fn
=
cmd
.
split
()[
-
1
]
assert
op
.
exists
(
fn
)
stdout
,
stderr
,
exitcode
=
run
.
run
(
cmd
,
exitcode
=
True
,
stdout
=
True
,
stderr
=
True
,
env
=
{
"PYTHONPATH"
:
cwd
})
assert
exitcode
==
0
if
clean
==
'never'
:
assert
op
.
exists
(
fn
),
"Successful job got removed, even though this was not requested"
else
:
assert
not
op
.
exists
(
fn
),
f
"Successful job did not get removed after run for clean =
{
clean
}
"
if
verbose
:
assert
stdout
.
strip
()
==
f
'running
{
fn
}
\n
hello'
else
:
assert
stdout
.
strip
()
==
'hello'
cmd
=
fslsub
.
func_to_cmd
(
_bad_func
,
clean
=
clean
,
tmp_dir
=
tmp_dir
)
fn
=
cmd
.
split
()[
-
1
]
assert
op
.
exists
(
fn
)
stdout
,
stderr
,
exitcode
=
run
.
run
(
cmd
,
exitcode
=
True
,
stdout
=
True
,
stderr
=
True
,
env
=
{
'PYTHONPATH'
:
cwd
})
assert
exitcode
!=
0
if
clean
==
'always'
:
assert
not
op
.
exists
(
fn
),
"Failing job should always be removed if requested"
else
:
assert
op
.
exists
(
fn
),
f
"Failing job got removed even with clean =
{
clean
}
"
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment