Commit aa198fc7 authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

Merge branch 'rel/3.2.2' into 'v3.2'

Rel/3.2.2

See merge request fsl/fslpy!244
parents 33e55210 2e1d49b5
Pipeline #5551 passed with stages
in 1 minute and 52 seconds
...@@ -2,6 +2,17 @@ This document contains the ``fslpy`` release history in reverse chronological ...@@ -2,6 +2,17 @@ This document contains the ``fslpy`` release history in reverse chronological
order. order.
3.2.2 (Thursday 9th July 2020)
------------------------------
Changed
^^^^^^^
* The :func:`.fslsub.func_to_cmd` function allows more fine-grained control
over whether the script file is removed after the job has finished running.
3.2.1 (Tuesday 23rd June 2020) 3.2.1 (Tuesday 23rd June 2020)
------------------------------ ------------------------------
......
...@@ -435,14 +435,13 @@ def hold(job_ids, hold_filename=None): ...@@ -435,14 +435,13 @@ def hold(job_ids, hold_filename=None):
os.remove(hold_filename) os.remove(hold_filename)
_external_job = """#!{} _external_job = ("""#!{}
# This is a temporary file designed to run the python function {}, # This is a temporary file designed to run the python function {},
# so that it can be submitted to the cluster # so that it can be submitted to the cluster
import pickle import pickle
from six import BytesIO from six import BytesIO
from importlib import import_module from importlib import import_module
{}
pickle_bytes = BytesIO({}) pickle_bytes = BytesIO({})
name_type, name, func_name, args, kwargs = pickle.load(pickle_bytes) name_type, name, func_name, args, kwargs = pickle.load(pickle_bytes)
...@@ -457,14 +456,12 @@ elif name_type == 'script': ...@@ -457,14 +456,12 @@ elif name_type == 'script':
else: else:
raise ValueError('Unknown name_type: %r' % name_type) raise ValueError('Unknown name_type: %r' % name_type)
res = func(*args, **kwargs) {}
if res is not None:
with open(__file__ + '_out.pickle', 'w') as f:
pickle.dump(f, res)
"""
""")
def func_to_cmd(func, args, kwargs, tmp_dir=None, clean=False):
def func_to_cmd(func, args=None, kwargs=None, tmp_dir=None, clean="never", verbose=False):
"""Defines the command needed to run the function from the command line """Defines the command needed to run the function from the command line
WARNING: if submitting a function defined in the __main__ script, WARNING: if submitting a function defined in the __main__ script,
...@@ -475,9 +472,21 @@ def func_to_cmd(func, args, kwargs, tmp_dir=None, clean=False): ...@@ -475,9 +472,21 @@ def func_to_cmd(func, args, kwargs, tmp_dir=None, clean=False):
:arg args: positional arguments :arg args: positional arguments
:arg kwargs: keyword arguments :arg kwargs: keyword arguments
:arg tmp_dir: directory where to store the temporary file :arg tmp_dir: directory where to store the temporary file
:arg clean: if True removes the submitted script after running it :arg clean: Whether the script should be removed after running. There are three options:
- "never" (default): Script is kept
- "on_success": only remove if script successfully finished (i.e., no error is raised)
- "always": always remove the script, even if it raises an error
:arg verbose: If set to True, the script will print its own filename before running
:return: string which will run the function :return: string which will run the function
""" """
if clean not in ('never', 'always', 'on_success'):
raise ValueError(f"Clean should be one of 'never', 'always', or 'on_success', not {clean}")
if args is None:
args = ()
if kwargs is None:
kwargs = {}
pickle_bytes = BytesIO() pickle_bytes = BytesIO()
if func.__module__ == '__main__': if func.__module__ == '__main__':
pickle.dump(('script', importlib.import_module('__main__').__file__, func.__name__, pickle.dump(('script', importlib.import_module('__main__').__file__, func.__name__,
...@@ -485,15 +494,29 @@ def func_to_cmd(func, args, kwargs, tmp_dir=None, clean=False): ...@@ -485,15 +494,29 @@ def func_to_cmd(func, args, kwargs, tmp_dir=None, clean=False):
else: else:
pickle.dump(('module', func.__module__, func.__name__, pickle.dump(('module', func.__module__, func.__name__,
args, kwargs), pickle_bytes) args, kwargs), pickle_bytes)
python_cmd = _external_job.format(sys.executable,
func.__name__,
pickle_bytes.getvalue())
_, filename = tempfile.mkstemp(prefix=func.__name__ + '_', _, filename = tempfile.mkstemp(prefix=func.__name__ + '_',
suffix='.py', suffix='.py',
dir=tmp_dir) dir=tmp_dir)
verbose_script = f'\nprint("running {filename}")\n' if verbose else ''
if clean == 'never':
run_script = "res = func(*args, **kwargs)"
elif clean == 'always':
run_script = f"""try:
res = func(*args, **kwargs)
finally:
import os; os.remove("{filename}")"""
elif clean == 'on_success':
run_script = f"""res = func(*args, **kwargs)
import os; os.remove("{filename}")"""
python_cmd = _external_job.format(sys.executable,
func.__name__,
verbose_script,
pickle_bytes.getvalue(),
run_script)
with open(filename, 'w') as python_file: with open(filename, 'w') as python_file:
python_file.write(python_cmd) python_file.write(python_cmd)
return sys.executable + " " + filename + ('; rm ' + filename if clean else '') return sys.executable + " " + filename
...@@ -47,7 +47,7 @@ import re ...@@ -47,7 +47,7 @@ import re
import string import string
__version__ = '3.2.1' __version__ = '3.2.2'
"""Current version number, as a string. """ """Current version number, as a string. """
......
...@@ -16,7 +16,7 @@ import argparse ...@@ -16,7 +16,7 @@ import argparse
import pytest import pytest
import fsl import fsl
from fsl.utils import fslsub from fsl.utils import fslsub, run
from fsl.utils.tempdir import tempdir from fsl.utils.tempdir import tempdir
from . import mockFSLDIR from . import mockFSLDIR
...@@ -256,3 +256,44 @@ def test_info(): ...@@ -256,3 +256,44 @@ def test_info():
with pytest.raises(ValueError): with pytest.raises(ValueError):
fslsub._parse_qstat(valid_job_ids[0], example_qstat_reply) fslsub._parse_qstat(valid_job_ids[0], example_qstat_reply)
def _good_func():
print('hello')
def _bad_func():
1/0
def test_func_to_cmd():
cwd = os.getcwd()
with tempdir():
for tmp_dir in (None, '.'):
for clean in ('never', 'on_success', 'always'):
for verbose in (False, True):
cmd = fslsub.func_to_cmd(_good_func, clean=clean, tmp_dir=tmp_dir, verbose=verbose)
fn = cmd.split()[-1]
assert op.exists(fn)
stdout, stderr, exitcode = run.run(cmd, exitcode=True, stdout=True, stderr=True,
env={"PYTHONPATH": cwd})
assert exitcode == 0
if clean == 'never':
assert op.exists(fn), "Successful job got removed, even though this was not requested"
else:
assert not op.exists(fn), f"Successful job did not get removed after run for clean = {clean}"
if verbose:
assert stdout.strip() == f'running {fn}\nhello'
else:
assert stdout.strip() == 'hello'
cmd = fslsub.func_to_cmd(_bad_func, clean=clean, tmp_dir=tmp_dir)
fn = cmd.split()[-1]
assert op.exists(fn)
stdout, stderr, exitcode = run.run(cmd, exitcode=True, stdout=True, stderr=True,
env={'PYTHONPATH': cwd})
assert exitcode != 0
if clean == 'always':
assert not op.exists(fn), "Failing job should always be removed if requested"
else:
assert op.exists(fn), f"Failing job got removed even with clean = {clean}"
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment