Commit 3287a371 authored by Michiel Cottaar's avatar Michiel Cottaar Committed by Paul McCarthy
Browse files

Have fsl_sub return just the job id.

The functionality of merging multiple jobs is now ensured by allowing
for nested sequences of job-ids.
parent 9d5c986e
......@@ -20,9 +20,10 @@ Example usage, building a short pipeline::
other_job = submit('some other pre-processing step', queue='short.q')
# submits cuda job, that should only start after both preparatory jobs are
# finished
# finished. This will work if bet_job and other_job are single job-ids
# (i.e., strings) or a sequence of multiple job-ids
cuda_job = submit('expensive job',
wait_for=bet_job + other_job,
wait_for=(bet_job, other_job),
queue='cuda.q')
# waits for the cuda job to finish
......@@ -125,9 +126,7 @@ def submit(command,
base_cmd.append('-v')
if wait_for:
if not isinstance(wait_for, string_types):
wait_for = ','.join(wait_for)
base_cmd.extend(['-j', wait_for])
base_cmd.extend(['-j', _flatten_job_ids(wait_for)])
if multi_threaded:
base_cmd.append('-s')
......@@ -135,7 +134,7 @@ def submit(command,
base_cmd.append(command)
return (runfsl(*base_cmd).strip(), )
return runfsl(*base_cmd).strip()
def info(job_id):
......@@ -215,6 +214,26 @@ def wait(job_ids):
log.debug('All jobs have finished')
def _flatten_job_ids(job_ids):
"""
Returns a potentially nested sequence of job ids as a single comma-separated string
:param job_ids: possibly nested sequence of job ids. The job ids themselves should be strings.
:return: comma-separated string of job ids
"""
def unpack(job_ids):
"""Unpack the (nested) job-ids in a single set"""
if isinstance(job_ids, str):
return {job_ids}
else:
res = set()
for job_id in job_ids:
res.update(unpack(job_id))
return res
return ','.join(sorted(unpack(job_ids)))
_external_job = """#!{}
# This is a temporary file designed to run the python function {},
# so that it can be submitted to the cluster
......
#!/usr/bin/env python
#
# test_fslsub.py - Tests functions in the fsl.utils.fslsub module.
#
# Author: Michiel Cottaar <Michiel.Cottaar@ndcn.ox.ac.uk>
#
from fsl.utils import fslsub
def test_flatten_jobids():
job_ids = ('12', '27', '35', '41', '721')
res = '12,27,35,41,721'
assert fslsub._flatten_job_ids(job_ids) == res
assert fslsub._flatten_job_ids(job_ids[::-1]) == res
assert fslsub._flatten_job_ids('12') == '12'
assert fslsub._flatten_job_ids([job_ids[:2], job_ids[2:]]) == res
assert fslsub._flatten_job_ids([set(job_ids[:2]), job_ids[2:]]) == res
assert fslsub._flatten_job_ids(((job_ids, ), job_ids + job_ids)) == res
......@@ -249,7 +249,7 @@ def mock_submit(cmd, **kwargs):
for k in sorted(kwargs.keys()):
f.write('{}: {}\n'.format(k, kwargs[k]))
return (jid,)
return jid
def test_run_submit():
......@@ -271,7 +271,7 @@ def test_run_submit():
mkexec(op.expandvars('$FSLDIR/bin/fsltest'), test_script)
jid = run.run('fsltest', submit=True)[0]
jid = run.run('fsltest', submit=True)
assert jid == '12345'
......@@ -282,7 +282,7 @@ def test_run_submit():
kwargs = {'name' : 'abcde', 'ram' : '4GB'}
jid = run.run('fsltest', submit=kwargs)[0]
jid = run.run('fsltest', submit=kwargs)
assert jid == '12345'
......
......@@ -630,7 +630,7 @@ def test_cmdwrapper_submit():
jid = test_func(1, 2, submit=True)
assert jid == ('12345',)
assert jid == '12345'
stdout, stderr = fslsub.output('12345')
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment