diff --git a/fsl/utils/fslsub.py b/fsl/utils/fslsub.py index 6b87c4bbf8110bf7b31b42e550b52273b8beb7a3..5957ba6f387d6e46190407e01b733f81fcd025c9 100644 --- a/fsl/utils/fslsub.py +++ b/fsl/utils/fslsub.py @@ -20,9 +20,10 @@ Example usage, building a short pipeline:: other_job = submit('some other pre-processing step', queue='short.q') # submits cuda job, that should only start after both preparatory jobs are - # finished + # finished. This will work if bet_job and other_job are single job-ids + # (i.e., strings) or a sequence of multiple job-ids cuda_job = submit('expensive job', - wait_for=bet_job + other_job, + wait_for=(bet_job, other_job), queue='cuda.q') # waits for the cuda job to finish @@ -125,9 +126,7 @@ def submit(command, base_cmd.append('-v') if wait_for: - if not isinstance(wait_for, string_types): - wait_for = ','.join(wait_for) - base_cmd.extend(['-j', wait_for]) + base_cmd.extend(['-j', _flatten_job_ids(wait_for)]) if multi_threaded: base_cmd.append('-s') @@ -135,7 +134,7 @@ def submit(command, base_cmd.append(command) - return (runfsl(*base_cmd).strip(), ) + return runfsl(*base_cmd).strip() def info(job_id): @@ -215,6 +214,26 @@ def wait(job_ids): log.debug('All jobs have finished') +def _flatten_job_ids(job_ids): + """ + Returns a potentially nested sequence of job ids as a single comma-separated string + + :param job_ids: possibly nested sequence of job ids. The job ids themselves should be strings. + :return: comma-separated string of job ids + """ + def unpack(job_ids): + """Unpack the (nested) job-ids in a single set""" + if isinstance(job_ids, str): + return {job_ids} + else: + res = set() + for job_id in job_ids: + res.update(unpack(job_id)) + return res + + return ','.join(sorted(unpack(job_ids))) + + _external_job = """#!{} # This is a temporary file designed to run the python function {}, # so that it can be submitted to the cluster diff --git a/tests/test_fslsub.py b/tests/test_fslsub.py new file mode 100644 index 0000000000000000000000000000000000000000..45b4d9e0eaa8a0eeda55db1172be29f162de9ce6 --- /dev/null +++ b/tests/test_fslsub.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# +# test_fslsub.py - Tests functions in the fsl.utils.fslsub module. +# +# Author: Michiel Cottaar <Michiel.Cottaar@ndcn.ox.ac.uk> +# + +from fsl.utils import fslsub + + +def test_flatten_jobids(): + job_ids = ('12', '27', '35', '41', '721') + res = '12,27,35,41,721' + + assert fslsub._flatten_job_ids(job_ids) == res + assert fslsub._flatten_job_ids(job_ids[::-1]) == res + assert fslsub._flatten_job_ids('12') == '12' + assert fslsub._flatten_job_ids([job_ids[:2], job_ids[2:]]) == res + assert fslsub._flatten_job_ids([set(job_ids[:2]), job_ids[2:]]) == res + assert fslsub._flatten_job_ids(((job_ids, ), job_ids + job_ids)) == res diff --git a/tests/test_run.py b/tests/test_run.py index 422222fb9b6454982197ee04720a561bf1da5868..8e7a7a8b1327508cb9b39ba6f2a84f8867be38fc 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -249,7 +249,7 @@ def mock_submit(cmd, **kwargs): for k in sorted(kwargs.keys()): f.write('{}: {}\n'.format(k, kwargs[k])) - return (jid,) + return jid def test_run_submit(): @@ -271,7 +271,7 @@ def test_run_submit(): mkexec(op.expandvars('$FSLDIR/bin/fsltest'), test_script) - jid = run.run('fsltest', submit=True)[0] + jid = run.run('fsltest', submit=True) assert jid == '12345' @@ -282,7 +282,7 @@ def test_run_submit(): kwargs = {'name' : 'abcde', 'ram' : '4GB'} - jid = run.run('fsltest', submit=kwargs)[0] + jid = run.run('fsltest', submit=kwargs) assert jid == '12345' diff --git a/tests/test_wrapperutils.py b/tests/test_wrapperutils.py index 54cf16f4c44b06af65dae7df906f095768ef1388..d24c89fc55a57c097e972a4a6f4bfc7a5c9444e5 100644 --- a/tests/test_wrapperutils.py +++ b/tests/test_wrapperutils.py @@ -630,7 +630,7 @@ def test_cmdwrapper_submit(): jid = test_func(1, 2, submit=True) - assert jid == ('12345',) + assert jid == '12345' stdout, stderr = fslsub.output('12345')