Have fsl_sub return just the job id.

The functionality of merging multiple jobs is now ensured by allowing
for nested sequences of job-ids.

fsl/utils/fslsub.py

@@ -20,9 +20,10 @@ Example usage, building a short pipeline::
     other_job = submit('some other pre-processing step', queue='short.q')
 
     # submits cuda job, that should only start after both preparatory jobs are
-    # finished
+    # finished. This will work if bet_job and other_job are single job-ids
+    # (i.e., strings) or a sequence of multiple job-ids
     cuda_job = submit('expensive job',
-                      wait_for=bet_job + other_job,
+                      wait_for=(bet_job, other_job),
                       queue='cuda.q')
 
     # waits for the cuda job to finish
@@ -125,9 +126,7 @@ def submit(command,
         base_cmd.append('-v')
 
     if wait_for:
-        if not isinstance(wait_for, string_types):
-            wait_for = ','.join(wait_for)
-        base_cmd.extend(['-j', wait_for])
+        base_cmd.extend(['-j', _flatten_job_ids(wait_for)])
 
     if multi_threaded:
         base_cmd.append('-s')
@@ -135,7 +134,7 @@ def submit(command,
 
     base_cmd.append(command)
 
-    return (runfsl(*base_cmd).strip(), )
+    return runfsl(*base_cmd).strip()
 
 
 def info(job_id):
@@ -215,6 +214,26 @@ def wait(job_ids):
     log.debug('All jobs have finished')
 
 
+def _flatten_job_ids(job_ids):
+    """
+    Returns a potentially nested sequence of job ids as a single comma-separated string
+
+    :param job_ids: possibly nested sequence of job ids. The job ids themselves should be strings.
+    :return: comma-separated string of job ids
+    """
+    def unpack(job_ids):
+        """Unpack the (nested) job-ids in a single set"""
+        if isinstance(job_ids, str):
+            return {job_ids}
+        else:
+            res = set()
+            for job_id in job_ids:
+                res.update(unpack(job_id))
+            return res
+
+    return ','.join(sorted(unpack(job_ids)))
+
+
 _external_job = """#!{}
 # This is a temporary file designed to run the python function {},
 # so that it can be submitted to the cluster

tests/test_fslsub.py

+#!/usr/bin/env python
+#
+# test_fslsub.py - Tests functions in the fsl.utils.fslsub module.
+#
+# Author: Michiel Cottaar <Michiel.Cottaar@ndcn.ox.ac.uk>
+#
+
+from fsl.utils import fslsub
+
+
+def test_flatten_jobids():
+    job_ids = ('12', '27', '35', '41', '721')
+    res = '12,27,35,41,721'
+
+    assert fslsub._flatten_job_ids(job_ids) == res
+    assert fslsub._flatten_job_ids(job_ids[::-1]) == res
+    assert fslsub._flatten_job_ids('12') == '12'
+    assert fslsub._flatten_job_ids([job_ids[:2], job_ids[2:]]) == res
+    assert fslsub._flatten_job_ids([set(job_ids[:2]), job_ids[2:]]) == res
+    assert fslsub._flatten_job_ids(((job_ids, ), job_ids + job_ids)) == res

tests/test_run.py

@@ -249,7 +249,7 @@ def mock_submit(cmd, **kwargs):
         for k in sorted(kwargs.keys()):
             f.write('{}: {}\n'.format(k, kwargs[k]))
 
-    return (jid,)
+    return jid
 
 
 def test_run_submit():
@@ -271,7 +271,7 @@ def test_run_submit():
 
         mkexec(op.expandvars('$FSLDIR/bin/fsltest'), test_script)
 
-        jid = run.run('fsltest', submit=True)[0]
+        jid = run.run('fsltest', submit=True)
 
         assert jid == '12345'
 
@@ -282,7 +282,7 @@ def test_run_submit():
 
         kwargs = {'name' : 'abcde', 'ram' : '4GB'}
 
-        jid = run.run('fsltest', submit=kwargs)[0]
+        jid = run.run('fsltest', submit=kwargs)
 
         assert jid == '12345'
 

tests/test_wrapperutils.py

@@ -630,7 +630,7 @@ def test_cmdwrapper_submit():
 
         jid = test_func(1, 2, submit=True)
 
-        assert jid == ('12345',)
+        assert jid == '12345'
 
         stdout, stderr = fslsub.output('12345')