Merge branch 'hold' into 'master'

Added new wait function called hold

See merge request fsl/fslpy!236

fsl/utils/fslsub.py

@@ -33,6 +33,7 @@ Example usage, building a short pipeline::
    info
    output
    func_to_cmd
+   hold
 """
 
 
@@ -49,6 +50,7 @@ from dataclasses import dataclass, asdict
 from typing import Optional, Collection, Union, Tuple, Dict
 import argparse
 import warnings
+import os
 
 
 log = logging.getLogger(__name__)
@@ -123,7 +125,7 @@ class SubmitParams(object):
         If not set explicitly by the user don't alter the environment in which the script will be submitted
         """
         if self.env is None:
-            self.env = {}
+            self.env = dict(os.environ)
 
     def as_flags(self, ):
         """
@@ -402,6 +404,37 @@ def _flatten_job_ids(job_ids):
     return ','.join(sorted(unpack(job_ids)))
 
 
+def hold(job_ids, hold_filename=None):
+    """
+    Waits until all jobs have finished
+
+    Internally works by submitting a new job, which creates a file named `hold_filename`,
+    which will only run after all jobs in `job_ids` finished.
+
+    This function will only return once `hold_filename` has been created
+
+    :param job_ids: possibly nested sequence of job ids. The job ids themselves should be strings.
+    :param hold_filename: filename to use as a hold file.
+        The containing directory should exist, but the file itself should not.
+        Defaults to a ./.<random characters>.hold in the current directory.
+    :return: only returns when all the jobs have finished
+    """
+    if hold_filename is None:
+        with tempfile.NamedTemporaryFile(prefix='.', suffix='.hold', dir='.') as f:
+            hold_filename = f.name
+    if op.exists(hold_filename):
+        raise IOError(f"Hold file ({hold_filename}) already exists")
+    elif not op.isdir(op.split(op.abspath(hold_filename))[0]):
+        raise IOError(f"Hold file ({hold_filename}) can not be created in non-existent directory")
+
+    submit(('touch', hold_filename), wait_for=job_ids, minutes=1, job_name='.hold')
+
+    while not op.exists(hold_filename):
+        time.sleep(10)
+
+    os.remove(hold_filename)
+
+
 _external_job = """#!{}
 # This is a temporary file designed to run the python function {},
 # so that it can be submitted to the cluster

fsl/utils/run.py

@@ -16,6 +16,7 @@
    run
    runfsl
    dryrun
+   hold
 """
 
 
@@ -422,3 +423,16 @@ def wslcmd(cmdpath, *args):
     else:
         # Command was not found in WSL with this path
         return None
+
+
+def hold(job_ids, hold_filename=None):
+    """
+    Waits until all jobs have finished
+
+    :param job_ids: possibly nested sequence of job ids. The job ids themselves should be strings.
+    :param hold_filename: filename to use as a hold file.
+        The containing directory should exist, but the file itself should not.
+        Defaults to a ./.<random characters>.hold in the current directory.
+    :return: only returns when all the jobs have finished
+    """
+    fslsub.hold(job_ids, hold_filename)