diff --git a/fsl/utils/fslsub.py b/fsl/utils/fslsub.py index c39723700d986a82d22927eb252038747347c160..30ffc0b72d901f774dd722649dc495f45d5668d8 100644 --- a/fsl/utils/fslsub.py +++ b/fsl/utils/fslsub.py @@ -33,6 +33,7 @@ Example usage, building a short pipeline:: info output func_to_cmd + hold """ @@ -49,6 +50,7 @@ from dataclasses import dataclass, asdict from typing import Optional, Collection, Union, Tuple, Dict import argparse import warnings +import os log = logging.getLogger(__name__) @@ -123,7 +125,7 @@ class SubmitParams(object): If not set explicitly by the user don't alter the environment in which the script will be submitted """ if self.env is None: - self.env = {} + self.env = dict(os.environ) def as_flags(self, ): """ @@ -402,6 +404,37 @@ def _flatten_job_ids(job_ids): return ','.join(sorted(unpack(job_ids))) +def hold(job_ids, hold_filename=None): + """ + Waits until all jobs have finished + + Internally works by submitting a new job, which creates a file named `hold_filename`, + which will only run after all jobs in `job_ids` finished. + + This function will only return once `hold_filename` has been created + + :param job_ids: possibly nested sequence of job ids. The job ids themselves should be strings. + :param hold_filename: filename to use as a hold file. + The containing directory should exist, but the file itself should not. + Defaults to a ./.<random characters>.hold in the current directory. + :return: only returns when all the jobs have finished + """ + if hold_filename is None: + with tempfile.NamedTemporaryFile(prefix='.', suffix='.hold', dir='.') as f: + hold_filename = f.name + if op.exists(hold_filename): + raise IOError(f"Hold file ({hold_filename}) already exists") + elif not op.isdir(op.split(op.abspath(hold_filename))[0]): + raise IOError(f"Hold file ({hold_filename}) can not be created in non-existent directory") + + submit(('touch', hold_filename), wait_for=job_ids, minutes=1, job_name='.hold') + + while not op.exists(hold_filename): + time.sleep(10) + + os.remove(hold_filename) + + _external_job = """#!{} # This is a temporary file designed to run the python function {}, # so that it can be submitted to the cluster diff --git a/fsl/utils/run.py b/fsl/utils/run.py index 739d89ed2e6fb0e5c89356ef45f2176fd48c3228..3ed799ca8330f2c17585d8959da448c3b9ccf6eb 100644 --- a/fsl/utils/run.py +++ b/fsl/utils/run.py @@ -16,6 +16,7 @@ run runfsl dryrun + hold """ @@ -422,3 +423,16 @@ def wslcmd(cmdpath, *args): else: # Command was not found in WSL with this path return None + + +def hold(job_ids, hold_filename=None): + """ + Waits until all jobs have finished + + :param job_ids: possibly nested sequence of job ids. The job ids themselves should be strings. + :param hold_filename: filename to use as a hold file. + The containing directory should exist, but the file itself should not. + Defaults to a ./.<random characters>.hold in the current directory. + :return: only returns when all the jobs have finished + """ + fslsub.hold(job_ids, hold_filename)