Commit 60308c4d authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

Merge branch 'hold' into 'master'

Added new wait function called hold

See merge request fsl/fslpy!236
parents 1a506448 e138ecd2
Pipeline #5469 passed with stages
in 12 minutes and 29 seconds
......@@ -33,6 +33,7 @@ Example usage, building a short pipeline::
info
output
func_to_cmd
hold
"""
......@@ -49,6 +50,7 @@ from dataclasses import dataclass, asdict
from typing import Optional, Collection, Union, Tuple, Dict
import argparse
import warnings
import os
log = logging.getLogger(__name__)
......@@ -123,7 +125,7 @@ class SubmitParams(object):
If not set explicitly by the user don't alter the environment in which the script will be submitted
"""
if self.env is None:
self.env = {}
self.env = dict(os.environ)
def as_flags(self, ):
"""
......@@ -402,6 +404,37 @@ def _flatten_job_ids(job_ids):
return ','.join(sorted(unpack(job_ids)))
def hold(job_ids, hold_filename=None):
"""
Waits until all jobs have finished
Internally works by submitting a new job, which creates a file named `hold_filename`,
which will only run after all jobs in `job_ids` finished.
This function will only return once `hold_filename` has been created
:param job_ids: possibly nested sequence of job ids. The job ids themselves should be strings.
:param hold_filename: filename to use as a hold file.
The containing directory should exist, but the file itself should not.
Defaults to a ./.<random characters>.hold in the current directory.
:return: only returns when all the jobs have finished
"""
if hold_filename is None:
with tempfile.NamedTemporaryFile(prefix='.', suffix='.hold', dir='.') as f:
hold_filename = f.name
if op.exists(hold_filename):
raise IOError(f"Hold file ({hold_filename}) already exists")
elif not op.isdir(op.split(op.abspath(hold_filename))[0]):
raise IOError(f"Hold file ({hold_filename}) can not be created in non-existent directory")
submit(('touch', hold_filename), wait_for=job_ids, minutes=1, job_name='.hold')
while not op.exists(hold_filename):
time.sleep(10)
os.remove(hold_filename)
_external_job = """#!{}
# This is a temporary file designed to run the python function {},
# so that it can be submitted to the cluster
......
......@@ -16,6 +16,7 @@
run
runfsl
dryrun
hold
"""
......@@ -422,3 +423,16 @@ def wslcmd(cmdpath, *args):
else:
# Command was not found in WSL with this path
return None
def hold(job_ids, hold_filename=None):
"""
Waits until all jobs have finished
:param job_ids: possibly nested sequence of job ids. The job ids themselves should be strings.
:param hold_filename: filename to use as a hold file.
The containing directory should exist, but the file itself should not.
Defaults to a ./.<random characters>.hold in the current directory.
:return: only returns when all the jobs have finished
"""
fslsub.hold(job_ids, hold_filename)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment