From f881b286a8c682b3fc8ea81495b8a92280ae2d48 Mon Sep 17 00:00:00 2001
From: Paul McCarthy <pauldmccarthy@gmail.com>
Date: Thu, 17 Mar 2022 12:14:49 +0000
Subject: [PATCH] MNT: Allow fsl_sub to fail, e.g. when trying to run eddy_cuda
on a non-cuda machine. Save runEddy output to file so we can inspect it on
failures
---
eddy/feedsRun.EddyHigh_b_Test | 20 +++++++++++---------
eddy/feedsRun.EddyLSRTest | 20 +++++++++++---------
eddy/feedsRun.EddyMBSTest | 23 ++++++++++++-----------
eddy/feedsRun.EddyS2VTest | 23 ++++++++++++-----------
eddy/feedsRun.EddyTest | 15 ++++++++-------
eddy/runEddy | 17 +++++++++++++----
6 files changed, 67 insertions(+), 51 deletions(-)
diff --git a/eddy/feedsRun.EddyHigh_b_Test b/eddy/feedsRun.EddyHigh_b_Test
index e04dd59..627b423 100755
--- a/eddy/feedsRun.EddyHigh_b_Test
+++ b/eddy/feedsRun.EddyHigh_b_Test
@@ -41,17 +41,19 @@ if [ ! -d "$indir" ]; then
fi
# Prepare arguments to pass to eddy
-eddy_args="--imain=${indir}/EddyHigh_b_TestData/eddyData/testData
- --mask=${indir}/EddyHigh_b_TestData/eddyData/testMask
- --bvals=${indir}/EddyHigh_b_TestData/eddyData/testBvals
- --bvecs=${indir}/EddyHigh_b_TestData/eddyData/testBvecs
- --index=${indir}/EddyHigh_b_TestData/eddyData/testIndex
- --acqp=${indir}/EddyHigh_b_TestData/eddyData/testAcqparams
- --topup=${indir}/EddyHigh_b_TestData/eddyData/testTopup
+eddy_args="--imain=${indir}/EddyHigh_b_TestData/eddyData/testData
+ --mask=${indir}/EddyHigh_b_TestData/eddyData/testMask
+ --bvals=${indir}/EddyHigh_b_TestData/eddyData/testBvals
+ --bvecs=${indir}/EddyHigh_b_TestData/eddyData/testBvecs
+ --index=${indir}/EddyHigh_b_TestData/eddyData/testIndex
+ --acqp=${indir}/EddyHigh_b_TestData/eddyData/testAcqparams
+ --topup=${indir}/EddyHigh_b_TestData/eddyData/testTopup
--nvoxhp=5000 --repol --fwhm=10,0,0,0,0 --dont_peas --very_verbose"
# run eddy
-output_prefixes=$(${thisdir}/runEddy ${exedir} ${odir} ${eddy_args} | tail -n1)
+${thisdir}/runEddy ${exedir} ${odir} ${eddy_args} > ${odir}/EddyHigh_b_Test.runEddy.o$$
+output_prefixes=$(tail -n1 ${odir}/EddyHigh_b_Test.runEddy.o$$)
+
# Define some constants
max_mean_ima_diff=15.0
@@ -65,7 +67,7 @@ for prefix in ${output_prefixes}; do
else
precomputed="${indir}/EddyHigh_b_TestData/eddyData/Precomputed/eddy_openmp_output"
fi
-
+
${thisdir}/EddyHigh_b_Feeds.py ${odir} `basename ${prefix}` \
`imglob -extension ${indir}/EddyHigh_b_TestData/eddyData/testMask` \
`imglob -extension ${prefix}` \
diff --git a/eddy/feedsRun.EddyLSRTest b/eddy/feedsRun.EddyLSRTest
index 0e0fdaa..fd8212f 100755
--- a/eddy/feedsRun.EddyLSRTest
+++ b/eddy/feedsRun.EddyLSRTest
@@ -38,18 +38,20 @@ if [ ! -d "$indir" ]; then
fi
# Prepare arguments to pass to eddy
-eddy_args="--imain=${indir}/EddyLSRTestData/eddyData/testData
- --mask=${indir}/EddyLSRTestData/eddyData/testMask
- --bvals=${indir}/EddyLSRTestData/eddyData/testBvals
- --bvecs=${indir}/EddyLSRTestData/eddyData/testBvecs
- --index=${indir}/EddyLSRTestData/eddyData/testIndex
- --acqp=${indir}/EddyLSRTestData/eddyData/testAcqparams
- --topup=${indir}/EddyLSRTestData/eddyData/testTopup
- --resamp=lsr --fep --nvoxhp=5000 --repol --fwhm=10,0,0,0,0
+eddy_args="--imain=${indir}/EddyLSRTestData/eddyData/testData
+ --mask=${indir}/EddyLSRTestData/eddyData/testMask
+ --bvals=${indir}/EddyLSRTestData/eddyData/testBvals
+ --bvecs=${indir}/EddyLSRTestData/eddyData/testBvecs
+ --index=${indir}/EddyLSRTestData/eddyData/testIndex
+ --acqp=${indir}/EddyLSRTestData/eddyData/testAcqparams
+ --topup=${indir}/EddyLSRTestData/eddyData/testTopup
+ --resamp=lsr --fep --nvoxhp=5000 --repol --fwhm=10,0,0,0,0
--dont_peas --very_verbose"
# run eddy
-output_prefixes=$(${thisdir}/runEddy ${exedir} ${odir} ${eddy_args} | tail -n1)
+${thisdir}/runEddy ${exedir} ${odir} ${eddy_args} > ${odir}/EddyLSRTest.runEddy.o$$
+output_prefixes=$(tail -n1 ${odir}/EddyLSRTest.runEddy.o$$)
+
# Define some constants
max_mean_ima_diff=15.0
diff --git a/eddy/feedsRun.EddyMBSTest b/eddy/feedsRun.EddyMBSTest
index 273f2ad..488fdf8 100755
--- a/eddy/feedsRun.EddyMBSTest
+++ b/eddy/feedsRun.EddyMBSTest
@@ -43,19 +43,20 @@ if [ ! -d "$indir" ]; then
fi
# Prepare arguments to pass to eddy
-eddy_args="--imain=${indir}/EddyMBSTestData/eddyData/testData
- --acqp=${indir}/EddyMBSTestData/eddyData/testAcqparams
- --mask=${indir}/EddyMBSTestData/eddyData/testMask
- --index=${indir}/EddyMBSTestData/eddyData/testIndex
- --bvecs=${indir}/EddyMBSTestData/eddyData/testBvecs
- --bvals=${indir}/EddyMBSTestData/eddyData/testBvals
- --topup=${indir}/EddyMBSTestData/eddyData/testTopup
- --fwhm=10,5,2,0,0,0,0,0 --niter=8 --nvoxhp=2000
- --flm=quadratic --dont_peas --estimate_move_by_susceptibility
+eddy_args="--imain=${indir}/EddyMBSTestData/eddyData/testData
+ --acqp=${indir}/EddyMBSTestData/eddyData/testAcqparams
+ --mask=${indir}/EddyMBSTestData/eddyData/testMask
+ --index=${indir}/EddyMBSTestData/eddyData/testIndex
+ --bvecs=${indir}/EddyMBSTestData/eddyData/testBvecs
+ --bvals=${indir}/EddyMBSTestData/eddyData/testBvals
+ --topup=${indir}/EddyMBSTestData/eddyData/testTopup
+ --fwhm=10,5,2,0,0,0,0,0 --niter=8 --nvoxhp=2000
+ --flm=quadratic --dont_peas --estimate_move_by_susceptibility
--mbs_niter=20 --mbs_lambda=10 --mbs_ksp=10 --very_verbose"
# run eddy
-output_prefixes=$(${thisdir}/runEddy ${exedir} ${odir} ${eddy_args} | tail -n1)
+${thisdir}/runEddy ${exedir} ${odir} ${eddy_args} > ${odir}/EddyMBSTest.runEddy.o$$
+output_prefixes=$(tail -n1 ${odir}/EddyMBSTest.runEddy.o$$)
# Define some constants
max_mean_dfield_diff=0.03
@@ -66,7 +67,7 @@ max_max_ima_diff=0.6
# Check the results against precomputed results
-for prefix in ${output_prefixes}; do
+for prefix in ${output_prefixes}; do
${thisdir}/MoveBySuscFeeds.py ${odir} `basename ${prefix}` \
`imglob -extension ${indir}/EddyMBSTestData/eddyData/Precomputed/BrainMaskForComparison` \
`imglob -extension ${prefix}` \
diff --git a/eddy/feedsRun.EddyS2VTest b/eddy/feedsRun.EddyS2VTest
index c2ae886..6d79f0d 100755
--- a/eddy/feedsRun.EddyS2VTest
+++ b/eddy/feedsRun.EddyS2VTest
@@ -40,20 +40,21 @@ if [ ! -d "$indir" ]; then
fi
# Prepare arguments to pass to eddy
-eddy_args="--imain=${indir}/EddyS2VTestData/eddyData/testData
- --mask=${indir}/EddyS2VTestData/eddyData/testMask
- --bvals=${indir}/EddyS2VTestData/eddyData/testBvals
- --bvecs=${indir}/EddyS2VTestData/eddyData/testBvecs
- --index=${indir}/EddyS2VTestData/eddyData/testIndex
- --acqp=${indir}/EddyS2VTestData/eddyData/testAcqparams
- --repol --ol_type=slice --ol_nstd=6 --ol_nvox=500
- --niter=8 --fwhm=10,6,4,2,0,0,0,0 --nvoxhp=5000
- --mporder=16 --s2v_niter=10 --s2v_interp=trilinear
- --s2v_lambda=1 --very_verbose
+eddy_args="--imain=${indir}/EddyS2VTestData/eddyData/testData
+ --mask=${indir}/EddyS2VTestData/eddyData/testMask
+ --bvals=${indir}/EddyS2VTestData/eddyData/testBvals
+ --bvecs=${indir}/EddyS2VTestData/eddyData/testBvecs
+ --index=${indir}/EddyS2VTestData/eddyData/testIndex
+ --acqp=${indir}/EddyS2VTestData/eddyData/testAcqparams
+ --repol --ol_type=slice --ol_nstd=6 --ol_nvox=500
+ --niter=8 --fwhm=10,6,4,2,0,0,0,0 --nvoxhp=5000
+ --mporder=16 --s2v_niter=10 --s2v_interp=trilinear
+ --s2v_lambda=1 --very_verbose
--slspec=${indir}/EddyS2VTestData/eddyData/testSlspec "
# run eddy
-output_prefixes=$(${thisdir}/runEddy ${exedir} ${odir} ${eddy_args} | tail -n1)
+${thisdir}/runEddy ${exedir} ${odir} ${eddy_args} > ${odir}/EddyS2VTest.runEddy.o$$
+output_prefixes=$(tail -n1 ${odir}/EddyS2VTest.runEddy.o$$)
# Define some constants
# These are for the comparison of output images divided on b=0, b=700 and b=2000
diff --git a/eddy/feedsRun.EddyTest b/eddy/feedsRun.EddyTest
index 3c0d2b2..6edd25f 100755
--- a/eddy/feedsRun.EddyTest
+++ b/eddy/feedsRun.EddyTest
@@ -37,16 +37,17 @@ if [ ! -d "$indir" ]; then
fi
# Prepare arguments to pass to eddy
-eddy_args="--imain=${indir}/EddyTestData/eddyData/testData
- --mask=${indir}/EddyTestData/eddyData/testMask
- --bvals=${indir}/EddyTestData/eddyData/testBvals
- --bvecs=${indir}/EddyTestData/eddyData/testBvecs
- --index=${indir}/EddyTestData/eddyData/testIndex
- --acqp=${indir}/EddyTestData/eddyData/testAcqparams
+eddy_args="--imain=${indir}/EddyTestData/eddyData/testData
+ --mask=${indir}/EddyTestData/eddyData/testMask
+ --bvals=${indir}/EddyTestData/eddyData/testBvals
+ --bvecs=${indir}/EddyTestData/eddyData/testBvecs
+ --index=${indir}/EddyTestData/eddyData/testIndex
+ --acqp=${indir}/EddyTestData/eddyData/testAcqparams
--repol --fwhm=10,0,0,0,0 --dfields -v"
# run eddy
-output_prefixes=$(${thisdir}/runEddy ${exedir} ${odir} ${eddy_args} | tail -n1)
+${thisdir}/runEddy ${exedir} ${odir} ${eddy_args} > ${odir}/EddyTest.runEddy.o$$
+output_prefixes=$(tail -n1 ${odir}/EddyTest.runEddy.o$$)
# Define some constants
max_mean_dfield_diff=0.5
diff --git a/eddy/runEddy b/eddy/runEddy
index 3dba71d..8e583c1 100755
--- a/eddy/runEddy
+++ b/eddy/runEddy
@@ -67,10 +67,14 @@ do
fsl_sub="fsl_sub -l ${outdir} -q long.q -s openmp,6"
fi
- jid=`${fsl_sub} ${exe} --out=${outdir}/eddyOutput_${variant} ${eddy_args}`
- if [ $? -ne 0 ]; then
- echo "Error submitting ${exe} ${eddy_args}"
- exit 1
+ # fsl_sub will return an error (and not
+ # output a job id) if we try to run a
+ # cuda exe on a non-cuda-capable machine
+ # or queue. So we allow it to fail.
+ jid=$(${fsl_sub} ${exe} --out=${outdir}/eddyOutput_${variant} ${eddy_args} || true)
+ if [ "${jid}" == "" ]; then
+ echo "Error submitting ${exe} - skipping"
+ continue
fi
if [[ "${exe}" == *"cuda"* ]]; then
@@ -80,6 +84,11 @@ do
fi
done
+if [ "${cuda_jids}" == "" ] && [ "${cpu_jids}" == "" ]; then
+ echo "Unable to submit any eddy executables!"
+ exit 1
+fi
+
# If running on a cluster, wait
# until all jobs have finished.
# If not running on a cluster,
--
GitLab