eddy/runEddy

@@ -10,12 +10,18 @@
 # output as the final line of output. Thne script returns
 # an exit code of non-0 if something goes wrong.
 #
+# If called from pyfeeds, the eddy_cpu executable is
+# executed twice - once with --nthr=1, and again with
+# --nthr=6. If not called from pyfeeds, the eddy_cpu
+# executable is executed only once, with --nthr=6.
+#
 # Outputs of each eddy_<variant> executable is saved with
 # prefix ${outdir}/eddyOutput_<variant>. For example, the
 # output of eddy_cuda9.2 will be saved with prefix
 # ${outdir}/eddyOutput_cuda9.2, and the output of
-# eddy_openmp will be saved with prefix
-# ${outdir}/eddyOutput_openmp.
+# eddy_cpu --nthr=1 will be saved with prefix
+# ${outdir}/eddyOutput_cpu_nthr_1.
+#
 
 set -e
 
@@ -26,7 +32,7 @@ set -e
 # added by this script.
 exedir="$1"; shift;
 outdir="$1"; shift;
-eddy_args="$@"
+eddy_args=("$@")
 
 # Find all eddy_cuda* executables
 cuda_exes=""
@@ -37,13 +43,24 @@ do
   fi
 done
 
-# Find all eddy_cpu/eddy_openmp executables
+# Find all eddy_cpu executables. We run
+# eddy_cpu twice - here, we add
+# "eddy_cpu_nthr_X" to the list of
+# executables to run. The name is unpacked
+# in the loop below.
+#
+# We're using 6 threads here for the multi-
+# threaded test, as it is the best option on
+# the FMRIB cluster
 cpu_exes=""
-if [ -x "${exedir}/eddy_openmp" ]; then
-  cpu_exes="${exedir}/eddy_openmp"
-fi
 if [ -x "${exedir}/eddy_cpu" ]; then
-  cpu_exes="${cpu_exes} ${exedir}/eddy_cpu"
+
+  # Only run single-threaded when running
+  # through pyfeeds
+  if [ "$PYFEEDS_TESTING" = "1" ]; then
+    cpu_exes="${cpu_exes} ${exedir}/eddy_cpu_nthr_1"
+  fi
+  cpu_exes="${cpu_exes} ${exedir}/eddy_cpu_nthr_6"
 fi
 
 if [ "${cuda_exes}" == "" ] && [ "${cpu_exes}" == "" ]; then
@@ -57,28 +74,37 @@ fi
 cuda_jids=""
 cpu_jids=""
 submitted=""
-for exe in ${cuda_exes} ${cpu_exes};
+for exe_name in ${cuda_exes} ${cpu_exes};
 do
-  variant=`basename ${exe}`
+  variant=`basename ${exe_name}`
   variant=`echo ${variant} | sed 's/eddy_//'`
 
-  if [[ "${exe}" == *"cuda"* ]]; then
+  if [[ "${exe_name}" == *"cuda"* ]]; then
     fsl_sub="fsl_sub -l ${outdir} --coprocessor=cuda"
+    exe="${exe_name}"
   else
-    fsl_sub="fsl_sub -l ${outdir} -q long.q -s openmp,6"
+    # unpack eddy_cpu_nthr_N
+    # into eddy_cpu --nthr=N
+    fsl_sub="fsl_sub -l ${outdir} -q long.q"
+    exe="${exe_name%_nthr*}"
+    nthr="${variant#cpu_nthr_}"
+    eddy_args+=("--nthr=${nthr}")
+    if [ "${nthr}" != "1" ]; then
+      fsl_sub="${fsl_sub} -s openmp,${nthr}"
+    fi
   fi
 
   # fsl_sub will return an error (and not
   # output a job id) if we try to run a
   # cuda exe on a non-cuda-capable machine
   # or queue. So we allow it to fail.
-  jid=$(${fsl_sub} ${exe} --out=${outdir}/eddyOutput_${variant} ${eddy_args} || true)
+  jid=$(${fsl_sub} ${exe} --out=${outdir}/eddyOutput_${variant} ${eddy_args[@]} || true)
   if [ "${jid}" == "" ]; then
-    echo "Error submitting ${exe} - skipping"
+    echo "Error submitting ${exe_name} - skipping"
     continue
   fi
 
-  submitted="${submitted} ${exe}"
+  submitted="${submitted} ${exe_name}"
 
   if [[ "${exe}" == *"cuda"* ]]; then
     cuda_jids="${cuda_jids} ${jid}"