#!/usr/bin/env bash
#
# This script is shared by all of the feedsRun.<eddytest>
# scripts. It runs eddy with a set of arguments, either
# directly, or by submitting to a cluster with fsl_sub,
# and then waits for eddy to finish. All eddy executables
# that are installed in $FSLDIR/bin/ are executed, and
# the outputs for each are directed to a different output
# prefix. All output prefixes are printed to standard
# output as the final line of output. Thne script returns
# an exit code of non-0 if something goes wrong.
#
# Outputs of each eddy_<variant> executable is saved with
# prefix ${outdir}/eddyOutput_<variant>. For example, the
# output of eddy_cuda9.2 will be saved with prefix
# ${outdir}/eddyOutput_cuda9.2, and the output of
# eddy_openmp will be saved with prefix
# ${outdir}/eddyOutput_openmp.

set -e

# First argument is directory in which to search 
# for eddy executables (typically $FSLDIR/bin).
# All remaining arguments are to be passed to
# eddy. Don't pass the --out option, as that is
# added by this script.
exedir="$1"; shift;
outdir="$2"; shift;
eddy_args="$@"

# Find all eddy_cuda* executables
cuda_exes=""
for cuda_exe in ${exedir}/eddy_cuda*;
do
  if [ -x "${cuda_exe}" ]; then
    cuda_exes="${cuda_exes} ${cuda_exe}"
  fi
done

# Find all eddy_cpu/eddy_openmp executables
cpu_exes=""
if [ -x "${exedir}/eddy_openmp" ]; then
  cpu_exes="${exedir}/eddy_openmp"
fi
if [ -x "${exedir}/eddy_cpu" ]; then
  cpu_exes="${cpu_exes} ${exedir}/eddy_cpu"
fi

if [ "${cuda_exes}" == "" ] && [ "${cpu_exes}" == "" ]; then
  echo "Cannot find any eddy executables in ${exedir}!"
  exit 1
fi

# Launch both GPU and CPU versions
cuda_jids=""
for cuda_exe in ${cuda_exes};
do
  tmp=`basename ${cuda_exe}`
  variant=`echo ${tmp} | sed 's/eddy_//'`
  jid=`fsl_sub -l ${outdir} -q cuda.q ${cuda_exe} --out=${outdir}/eddyOutput_${variant} ${eddy_args}`
  cuda_jids="${cuda_jids} ${jid}"
done

cpu_jids=""
for cpu_exe in ${cpu_exes};
do
  tmp=`basename ${cuda_exe}`
  variant=`echo ${tmp} | sed 's/eddy_//'`
  jid=`fsl_sub -l ${outdir} -q long.q -s openmp,6 ${cpu_exe} --out=${outdir}/eddyOutput_${variant} ${eddy_args}`
  cpu_jids="${cpu_jids} ${jid}"
done

# If running on a cluster, wait
# until all jobs have finished.
# If not running on a cluster,
# the above fsl_sub calls will
# have blocked until completion.
if [ ! -z "${SGE_ROOT}" ]; then

  # Ensure that slots are being reserved 
  # on the queue for CPU jobs
  for jid in ${cpu_jids}; do
    qalter ${jid} -R y
  fi

  # wait for all jobs to finish
  while [ : ]; do
    for jid in ${cuda_jids} ${cpu_jids};
    do
      tmp=`qstat -j ${jid} | wc`
      tmp=`echo $tmp | awk '{print $1}'`
      if [ $tmp -ne 0 ]; then
    	break
      fi
    done
    if [ $tmp -eq 0 ]; then
      break
    fi
    sleep 5m
  done
fi

# Gather output prefixes for each run,
# and check that the main output file 
# was created
outputs=""
for exe in ${cuda_exes} ${cpu_exes};
do
  tmp=`basename ${cuda_exe}`
  variant=`echo ${tmp} | sed 's/eddy_//'`
  prefix="${outdir}/eddyOutput_${variant}"
  outputs="${outputs} ${prefix}"
  if [ ! -f ${prefix}.nii* ]; then
    echo "${prefix} is missing"
	exit 1
  fi
done

echo ${outputs}
exit 0