bedpostx_gpu

#!/bin/sh

#   Copyright (C) 2004 University of Oxford
#
#   SHCOPYRIGHT

export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${FSLDIR}/lib

if [ "x$SGE_ROOT" = "x" ] ; then
    if [ -f /usr/local/share/sge/default/common/settings.sh ] ; then
	. /usr/local/share/sge/default/common/settings.sh
    elif [ -f /usr/local/sge/default/common/settings.sh ] ; then
	. /usr/local/sge/default/common/settings.sh
    fi
fi

Usage() {
    echo ""
    echo "Usage: bedpostx <subject directory> [options]"
    echo ""
    echo "expects to find bvals and bvecs in subject directory"
    echo "expects to find data and nodif_brain_mask in subject directory"
    echo "expects to find grad_dev in subject directory, if -g is set"
    echo ""
    echo "<options>:"
    echo "-NJOBS (number of jobs to queue, the data is divided in NJOBS parts, usefull for a GPU cluster, default 1)"
    echo "-Q (name of the GPU(s) queue, default cuda.q)"
    echo "-n (number of fibres per voxel, default 2)"
    echo "-w (ARD weight, more weight means less secondary fibres per voxel, default 1)"
    echo "-b (burnin period, default 1000)"
    echo "-j (number of jumps, default 1250)"
    echo "-s (sample every, default 25)"
    echo "-model (1 for monoexponential, 2 for multiexponential, default 1)"
    echo "-g (consider gradient nonlinearities, default off)"
    echo ""
    echo ""
    echo "ALTERNATIVELY: you can pass on xfibres options onto directly bedpostx"
    echo " For example:  bedpostx <subject directory> --noard --cnonlinear"
    echo " Type 'xfibres --help' for a list of available options "
    echo " Default options will be bedpostx default (see above), and not xfibres default."
    echo ""
    echo "Note: Use EITHER old OR new syntax."
    exit 1
}

make_absolute(){
    dir=$1;
    if [ -d ${dir} ]; then
	OLDWD=`pwd`
	cd ${dir}
	dir_all=`pwd`
	cd $OLDWD
    else
	dir_all=${dir}
    fi
    echo ${dir_all}
}

[ "$1" = "" ] && Usage

subjdir=`make_absolute $1`
subjdir=`echo $subjdir | sed 's/\/$/$/g'`

echo subjectdir is $subjdir

#parse option arguments
queue="-q cuda.q"
njobs=1
nfibres=2
fudge=1
burnin=1000
njumps=1250
sampleevery=25
model=1
gflag=0
if [ "x$SGE_ROOT" = "x" ]; then
	queue=''
fi

shift
while [ ! -z "$1" ]
do
  case "$1" in
      -Q) queue=-q $2;shift;;
      -NJOBS) njobs=$2;shift;;
      -n) nfibres=$2;shift;;
      -w) fudge=$2;shift;;
      -b) burnin=$2;shift;;
      -j) njumps=$2;shift;;
      -s) sampleevery=$2;shift;;
      -model) model=$2;shift;;
      -g) gflag=1;; 
      *) break;;
  esac
  shift
done
opts="--nf=$nfibres --fudge=$fudge --bi=$burnin --nj=$njumps --se=$sampleevery --model=$model"
defopts="--cnonlinear"
opts="$opts $defopts $*"  

#check that all required files exist

if [ ! -d $subjdir ]; then
	echo "subject directory $1 not found"
	exit 1
fi

if [ ! -e ${subjdir}/bvecs ]; then
	echo "${subjdir}/bvecs not found"
	exit 1
fi

if [ ! -e ${subjdir}/bvals ]; then
	echo "${subjdir}/bvals not found"
	exit 1
fi

if [ `${FSLDIR}/bin/imtest ${subjdir}/data` -eq 0 ]; then
	echo "${subjdir}/data not found"
	exit 1
fi

if [ ${gflag} -eq 1 ]; then
    if [ `${FSLDIR}/bin/imtest ${subjdir}/grad_dev` -eq 0 ]; then
	echo "${subjdir}/grad_dev not found"
	exit 1
    fi
fi

if [ `${FSLDIR}/bin/imtest ${subjdir}/nodif_brain_mask` -eq 0 ]; then
	echo "${subjdir}/nodif_brain_mask not found"
	exit 1
fi

echo Making bedpostx directory structure

mkdir -p ${subjdir}.bedpostX/
mkdir -p ${subjdir}.bedpostX/diff_parts
mkdir -p ${subjdir}.bedpostX/logs
mkdir -p ${subjdir}.bedpostX/logs/logs_gpu
mkdir -p ${subjdir}.bedpostX/logs/pid_${$}
mkdir -p ${subjdir}.bedpostX/xfms

mailto=`whoami`@fmrib.ox.ac.uk

echo Copying files to bedpost directory
cp ${subjdir}/bvecs ${subjdir}/bvals ${subjdir}.bedpostX
${FSLDIR}/bin/imcp ${subjdir}/nodif_brain_mask ${subjdir}.bedpostX
if [ `${FSLDIR}/bin/imtest ${subjdir}/nodif` = 1 ] ; then
    ${FSLDIR}/bin/fslmaths ${subjdir}/nodif -mas ${subjdir}/nodif_brain_mask ${subjdir}.bedpostX/nodif_brain
fi

nvox=`${FSLDIR}/bin/fslstats $subjdir.bedpostX/nodif_brain_mask -V  | cut -d ' ' -f1 `

echo Queuing parallel processing stage
[ -f ${subjdir}.bedpostX/commands.txt ] && rm ${subjdir}.bedpostX/commands.txt
if [ "x$SGE_ROOT" = "x" ]; then

# 
# Need to set off a backgrounded monitoring task
# 
    cat <<EOM > ${subjdir}.bedpostX/monitor
#!/bin/sh
finished=0
logdir=${subjdir}.bedpostX/logs
#voxels processed in each subpart are 12800 or more if the last one is less than 6400 (1 part less)
nsubparts=\$(($nvox/12800))
if [ \$nsubparts%12800 != 0 ];then 
	nsubparts=\$((\$nsubparts + 1)) 
fi
last_part=\$(($nvox-(((\$nsubparts-1))*12800)))
if [ \$last_part -lt 6400 ];then 
	nsubparts=\$((\$nsubparts - 1)) 
fi

check() {
    if [ \$finished -eq 1 ];then
        echo "\$nfin parts processed of "$nsubparts
    else
        nfin=0
        subpart=0
         while [ "\$subpart" -le \$nsubparts ];do
            partzp=\`\$FSLDIR/bin/zeropad \$part 4\`
            if [ -f ${subjdir}.bedpostX/diff_parts/data_part_0000/mean_S0samples_\$subpart ];then
                nfin=\$((\$nfin + 1))
            fi
                subpart=\$((\$subpart + 1))
        done
        if [ \$nfin -eq \$nsubparts ] ; then
            finished=1
        fi
        echo "\$nfin parts processed of "\$nsubparts
    fi
}
exit_test() {
    isimg=\`\$FSLDIR/bin/imtest ${subjdir}.bedpostX/dyads1\`
    if [ \$isimg -eq 0 ] ; then
	echo For some reason the bedpostX process DOES NOT appear
	echo to have successfully completed. Please examine your
	echo results carefully.
    else
	echo All parts processed
    fi
}

check
n=0
while [ \$finished -eq 0 ] ; do
    sleep 10
    if [ \$n -eq 6 ] ; then
        check
        n=0
    else
        n=\$((\$n + 1))
    fi
done
exit_test
EOM
    chmod +x ${subjdir}.bedpostX/monitor
    ${subjdir}.bedpostX/monitor&
    monitorpid=$!
fi
part=0
while [ $part -lt $njobs ]
do
    	partzp=`$FSLDIR/bin/zeropad $part 4`
    
	if [ ${gflag} -eq 1 ]; then
	    gopts="$opts --gradnonlin=${subjdir}/grad_dev"
	else
	    gopts=$opts
	fi    
	echo "${FSLDIR}/bin/xfibres_gpu --data=${subjdir}/data --mask=$subjdir.bedpostX/nodif_brain_mask -b ${subjdir}.bedpostX/bvals -r ${subjdir}.bedpostX/bvecs --forcedir --logdir=$subjdir.bedpostX/diff_parts/data_part_$partzp $gopts ${subjdir} $part $njobs > $subjdir.bedpostX/logs/log$partzp  && echo Done" >> ${subjdir}.bedpostX/commands.txt
    
    	part=$(($part + 1))
done

bedpostid=`${FSLDIR}/bin/fsl_sub $queue -l ${subjdir}.bedpostX/logs -M $mailto -N bedpostx_gpu -t ${subjdir}.bedpostX/commands.txt`

echo Queuing post processing stage
mergeid=`${FSLDIR}/bin/fsl_sub $queue -j $bedpostid -m ase -N bpx_postproc -l ${subjdir}.bedpostX/logs ${FSLDIR}/bin/bedpostx_postproc_gpu.sh --data=${subjdir}/data --mask=$subjdir.bedpostX/nodif_brain_mask -b ${subjdir}.bedpostX/bvals -r ${subjdir}.bedpostX/bvecs  --forcedir --logdir=$subjdir.bedpostX/diff_parts $gopts $nvox $njobs ${subjdir}`

[ "x$SGE_ROOT" = "x" ] && kill -USR1 $monitorpid
if [ "x$SGE_ROOT" != "x" ]; then
    echo
    echo Type ${subjdir}.bedpostX/monitor to show progress.
    cat <<EOM > ${subjdir}.bedpostX/monitor
#!/bin/sh
finished=0
logdir=${subjdir}.bedpostX/logs
if [ $njobs -eq 1 ]; then
	nsubparts=\$(($nvox/12800))
	if [ \$nsubparts%12800 != 0 ];then 
		nsubparts=\$((\$nsubparts + 1)) 
	fi
	last_part=\$(($nvox-(((\$nsubparts-1))*12800)))
	if [ \$last_part -lt 6400 ];then 
		nsubparts=\$((\$nsubparts - 1)) 
	fi
	while [ \$finished -eq 0 ] ; do
           nfin=0
           subpart=0
           while [ "\$subpart" -le \$nsubparts ];do
              partzp=\`\$FSLDIR/bin/zeropad \$part 4\`
              if [ -f ${subjdir}.bedpostX/diff_parts/data_part_0000/mean_S0samples_\$subpart ];then
                 nfin=\$((\$nfin + 1))
              fi
              subpart=\$((\$subpart + 1))
           done
           echo "\$nfin parts processed of "\$nsubparts

           if [ -f ${subjdir}.bedpostX/xfms/eye.mat ] ; then
              finished=1
	      echo "All parts processed"
           fi

	   sleep 60;
         done
else
	while [ \$finished -eq 0 ] ; do
	    nfin=0
	    part=0
	    while [ "\$part" -le $njobs ];do
		partzp=\`\$FSLDIR/bin/zeropad \$part 4\`
		if [ -f ${subjdir}.bedpostX/diff_parts/data_part_\$partzp/mean_S0samplesJ ];then
		  nfin=\$((\$nfin + 1))
		fi
		part=\$((\$part + 1))
	    done
	    echo "\$nfin parts processed of "$njobs

	    if [ -f ${subjdir}.bedpostX/xfms/eye.mat ] ; then
		finished=1
		echo "All parts processed"
	    fi

	    sleep 60;
	done
fi

EOM
    chmod +x ${subjdir}.bedpostX/monitor

    echo Type ${subjdir}.bedpostX/cancel to terminate all the queued tasks.
    cat <<EOC > ${subjdir}.bedpostX/cancel
#!/bin/sh
qdel $mergeid $bedpostid
EOC
    chmod +x ${subjdir}.bedpostX/cancel

    echo
    echo You will get an email at the end of the post-processing stage.
    echo
fi