bedpostx_gpu

#!/bin/sh

#   Copyright (C) 2004 University of Oxford
#
#   SHCOPYRIGHT

beforet=`date +%s`
if [ "x$SGE_ROOT" = "x" ] ; then
    if [ -f /usr/local/share/sge/default/common/settings.sh ] ; then
	. /usr/local/share/sge/default/common/settings.sh
    elif [ -f /usr/local/sge/default/common/settings.sh ] ; then
	. /usr/local/sge/default/common/settings.sh
    fi
fi

Usage() {
    echo ""
    echo "Usage: bedpostx <subject directory> [options]"
    echo ""
    echo "expects to find bvals and bvecs in subject directory"
    echo "expects to find data and nodif_brain_mask in subject directory"
    echo "expects to find grad_dev in subject directory, if -g is set"
    echo "options (old syntax)"
    echo "-n (number of fibres per voxel, default 2)"
    echo "-w (ARD weight, more weight means less secondary fibres per voxel, default 1)"
    echo "-b (burnin period, default 1000)"
    echo "-j (number of jumps, default 1250)"
    echo "-s (sample every, default 25)"
    echo "-model (1 for monoexponential, 2 for multiexponential, default 1)"
    echo "-g (consider gradient nonlinearities, default off)"
    echo ""
    echo ""
    echo "ALTERNATIVELY: you can pass on xfibres options onto directly bedpostx"
    echo " For example:  bedpostx <subject directory> --noard --cnonlinear"
    echo " Type 'xfibres --help' for a list of available options "
    echo " Default options will be bedpostx default (see above), and not xfibres default."
    echo ""
    echo "Note: Use EITHER old OR new syntax."
    exit 1
}

make_absolute(){
    dir=$1;
    if [ -d ${dir} ]; then
	OLDWD=`pwd`
	cd ${dir}
	dir_all=`pwd`
	cd $OLDWD
    else
	dir_all=${dir}
    fi
    echo ${dir_all}
}

[ "$1" = "" ] && Usage

subjdir=`make_absolute $1`
subjdir=`echo $subjdir | sed 's/\/$/$/g'`

echo subjectdir is $subjdir

#parse option arguments
nfibres=2
fudge=1
burnin=1000
njumps=1250
sampleevery=25
model=1
gflag=0
f0=0
rician=0

shift
while [ ! -z "$1" ]
do
  case "$1" in
      -n) nfibres=$2;shift;;
      --nf=?) nfibres=`echo $1 | sed s/"--nf="//g`;;
      --model=?) model=`echo $1 | sed s/"--model="//g`;;
      -w) fudge=$2;shift;;
      -b) burnin=$2;shift;;
      -j) njumps=$2;shift;;
      -s) sampleevery=$2;shift;;
      --f0) f0=1 others="$others $1";;
      --rician) rician=1 others="$others $1";; 	
      -model) model=$2;shift;;
      -g) gflag=1;; 
      *) others="$others $1";;
  esac
  shift
done

opts="--nf=$nfibres --fudge=$fudge --bi=$burnin --nj=$njumps --se=$sampleevery --model=$model"
defopts="--cnonlinear"
opts="$opts $defopts $others"  

#check that all required files exist

if [ ! -d $subjdir ]; then
	echo "subject directory $1 not found"
	exit 1
fi

if [ ! -e ${subjdir}/bvecs ]; then
	echo "${subjdir}/bvecs not found"
	exit 1
fi

if [ ! -e ${subjdir}/bvals ]; then
	echo "${subjdir}/bvals not found"
	exit 1
fi

if [ `${FSLDIR}/bin/imtest ${subjdir}/data` -eq 0 ]; then
	echo "${subjdir}/data not found"
	exit 1
fi

if [ ${gflag} -eq 1 ]; then
    if [ `${FSLDIR}/bin/imtest ${subjdir}/grad_dev` -eq 0 ]; then
	echo "${subjdir}/grad_dev not found"
	exit 1
    fi
fi

if [ `${FSLDIR}/bin/imtest ${subjdir}/nodif_brain_mask` -eq 0 ]; then
	echo "${subjdir}/nodif_brain_mask not found"
	exit 1
fi

echo Making bedpostx directory structure

mkdir -p ${subjdir}.bedpostX/
mkdir -p ${subjdir}.bedpostX/diff_slices
mkdir -p ${subjdir}.bedpostX/logs
mkdir -p ${subjdir}.bedpostX/logs/pid_${$}
mkdir -p ${subjdir}.bedpostX/xfms

if [ ${gflag} -eq 1 ]; then
    echo "bedpostx_gpu "${subjdir}  $opts "-g" >> ${subjdir}.bedpostX/commands.txt
else
    echo "bedpostx_gpu "${subjdir}  $opts >> ${subjdir}.bedpostX/commands.txt
fi


echo "..............Compiling CUDA-gpu code .................."

export LD_LIBRARY_PATH=${CUDA}/lib64:${subjdir}.bedpostX:${CUDA}/lib:$LD_LIBRARY_PATH
export DYLD_LIBRARY_PATH=${CUDA}/lib64:${subjdir}.bedpostX:${CUDA}/lib:$LD_LIBRARY_PATH
export PATH=${FSLDIR}/bin:$PATH

volumes=`${FSLDIR}/bin/fslval ${subjdir}/data dim4`
echo "DIRECTIONS NUMBER: $volumes"

filename="${subjdir}.bedpostX/options.h"

if [ $volumes -le 64 ]; then
	threadsBlock=$volumes
else 
	threadsBlock=64
fi

params=$((2+3*$nfibres)) 

if [ $f0 -eq 1 ]; then
	params=$(($params+1))
fi

if [ $model -eq 2 ]; then
	params=$(($params+1))
fi

if [ $rician -eq 1 ]; then
	params=$(($params+1))
fi

rem=$(($volumes%$threadsBlock))
maxndirs=$(($volumes/$threadsBlock))

if [ $rem -ne 0 ]; then
	maxndirs=$(($maxndirs+1))
fi

dstd=$(($model-1));

params_rep=$(($params-1))
params_step0_m2_rep=$(($params_step0_m2-1))

echo "#define NDIRECTIONS $volumes 
#define NFIBRES $nfibres 
#define NPARAMS $params
#define MAXNDIRS_PER_THREAD $maxndirs
#define THREADS_BLOCK $threadsBlock">$filename

before=`date +%s`

${CUDA}/bin/nvcc --shared --compiler-options '-fPIC' -o ${subjdir}.bedpostX/libbedpostx_cuda.so ${FSLDIR}/src/fdt/CUDA/init_gpu.cu ${FSLDIR}/src/fdt/CUDA/samples.cu ${FSLDIR}/src/fdt/CUDA/diffmodels.cu ${FSLDIR}/src/fdt/CUDA/runmcmc.cu ${FSLDIR}/src/fdt/CUDA/xfibres_gpu.cu -O3 -arch sm_20  -lcudart -lcuda -lcurand -L${CUDA}/lib64 -L${CUDA}/lib -I${subjdir}.bedpostX -I${FSLDIR}/extras/include/newmat -I${FSLDIR}/extras/include/boost -I${FSLDIR}/include -I${CUDA}/include/thrust -I${FSLDIR}/src/fdt 

file=${subjdir}.bedpostX/libbedpostx_cuda.so

if [ -f $file ];
then
after=`date +%s`
diff=$(($after-$before))
echo "Compilation Time: $diff seconds" >> ${subjdir}.bedpostX/times

echo "CUDA CODE compiled successfully"

mailto=`whoami`@fmrib.ox.ac.uk
before=`date +%s`
echo Preprocessing stages
${FSLDIR}/bin/bedpostx_preproc.sh ${subjdir} ${gflag}

after=`date +%s`
diff=$(($after-$before))
echo "Preprocessing Time: $diff seconds" >> ${subjdir}.bedpostX/times

before=`date +%s`
nslices=`${FSLDIR}/bin/fslval ${subjdir}/data dim3`

${FSLDIR}/bin/splitter_gpu $subjdir $gflag $nfibres $nslices $opts

after=`date +%s`
diff=$(($after-$before))
echo "Execution Time: $diff seconds" >> ${subjdir}.bedpostX/times

before=`date +%s`


echo Post processing stage
${FSLDIR}/bin/bedpostx_postproc.sh $subjdir

after=`date +%s`
difd=$(($after-$before))
echo "Post-Processing Time : $diff seconds" >> ${subjdir}.bedpostX/times

aftert=`date +%s`
diff=$(($aftert-$beforet))
echo "TOTAL Time: $diff seconds" >> ${subjdir}.bedpostX/times

else
	echo
	echo
	echo "CUDA COMPILER ERROR. Please be sure that you have set correctly CUDA environment variable with your CUDA path"
	echo 
fi