bedpostx_gpu

#!/bin/sh

#   Copyright (C) 2004 University of Oxford
#
#   Part of FSL - FMRIB's Software Library
#   http://www.fmrib.ox.ac.uk/fsl
#   fsl@fmrib.ox.ac.uk
#   
#   Developed at FMRIB (Oxford Centre for Functional Magnetic Resonance
#   Imaging of the Brain), Department of Clinical Neurology, Oxford
#   University, Oxford, UK
#   
#   
#   LICENCE
#   
#   FMRIB Software Library, Release 4.0 (c) 2007, The University of
#   Oxford (the "Software")
#   
#   The Software remains the property of the University of Oxford ("the
#   University").
#   
#   The Software is distributed "AS IS" under this Licence solely for
#   non-commercial use in the hope that it will be useful, but in order
#   that the University as a charitable foundation protects its assets for
#   the benefit of its educational and research purposes, the University
#   makes clear that no condition is made or to be implied, nor is any
#   warranty given or to be implied, as to the accuracy of the Software,
#   or that it will be suitable for any particular purpose or for use
#   under any specific conditions. Furthermore, the University disclaims
#   all responsibility for the use which is made of the Software. It
#   further disclaims any liability for the outcomes arising from using
#   the Software.
#   
#   The Licensee agrees to indemnify the University and hold the
#   University harmless from and against any and all claims, damages and
#   liabilities asserted by third parties (including claims for
#   negligence) which arise directly or indirectly from the use of the
#   Software or the sale of any products based on the Software.
#   
#   No part of the Software may be reproduced, modified, transmitted or
#   transferred in any form or by any means, electronic or mechanical,
#   without the express permission of the University. The permission of
#   the University is not required if the said reproduction, modification,
#   transmission or transference is done without financial return, the
#   conditions of this Licence are imposed upon the receiver of the
#   product, and all original and amended source code is included in any
#   transmitted product. You may be held legally responsible for any
#   copyright infringement that is caused or encouraged by your failure to
#   abide by these terms and conditions.
#   
#   You are not permitted under this Licence to use this Software
#   commercially. Use for which any financial return is received shall be
#   defined as commercial use, and includes (1) integration of all or part
#   of the source code or the Software into a product for sale or license
#   by or on behalf of Licensee to third parties or (2) use of the
#   Software or any derivative of it for research with the final aim of
#   developing software products for sale or license to a third party or
#   (3) use of the Software or any derivative of it for research with the
#   final aim of developing non-software products for sale or license to a
#   third party, or (4) use of the Software to provide any service to an
#   external organisation for which payment is received. If you are
#   interested in using the Software commercially, please contact Isis
#   Innovation Limited ("Isis"), the technology transfer company of the
#   University, to negotiate a licence. Contact details are:
#   innovation@isis.ox.ac.uk quoting reference DE/1112.

beforet=`date +%s`
if [ "x$SGE_ROOT" = "x" ] ; then
    if [ -f /usr/local/share/sge/default/common/settings.sh ] ; then
	. /usr/local/share/sge/default/common/settings.sh
    elif [ -f /usr/local/sge/default/common/settings.sh ] ; then
	. /usr/local/sge/default/common/settings.sh
    fi
fi

Usage() {
    echo ""
    echo "Usage: bedpostx <subject directory> [options]"
    echo ""
    echo "expects to find bvals and bvecs in subject directory"
    echo "expects to find data and nodif_brain_mask in subject directory"
    echo "expects to find grad_dev in subject directory, if -g is set"
    echo "options (old syntax)"
    echo "-n (number of fibres per voxel, default 2)"
    echo "-w (ARD weight, more weight means less secondary fibres per voxel, default 1)"
    echo "-b (burnin period, default 1000)"
    echo "-j (number of jumps, default 1250)"
    echo "-s (sample every, default 25)"
    echo "-model (1 for monoexponential, 2 for multiexponential, default 1)"
    echo "-g (consider gradient nonlinearities, default off)"
    echo ""
    echo ""
    echo "ALTERNATIVELY: you can pass on xfibres options onto directly bedpostx"
    echo " For example:  bedpostx <subject directory> --noard --cnonlinear"
    echo " Type 'xfibres --help' for a list of available options "
    echo " Default options will be bedpostx default (see above), and not xfibres default."
    echo ""
    echo "Note: Use EITHER old OR new syntax."
    exit 1
}

make_absolute(){
    dir=$1;
    if [ -d ${dir} ]; then
	OLDWD=`pwd`
	cd ${dir}
	dir_all=`pwd`
	cd $OLDWD
    else
	dir_all=${dir}
    fi
    echo ${dir_all}
}

[ "$1" = "" ] && Usage

subjdir=`make_absolute $1`
subjdir=`echo $subjdir | sed 's/\/$/$/g'`

echo subjectdir is $subjdir

#parse option arguments
nfibres=2
fudge=1
burnin=1000
njumps=1250
sampleevery=25
model=1
gflag=0
f0=0
rician=0

shift
while [ ! -z "$1" ]
do
  case "$1" in
      -n) nfibres=$2;shift;;
      --nf=?) nfibres=`echo $1 | sed s/"--nf="//g`;;
      --model=?) model=`echo $1 | sed s/"--model="//g`;;
      -w) fudge=$2;shift;;
      -b) burnin=$2;shift;;
      -j) njumps=$2;shift;;
      -s) sampleevery=$2;shift;;
      --f0) f0=1 others="$others $1";;
      --rician) rician=1 others="$others $1";; 	
      -model) model=$2;shift;;
      -g) gflag=1;; 
      *) others="$others $1";;
  esac
  shift
done

opts="--nf=$nfibres --fudge=$fudge --bi=$burnin --nj=$njumps --se=$sampleevery --model=$model"
defopts="--cnonlinear"
opts="$opts $defopts $others"  

#check that all required files exist

if [ ! -d $subjdir ]; then
	echo "subject directory $1 not found"
	exit 1
fi

if [ ! -e ${subjdir}/bvecs ]; then
	echo "${subjdir}/bvecs not found"
	exit 1
fi

if [ ! -e ${subjdir}/bvals ]; then
	echo "${subjdir}/bvals not found"
	exit 1
fi

if [ `${FSLDIR}/bin/imtest ${subjdir}/data` -eq 0 ]; then
	echo "${subjdir}/data not found"
	exit 1
fi

if [ ${gflag} -eq 1 ]; then
    if [ `${FSLDIR}/bin/imtest ${subjdir}/grad_dev` -eq 0 ]; then
	echo "${subjdir}/grad_dev not found"
	exit 1
    fi
fi

if [ `${FSLDIR}/bin/imtest ${subjdir}/nodif_brain_mask` -eq 0 ]; then
	echo "${subjdir}/nodif_brain_mask not found"
	exit 1
fi

echo Making bedpostx directory structure

mkdir -p ${subjdir}.bedpostX/
mkdir -p ${subjdir}.bedpostX/diff_slices
mkdir -p ${subjdir}.bedpostX/logs
mkdir -p ${subjdir}.bedpostX/logs/pid_${$}
mkdir -p ${subjdir}.bedpostX/xfms

if [ ${gflag} -eq 1 ]; then
    echo "bedpostx_gpu "${subjdir}  $opts "-g" >> ${subjdir}.bedpostX/commands.txt
else
    echo "bedpostx_gpu "${subjdir}  $opts >> ${subjdir}.bedpostX/commands.txt
fi


echo "..............Compiling CUDA-gpu code .................."

export LD_LIBRARY_PATH=${CUDA}/lib64:${subjdir}.bedpostX:${CUDA}/lib:$LD_LIBRARY_PATH
export DYLD_LIBRARY_PATH=${CUDA}/lib64:${subjdir}.bedpostX:${CUDA}/lib:$LD_LIBRARY_PATH
export PATH=${FSLDIR}/bin:$PATH

volumes=`${FSLDIR}/bin/fslval ${subjdir}/data dim4`
echo "DIRECTIONS NUMBER: $volumes"

filename="${subjdir}.bedpostX/options.h"

if [ $volumes -le 64 ]; then
	threadsBlock=$volumes
else 
	threadsBlock=64
fi

params=$((2+3*$nfibres)) 

if [ $f0 -eq 1 ]; then
	params=$(($params+1))
fi

if [ $model -eq 2 ]; then
	params=$(($params+1))
fi

if [ $rician -eq 1 ]; then
	params=$(($params+1))
fi

rem=$(($volumes%$threadsBlock))
maxndirs=$(($volumes/$threadsBlock))

if [ $rem -ne 0 ]; then
	maxndirs=$(($maxndirs+1))
fi

dstd=$(($model-1));

params_rep=$(($params-1))
params_step0_m2_rep=$(($params_step0_m2-1))

echo "#define NDIRECTIONS $volumes 
#define NFIBRES $nfibres 
#define NPARAMS $params
#define MAXNDIRS_PER_THREAD $maxndirs
#define THREADS_BLOCK $threadsBlock">$filename

before=`date +%s`

${CUDA}/bin/nvcc --shared --compiler-options '-fPIC' -o ${subjdir}.bedpostX/libbedpostx_cuda.so ${FSLDIR}/src/fdt/CUDA/init_gpu.cu ${FSLDIR}/src/fdt/CUDA/samples.cu ${FSLDIR}/src/fdt/CUDA/diffmodels.cu ${FSLDIR}/src/fdt/CUDA/runmcmc.cu ${FSLDIR}/src/fdt/CUDA/xfibres_gpu.cu -O3 -arch sm_20  -lcudart -lcuda -lcurand -L${CUDA}/lib64 -L${CUDA}/lib -I${subjdir}.bedpostX -I${FSLDIR}/extras/include/newmat -I${FSLDIR}/extras/include/boost -I${FSLDIR}/include -I${CUDA}/include/thrust -I${FSLDIR}/src/fdt 

file=${subjdir}.bedpostX/libbedpostx_cuda.so

if [ -f $file ];
then
after=`date +%s`
diff=$(($after-$before))
echo "Compilation Time: $diff seconds" >> ${subjdir}.bedpostX/times

echo "CUDA CODE compiled successfully"

mailto=`whoami`@fmrib.ox.ac.uk
before=`date +%s`
echo Preprocessing stages
${FSLDIR}/bin/bedpostx_preproc.sh ${subjdir} ${gflag}

after=`date +%s`
diff=$(($after-$before))
echo "Preprocessing Time: $diff seconds" >> ${subjdir}.bedpostX/times

before=`date +%s`
nslices=`${FSLDIR}/bin/fslval ${subjdir}/data dim3`

${FSLDIR}/bin/splitter_gpu $subjdir $gflag $nfibres $nslices $opts

after=`date +%s`
diff=$(($after-$before))
echo "Execution Time: $diff seconds" >> ${subjdir}.bedpostX/times

before=`date +%s`


echo Post processing stage
${FSLDIR}/bin/bedpostx_postproc.sh $subjdir

after=`date +%s`
difd=$(($after-$before))
echo "Post-Processing Time : $diff seconds" >> ${subjdir}.bedpostX/times

aftert=`date +%s`
diff=$(($aftert-$beforet))
echo "TOTAL Time: $diff seconds" >> ${subjdir}.bedpostX/times

else
	echo
	echo
	echo "CUDA COMPILER ERROR. Please be sure that you have set correctly CUDA environment variable with your CUDA path"
	echo 
fi