Main script that compiles Bedpostx CUDA version and divide the slices between...

Main script that compiles Bedpostx CUDA version and divide the slices between several GPUs for a LSF Cluster

CUDA/bedpostx_multigpu_LSF

+#!/bin/sh
+
+#   Copyright (C) 2004 University of Oxford
+#
+#   Part of FSL - FMRIB's Software Library
+#   http://www.fmrib.ox.ac.uk/fsl
+#   fsl@fmrib.ox.ac.uk
+#   
+#   Developed at FMRIB (Oxford Centre for Functional Magnetic Resonance
+#   Imaging of the Brain), Department of Clinical Neurology, Oxford
+#   University, Oxford, UK
+#   
+#   
+#   LICENCE
+#   
+#   FMRIB Software Library, Release 4.0 (c) 2007, The University of
+#   Oxford (the "Software")
+#   
+#   The Software remains the property of the University of Oxford ("the
+#   University").
+#   
+#   The Software is distributed "AS IS" under this Licence solely for
+#   non-commercial use in the hope that it will be useful, but in order
+#   that the University as a charitable foundation protects its assets for
+#   the benefit of its educational and research purposes, the University
+#   makes clear that no condition is made or to be implied, nor is any
+#   warranty given or to be implied, as to the accuracy of the Software,
+#   or that it will be suitable for any particular purpose or for use
+#   under any specific conditions. Furthermore, the University disclaims
+#   all responsibility for the use which is made of the Software. It
+#   further disclaims any liability for the outcomes arising from using
+#   the Software.
+#   
+#   The Licensee agrees to indemnify the University and hold the
+#   University harmless from and against any and all claims, damages and
+#   liabilities asserted by third parties (including claims for
+#   negligence) which arise directly or indirectly from the use of the
+#   Software or the sale of any products based on the Software.
+#   
+#   No part of the Software may be reproduced, modified, transmitted or
+#   transferred in any form or by any means, electronic or mechanical,
+#   without the express permission of the University. The permission of
+#   the University is not required if the said reproduction, modification,
+#   transmission or transference is done without financial return, the
+#   conditions of this Licence are imposed upon the receiver of the
+#   product, and all original and amended source code is included in any
+#   transmitted product. You may be held legally responsible for any
+#   copyright infringement that is caused or encouraged by your failure to
+#   abide by these terms and conditions.
+#   
+#   You are not permitted under this Licence to use this Software
+#   commercially. Use for which any financial return is received shall be
+#   defined as commercial use, and includes (1) integration of all or part
+#   of the source code or the Software into a product for sale or license
+#   by or on behalf of Licensee to third parties or (2) use of the
+#   Software or any derivative of it for research with the final aim of
+#   developing software products for sale or license to a third party or
+#   (3) use of the Software or any derivative of it for research with the
+#   final aim of developing non-software products for sale or license to a
+#   third party, or (4) use of the Software to provide any service to an
+#   external organisation for which payment is received. If you are
+#   interested in using the Software commercially, please contact Isis
+#   Innovation Limited ("Isis"), the technology transfer company of the
+#   University, to negotiate a licence. Contact details are:
+#   innovation@isis.ox.ac.uk quoting reference DE/1112.
+
+beforet=`date +%s`
+if [ "x$SGE_ROOT" = "x" ] ; then
+    if [ -f /usr/local/share/sge/default/common/settings.sh ] ; then
+	. /usr/local/share/sge/default/common/settings.sh
+    elif [ -f /usr/local/sge/default/common/settings.sh ] ; then
+	. /usr/local/sge/default/common/settings.sh
+    fi
+fi
+
+Usage() {
+    echo ""
+    echo "Usage: bedpostx <subject directory> [options]"
+    echo ""
+    echo "expects to find bvals and bvecs in subject directory"
+    echo "expects to find data and nodif_brain_mask in subject directory"
+    echo "expects to find grad_dev in subject directory, if -g is set"
+    echo "options (old syntax)"
+    echo "-n (number of fibres per voxel, default 2)"
+    echo "-w (ARD weight, more weight means less secondary fibres per voxel, default 1)"
+    echo "-b (burnin period, default 1000)"
+    echo "-j (number of jumps, default 1250)"
+    echo "-s (sample every, default 25)"
+    echo "-model (1 for monoexponential, 2 for multiexponential, default 1)"
+    echo "-g (consider gradient nonlinearities, default off)"
+    echo ""
+    echo ""
+    echo "ALTERNATIVELY: you can pass on xfibres options onto directly bedpostx"
+    echo " For example:  bedpostx <subject directory> --noard --cnonlinear"
+    echo " Type 'xfibres --help' for a list of available options "
+    echo " Default options will be bedpostx default (see above), and not xfibres default."
+    echo ""
+    echo "Note: Use EITHER old OR new syntax."
+    exit 1
+}
+
+make_absolute(){
+    dir=$1;
+    if [ -d ${dir} ]; then
+	OLDWD=`pwd`
+	cd ${dir}
+	dir_all=`pwd`
+	cd $OLDWD
+    else
+	dir_all=${dir}
+    fi
+    echo ${dir_all}
+}
+
+[ "$1" = "" ] && Usage
+
+subjdir=`make_absolute $1`
+subjdir=`echo $subjdir | sed 's/\/$/$/g'`
+
+echo subjectdir is $subjdir
+
+#parse option arguments
+nfibres=2
+fudge=1
+burnin=1000
+njumps=1250
+sampleevery=25
+model=1
+gflag=0
+f0=0
+rician=0
+
+shift
+while [ ! -z "$1" ]
+do
+  case "$1" in
+      -n) nfibres=$2;shift;;
+      --nf=?) nfibres=`echo $1 | sed s/"--nf="//g`;;
+      --model=?) model=`echo $1 | sed s/"--model="//g`;;
+      -w) fudge=$2;shift;;
+      -b) burnin=$2;shift;;
+      -j) njumps=$2;shift;;
+      -s) sampleevery=$2;shift;;
+      --f0) f0=1 others="$others $1";;
+      --rician) rician=1 others="$others $1";; 	
+      -model) model=$2;shift;;
+      -g) gflag=1;; 
+      *) others="$others $1";;
+  esac
+  shift
+done
+
+opts="--nf=$nfibres --fudge=$fudge --bi=$burnin --nj=$njumps --se=$sampleevery --model=$model"
+defopts="--cnonlinear"
+opts="$opts $defopts $others"  
+
+#check that all required files exist
+
+if [ ! -d $subjdir ]; then
+	echo "subject directory $1 not found"
+	exit 1
+fi
+
+if [ ! -e ${subjdir}/bvecs ]; then
+	echo "${subjdir}/bvecs not found"
+	exit 1
+fi
+
+if [ ! -e ${subjdir}/bvals ]; then
+	echo "${subjdir}/bvals not found"
+	exit 1
+fi
+
+if [ `${FSLDIR}/bin/imtest ${subjdir}/data` -eq 0 ]; then
+	echo "${subjdir}/data not found"
+	exit 1
+fi
+
+if [ ${gflag} -eq 1 ]; then
+    if [ `${FSLDIR}/bin/imtest ${subjdir}/grad_dev` -eq 0 ]; then
+	echo "${subjdir}/grad_dev not found"
+	exit 1
+    fi
+fi
+
+if [ `${FSLDIR}/bin/imtest ${subjdir}/nodif_brain_mask` -eq 0 ]; then
+	echo "${subjdir}/nodif_brain_mask not found"
+	exit 1
+fi
+
+echo Making bedpostx directory structure
+
+mkdir -p ${subjdir}.bedpostX/
+mkdir -p ${subjdir}.bedpostX/diff_slices
+mkdir -p ${subjdir}.bedpostX/logs
+mkdir -p ${subjdir}.bedpostX/logs/pid_${$}
+mkdir -p ${subjdir}.bedpostX/xfms
+
+if [ ${gflag} -eq 1 ]; then
+    echo "bedpostx_gpu "${subjdir}  $opts "-g" >> ${subjdir}.bedpostX/commands.txt
+else
+    echo "bedpostx_gpu "${subjdir}  $opts >> ${subjdir}.bedpostX/commands.txt
+fi
+
+echo "..............Compiling CUDA-gpu code .................."
+
+export LD_LIBRARY_PATH=${CUDA}/lib64:${subjdir}.bedpostX:${CUDA}/lib:$LD_LIBRARY_PATH
+export DYLD_LIBRARY_PATH=${CUDA}/lib64:${subjdir}.bedpostX:${CUDA}/lib:$LD_LIBRARY_PATH
+export PATH=${FSLDIR}/bin:$PATH
+
+volumes=`${FSLDIR}/bin/fslval ${subjdir}/data dim4`
+echo "DIRECTIONS NUMBER: $volumes"
+
+filename="${subjdir}.bedpostX/options.h"
+
+if [ $volumes -le 64 ]; then
+	threadsBlock=$volumes
+else 
+	threadsBlock=64
+fi
+
+params=$((2+3*$nfibres)) 
+
+if [ $f0 -eq 1 ]; then
+	params=$(($params+1))
+fi
+
+if [ $model -eq 2 ]; then
+	params=$(($params+1))
+fi
+
+if [ $rician -eq 1 ]; then
+	params=$(($params+1))
+fi
+
+rem=$(($volumes%$threadsBlock))
+maxndirs=$(($volumes/$threadsBlock))
+
+if [ $rem -ne 0 ]; then
+	maxndirs=$(($maxndirs+1))
+fi
+
+dstd=$(($model-1));
+
+params_rep=$(($params-1))
+params_step0_m2_rep=$(($params_step0_m2-1))
+
+echo "#define NDIRECTIONS $volumes 
+#define NFIBRES $nfibres 
+#define NPARAMS $params
+#define MAXNDIRS_PER_THREAD $maxndirs
+#define THREADS_BLOCK $threadsBlock">$filename
+
+before=`date +%s`
+
+${CUDA}/bin/nvcc --shared --compiler-options '-fPIC' -o ${subjdir}.bedpostX/libbedpostx_cuda.so ${FSLDIR}/src/fdt/CUDA/init_gpu.cu ${FSLDIR}/src/fdt/CUDA/samples.cu ${FSLDIR}/src/fdt/CUDA/diffmodels.cu ${FSLDIR}/src/fdt/CUDA/runmcmc.cu ${FSLDIR}/src/fdt/CUDA/xfibres_gpu.cu -O3 -arch sm_20  -lcudart -lcuda -lcurand -L${CUDA}/lib64 -L${CUDA}/lib -I${subjdir}.bedpostX -I${FSLDIR}/extras/include/newmat -I${FSLDIR}/extras/include/boost -I${FSLDIR}/include -I${CUDA}/include/thrust -I${FSLDIR}/src/fdt 
+
+file=${subjdir}.bedpostX/libbedpostx_cuda.so
+
+if [ -f $file ];
+then
+after=`date +%s`
+diff=$(($after-$before))
+echo "Compilation Time: $diff seconds" >> ${subjdir}.bedpostX/times
+
+echo "CUDA CODE compiled successfully"
+
+mailto=`whoami`@fmrib.ox.ac.uk
+before=`date +%s`
+echo Preprocessing stages
+${FSLDIR}/bin/bedpostx_preproc.sh ${subjdir} ${gflag}
+
+after=`date +%s`
+diff=$(($after-$before))
+echo "Preprocessing Time: $diff seconds" >> ${subjdir}.bedpostX/times
+
+before=`date +%s`
+nslices=`${FSLDIR}/bin/fslval ${subjdir}/data dim3`
+
+slice=0
+
+post_string="bsub -o ${subjdir}'.bedpostX/logs/output.log -e ${subjdir}.bedpostX/logs/error.log -W 0:50"
+
+post_string=$post_string" -w "
+
+echo Submitting slices to GPUs
+while [ $slice -lt $nslices ];do
+
+    string=$(bsub -o ${subjdir}'.bedpostX/logs/output.log' -e ${subjdir}'.bedpostX/logs/error.log' -W 0:15 -n 1  ${FSLDIR}'/bin/splitter_multigpu' $subjdir $gflag $nfibres $slice $opts)
+    array=(`echo $string | tr "<" ' '`)
+    aux=${array[1]}
+    array=(`echo $aux | tr ">" ' '`)
+    job=${array[0]}
+    #echo $job
+
+if [ $slice -eq $(($nslices-1)) ];then
+     post_string=$post_string"done("${job}")"
+else
+     post_string=$post_string"done("${job}")&&"
+fi
+    slice=$(($slice + 1))
+done
+
+post_string=$post_string" ${FSLDIR}/bin/bedpostx_postproc.sh $subjdir"
+#echo $post_string
+$post_string
+
+echo "All jobs submitted"
+
+finished=0
+logdir=${subjdir}.bedpostX/logs
+tim=0
+
+while [ $finished -eq 0 ] ; do
+    nfin=0
+    pslice=0
+    while [ $pslice -lt $nslices ];do
+        slicezp=`${FSLDIR}/bin/zeropad $pslice 4`
+        isimg=`${FSLDIR}/bin/imtest ${subjdir}.bedpostX/diff_slices/data_slice_$slicezp/dyads1`
+        if [ $isimg -eq 1 ];then
+          nfin=$(($nfin + 1))
+        fi
+        pslice=$(($pslice + 1))
+    done
+    echo $nfin "slices processed"
+	
+    if [ $tim -eq 0 ] ; then
+	    if [ $nfin -eq $nslices ] ; then
+		after=`date +%s`
+		diff=$(($after-$before))
+		echo "Execution Time: $diff seconds" >> ${subjdir}.bedpostX/times
+		tim=1
+	    fi
+    fi
+
+
+
+    if [ -f ${subjdir}.bedpostX/xfms/eye.mat ] ; then
+        finished=1
+        echo "All slices processed"
+    fi
+    sleep 15;
+done
+
+
+
+aftert=`date +%s`
+diff=$(($aftert-$beforet))
+echo "TOTAL Time: $diff seconds" >> ${subjdir}.bedpostX/times
+
+else
+	echo
+	echo
+	echo "CUDA COMPILER ERROR. Please be sure that you have set correctly CUDALIBDIR in first line in file bin/bedpostx_cuda"
+	echo 
+fi
+
+