From d665829f98a70ed2d8ef2e926344aa8cdfc9ade6 Mon Sep 17 00:00:00 2001
From: Moises Fernandez <moisesf@fmrib.ox.ac.uk>
Date: Fri, 7 Dec 2012 20:19:13 +0000
Subject: [PATCH] Main script that compiles Bedpostx CUDA version and divide
the slices between several GPUs for a LSF Cluster
---
CUDA/bedpostx_multigpu_LSF | 358 +++++++++++++++++++++++++++++++++++++
1 file changed, 358 insertions(+)
create mode 100755 CUDA/bedpostx_multigpu_LSF
diff --git a/CUDA/bedpostx_multigpu_LSF b/CUDA/bedpostx_multigpu_LSF
new file mode 100755
index 0000000..9872d6d
--- /dev/null
+++ b/CUDA/bedpostx_multigpu_LSF
@@ -0,0 +1,358 @@
+#!/bin/sh
+
+# Copyright (C) 2004 University of Oxford
+#
+# Part of FSL - FMRIB's Software Library
+# http://www.fmrib.ox.ac.uk/fsl
+# fsl@fmrib.ox.ac.uk
+#
+# Developed at FMRIB (Oxford Centre for Functional Magnetic Resonance
+# Imaging of the Brain), Department of Clinical Neurology, Oxford
+# University, Oxford, UK
+#
+#
+# LICENCE
+#
+# FMRIB Software Library, Release 4.0 (c) 2007, The University of
+# Oxford (the "Software")
+#
+# The Software remains the property of the University of Oxford ("the
+# University").
+#
+# The Software is distributed "AS IS" under this Licence solely for
+# non-commercial use in the hope that it will be useful, but in order
+# that the University as a charitable foundation protects its assets for
+# the benefit of its educational and research purposes, the University
+# makes clear that no condition is made or to be implied, nor is any
+# warranty given or to be implied, as to the accuracy of the Software,
+# or that it will be suitable for any particular purpose or for use
+# under any specific conditions. Furthermore, the University disclaims
+# all responsibility for the use which is made of the Software. It
+# further disclaims any liability for the outcomes arising from using
+# the Software.
+#
+# The Licensee agrees to indemnify the University and hold the
+# University harmless from and against any and all claims, damages and
+# liabilities asserted by third parties (including claims for
+# negligence) which arise directly or indirectly from the use of the
+# Software or the sale of any products based on the Software.
+#
+# No part of the Software may be reproduced, modified, transmitted or
+# transferred in any form or by any means, electronic or mechanical,
+# without the express permission of the University. The permission of
+# the University is not required if the said reproduction, modification,
+# transmission or transference is done without financial return, the
+# conditions of this Licence are imposed upon the receiver of the
+# product, and all original and amended source code is included in any
+# transmitted product. You may be held legally responsible for any
+# copyright infringement that is caused or encouraged by your failure to
+# abide by these terms and conditions.
+#
+# You are not permitted under this Licence to use this Software
+# commercially. Use for which any financial return is received shall be
+# defined as commercial use, and includes (1) integration of all or part
+# of the source code or the Software into a product for sale or license
+# by or on behalf of Licensee to third parties or (2) use of the
+# Software or any derivative of it for research with the final aim of
+# developing software products for sale or license to a third party or
+# (3) use of the Software or any derivative of it for research with the
+# final aim of developing non-software products for sale or license to a
+# third party, or (4) use of the Software to provide any service to an
+# external organisation for which payment is received. If you are
+# interested in using the Software commercially, please contact Isis
+# Innovation Limited ("Isis"), the technology transfer company of the
+# University, to negotiate a licence. Contact details are:
+# innovation@isis.ox.ac.uk quoting reference DE/1112.
+
+beforet=`date +%s`
+if [ "x$SGE_ROOT" = "x" ] ; then
+ if [ -f /usr/local/share/sge/default/common/settings.sh ] ; then
+ . /usr/local/share/sge/default/common/settings.sh
+ elif [ -f /usr/local/sge/default/common/settings.sh ] ; then
+ . /usr/local/sge/default/common/settings.sh
+ fi
+fi
+
+Usage() {
+ echo ""
+ echo "Usage: bedpostx <subject directory> [options]"
+ echo ""
+ echo "expects to find bvals and bvecs in subject directory"
+ echo "expects to find data and nodif_brain_mask in subject directory"
+ echo "expects to find grad_dev in subject directory, if -g is set"
+ echo "options (old syntax)"
+ echo "-n (number of fibres per voxel, default 2)"
+ echo "-w (ARD weight, more weight means less secondary fibres per voxel, default 1)"
+ echo "-b (burnin period, default 1000)"
+ echo "-j (number of jumps, default 1250)"
+ echo "-s (sample every, default 25)"
+ echo "-model (1 for monoexponential, 2 for multiexponential, default 1)"
+ echo "-g (consider gradient nonlinearities, default off)"
+ echo ""
+ echo ""
+ echo "ALTERNATIVELY: you can pass on xfibres options onto directly bedpostx"
+ echo " For example: bedpostx <subject directory> --noard --cnonlinear"
+ echo " Type 'xfibres --help' for a list of available options "
+ echo " Default options will be bedpostx default (see above), and not xfibres default."
+ echo ""
+ echo "Note: Use EITHER old OR new syntax."
+ exit 1
+}
+
+make_absolute(){
+ dir=$1;
+ if [ -d ${dir} ]; then
+ OLDWD=`pwd`
+ cd ${dir}
+ dir_all=`pwd`
+ cd $OLDWD
+ else
+ dir_all=${dir}
+ fi
+ echo ${dir_all}
+}
+
+[ "$1" = "" ] && Usage
+
+subjdir=`make_absolute $1`
+subjdir=`echo $subjdir | sed 's/\/$/$/g'`
+
+echo subjectdir is $subjdir
+
+#parse option arguments
+nfibres=2
+fudge=1
+burnin=1000
+njumps=1250
+sampleevery=25
+model=1
+gflag=0
+f0=0
+rician=0
+
+shift
+while [ ! -z "$1" ]
+do
+ case "$1" in
+ -n) nfibres=$2;shift;;
+ --nf=?) nfibres=`echo $1 | sed s/"--nf="//g`;;
+ --model=?) model=`echo $1 | sed s/"--model="//g`;;
+ -w) fudge=$2;shift;;
+ -b) burnin=$2;shift;;
+ -j) njumps=$2;shift;;
+ -s) sampleevery=$2;shift;;
+ --f0) f0=1 others="$others $1";;
+ --rician) rician=1 others="$others $1";;
+ -model) model=$2;shift;;
+ -g) gflag=1;;
+ *) others="$others $1";;
+ esac
+ shift
+done
+
+opts="--nf=$nfibres --fudge=$fudge --bi=$burnin --nj=$njumps --se=$sampleevery --model=$model"
+defopts="--cnonlinear"
+opts="$opts $defopts $others"
+
+#check that all required files exist
+
+if [ ! -d $subjdir ]; then
+ echo "subject directory $1 not found"
+ exit 1
+fi
+
+if [ ! -e ${subjdir}/bvecs ]; then
+ echo "${subjdir}/bvecs not found"
+ exit 1
+fi
+
+if [ ! -e ${subjdir}/bvals ]; then
+ echo "${subjdir}/bvals not found"
+ exit 1
+fi
+
+if [ `${FSLDIR}/bin/imtest ${subjdir}/data` -eq 0 ]; then
+ echo "${subjdir}/data not found"
+ exit 1
+fi
+
+if [ ${gflag} -eq 1 ]; then
+ if [ `${FSLDIR}/bin/imtest ${subjdir}/grad_dev` -eq 0 ]; then
+ echo "${subjdir}/grad_dev not found"
+ exit 1
+ fi
+fi
+
+if [ `${FSLDIR}/bin/imtest ${subjdir}/nodif_brain_mask` -eq 0 ]; then
+ echo "${subjdir}/nodif_brain_mask not found"
+ exit 1
+fi
+
+echo Making bedpostx directory structure
+
+mkdir -p ${subjdir}.bedpostX/
+mkdir -p ${subjdir}.bedpostX/diff_slices
+mkdir -p ${subjdir}.bedpostX/logs
+mkdir -p ${subjdir}.bedpostX/logs/pid_${$}
+mkdir -p ${subjdir}.bedpostX/xfms
+
+if [ ${gflag} -eq 1 ]; then
+ echo "bedpostx_gpu "${subjdir} $opts "-g" >> ${subjdir}.bedpostX/commands.txt
+else
+ echo "bedpostx_gpu "${subjdir} $opts >> ${subjdir}.bedpostX/commands.txt
+fi
+
+echo "..............Compiling CUDA-gpu code .................."
+
+export LD_LIBRARY_PATH=${CUDA}/lib64:${subjdir}.bedpostX:${CUDA}/lib:$LD_LIBRARY_PATH
+export DYLD_LIBRARY_PATH=${CUDA}/lib64:${subjdir}.bedpostX:${CUDA}/lib:$LD_LIBRARY_PATH
+export PATH=${FSLDIR}/bin:$PATH
+
+volumes=`${FSLDIR}/bin/fslval ${subjdir}/data dim4`
+echo "DIRECTIONS NUMBER: $volumes"
+
+filename="${subjdir}.bedpostX/options.h"
+
+if [ $volumes -le 64 ]; then
+ threadsBlock=$volumes
+else
+ threadsBlock=64
+fi
+
+params=$((2+3*$nfibres))
+
+if [ $f0 -eq 1 ]; then
+ params=$(($params+1))
+fi
+
+if [ $model -eq 2 ]; then
+ params=$(($params+1))
+fi
+
+if [ $rician -eq 1 ]; then
+ params=$(($params+1))
+fi
+
+rem=$(($volumes%$threadsBlock))
+maxndirs=$(($volumes/$threadsBlock))
+
+if [ $rem -ne 0 ]; then
+ maxndirs=$(($maxndirs+1))
+fi
+
+dstd=$(($model-1));
+
+params_rep=$(($params-1))
+params_step0_m2_rep=$(($params_step0_m2-1))
+
+echo "#define NDIRECTIONS $volumes
+#define NFIBRES $nfibres
+#define NPARAMS $params
+#define MAXNDIRS_PER_THREAD $maxndirs
+#define THREADS_BLOCK $threadsBlock">$filename
+
+before=`date +%s`
+
+${CUDA}/bin/nvcc --shared --compiler-options '-fPIC' -o ${subjdir}.bedpostX/libbedpostx_cuda.so ${FSLDIR}/src/fdt/CUDA/init_gpu.cu ${FSLDIR}/src/fdt/CUDA/samples.cu ${FSLDIR}/src/fdt/CUDA/diffmodels.cu ${FSLDIR}/src/fdt/CUDA/runmcmc.cu ${FSLDIR}/src/fdt/CUDA/xfibres_gpu.cu -O3 -arch sm_20 -lcudart -lcuda -lcurand -L${CUDA}/lib64 -L${CUDA}/lib -I${subjdir}.bedpostX -I${FSLDIR}/extras/include/newmat -I${FSLDIR}/extras/include/boost -I${FSLDIR}/include -I${CUDA}/include/thrust -I${FSLDIR}/src/fdt
+
+file=${subjdir}.bedpostX/libbedpostx_cuda.so
+
+if [ -f $file ];
+then
+after=`date +%s`
+diff=$(($after-$before))
+echo "Compilation Time: $diff seconds" >> ${subjdir}.bedpostX/times
+
+echo "CUDA CODE compiled successfully"
+
+mailto=`whoami`@fmrib.ox.ac.uk
+before=`date +%s`
+echo Preprocessing stages
+${FSLDIR}/bin/bedpostx_preproc.sh ${subjdir} ${gflag}
+
+after=`date +%s`
+diff=$(($after-$before))
+echo "Preprocessing Time: $diff seconds" >> ${subjdir}.bedpostX/times
+
+before=`date +%s`
+nslices=`${FSLDIR}/bin/fslval ${subjdir}/data dim3`
+
+slice=0
+
+post_string="bsub -o ${subjdir}'.bedpostX/logs/output.log -e ${subjdir}.bedpostX/logs/error.log -W 0:50"
+
+post_string=$post_string" -w "
+
+echo Submitting slices to GPUs
+while [ $slice -lt $nslices ];do
+
+ string=$(bsub -o ${subjdir}'.bedpostX/logs/output.log' -e ${subjdir}'.bedpostX/logs/error.log' -W 0:15 -n 1 ${FSLDIR}'/bin/splitter_multigpu' $subjdir $gflag $nfibres $slice $opts)
+ array=(`echo $string | tr "<" ' '`)
+ aux=${array[1]}
+ array=(`echo $aux | tr ">" ' '`)
+ job=${array[0]}
+ #echo $job
+
+if [ $slice -eq $(($nslices-1)) ];then
+ post_string=$post_string"done("${job}")"
+else
+ post_string=$post_string"done("${job}")&&"
+fi
+ slice=$(($slice + 1))
+done
+
+post_string=$post_string" ${FSLDIR}/bin/bedpostx_postproc.sh $subjdir"
+#echo $post_string
+$post_string
+
+echo "All jobs submitted"
+
+finished=0
+logdir=${subjdir}.bedpostX/logs
+tim=0
+
+while [ $finished -eq 0 ] ; do
+ nfin=0
+ pslice=0
+ while [ $pslice -lt $nslices ];do
+ slicezp=`${FSLDIR}/bin/zeropad $pslice 4`
+ isimg=`${FSLDIR}/bin/imtest ${subjdir}.bedpostX/diff_slices/data_slice_$slicezp/dyads1`
+ if [ $isimg -eq 1 ];then
+ nfin=$(($nfin + 1))
+ fi
+ pslice=$(($pslice + 1))
+ done
+ echo $nfin "slices processed"
+
+ if [ $tim -eq 0 ] ; then
+ if [ $nfin -eq $nslices ] ; then
+ after=`date +%s`
+ diff=$(($after-$before))
+ echo "Execution Time: $diff seconds" >> ${subjdir}.bedpostX/times
+ tim=1
+ fi
+ fi
+
+
+
+ if [ -f ${subjdir}.bedpostX/xfms/eye.mat ] ; then
+ finished=1
+ echo "All slices processed"
+ fi
+ sleep 15;
+done
+
+
+
+aftert=`date +%s`
+diff=$(($aftert-$beforet))
+echo "TOTAL Time: $diff seconds" >> ${subjdir}.bedpostX/times
+
+else
+ echo
+ echo
+ echo "CUDA COMPILER ERROR. Please be sure that you have set correctly CUDALIBDIR in first line in file bin/bedpostx_cuda"
+ echo
+fi
+
+
--
GitLab