Skip to content
Snippets Groups Projects
Commit 557802c3 authored by Moises Fernandez's avatar Moises Fernandez
Browse files

Deleted bedpostx_multigpu_SGE because now the MultiGPU version in SGE will be... 

Deleted bedpostx_multigpu_SGE because now the MultiGPU version in SGE will be controlled by fsl_sub and NGPU parameter in bedpostx_gpu
parent 0c2259f2
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
CUDA/bedpostx_multigpu_SGE deleted 100755 → 0
+ 0
− 315
View file @ 0c2259f2
#!/bin/bash
# Copyright (C) 2004 University of Oxford
#
# SHCOPYRIGHT
if [ -z $PE_NAME ] ; then
echo "Please set PE_NAME environment variable with your Parallel Environment List name used in SGE for GPUs slots"
exit 1
fi
if [ -z $QUEUE_GPU ] ; then
echo "Please set QUEUE_GPU environment variable with a Queue Instance name used in SGE for process slices in GPUs"
exit 1
fi
if [ -z $QUEUE_GENERAL ] ; then
echo "Please set QUEUE_GENERAL environment variable with a Queue Instance name used in SGE for process the post processing stage in CPU"
exit 1
fi
beforet=`date +%s`
if [ "x$SGE_ROOT" = "x" ] ; then
if [ -f /usr/local/share/sge/default/common/settings.sh ] ; then
. /usr/local/share/sge/default/common/settings.sh
elif [ -f /usr/local/sge/default/common/settings.sh ] ; then
. /usr/local/sge/default/common/settings.sh
fi
fi
Usage() {
echo ""
echo "Usage: bedpostx <subject directory> [options]"
echo ""
echo "expects to find bvals and bvecs in subject directory"
echo "expects to find data and nodif_brain_mask in subject directory"
echo "expects to find grad_dev in subject directory, if -g is set"
echo "options (old syntax)"
echo "-n (number of fibres per voxel, default 2)"
echo "-w (ARD weight, more weight means less secondary fibres per voxel, default 1)"
echo "-b (burnin period, default 1000)"
echo "-j (number of jumps, default 1250)"
echo "-s (sample every, default 25)"
echo "-model (1 for monoexponential, 2 for multiexponential, default 1)"
echo "-g (consider gradient nonlinearities, default off)"
echo ""
echo ""
echo "ALTERNATIVELY: you can pass on xfibres options onto directly bedpostx"
echo " For example: bedpostx <subject directory> --noard --cnonlinear"
echo " Type 'xfibres --help' for a list of available options "
echo " Default options will be bedpostx default (see above), and not xfibres default."
echo ""
echo "Note: Use EITHER old OR new syntax."
exit 1
}
make_absolute(){
dir=$1;
if [ -d ${dir} ]; then
OLDWD=`pwd`
cd ${dir}
dir_all=`pwd`
cd $OLDWD
else
dir_all=${dir}
fi
echo ${dir_all}
}
[ "$1" = "" ] && Usage
subjdir=`make_absolute $1`
subjdir=`echo $subjdir | sed 's/\/$/$/g'`
echo subjectdir is $subjdir
# this is the name of cuda queue
queue=${QUEUE_GPU}
#this is the name of general queue
queue2=${QUEUE_GENERAL}
#parse option arguments
nfibres=2
fudge=1
burnin=1000
njumps=1250
sampleevery=25
model=1
gflag=0
f0=0
rician=0
shift
while [ ! -z "$1" ]
do
case "$1" in
-n) nfibres=$2;shift;;
--nf=?) nfibres=`echo $1 | sed s/"--nf="//g`;;
--model=?) model=`echo $1 | sed s/"--model="//g`;;
-w) fudge=$2;shift;;
-b) burnin=$2;shift;;
-j) njumps=$2;shift;;
-s) sampleevery=$2;shift;;
--f0) f0=1 others="$others $1";;
--rician) rician=1 others="$others $1";;
-model) model=$2;shift;;
-g) gflag=1;;
*) others="$others $1";;
esac
shift
done
opts="--nf=$nfibres --fudge=$fudge --bi=$burnin --nj=$njumps --se=$sampleevery --model=$model"
defopts="--cnonlinear"
opts="$opts $defopts $others"
#check that all required files exist
if [ ! -d $subjdir ]; then
echo "subject directory $1 not found"
exit 1
fi
if [ ! -e ${subjdir}/bvecs ]; then
echo "${subjdir}/bvecs not found"
exit 1
fi
if [ ! -e ${subjdir}/bvals ]; then
echo "${subjdir}/bvals not found"
exit 1
fi
if [ `${FSLDIR}/bin/imtest ${subjdir}/data` -eq 0 ]; then
echo "${subjdir}/data not found"
exit 1
fi
if [ ${gflag} -eq 1 ]; then
if [ `${FSLDIR}/bin/imtest ${subjdir}/grad_dev` -eq 0 ]; then
echo "${subjdir}/grad_dev not found"
exit 1
fi
fi
if [ `${FSLDIR}/bin/imtest ${subjdir}/nodif_brain_mask` -eq 0 ]; then
echo "${subjdir}/nodif_brain_mask not found"
exit 1
fi
echo Making bedpostx directory structure
mkdir -p ${subjdir}.bedpostX/
mkdir -p ${subjdir}.bedpostX/diff_slices
mkdir -p ${subjdir}.bedpostX/logs
mkdir -p ${subjdir}.bedpostX/logs/pid_${$}
mkdir -p ${subjdir}.bedpostX/xfms
if [ ${gflag} -eq 1 ]; then
echo "bedpostx_multigpu_SGE "${subjdir} $opts "-g" >> ${subjdir}.bedpostX/commands.txt
else
echo "bedpostx_multigpu_SGE "${subjdir} $opts >> ${subjdir}.bedpostX/commands.txt
fi
echo "..............Compiling CUDA-gpu code .................."
export LD_LIBRARY_PATH=${CUDA}/lib64:${subjdir}.bedpostX:${CUDA}/lib:$LD_LIBRARY_PATH
export DYLD_LIBRARY_PATH=${CUDA}/lib64:${subjdir}.bedpostX:${CUDA}/lib:$LD_LIBRARY_PATH
export PATH=${FSLDIR}/bin:$PATH
volumes=`${FSLDIR}/bin/fslval ${subjdir}/data dim4`
echo "DIRECTIONS NUMBER: $volumes"
filename="${subjdir}.bedpostX/options.h"
if [ $volumes -le 64 ]; then
threadsBlock=$volumes
else
threadsBlock=64
fi
params=$((2+3*$nfibres))
if [ $f0 -eq 1 ]; then
params=$(($params+1))
fi
if [ $model -eq 2 ]; then
params=$(($params+1))
fi
if [ $rician -eq 1 ]; then
params=$(($params+1))
fi
rem=$(($volumes%$threadsBlock))
maxndirs=$(($volumes/$threadsBlock))
if [ $rem -ne 0 ]; then
maxndirs=$(($maxndirs+1))
fi
dstd=$(($model-1));
params_rep=$(($params-1))
params_step0_m2_rep=$(($params_step0_m2-1))
echo "#define NDIRECTIONS $volumes
#define NFIBRES $nfibres
#define NPARAMS $params
#define MAXNDIRS_PER_THREAD $maxndirs
#define THREADS_BLOCK $threadsBlock">$filename
before=`date +%s`
${CUDA}/bin/nvcc --shared --compiler-options '-fPIC' -o ${subjdir}.bedpostX/libbedpostx_cuda.so ${FSLDIR}/src/fdt/CUDA/init_gpu.cu ${FSLDIR}/src/fdt/CUDA/samples.cu ${FSLDIR}/src/fdt/CUDA/diffmodels.cu ${FSLDIR}/src/fdt/CUDA/runmcmc.cu ${FSLDIR}/src/fdt/CUDA/xfibres_gpu.cu -O3 -arch sm_20 -lcudart -lcuda -lcurand -L${CUDA}/lib64 -L${CUDA}/lib -I${subjdir}.bedpostX -I${FSLDIR}/extras/include/newmat -I${FSLDIR}/extras/include/boost -I${FSLDIR}/include -I${CUDA}/include/thrust -I${FSLDIR}/src/fdt
file=${subjdir}.bedpostX/libbedpostx_cuda.so
if [ -f $file ];
then
after=`date +%s`
diff=$(($after-$before))
echo "Compilation Time: $diff seconds" >> ${subjdir}.bedpostX/times
echo "CUDA CODE compiled successfully"
mailto=`whoami`@fmrib.ox.ac.uk
before=`date +%s`
echo Preprocessing stages
${FSLDIR}/bin/bedpostx_preproc.sh ${subjdir} ${gflag}
after=`date +%s`
diff=$(($after-$before))
echo "Preprocessing Time: $diff seconds" >> ${subjdir}.bedpostX/times
before=`date +%s`
nslices=`${FSLDIR}/bin/fslval ${subjdir}/data dim3`
slice=0
post_string="qsub -q $queue2 -N postproc -V -b y"
echo Submitting slices to GPUs
while [ $slice -lt $nslices ];do
job=`qsub -q $queue -pe $PE_NAME 1 -N "Bedpostx" -V -b y -o ${subjdir}.bedpostX/logs/output -e ${subjdir}.bedpostX/logs/error ${FSLDIR}/bin/splitter_multigpu $subjdir $gflag $nfibres $slice $opts`
IFS=" "
array=( $job )
job_id=${array[2]}
#echo "JOB: "$job_id
post_string="$post_string -hold_jid $job_id"
slice=$(($slice + 1))
done
post_string="$post_string -o ${subjdir}.bedpostX/logs/output -e ${subjdir}.bedpostX/logs/error ${FSLDIR}/bin/bedpostx_postproc.sh $subjdir"
#echo $post_string
$post_string
echo "All jobs submitted"
finished=0
logdir=${subjdir}.bedpostX/logs
tim=0
while [ $finished -eq 0 ] ; do
nfin=0
pslice=0
while [ $pslice -lt $nslices ];do
slicezp=`${FSLDIR}/bin/zeropad $pslice 4`
isimg=`${FSLDIR}/bin/imtest ${subjdir}.bedpostX/diff_slices/data_slice_$slicezp/dyads1`
if [ $isimg -eq 1 ];then
nfin=$(($nfin + 1))
fi
pslice=$(($pslice + 1))
done
echo $nfin "slices processed"
if [ $tim -eq 0 ] ; then
if [ $nfin -eq $nslices ] ; then
after=`date +%s`
diff=$(($after-$before))
echo "Execution Time: $diff seconds" >> ${subjdir}.bedpostX/times
tim=1
fi
fi
if [ -f ${subjdir}.bedpostX/xfms/eye.mat ] ; then
finished=1
echo "All slices processed"
fi
sleep 15;
done
aftert=`date +%s`
diff=$(($aftert-$beforet))
echo "TOTAL Time: $diff seconds" >> ${subjdir}.bedpostX/times
else
echo
echo
echo "CUDA COMPILER ERROR. Please be sure that you have set correctly CUDA environment variable with your CUDA path"
echo
fi
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment