First commit

COPYRIGHT

+Copyright 2017 University of Oxford, Oxford, UK

LICENSE

+UK_biobank_pipeline
+
+Copyright 2017 University of Oxford, Oxford, UK
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

README.md

+UK_biobank_pipeline
+===================
+
+The `UK_biobank_pipeline` project is a processing pipeline written mainly in Python and bash. It uses [FSL](http://fsl.fmrib.ox.ac.uk/fsl/fslwiki/) as the basic building blocks.
+
+
+Dependencies
+------------
+
+Most of the dependencies of `UK_biobank_pipeline` are listed in the [requirements.txt](requirements.txt) file.
+
+One additional dependency - [gradunwarp](bb_python/python_installation/gradunwarp_FMRIB.tar.gz) - can be installed with a provided [installation script](bb_python/python_installation/install_bb_python.sh) 
+
+
+Documentation
+-------------
+
+`UK_biobank_pipeline` is explained in detail in the paper [Image Processing and Quality Control for the first 10,000 Brain Imaging Datasets from UK Biobank](http://www.biorxiv.org/content/early/2017/04/24/130385).
+

bb_IDP/bb_IDP

+#!/bin/sh
+#
+# Script name: bb_IDP
+#
+# Description: Script to generate the IDPs for a certain subject.
+#
+# Authors: Fidel Alfaro-Almagro, Stephen M. Smith & Mark Jenkinson
+#
+# Copyright 2017 University of Oxford
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -x
+
+origDir=`pwd`
+
+dirScript=`dirname $0`
+
+dirSubject=`pwd`/$1
+
+if [[ "$dirSubject" == "" ]] ; then
+    echo "Error: The selected subject does not exist"
+    exit 0
+fi
+
+mkdir -p $dirSubject/IDP_files
+
+if [[ "$2" == "-remove" ]] ; then
+    rm -f $dirSubject/IDP_files/*
+fi
+
+result="$1"
+
+for elem in `cat $dirScript/list.txt | awk '{print $3}' | uniq` ; do
+    if [ -f $dirSubject/IDP_files/${elem}.txt ] ; then
+        result="$result `cat $dirSubject/IDP_files/${elem}.txt`"
+    else
+        result="$result `$dirScript/scripts/$elem $1`"
+    fi
+done
+
+result=`echo $result | sed 's/  / /g'`
+
+echo $result > $dirSubject/IDP_files/IDPs.txt
+echo $result
+

bb_IDP/bb_IDP.py

+#!/bin/env python
+#
+# Script name: bb_IDP.py
+#
+# Description: Script to run the IDP generation in a queueing system.
+#
+#
+# Authors: Fidel Alfaro-Almagro, Stephen M. Smith & Mark Jenkinson
+#
+# Copyright 2017 University of Oxford
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import bb_pipeline_tools.bb_logging_tool as LT
+import os.path
+
+def bb_IDP(subject, jobHold, fileConfiguration):
+
+    logger = LT.initLogging(__file__, subject)
+    logDir  = logger.logDir
+    baseDir = logDir[0:logDir.rfind('/logs/')]
+
+    jobIDP = LT.runCommand(logger, '${FSLDIR}/bin/fsl_sub -T 30 -N "bb_IDP_' + subject + '" -j ' + jobHold + '  -l ' + logDir + ' $BB_BIN_DIR/bb_IDP/bb_IDP ' + subject)
+    return jobIDP
+        
+

bb_IDP/scripts/bb_IDP_SWI_T2star

+#!/bin/sh
+#
+# Script name: bb_IDP_SWI_T2star
+#
+# Description: Script to generate the IDPs related to swMRI.
+#
+# Authors: Fidel Alfaro-Almagro, Stephen M. Smith & Mark Jenkinson
+#
+# Copyright 2017 University of Oxford
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+origDir=`pwd`
+scriptName=`basename "$0"`
+direc=$1
+
+cd $direc
+
+baseT1="T1"
+#if [ -d T1/unusable ] ; then
+#    baseT1="T1/unusable"
+#fi
+
+baseSWI="SWI"
+#if [ -d $baseSWI/unusable ] ; then
+#    baseSWI="$baseSWI/unusable"
+#fi
+
+#Setting the string of NaN in case there is a problem.
+numVars="14"
+result="";
+for i in $(seq 1 $numVars) ; do 
+    result="NaN $result" ; 
+done 
+
+if [ -f $baseSWI/T2star_to_T1.nii.gz ] ; then
+  if [ -f $baseT1/T1_first/T1_first_all_fast_firstseg.nii.gz ] ; then
+    vals=`fslstats -K $baseT1/T1_first/T1_first_all_fast_firstseg $baseSWI/T2star_to_T1 -p 50 `
+    if [ "`echo $vals | wc -w`" = "58" ] ; then
+        result=`echo $vals | awk '{ print $10 " " $49 " " $11 " " $50 " " $12 " " $51 " " $13 " " $52 " " $17 " " $53 " " $18 " " $54 " " $26 " " $58}'`
+    fi
+  fi
+fi
+
+mkdir -p IDP_files
+
+echo $result > IDP_files/$scriptName.txt
+echo $result
+
+cd $origDir
+

bb_IDP/scripts/bb_IDP_T1_FIRST_vols

+#!/bin/sh
+#
+# Script name: bb_IDP_T1_FIRST_vols
+#
+# Description: Script to generate the IDPs related to subcortical volumes
+#			   calculated with FIRST.
+#
+# Authors: Fidel Alfaro-Almagro, Stephen M. Smith & Mark Jenkinson
+#
+# Copyright 2017 University of Oxford
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+origDir=`pwd`
+scriptName=`basename "$0"`
+direc=$1
+
+cd $direc
+
+baseT1="T1"
+
+#Setting the string of NaN in case there is a problem.
+numVars="15"
+result="";
+for i in $(seq 1 $numVars) ; do 
+    result="NaN $result" ; 
+done 
+
+
+if [ -f $baseT1/T1_first/T1_first_all_fast_firstseg.nii.gz ] ; then
+    result=`${FSLDIR}/bin/fslstats $baseT1/T1_first/T1_first_all_fast_firstseg -H 58 0.5 58.5 | sed 's/\.000000//g' | awk 'BEGIN { ORS = " " } { print }'| awk '{print $10 " " $49 " " $11 " " $50 " " $12 " " $51 " " $13 " " $52 " " $17 " " $53 " " $18 " " $54 " " $26 " " $58 " " $16 }' `
+fi
+
+mkdir -p IDP_files
+
+echo $result > IDP_files/$scriptName.txt
+echo $result
+    
+cd $origDir
+
+

bb_IDP/scripts/bb_IDP_T1_GM_parcellation

+#!/bin/bash 
+#
+# Script name: bb_IDP_T1_GM_parcellation
+#
+# Description: Script to generate the IDPs related to Grey Matter parcellation.
+#
+# Authors: Fidel Alfaro-Almagro, Stephen M. Smith & Mark Jenkinson
+#
+# Copyright 2017 University of Oxford
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+origDir=`pwd`
+scriptName=`basename "$0"`
+direc=$1
+
+
+#Setting the string of NaN in case there is a problem.
+numVars="139"
+result="";
+for i in $(seq 1 $numVars) ; do 
+    result="NaN $result" ; 
+done 
+
+if [ -f $direc/T1/transforms/T1_to_MNI_warp_coef_inv.nii.gz ] ; then
+    if [ -f $direc/T1/T1_fast/T1_brain_pve_1.nii.gz ] ; then
+        ${FSLDIR}/bin/applywarp -i $templ/GMatlas/GMatlas -o $direc/T1/GMatlas_to_T1 -r $direc/T1/T1 -w $direc/T1/transforms/T1_to_MNI_warp_coef_inv.nii.gz --interp=nn
+        result=`${FSLDIR}/bin/fslstats -K $direc/T1/GMatlas_to_T1.nii.gz $direc/T1/T1_fast/T1_brain_pve_1.nii.gz -m -v | xargs -n 3 | awk '{print "("$1"*"$2")"}' | bc `
+    fi
+fi 
+
+mkdir -p $direc/IDP_files
+
+echo $result > $direc/IDP_files/$scriptName.txt
+echo $result
+    
+cd $origDir
+

bb_IDP/scripts/bb_IDP_T1_SIENAX

+#!/bin/sh
+#
+# Script name: bb_IDP_T1_SIENAX
+#
+# Description: Script to generate the IDPs related to SIENAX
+#
+# Authors: Fidel Alfaro-Almagro, Stephen M. Smith & Mark Jenkinson
+#
+# Copyright 2017 University of Oxford
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+origDir=`pwd`
+scriptName=`basename "$0"`
+direc=$1
+
+cd $direc
+
+baseT1="T1"
+#if [ -d T1/unusable ] ; then
+#    baseT1="T1/unusable"
+#fi
+
+#Setting the string of NaN in case there is a problem.
+numVars="11"
+result="";
+for i in $(seq 1 $numVars) ; do 
+    result="NaN $result" ; 
+done 
+
+
+if [ ! -f $baseT1/T1_sienax/T1_sienax.txt ] && [ -f $baseT1/T1_sienax/report.sienax ] ; then
+  echo `cat $baseT1/T1_sienax/report.sienax` | cut -d " " -f2,7,8,12,13,17,18,20,21,23,24 > $baseT1/T1_sienax/T1_sienax.txt
+  result=`cat $baseT1/T1_sienax/T1_sienax.txt` 
+
+elif [ -f $baseT1/T1_sienax/T1_sienax.txt ] ; then
+    result=`cat $baseT1/T1_sienax/T1_sienax.txt` 
+fi
+
+mkdir -p IDP_files
+
+echo $result > IDP_files/$scriptName.txt
+echo $result
+
+cd $origDir
+
+

bb_IDP/scripts/bb_IDP_T1_align_to_std

+#!/bin/sh
+#
+# Script name: bb_IDP_T1_SIENAX
+#
+# Description: Script to generate the IDPs related to the alginment between
+#			   T1 and MNI template.
+#
+# Authors: Fidel Alfaro-Almagro, Stephen M. Smith & Mark Jenkinson
+#
+# Copyright 2017 University of Oxford
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+ST=$FSLDIR/data/standard
+MC=$FSLDIR/etc/flirtsch/measurecost1.sch
+
+origDir=`pwd`
+scriptName=`basename "$0"`
+direc=$1
+
+cd $direc
+
+baseT1="T1"
+#if [ -d T1/unusable ] ; then
+#    baseT1="T1/unusable"
+#fi
+
+result1="NaN"
+result2="NaN"
+result3="NaN"
+
+if [ -f $baseT1/T1_brain.nii.gz ] && [ -f $baseT1/transforms/T1_to_MNI_linear.mat ] ; then
+  result1=`${FSLDIR}/bin/flirt -in $baseT1/T1_brain -ref $ST/MNI152_T1_1mm_brain -refweight $ST/MNI152_T1_1mm_brain_mask -init $baseT1/transforms/T1_to_MNI_linear.mat -schedule $MC | head -1 | cut -f1 -d' '`
+fi
+
+if [ -f $baseT1/T1_brain_to_MNI.nii.gz ] ; then
+  result2=`${FSLDIR}/bin/flirt -in $baseT1/T1_brain_to_MNI -ref $ST/MNI152_T1_1mm_brain -refweight $ST/MNI152_T1_1mm_brain_mask -schedule $MC | head -1 | cut -f1 -d' ' `
+fi
+
+
+if [ -f $baseT1/transforms/T1_to_MNI_warp_jac.nii.gz ] ; then
+  ${FSLDIR}/bin/fslmaths $baseT1/transforms/T1_to_MNI_warp_jac -sub 1 -sqr /tmp/tmpjac_$1
+  result3=`${FSLDIR}/bin/fslstats /tmp/tmpjac_$1 -k $ST/MNI152_T1_1mm_brain_mask -m `
+  ${FSLDIR}/bin/imrm /tmp/tmpjac_$1
+fi
+
+result="$result1 $result2 $result3"
+
+mkdir -p IDP_files
+
+echo $result > IDP_files/$scriptName.txt
+echo $result
+    
+cd $origDir
+
+

bb_IDP/scripts/bb_IDP_T1_noise_ratio

+#!/bin/sh
+#
+# Script name: bb_IDP_T1_noise_ratio
+#
+# Description: Script to generate the IDPs related to noise in T1.
+#
+# Authors: Fidel Alfaro-Almagro, Stephen M. Smith & Mark Jenkinson
+#
+# Copyright 2017 University of Oxford
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+origDir=`pwd`
+scriptName=`basename "$0"`
+direc=$1
+
+cd $direc
+
+baseT1="T1"
+#if [ -d T1/unusable ] ; then
+#    baseT1="T1/unusable"
+#fi
+
+#Setting the string of NaN in case there is a problem.
+numVars="2"
+result="";
+for i in $(seq 1 $numVars) ; do 
+    result="NaN $result" ; 
+done 
+
+if [ -f $baseT1/T1_fast/T1_brain_pveseg.nii.gz ] ; then
+  ${FSLDIR}/bin/fslmaths $baseT1/T1_fast/T1_brain_pveseg -thr 2 -uthr 2 -ero /tmp/tmpsnr_$1
+  TheGrey=`${FSLDIR}/bin/fslstats $baseT1/T1 -k /tmp/tmpsnr_$1 -m`
+  ${FSLDIR}/bin/fslmaths $baseT1/T1_fast/T1_brain_pveseg -thr 3 -uthr 3 -ero /tmp/tmpsnr_$1
+  TheWhite=`${FSLDIR}/bin/fslstats $baseT1/T1 -k /tmp/tmpsnr_$1 -m`
+  ${FSLDIR}/bin/imrm /tmp/tmpsnr_$1
+  TheBrain=`echo "1 k $TheGrey $TheWhite + 2 / p" | dc -`
+  TheContrast=`echo "1 k $TheWhite $TheGrey - p" | dc -`
+  TheThresh=`echo "$TheBrain 10 / p" | dc -`
+  TheNoise=`${FSLDIR}/bin/fslstats $baseT1/T1 -l 0.001 -u $TheThresh -s`
+  TheSNRrecip=`echo "10 k $TheNoise $TheBrain    / p" | dc -`
+  TheCNRrecip=`echo "10 k $TheNoise $TheContrast / p" | dc -`
+  result="$TheSNRrecip $TheCNRrecip" 
+fi
+
+mkdir -p IDP_files
+
+echo $result > IDP_files/$scriptName.txt
+echo $result
+    
+cd $origDir
+
+

bb_IDP/scripts/bb_IDP_T2_FLAIR_WMH

+#!/bin/sh
+#
+# Script name: bb_IDP_T2_FLAIR_WMH
+#
+# Description: Script to generate the IDPs related to White Matter Hyperintens.
+#
+# Authors: Fidel Alfaro-Almagro, Stephen M. Smith & Mark Jenkinson
+#
+# Copyright 2017 University of Oxford
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+origDir=`pwd`
+scriptName=`basename "$0"`
+direc=$1
+
+dirT2=$direc/T2_FLAIR
+
+#Setting the string of NaN in case there is a problem.
+result="NaN"
+
+if [ -f $dirT2/lesions/volume.txt ] ; then
+    result=`cat $dirT2/lesions/volume.txt`
+fi
+
+mkdir -p $direc/IDP_files
+
+echo $result > $direc/IDP_files/$scriptName.txt
+echo $result
+    
+cd $origDir
+
+

bb_IDP/scripts/bb_IDP_all_align_to_T1

+#!/bin/sh
+#
+# Script name: bb_IDP_all_align_to_T1
+#
+# Description: Script to generate the IDPs related to the alignment to T1.
+#
+# Authors: Fidel Alfaro-Almagro, Stephen M. Smith & Mark Jenkinson
+#
+# Copyright 2017 University of Oxford
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+origDir=`pwd`
+scriptName=`basename "$0"`
+direc=$1
+MC="-schedule $FSLDIR/etc/flirtsch/measurecost1.sch"
+
+cd $direc
+
+baseT1="T1"
+#if [ -d $baseT1/unusable ] ; then
+#    baseT1="$baseT1/unusable"
+#fi
+
+baseT2_FLAIR="T2_FLAIR"
+#if [ -d $baseT2_FLAIR/unusable ] ; then
+#    baseT2_FLAIR="$baseT2_FLAIR/unusable"
+#elif [ -d $baseT2_FLAIR/incompatible ] ; then
+#    baseT2_FLAIR="$baseT2_FLAIR/incompatible"
+#fi
+
+baseFieldmap="fieldmap"
+#if [ -d $baseFieldmap/unusable ] ; then
+#    baseFieldmap="$baseFieldmap/unusable"
+#elif [ -d $baseFieldmap/incompatible ] ; then
+#    baseFieldmap="$baseFieldmap/incompatible"
+#fi
+
+baseSWI="SWI"
+#if [ -d $baseSWI/unusable ] ; then
+#    baseSWI="$baseSWI/unusable"
+#fi
+
+basefMRI="fMRI/"
+#if [ -d $basefMRI/unusable ] ; then
+#    basefMRI="$basefMRI/unusable"
+#fi
+
+baserfMRI="$basefMRI/rfMRI.ica/"
+basetfMRI="$basefMRI/tfMRI.feat/"
+
+
+for i in $baseT2_FLAIR/T2_FLAIR_brain $baseFieldmap/fieldmap_iout_to_T1 $baseSWI/SWI_TOTAL_MAG_to_T1 $baserfMRI/reg/example_func2highres $basetfMRI/reg/example_func2highres ; do
+  if [ -f ${i}.nii.gz ] ; then
+    result="$result `flirt -ref $baseT1/T1_brain -in $i $MC -refweight $baseT1/T1_brain_mask | head -1 | cut -f1 -d' ' `"
+  else
+    result="$result NaN"
+  fi
+done
+
+
+mkdir -p IDP_files
+
+echo $result > IDP_files/$scriptName.txt
+echo $result