Advances in IDP generation

bip/pipelines/IDPs_gen/IDP_T1_FIRST_vols.py

+#!/usr/bin/env python
+#
+# IDP_T1_FIRST_vols.py - Generating IDP file with measure of FIRST volumes of T1
+#
+# Author: Fidel Alfaro Almagro <fidel.alfaroalmagro@ndcn.ox.ac.uk>
+# Author: Paul McCarthy <pauldmccarthy@gmail.com>
+# Author: Michiel Cottaar <michiel.cottaar@ndcn.ox.ac.uk>
+#
+# pylint: disable=C0103,E0602,C0114,C0115,C0116,R0913,R0914,R0915
+#
+
+import os
+import json
+import logging
+from fsl import wrappers
+from pipe_tree import In, Out, Ref
+from bip.utils.log_utils import redirect_logging
+
+log = logging.getLogger(__name__)
+
+def run(ctx,
+        T1_first_all_fast_firstseg: In(optional=True),
+        logs_dir:                   Ref,
+        IDP_T1_FIRST_vols:          Out):
+    
+    with redirect_logging('IDP_T1_FIRST_vols', outdir=logs_dir):
+        result = ("NaN " * 15).strip()
+
+        if os.path.exists(T1_first_all_fast_firstseg):
+            v=wrappers.fslstats(T1_first_all_fast_firstseg).H(58,0.5,58.5).run()
+            # indices that we are interested in
+            ind = [9, 48, 10, 49, 11, 50, 12, 51, 16, 52, 17, 53, 25, 57, 15]
+            result = [str(int(v[x])) for x in ind]
+            result = " ".join(result)
+
+
+        with open(IDP_T1_FIRST_vols, 'wt', encoding="utf-8") as f:
+            f.write(f'{result}\n')

bip/pipelines/IDPs_gen/IDP_T1_SIENAX.py

+#!/usr/bin/env python
+#
+# IDP_T1_SIENAX.py - Generating IDP file with SIENAX metrics of T1.
+#
+# Author: Fidel Alfaro Almagro <fidel.alfaroalmagro@ndcn.ox.ac.uk>
+# Author: Paul McCarthy <pauldmccarthy@gmail.com>
+# Author: Michiel Cottaar <michiel.cottaar@ndcn.ox.ac.uk>
+#
+# pylint: disable=C0103,E0602,C0114,C0115,C0116,R0913,R0914,R0915
+#
+
+import os
+import json
+import logging
+from pipe_tree import In, Out, Ref
+from bip.utils.log_utils import redirect_logging
+
+log = logging.getLogger(__name__)
+
+def run(ctx,
+        T1_sienax_report: In,
+        T1_sienax_txt:    Ref,
+        logs_dir:         Ref,
+        IDP_T1_SIENAX:    Out):
+    
+    with redirect_logging('IDP_T1_SIENAX', outdir=logs_dir):
+
+        result = ("NaN " * 11).strip()
+
+        if os.path.exists(T1_sienax_report):
+            with open(T1_sienax_report, "r",  encoding="utf-8") as f:
+                text = f.readlines()
+                # indices that we are interested in
+                result  = text[0].split()[1] + " "
+                result += text[2].split()[1] + " " + text[2].split()[2] + " "
+                result += text[3].split()[1] + " " + text[3].split()[2] + " "
+                result += text[4].split()[1] + " " + text[4].split()[2] + " "
+                result += text[5].split()[1] + " " + text[5].split()[2] + " "
+                result += text[6].split()[1] + " " + text[6].split()[2]
+
+        with open(IDP_T1_SIENAX, 'wt', encoding="utf-8") as f:
+            f.write(f'{result}\n')
+
+        copyfile(src=IDP_T1_SIENAX, dst=T1_sienax_txt)

bip/pipelines/IDPs_gen/IDP_T2_FLAIR_WMH.py

+#!/usr/bin/env python
+#
+# IDP_T2_FLAIR_WMH.py - Generating IDP file with WMH segmentation metrics
+#
+# Author: Fidel Alfaro Almagro <fidel.alfaroalmagro@ndcn.ox.ac.uk>
+# Author: Paul McCarthy <pauldmccarthy@gmail.com>
+# Author: Michiel Cottaar <michiel.cottaar@ndcn.ox.ac.uk>
+#
+# pylint: disable=C0103,E0602,C0114,C0115,C0116,R0913,R0914,R0915
+#
+
+import os
+import json
+import logging
+from shutil import copyfile
+from pipe_tree import In, Out, Ref
+from bip.utils.log_utils import redirect_logging
+
+log = logging.getLogger(__name__)
+
+def run(ctx,
+        T2_FLAIR_bianca_volume: In,
+        logs_dir:               Ref,
+        IDP_T2_FLAIR_WMH:       Out):
+    
+    with redirect_logging('IDP_T2_FLAIR_WMH', outdir=logs_dir):
+        copyfile(src=T2_FLAIR_bianca_volume, dst=IDP_T2_FLAIR_WMH)

bip/pipelines/IDPs_gen/IDP_diff_TBSS.py

+#!/usr/bin/env python
+#
+# IDP_diff_TBSS.py - Generating IDP file with TBSS metrics 
+#
+# Author: Fidel Alfaro Almagro <fidel.alfaroalmagro@ndcn.ox.ac.uk>
+# Author: Paul McCarthy <pauldmccarthy@gmail.com>
+# Author: Michiel Cottaar <michiel.cottaar@ndcn.ox.ac.uk>
+#
+# pylint: disable=C0103,E0602,C0114,C0115,C0116,R0913,R0914,R0915
+#
+
+import os
+import json
+import logging
+from pipe_tree import In, Out, Ref
+from bip.utils.log_utils import redirect_logging
+
+log = logging.getLogger(__name__)
+
+def run(ctx,
+        JHUrois_FA:     In(optional=True),
+        logs_dir:       Ref,
+        JHUrois_prefix: Ref,
+        IDP_diff_TBSS:  Out):
+    
+    with redirect_logging('IDP_diff_TBSS', outdir=logs_dir):
+
+        nan_result = ("NaN " * 48).strip()
+
+        result = ""
+
+        for mod in ["FA", "MD", "MO", "L1", "L2", "L3", "ICVF", "OD", "ISOVF"]:
+
+            file_name = JHUrois_prefix + mod + ".txt"
+            print(file_name)
+            if os.path.exists(file_name):
+                with open(file_name, "r") as f:
+                    mini_result = f.read()
+                    mini_result = mini_result.replace("\n", " ")
+                    mini_result = mini_result.replace("  ", " ")
+                    mini_result = mini_result.replace("   ", " ")
+                    mini_result = mini_result.replace("[", "")
+                    mini_result = mini_result.replace("]", "").strip()
+            else:
+                mini_result = nan_result
+
+            result += mini_result + " "
+
+        result = result.strip()
+
+        with open(IDP_diff_TBSS, 'wt', encoding="utf-8") as f:
+            f.write(f'{result}\n')

bip/pipelines/IDPs_gen/IDP_diff_eddy_outliers.py

+#!/usr/bin/env python
+#
+# IDP_diff_eddy_outliers.py - Generating IDP file with eddy outliers metrics 
+#
+# Author: Fidel Alfaro Almagro <fidel.alfaroalmagro@ndcn.ox.ac.uk>
+# Author: Paul McCarthy <pauldmccarthy@gmail.com>
+# Author: Michiel Cottaar <michiel.cottaar@ndcn.ox.ac.uk>
+#
+# pylint: disable=C0103,E0602,C0114,C0115,C0116,R0913,R0914,R0915
+#
+
+import os
+import json
+import logging
+from pipe_tree import In, Out, Ref
+from bip.utils.log_utils import redirect_logging
+
+log = logging.getLogger(__name__)
+
+def run(ctx,
+        eddy_outlier_report:    In(optional=True),
+        logs_dir:               Ref,
+        IDP_diff_eddy_outliers: Out):
+    
+    with redirect_logging('IDP_diff_eddy_outliers', outdir=logs_dir):
+
+        num_outliers = 0
+
+        if os.path.exists(eddy_outlier_report):
+            with open(eddy_outlier_report, "r", encoding="utf-8") as f:
+                num_outliers = str(len(f.readlines()))
+        else:
+            num_outliers = "NaN"
+
+        with open(IDP_diff_eddy_outliers, 'wt', encoding="utf-8") as f:
+            f.write(f'{num_outliers}\n')

bip/pipelines/IDPs_gen/IDP_func_TSNR.py

+#!/usr/bin/env python
+#
+# IDP_func_TSNR.py - Generating IDP file with measure of TSNR of fMRI
+#
+# Author: Fidel Alfaro Almagro <fidel.alfaroalmagro@ndcn.ox.ac.uk>
+# Author: Paul McCarthy <pauldmccarthy@gmail.com>
+# Author: Michiel Cottaar <michiel.cottaar@ndcn.ox.ac.uk>
+#
+# pylint: disable=C0103,E0602,C0114,C0115,C0116,R0913,R0914,R0915
+#
+
+import os
+import json
+import logging
+from fsl import wrappers
+from pipe_tree import In, Out, Ref
+from bip.utils.log_utils import redirect_logging, tempdir
+
+log = logging.getLogger(__name__)
+
+def run(ctx,
+        filtered_func_data:       In(optional=True),
+        filtered_func_data_clean: In(optional=True),
+        tfMRI_filtered_func_data: In(optional=True),
+        logs_dir:                 Ref,
+        tmp_dir:                  Ref,
+        IDP_func_TSNR:            Out):
+    
+    with redirect_logging('IDP_func_TSNR', outdir=logs_dir),\
+        tempdir(tmp_dir):
+
+        fMRI_SNR = tmp_dir + '/fMRI_SNR.nii.gz'
+
+        result =""
+
+        for file_name in [filtered_func_data, filtered_func_data_clean,
+                          tfMRI_filtered_func_data]:
+            if os.path.exists(file_name):
+                wrappers.fslmaths(file_name).Tstd().run(fMRI_SNR)
+                wrappers.fslmaths(file_name).Tmean().div(fMRI_SNR).run(fMRI_SNR)
+                v=wrappers.fslstats(fMRI_SNR).l(0.1).p(50).run()
+                print(v)
+                v = 1/v
+                print(v)
+                result += f"{v} "
+            else:
+                result += "NaN "
+
+        result = result.strip()
+
+        with open(IDP_func_TSNR, 'wt', encoding="utf-8") as f:
+            f.write(f'{result}\n')

bip/pipelines/IDPs_gen/IDP_func_head_motion.py

+#!/usr/bin/env python
+#
+# IDP_func_head_motion.py - Generating IDP file with measure of fMRI head motion
+#
+# Author: Fidel Alfaro Almagro <fidel.alfaroalmagro@ndcn.ox.ac.uk>
+# Author: Paul McCarthy <pauldmccarthy@gmail.com>
+# Author: Michiel Cottaar <michiel.cottaar@ndcn.ox.ac.uk>
+#
+# pylint: disable=C0103,E0602,C0114,C0115,C0116,R0913,R0914,R0915
+#
+
+import os
+import json
+import logging
+from pipe_tree import In, Out, Ref
+from bip.utils.log_utils import redirect_logging
+
+log = logging.getLogger(__name__)
+
+def run(ctx,
+        rfMRI_mc_rel_mean:    In(optional=True),
+        tfMRI_mc_rel_mean:    In(optional=True),
+        logs_dir:             Ref,
+        IDP_func_head_motion: Out):
+    
+    with redirect_logging('IDP_func_head_motion', outdir=logs_dir):
+        result=""
+
+        if os.path.exists(rfMRI_mc_rel_mean):
+            with open(rfMRI_mc_rel_mean, "r", encoding="utf-8") as f:
+                val_1 = json.load(f)
+        else:
+            val_1 = "NaN"
+
+        if os.path.exists(tfMRI_mc_rel_mean):
+            with open(tfMRI_mc_rel_mean, "r", encoding="utf-8") as f:
+                val_2 = json.load(f)
+        else:
+            val_2 = "NaN"
+
+        with open(IDP_func_head_motion, 'wt', encoding="utf-8") as f:
+            f.write(f'{val_1} {val_2}\n')

bip/pipelines/IDPs_gen/IDP_subject_COG_table.py

+#!/usr/bin/env python
+#
+# IDP_subject_centre.py - Generating IDP file with the acquisition centre.
+#
+# Author: Fidel Alfaro Almagro <fidel.alfaroalmagro@ndcn.ox.ac.uk>
+# Author: Paul McCarthy <pauldmccarthy@gmail.com>
+# Author: Michiel Cottaar <michiel.cottaar@ndcn.ox.ac.uk>
+#
+# pylint: disable=C0103,E0602,C0114,C0115,C0116,R0913,R0914,R0915
+#
+
+import os
+import json
+import logging
+from pipe_tree import In, Out, Ref
+from bip.utils.log_utils import redirect_logging
+
+log = logging.getLogger(__name__)
+
+def run(ctx,
+        T1_orig:            In,
+        T1_QC_COG:          In,
+        T1_dcm_txt:         In,
+        dMRI_dcm_txt:       Ref,
+        rfMRI_dcm_txt:      Ref,
+        SWI_dcm_txt:        Ref,
+        T2_dcm_txt:         Ref,
+        tfMRI_dcm_txt:      Ref,
+        logs_dir:           Ref,
+        IDP_subject_COG_table: Out):
+
+    with redirect_logging('IDP_subject_COG_table', outdir=logs_dir):
+
+        result = ""
+        with open(T1_QC_COG, "r", encoding="utf-8") as f:
+            COG  = f.read().strip()
+
+        table = "NaN"
+        table_text = "(0019, 1012) [TablePositionOrigin]"
+
+        for file_name in [T1_dcm_txt, T2_dcm_txt, rfMRI_dcm_txt, SWI_dcm_txt, 
+                          dMRI_dcm_txt, tfMRI_dcm_txt]:
+
+            if os.path.exists(file_name):
+                with open(file_name, 'rt', encoding="utf-8") as f:
+                    cont = f.readlines()
+                    for x in cont:
+                        if x.startswith(table_text):
+                            table = x.replace(table_text, "")
+                            table = table.strip().replace("SL: ", "").strip()
+                            table = table.split()[2].replace("]","")
+                            break
+            if table != "NaN":
+                break
+
+        with open(IDP_subject_COG_table, 'wt', encoding="utf-8") as f:
+            f.write(f'{COG} {table}\n')

bip/pipelines/IDPs_gen/IDP_subject_centre.py

@@ -27,31 +27,36 @@ def run(ctx,
         tfMRI_dcm_txt:      Ref,
         logs_dir:           Ref,
         IDP_subject_centre: Out):
+    
+    with redirect_logging('IDP_subject_centre', outdir=logs_dir):
 
-    line = ""
+        line = ""
+        address_text = "(0008, 0081) Institution Address"
 
-    for file_name in [T1_dcm_txt, T2_dcm_txt, rfMRI_dcm_txt, SWI_dcm_txt, 
-                      dMRI_dcm_txt, tfMRI_dcm_txt]:
+        for file_name in [T1_dcm_txt, T2_dcm_txt, rfMRI_dcm_txt, SWI_dcm_txt, 
+                          dMRI_dcm_txt, tfMRI_dcm_txt]:
 
-        if os.path.exists(file_name):
-            with open(file_name, 'rt', encoding="utf-8") as f:
-                cont = f.readlines()
-                for x in cont:
-                    if x.startswith("(0008, 0081) Institution Address"):
-                        line = x.replace("(0008, 0081) Institution Address", "")
-                        line = line.strip().replace("ST: ", "").strip()
-                        line = line.replace("'","").replace(",","__").replace(" ","_")
-                        break
-        if line != "":
-            break
+            if os.path.exists(file_name):
+                with open(file_name, 'rt', encoding="utf-8") as f:
+                    cont = f.readlines()
 
-    if line != "":
-        # TODO: Allow users to change the file with the sites
-        with open(ctx.get_data("IDPs/sites_UKBB.json"), "r", encoding="utf-8") as f:
-            sites = json.load(f)
+                    for x in cont:
+                        if x.startswith(address_text):
+                            line = x.replace(address_text, "")
+                            line = line.strip().replace("ST: ", "").strip()
+                            line = line.replace("'","").replace(",","__")
+                            line = line.replace(" ","_")
+                            break
+            if line != "":
+                break
 
-    centre = sites[line]
+        if line != "":
+            # TODO: Allow users to change the file with the sites
+            json_name = "IDPs/sites_UKBB.json"
+            with open(ctx.get_data(json_name), "r", encoding="utf-8") as f:
+                sites = json.load(f)
+
+        centre = sites[line]
 
-    with redirect_logging('IDP_subject_centre', outdir=logs_dir):
         with open(IDP_subject_centre, 'wt', encoding="utf-8") as f:
             f.write(f'{centre}\n')

bip/pipelines/IDPs_gen/IDPs_gen.py

@@ -10,9 +10,13 @@
 #
 
 import logging
-from bip.utils.log_utils     import redirect_logging
-from bip.pipelines.IDPs_gen import IDP_subject_ID
-from bip.pipelines.IDPs_gen import IDP_subject_centre
+from bip.utils.log_utils    import redirect_logging
+from bip.pipelines.IDPs_gen import IDP_subject_ID, IDP_subject_centre
+from bip.pipelines.IDPs_gen import IDP_subject_COG_table
+from bip.pipelines.IDPs_gen import IDP_T1_FIRST_vols, IDP_T1_SIENAX
+from bip.pipelines.IDPs_gen import IDP_T2_FLAIR_WMH
+from bip.pipelines.IDPs_gen import IDP_func_head_motion, IDP_func_TSNR
+from bip.pipelines.IDPs_gen import IDP_diff_eddy_outliers, IDP_diff_TBSS
 
 log = logging.getLogger(__name__)
 
@@ -24,9 +28,50 @@ def add_to_pipeline(ctx, pipe, tree, targets):
 
     with redirect_logging('pipe_IDPs_gen', outdir=logs_dir):
         pipe(IDP_subject_ID.run,
-             submit=dict(jobtime=200, name="BIP_IDPs_subject_ID_" + subj),
+             submit=dict(jobtime=200, name="BIP_IDP_subject_ID_" + subj),
              kwargs={'ctx' : ctx})
         targets.append('IDP_subject_ID')
+
+        pipe(IDP_func_head_motion.run,
+             submit=dict(jobtime=200, name="BIP_IDP_func_head_motion_" + subj),
+             kwargs={'ctx' : ctx})
+        targets.append('IDP_func_head_motion')
+
+        pipe(IDP_func_TSNR.run,
+             submit=dict(jobtime=200, name="BIP_IDP_func_TSNR_" + subj),
+             kwargs={'ctx' : ctx})
+        targets.append('IDP_func_TSNR')
+
+        pipe(IDP_diff_eddy_outliers.run,
+             submit=dict(jobtime=200, name="BIP_IDP_diff_eddy_outliers_"+ subj),
+             kwargs={'ctx' : ctx})
+        targets.append('IDP_diff_eddy_outliers')
+
+        pipe(IDP_T1_SIENAX.run,
+             submit=dict(jobtime=200, name="BIP_IDP_T1_SIENAX_" + subj),
+             kwargs={'ctx' : ctx})
+        targets.append('IDP_T1_SIENAX')
+
+        pipe(IDP_T1_FIRST_vols.run,
+             submit=dict(jobtime=200, name="BIP_IDP_T1_FIRST_vols_" + subj),
+             kwargs={'ctx' : ctx})
+        targets.append('IDP_T1_FIRST_vols')
+
+        pipe(IDP_T2_FLAIR_WMH.run,
+             submit=dict(jobtime=200, name="BIP_IDP_T2_FLAIR_WMH_" + subj),
+             kwargs={'ctx' : ctx})
+        targets.append('IDP_T2_FLAIR_WMH')
+
+        pipe(IDP_diff_TBSS.run,
+             submit=dict(jobtime=200, name="BIP_IDP_diff_TBSS_"+ subj),
+             kwargs={'ctx' : ctx})
+        targets.append('IDP_diff_TBSS')
+
+        pipe(IDP_subject_COG_table.run,
+             submit=dict(jobtime=200, name="BIP_IDP_subject_COG_table_" + subj),
+             kwargs={'ctx' : ctx})
+        targets.append('IDP_subject_COG_table')
+
         pipe(IDP_subject_centre.run,
              submit=dict(jobtime=200, name="BIP_IDP_subject_centre_" + subj),
              kwargs={'ctx' : ctx})

bip/pipelines/dMRI_diff/diff_eddy.py

@@ -39,7 +39,8 @@ def run(ctx,
         eddy_bvals:                     Out,
         eddy_bvecs:                     Out,
         eddy_index:                     Out,
-        eddy_data:                      Out):
+        eddy_data:                      Out,
+        eddy_outlier_report:            Out):
 
     with redirect_logging('diff_eddy', outdir=logs_dir):
 

bip/pipelines/dMRI_diff/diff_tbss.py

@@ -131,9 +131,11 @@ def run(ctx,
         wrappers.fslmaths(TBSS_mean_FA_skeleton).thr(thresh).bin().run(TBSS_mean_FA_skeleton_mask)
         wrappers.fslmaths(TBSS_all_FA).mas(TBSS_mean_FA_skeleton_mask).run(TBSS_all_FA_skeletonised)
 
-        atlas = ctx.get_atlas('JHU/JHU-ICBM-labels-1mm.nii.gz')
+        atlas = ctx.get_data('/dMRI/TBSS/JHU-ICBM-labels-1mm.nii.gz')
         mean = wrappers.fslstats(TBSS_all_FA_skeletonised, K=atlas).M.run()
 
+        # TODO: Funnily enough, this atlas has changed over time, so for v2
+        #       of the pipeline, we will need to change the IDPs
         with open(JHUrois_FA, 'wt', encoding="utf-8") as f:
             f.write(f'{mean}')
 

bip/pipelines/fMRI_rest/rfMRI_melodic.py

@@ -29,7 +29,8 @@ def run(ctx,
         rfMRI_standard:                   Out,
         filtered_func_data:               Out,
         melodic_mix:                      Out,
-        rfMRI_mc_parameters:              Out):
+        rfMRI_mc_parameters:              Out,
+        rfMRI_mc_rel_mean:                Out):
 
     with redirect_logging('rfMRI_melodic', outdir=logs_dir):
         wrappers.feat(fsf=rfMRI_fsf)

bip/pipelines/fMRI_task/tfMRI_feat.py

@@ -18,9 +18,10 @@ from pipe_tree import In, Out, Ref
 log = logging.getLogger(__name__)
 
 def run(ctx,
-        tfMRI_fsf:                               In,
-        logs_dir:                                Ref,
-        tfMRI_feat:                              Out):
+        tfMRI_fsf:                In,
+        logs_dir:                 Ref,
+        tfMRI_feat:               Out,
+        tfMRI_filtered_func_data: Out):
 
     with redirect_logging('tfMRI_feat', outdir=logs_dir):
         wrappers.feat(fsf=tfMRI_fsf)