General refactoring

bip/main.py

@@ -16,17 +16,11 @@ from dataclasses import dataclass, field
 from file_tree import FileTree
 from pipe_tree import Pipeline, Ref
 import bip
-from bip.utils.log_utils           import setup_logging
-from bip.pipelines.struct_T1       import struct_T1
-from bip.pipelines.struct_T2_FLAIR import struct_T2_FLAIR
-from bip.pipelines.struct_FS       import struct_FS
-from bip.pipelines.struct_swMRI    import struct_swMRI
-from bip.pipelines.struct_asl      import struct_asl
-from bip.pipelines.dMRI_fieldmap   import dMRI_fieldmap
-from bip.pipelines.fMRI_task       import fMRI_task
-from bip.pipelines.fMRI_rest       import fMRI_rest
-from bip.pipelines.dMRI_diff       import dMRI_diff
-from bip.pipelines.IDPs_gen        import IDPs_gen
+from bip.utils.log_utils import setup_logging
+from bip.pipelines import struct_T1, struct_T2_FLAIR, struct_FS
+from bip.pipelines import struct_swMRI, struct_asl, IDPs_gen
+from bip.pipelines import dMRI_fieldmap, dMRI_diff, fMRI_task, fMRI_rest
+
 
 log = logging.getLogger('bip.main')
 
@@ -43,6 +37,7 @@ class Usage(Exception):
 
 @dataclass
 class Context:
+    subject:    str = ""
     BB_BIN_DIR: str = bip.__path__[0]
     FSLDIR:     str = os.environ['FSLDIR']
     gdc:        str = field(init=False)
@@ -332,19 +327,19 @@ def main():
     #pipe = Pipeline(default_submit=dict(logdir=Ref("logs_dir")))
     #ctx.save_context(tree.get('config_file'))
 
-    # This list will be filled with all desired outputs
+    # This list will be filled with all desired outputs ... Later
     targets = []
 
-    pipe, targets = struct_T1.add_to_pipeline(ctx, pipe, tree, targets)
-    pipe, targets = struct_T2_FLAIR.add_to_pipeline(ctx, pipe, tree, targets)
-    pipe, targets = struct_FS.add_to_pipeline(ctx, pipe, tree, targets)
-    pipe, targets = struct_swMRI.add_to_pipeline(ctx, pipe, tree, targets)
-    pipe, targets = struct_asl.add_to_pipeline(ctx, pipe, tree, targets)
-    pipe, targets = dMRI_fieldmap.add_to_pipeline(ctx, pipe, tree, targets)
-    pipe, targets = fMRI_task.add_to_pipeline(ctx, pipe, tree, targets)
-    pipe, targets = fMRI_rest.add_to_pipeline(ctx, pipe, tree, targets)
-    pipe, targets = dMRI_diff.add_to_pipeline(ctx, pipe, tree, targets)
-    pipe, targets = IDPs_gen.add_to_pipeline(ctx, pipe, tree, targets)
+    struct_T1.add_to_pipeline      (ctx, pipe, tree)
+    struct_T2_FLAIR.add_to_pipeline(ctx, pipe, tree)
+    struct_FS.add_to_pipeline      (ctx, pipe, tree)
+    struct_swMRI.add_to_pipeline   (ctx, pipe, tree)
+    struct_asl.add_to_pipeline     (ctx, pipe, tree)
+    dMRI_fieldmap.add_to_pipeline  (ctx, pipe, tree)
+    fMRI_task.add_to_pipeline      (ctx, pipe, tree)
+    fMRI_rest.add_to_pipeline      (ctx, pipe, tree)
+    dMRI_diff.add_to_pipeline      (ctx, pipe, tree)
+    IDPs_gen.add_to_pipeline       (ctx, pipe, tree)
 
     # The skip-missing flag deals with cases where the subject is missing
     # the data of some modalities. For more details, check:

bip/pipelines/IDPs_gen/IDP_SWI_T2star.py

@@ -14,7 +14,7 @@ import os
 import logging
 from fsl import wrappers
 from pipe_tree import In, Out, Ref
-from bip.utils.log_utils import redirect_logging
+from bip.utils.log_utils import redirect_logging, job_name
 
 log = logging.getLogger(__name__)
 
@@ -24,7 +24,7 @@ def run(ctx,
         logs_dir:                   Ref,
         IDP_SWI_T2star:             Out):
 
-    with redirect_logging('IDP_SWI_T2star', outdir=logs_dir):
+    with redirect_logging(job_name(run), outdir=logs_dir):
 
         result = ("NaN " * 14).strip()
 

bip/pipelines/IDPs_gen/IDP_T1_FIRST_vols.py

@@ -14,7 +14,7 @@ import os
 import logging
 from fsl import wrappers
 from pipe_tree import In, Out, Ref
-from bip.utils.log_utils import redirect_logging
+from bip.utils.log_utils import redirect_logging, job_name
 
 log = logging.getLogger(__name__)
 
@@ -23,7 +23,7 @@ def run(ctx,
         logs_dir:                   Ref,
         IDP_T1_FIRST_vols:          Out):
 
-    with redirect_logging('IDP_T1_FIRST_vols', outdir=logs_dir):
+    with redirect_logging(job_name(run), outdir=logs_dir):
         result = ("NaN " * 15).strip()
 
         if os.path.exists(T1_first_all_fast_firstseg):

bip/pipelines/IDPs_gen/IDP_T1_GM_parcellation.py

@@ -12,7 +12,7 @@
 import logging
 from fsl import wrappers
 from pipe_tree import In, Out, Ref
-from bip.utils.log_utils import redirect_logging, tempdir
+from bip.utils.log_utils import redirect_logging, tempdir, job_name
 
 log = logging.getLogger(__name__)
 
@@ -24,7 +24,7 @@ def run(ctx,
         logs_dir:                Ref,
         IDP_T1_GM_parcellation:  Out):
 
-    with redirect_logging('IDP_T1_GM_parcellation', outdir=logs_dir),\
+    with redirect_logging(job_name(run), outdir=logs_dir),\
         tempdir(tmp_dir):
         result = ("NaN " * 139).strip()
 

bip/pipelines/IDPs_gen/IDP_T1_SIENAX.py

@@ -14,7 +14,7 @@ import os
 import logging
 from shutil import copyfile
 from pipe_tree import In, Out, Ref
-from bip.utils.log_utils import redirect_logging
+from bip.utils.log_utils import redirect_logging, job_name
 
 log = logging.getLogger(__name__)
 
@@ -24,7 +24,7 @@ def run(ctx,
         logs_dir:         Ref,
         IDP_T1_SIENAX:    Out):
 
-    with redirect_logging('IDP_T1_SIENAX', outdir=logs_dir):
+    with redirect_logging(job_name(run), outdir=logs_dir):
 
         result = ("NaN " * 11).strip()
 

bip/pipelines/IDPs_gen/IDP_T1_align_to_std.py

@@ -12,7 +12,7 @@
 import logging
 from fsl import wrappers
 from pipe_tree import In, Out, Ref
-from bip.utils.log_utils import redirect_logging, tempdir
+from bip.utils.log_utils import redirect_logging, tempdir, job_name
 
 log = logging.getLogger(__name__)
 
@@ -25,7 +25,7 @@ def run(ctx,
         logs_dir:             Ref,
         IDP_T1_align_to_std:  Out):
 
-    with redirect_logging('IDP_T1_align_to_std', outdir=logs_dir),\
+    with redirect_logging(job_name(run), outdir=logs_dir),\
         tempdir(tmp_dir):
 
         tmp_jac = tmp_dir + '/tmpjac.nii.gz'

bip/pipelines/IDPs_gen/IDP_T1_noise_ratio.py

@@ -13,7 +13,7 @@
 import logging
 from fsl import wrappers
 from pipe_tree import In, Out, Ref
-from bip.utils.log_utils import redirect_logging, tempdir
+from bip.utils.log_utils import redirect_logging, tempdir, job_name
 
 log = logging.getLogger(__name__)
 
@@ -24,7 +24,7 @@ def run(ctx,
         logs_dir:           Ref,
         IDP_T1_noise_ratio: Out):
 
-    with redirect_logging('IDP_T1_noise_ratio', outdir=logs_dir),\
+    with redirect_logging(job_name(run), outdir=logs_dir),\
         tempdir(tmp_dir):
 
         tmp_SNR = tmp_dir + '/tmp_SNR.nii.gz'

bip/pipelines/IDPs_gen/IDP_T2_FLAIR_WMH.py

@@ -13,7 +13,7 @@
 import logging
 from shutil import copyfile
 from pipe_tree import In, Out, Ref
-from bip.utils.log_utils import redirect_logging
+from bip.utils.log_utils import redirect_logging, job_name
 
 log = logging.getLogger(__name__)
 
@@ -22,5 +22,5 @@ def run(ctx,
         logs_dir:               Ref,
         IDP_T2_FLAIR_WMH:       Out):
 
-    with redirect_logging('IDP_T2_FLAIR_WMH', outdir=logs_dir):
+    with redirect_logging(job_name(run), outdir=logs_dir):
         copyfile(src=T2_FLAIR_bianca_volume, dst=IDP_T2_FLAIR_WMH)

bip/pipelines/IDPs_gen/IDP_all_align_to_T1.py

@@ -13,7 +13,7 @@ import os
 import logging
 from fsl import wrappers
 from pipe_tree import In, Out, Ref
-from bip.utils.log_utils import redirect_logging, tempdir
+from bip.utils.log_utils import redirect_logging, tempdir, job_name
 
 log = logging.getLogger(__name__)
 
@@ -29,7 +29,7 @@ def run(ctx,
         logs_dir:                   Ref,
         IDP_all_align_to_T1:        Out):
 
-    with redirect_logging('IDP_all_align_to_T1', outdir=logs_dir),\
+    with redirect_logging(job_name(run), outdir=logs_dir),\
         tempdir(tmp_dir):
 
         tmp_mat = tmp_dir + '/tmp_mat.mat'

bip/pipelines/IDPs_gen/IDP_diff_TBSS.py

@@ -13,7 +13,7 @@
 import os
 import logging
 from pipe_tree import In, Out, Ref
-from bip.utils.log_utils import redirect_logging
+from bip.utils.log_utils import redirect_logging, job_name
 
 log = logging.getLogger(__name__)
 
@@ -23,7 +23,7 @@ def run(ctx,
         JHUrois_prefix: Ref,
         IDP_diff_TBSS:  Out):
 
-    with redirect_logging('IDP_diff_TBSS', outdir=logs_dir):
+    with redirect_logging(job_name(run), outdir=logs_dir):
 
         nan_result = ("NaN " * 48).strip()
 

bip/pipelines/IDPs_gen/IDP_diff_autoPtx.py

@@ -15,7 +15,7 @@ import logging
 import nibabel as nib
 from fsl import wrappers
 from pipe_tree import In, Out, Ref, Var
-from bip.utils.log_utils import redirect_logging, tempdir
+from bip.utils.log_utils import redirect_logging, tempdir, job_name
 
 log = logging.getLogger(__name__)
 
@@ -28,8 +28,8 @@ def run(ctx,
         autoptx_tract:      Var(no_iter=True),
         IDP_diff_autoPtx:   Out):
 
-    with redirect_logging('IDP_diff_autoPtx', outdir=logs_dir),\
-        tempdir(tmp_dir):
+    with redirect_logging(job_name(run), outdir=logs_dir),\
+        tempdir(tmp_dir + "/" + __name__):
 
         autoPtx_all = tmp_dir + '/autoPtx_all.nii.gz'
         autoPtx_tmp = tmp_dir + '/autoPtx_tmp.nii.gz'

bip/pipelines/IDPs_gen/IDP_diff_eddy_outliers.py

@@ -13,7 +13,7 @@
 import os
 import logging
 from pipe_tree import In, Out, Ref
-from bip.utils.log_utils import redirect_logging
+from bip.utils.log_utils import redirect_logging, job_name
 
 log = logging.getLogger(__name__)
 
@@ -22,7 +22,7 @@ def run(ctx,
         logs_dir:               Ref,
         IDP_diff_eddy_outliers: Out):
 
-    with redirect_logging('IDP_diff_eddy_outliers', outdir=logs_dir):
+    with redirect_logging(job_name(run), outdir=logs_dir):
 
         num_outliers = 0
 

bip/pipelines/IDPs_gen/IDP_func_TSNR.py

@@ -14,7 +14,7 @@ import os
 import logging
 from fsl import wrappers
 from pipe_tree import In, Out, Ref
-from bip.utils.log_utils import redirect_logging, tempdir
+from bip.utils.log_utils import redirect_logging, tempdir, job_name
 
 log = logging.getLogger(__name__)
 
@@ -26,7 +26,7 @@ def run(ctx,
         tmp_dir:                  Ref,
         IDP_func_TSNR:            Out):
 
-    with redirect_logging('IDP_func_TSNR', outdir=logs_dir),\
+    with redirect_logging(job_name(run), outdir=logs_dir),\
         tempdir(tmp_dir):
 
         fMRI_SNR = tmp_dir + '/fMRI_SNR.nii.gz'

bip/pipelines/IDPs_gen/IDP_func_head_motion.py

@@ -14,7 +14,7 @@ import os
 import json
 import logging
 from pipe_tree import In, Out, Ref
-from bip.utils.log_utils import redirect_logging
+from bip.utils.log_utils import redirect_logging, job_name
 
 log = logging.getLogger(__name__)
 
@@ -24,7 +24,7 @@ def run(ctx,
         logs_dir:             Ref,
         IDP_func_head_motion: Out):
 
-    with redirect_logging('IDP_func_head_motion', outdir=logs_dir):
+    with redirect_logging(job_name(run), outdir=logs_dir):
 
         if os.path.exists(rfMRI_mc_rel_mean):
             with open(rfMRI_mc_rel_mean, "r", encoding="utf-8") as f:

bip/pipelines/IDPs_gen/IDP_func_task_activation.py

@@ -14,7 +14,7 @@ import shutil
 import logging
 from fsl import wrappers
 from pipe_tree import In, Out, Ref
-from bip.utils.log_utils import redirect_logging
+from bip.utils.log_utils import redirect_logging, job_name
 
 log = logging.getLogger(__name__)
 
@@ -41,7 +41,7 @@ def run(ctx,
         tfMRI_featquery_5a_report: Out,
         IDP_func_task_activation:  Out):
 
-    with redirect_logging('IDP_func_task_activation', outdir=logs_dir):
+    with redirect_logging(job_name(run), outdir=logs_dir):
         if not os.path.exists(highres2standard_warp):
             os.symlink(src=os.getcwd() + "/" + T1_to_MNI_warp,
                        dst=highres2standard_warp)

bip/pipelines/IDPs_gen/IDP_subject_COG_table.py

@@ -13,7 +13,7 @@
 import os
 import logging
 from pipe_tree import In, Out, Ref
-from bip.utils.log_utils import redirect_logging
+from bip.utils.log_utils import redirect_logging, job_name
 
 log = logging.getLogger(__name__)
 
@@ -28,7 +28,7 @@ def run(ctx,
         logs_dir:           Ref,
         IDP_subject_COG_table: Out):
 
-    with redirect_logging('IDP_subject_COG_table', outdir=logs_dir):
+    with redirect_logging(job_name(run), outdir=logs_dir):
 
         with open(T1_QC_COG, "r", encoding="utf-8") as f:
             COG  = f.read().strip()

bip/pipelines/IDPs_gen/IDP_subject_ID.py

@@ -12,7 +12,7 @@
 
 import logging
 from pipe_tree import In, Out, Ref
-from bip.utils.log_utils import redirect_logging
+from bip.utils.log_utils import redirect_logging, job_name
 
 log = logging.getLogger(__name__)
 
@@ -21,6 +21,6 @@ def run(ctx,
         IDP_subject_ID: Out,
         logs_dir:       Ref):
 
-    with redirect_logging('IDP_subject_ID', outdir=logs_dir):
+    with redirect_logging(job_name(run), outdir=logs_dir):
         with open(IDP_subject_ID, 'wt', encoding="utf-8") as f:
             f.write(f'{ctx.subject}\n')

bip/pipelines/IDPs_gen/IDP_subject_centre.py

@@ -13,7 +13,7 @@ import os
 import json
 import logging
 from pipe_tree import In, Out, Ref
-from bip.utils.log_utils import redirect_logging
+from bip.utils.log_utils import redirect_logging, job_name
 
 log = logging.getLogger(__name__)
 
@@ -27,7 +27,7 @@ def run(ctx,
         logs_dir:           Ref,
         IDP_subject_centre: Out):
 
-    with redirect_logging('IDP_subject_centre', outdir=logs_dir):
+    with redirect_logging(job_name(run), outdir=logs_dir):
 
         line = ""
         address_text = "(0008, 0081) Institution Address"

bip/pipelines/IDPs_gen/IDPs_gen.py

-#!/usr/bin/env python
-#
-# struct_T1.py - Pipeline with the T1w processing.
-#
-# Author: Fidel Alfaro Almagro <fidel.alfaroalmagro@ndcn.ox.ac.uk>
-# Author: Paul McCarthy <pauldmccarthy@gmail.com>
-# Author: Michiel Cottaar <michiel.cottaar@ndcn.ox.ac.uk>
-#
-# pylint: disable=C0103,E0602,C0114,C0115,C0116,R0913,R0914,R0915
-#
-
-import logging
-from bip.utils.log_utils    import redirect_logging
-from bip.pipelines.IDPs_gen import IDP_subject_ID
-from bip.pipelines.IDPs_gen import IDP_subject_centre
-from bip.pipelines.IDPs_gen import IDP_subject_COG_table
-from bip.pipelines.IDPs_gen import IDP_all_align_to_T1
-from bip.pipelines.IDPs_gen import IDP_T1_FIRST_vols
-from bip.pipelines.IDPs_gen import IDP_T1_SIENAX
-from bip.pipelines.IDPs_gen import IDP_T1_GM_parcellation
-from bip.pipelines.IDPs_gen import IDP_T1_align_to_std
-from bip.pipelines.IDPs_gen import IDP_T1_noise_ratio
-from bip.pipelines.IDPs_gen import IDP_T2_FLAIR_WMH
-from bip.pipelines.IDPs_gen import IDP_SWI_T2star
-from bip.pipelines.IDPs_gen import IDP_func_head_motion
-from bip.pipelines.IDPs_gen import IDP_func_TSNR
-from bip.pipelines.IDPs_gen import IDP_func_task_activation
-from bip.pipelines.IDPs_gen import IDP_diff_eddy_outliers
-from bip.pipelines.IDPs_gen import IDP_diff_TBSS
-from bip.pipelines.IDPs_gen import IDP_diff_autoPtx
-from bip.pipelines.IDPs_gen import IDPs_generator
-
-log = logging.getLogger(__name__)
-
-def add_to_pipeline(ctx, pipe, tree, targets):
-
-    logs_dir=tree.get('logs_dir')
-
-    subj = ctx.subject
-
-    with redirect_logging('pipe_IDPs_gen', outdir=logs_dir):
-        pipe(IDP_subject_ID.run,
-             submit=dict(jobtime=200, name="BIP_IDP_subject_ID_" + subj),
-             kwargs={'ctx' : ctx})
-        targets.append('IDP_subject_ID')
-
-        pipe(IDP_T1_align_to_std.run,
-             submit=dict(jobtime=200, name="BIP_IDP_T1_align_to_std_" + subj),
-             kwargs={'ctx' : ctx})
-        targets.append('IDP_T1_align_to_std')
-
-        pipe(IDP_T1_noise_ratio.run,
-             submit=dict(jobtime=200, name="BIP_IDP_T1_noise_ratio_" + subj),
-             kwargs={'ctx' : ctx})
-        targets.append('IDP_T1_noise_ratio')
-
-        pipe(IDP_all_align_to_T1.run,
-             submit=dict(jobtime=200, name="BIP_IDP_all_align_to_T1_" + subj),
-             kwargs={'ctx' : ctx})
-        targets.append('IDP_all_align_to_T1')
-
-        pipe(IDP_func_head_motion.run,
-             submit=dict(jobtime=200, name="BIP_IDP_func_head_motion_" + subj),
-             kwargs={'ctx' : ctx})
-        targets.append('IDP_func_head_motion')
-
-        pipe(IDP_func_TSNR.run,
-             submit=dict(jobtime=200, name="BIP_IDP_func_TSNR_" + subj),
-             kwargs={'ctx' : ctx})
-        targets.append('IDP_func_TSNR')
-
-        pipe(IDP_diff_eddy_outliers.run,
-             submit=dict(jobtime=200, name="BIP_IDP_diff_eddy_outliers_"+ subj),
-             kwargs={'ctx' : ctx})
-        targets.append('IDP_diff_eddy_outliers')
-
-        pipe(IDP_T1_SIENAX.run,
-             submit=dict(jobtime=200, name="BIP_IDP_T1_SIENAX_" + subj),
-             kwargs={'ctx' : ctx})
-        targets.append('IDP_T1_SIENAX')
-
-        pipe(IDP_T1_FIRST_vols.run,
-             submit=dict(jobtime=200, name="BIP_IDP_T1_FIRST_vols_" + subj),
-             kwargs={'ctx' : ctx})
-        targets.append('IDP_T1_FIRST_vols')
-
-        pipe(IDP_T1_GM_parcellation.run,
-             submit=dict(jobtime=200, name="BIP_IDP_T1_GM_parcellation_" +subj),
-             kwargs={'ctx' : ctx})
-        targets.append('IDP_T1_GM_parcellation')
-
-        pipe(IDP_T2_FLAIR_WMH.run,
-             submit=dict(jobtime=200, name="BIP_IDP_T2_FLAIR_WMH_" + subj),
-             kwargs={'ctx' : ctx})
-        targets.append('IDP_T2_FLAIR_WMH')
-
-        pipe(IDP_SWI_T2star.run,
-             submit=dict(jobtime=200, name="BIP_IDP_SWI_T2star_" + subj),
-             kwargs={'ctx' : ctx})
-        targets.append('IDP_SWI_T2star')
-
-        pipe(IDP_func_task_activation.run,
-             submit=dict(jobtime=200,name="BIP_IDP_func_task_activation_"+subj),
-             kwargs={'ctx' : ctx})
-        targets.append('IDP_func_task_activation')
-
-        pipe(IDP_diff_TBSS.run,
-             submit=dict(jobtime=200, name="BIP_IDP_diff_TBSS_"+ subj),
-             kwargs={'ctx' : ctx})
-        targets.append('IDP_diff_TBSS')
-
-        pipe(IDP_diff_autoPtx.run,
-             submit=dict(jobtime=200, name="BIP_IDP_diff_autoPtx_"+ subj),
-             kwargs={'ctx' : ctx})
-        targets.append('IDP_diff_autoPtx')
-
-        pipe(IDP_subject_COG_table.run,
-             submit=dict(jobtime=200, name="BIP_IDP_subject_COG_table_" + subj),
-             kwargs={'ctx' : ctx})
-        targets.append('IDP_subject_COG_table')
-
-        pipe(IDP_subject_centre.run,
-             submit=dict(jobtime=200, name="BIP_IDP_subject_centre_" + subj),
-             kwargs={'ctx' : ctx})
-        targets.append('IDP_subject_centre')
-
-        pipe(IDPs_generator.run,
-             submit=dict(jobtime=200, name="BIP_IDPs_generator_" + subj),
-             kwargs={'ctx' : ctx})
-        targets.append('IDPs')
-
-    return pipe, targets

bip/pipelines/IDPs_gen/IDPs_generator.py

@@ -13,7 +13,7 @@ import os
 import json
 import logging
 from pipe_tree import In, Out, Ref
-from bip.utils.log_utils import redirect_logging
+from bip.utils.log_utils import redirect_logging, job_name
 
 log = logging.getLogger(__name__)
 
@@ -38,7 +38,7 @@ def run(ctx,
         logs_dir:                 Ref,
         IDPs:                     Out):
 
-    with redirect_logging('IDPs_generator', outdir=logs_dir):
+    with redirect_logging(job_name(run), outdir=logs_dir):
 
         result=""
         IDPs_json_file = ctx.get_data("IDPs/IDPs.json")