Remove bugs that prevented the pipeline from running

bip/main.py

@@ -12,7 +12,6 @@ import sys
 import logging
 import argparse
 import json
-from dataclasses import dataclass, field
 from file_tree import FileTree
 from pipe_tree import Pipeline, Ref
 import bip
@@ -35,18 +34,17 @@ class Usage(Exception):
     def __init__(self, msg):
         self.msg = msg
 
-@dataclass
 class Context:
-    subject:    str = ""
-    BB_BIN_DIR: str = bip.__path__[0]
-    FSLDIR:     str = os.environ['FSLDIR']
-    gdc:        str = field(init=False)
 
-    with open(self.get_data('dMRI/autoptx/struct_list.json'), 'r') as f:
-        tract_struct = json.load(f)
+    def __init__(self, subject=""):
 
-    def __post_init__(self):
-        self.gdc = self.get_data('GDC/UKB.txt')
+        self.subject    = subject
+        self.BB_BIN_DIR = bip.__path__[0]
+        self.FSLDIR     = os.environ['FSLDIR']
+        self.gdc        = self.get_data('GDC/UKB.txt')
+
+        with open(self.get_data('dMRI/autoptx/struct_list.json'), 'r') as f:
+            self.tract_struct = json.load(f)
 
     @property
     def MNI(self):
@@ -84,54 +82,67 @@ def parseArguments(ctx):
     parser = MyParser(description='BioBank Pipeline Manager V. 2.0')
     parser.add_argument("subjectFolder", help='Subject Folder', action="store")
     parser.add_argument("-q", "--queue", help='Queue modifier (default: normal)',
-                        action="store", nargs="?", dest="queue", default="normal")
+                      action="store", nargs="?", dest="queue", default="normal")
     parser.add_argument("-n", "--normcheck", action="store_false", default=True,
-                        help='Do NOT check Normalisation in structural image (default if flag not used: Check normalisation)',
+                        help='Do NOT check Normalisation in structural image '+\
+                             '(default if flag not used: Check normalisation)',
                         dest="norm_check")
     parser.add_argument("-P", "--namingPatterns", action="store", nargs="?",
                         default=ctx.get_data("config/naming_pattern_UKBB.json"),
-                        help='File with the naming patterns coming from dcm2niix (default: UKB naming patterns)',
+                        help='File with the naming patterns coming from ' +\
+                             'dcm2niix (default: UKB naming patterns)',
                         dest="naming_patterns")
     parser.add_argument("-c", "--coeff", action="store", nargs="?",
                         default=ctx.get_data("GDC/UKB.txt"),
-                        help='Coefficient file for the GDC (Gradient Distiortion Correction). \n' +
+                        help='Coefficient file for the GDC (Gradient ' +\
+                            'Distiortion Correction). \n' +
                               '   Options:  \n' +
-                              '           none--> No GDC is performed' +
-                              '           <path to file> --> Uses the user-pecified gradients file' +
-                              '            --> If this option is not used, the default is the UKB',
+                              '    none--> No GDC is performed' +
+                              '    <path to file> --> Uses the user-pecified gradients file' +
+                              '    --> If this option is not used, the default is the UKB',
                         dest="coeff")
-    parser.add_argument("-C", "--coils_SWI", action="store", nargs="?", default=32,
-                        help='Number of coils for SWI data. Default: 32. 0 means "no separate coil data"',
+    parser.add_argument("-C", "--coils_SWI",action="store",nargs="?",default=32,
+                        help='Number of coils for SWI data. Default: 32. 0 ' +\
+                             'means "no separate coil data"',
                         dest="coils_SWI")
-    parser.add_argument("-E", "--echoes_SWI", action="store", nargs="?", default=2,
+    parser.add_argument("-E", "--echoes_SWI",action="store",nargs="?",default=2,
                         help='Number of echo times for SWI data (default: 2)',
                         dest="echoes_SWI")
     parser.add_argument("-Q", "--basic_QC_file", action="store", nargs="?",
                         default=ctx.get_data("config/ideal_config_sizes_UKBB.json"),
-                        help='File with the ideal configuration of the files (default: UKB)',
+                        help='File with the ideal configuration of the files +'\
+                             '(default: UKB)',
                         dest="basic_QC_file")
-    parser.add_argument("-p", "--complex_phase", action="store_true", default=False,
+    parser.add_argument("-p", "--complex_phase", action="store_true",
+                        default=False,
                         help='SWI phase is complex (default: False)',
                         dest="complex_phase")
-    parser.add_argument("-i", "--inverted_PED", action="store_true", default=False,
-                        help='Use if the dMRI Phase Encoding Direction is inverted (default if flag not used: False)',
+    parser.add_argument("-i", "--inverted_PED", action="store_true",
+                        default=False,
+                        help='Use if the dMRI Phase Encoding Direction is '+\
+                             'inverted (default if flag not used: False)',
                         dest="inverted_PED")
     parser.add_argument("-b", "--b_value_shell", action="store", nargs="?",
                         default=1000,
-                        help='B value for the single shell data to be used (default: 1000)',
+                        help='B value for the single shell data to be used '+\
+                             '(default: 1000)',
                         dest="b_value_shell")
     parser.add_argument("-B", "--B_files", action="store", nargs="?",
                         default="",
-                        help='Name of the directory with the bval and bvec files for dMRI data.\n' +
-                             'If specified, the directory needs to have these 4 files: \n' +
+                        help='Name of the directory with the bval and bvec '+\
+                             'files for dMRI data.\n' +
+                             'If specified, the directory needs to have these'+\
+                             ' 4 files: \n' +
                              '    - AP.bval\n ' +
                              '    - AP.bvec\n ' +
                              '    - PA.bval\n ' +
                              '    - PA.bvec\n ' +
-                             'If not set, the pipeline will use the dcm2niix generated files',
+                             'If not set, the pipeline will use the dcm2niix '+\
+                             'generated files',
                         dest="B_files")
-    parser.add_argument("-S", "--num_shells", action="store", nargs="?", default=2,
-                        help='Number of different shells (B-values) for dMRI data.',
+    parser.add_argument("-S", "--num_shells",action="store",nargs="?",default=2,
+                        help='Number of different shells (B-values) '+\
+                             'for dMRI data.',
                         dest="num_shells")
     parser.add_argument("-X", "--exec_block", action="store", nargs="?", default=0,
                         help='Pipeline block(s) to execute: Option (default 0): \n' +
@@ -320,12 +331,12 @@ def main():
 
     setup_logging(op.join(ctx.subject, 'logs', 'main.log'))
 
-    tree = FileTree.read(ctx.get_data('FileTree'), 
+    tree = FileTree.read(ctx.get_data('FileTree'),
                          subject=ctx.subject,
                          autoptx_tract=list(ctx.tract_struct.keys()),
                          netmats_dim=["25", "100"])
 
-    # add submit parameters here that should 
+    # add submit parameters here that should
     # be applied to all jobs in the pipeline
     pipe = Pipeline(default_submit=dict(logdir=Ref("logs_dir"),
                     export_vars=list(os.environ.keys())))
@@ -334,7 +345,7 @@ def main():
     #ctx.save_context(tree.get('config_file'))
 
     # This list will be filled with all desired outputs ... Later
-    targets = []
+    #targets = []
 
     struct_T1.add_to_pipeline      (ctx, pipe, tree)
     struct_T2_FLAIR.add_to_pipeline(ctx, pipe, tree)
@@ -349,8 +360,8 @@ def main():
 
     # The skip-missing flag deals with cases where the subject is missing
     # the data of some modalities. For more details, check:
-    # https://open.win.ox.ac.uk/pages/ndcn0236/pipe-tree/tutorial.html#dealing-with-missing-data 
-    jobs = pipe.generate_jobs(tree).filter(targets, skip_missing=True)
+    # https://open.win.ox.ac.uk/pages/ndcn0236/pipe-tree/tutorial.html#dealing-with-missing-data
+    jobs = pipe.generate_jobs(tree).filter(None, skip_missing=True)
 
     jobs.report()
 

bip/pipelines/IDPs_gen/IDP_SWI_T2star.py

@@ -10,7 +10,6 @@
 # pylint: disable=W0613
 #
 
-import os
 import os.path as op
 import logging
 from fsl import wrappers

bip/pipelines/IDPs_gen/IDP_T1_FIRST_vols.py

@@ -10,7 +10,6 @@
 # pylint: disable=W0613
 #
 
-import os
 import os.path as op
 import logging
 from fsl import wrappers

bip/pipelines/IDPs_gen/IDP_T1_SIENAX.py

@@ -10,7 +10,6 @@
 # pylint: disable=W0613
 #
 
-import os
 import os.path as op
 import logging
 from shutil import copyfile

bip/pipelines/IDPs_gen/IDP_T1_align_to_std.py

@@ -9,6 +9,7 @@
 # pylint: disable=C0103,E0602,C0114,C0115,C0116,R0913,R0914,R0915
 #
 
+import os
 import os.path as op
 import logging
 from fsl import wrappers
@@ -32,7 +33,7 @@ def run(ctx,
         tmp_jac = op.join(tmp_dir, 'tmpjac.nii.gz')
         tmp_mat = op.join(tmp_dir, 'tmp_mat.mat')
 
-        MC=ctx.FSLDIR + '/etc/flirtschop.join(tmp_dir, 'op.join(tmp_dir, 'measurecost1.sch'
+        MC=ctx.FSLDIR + os.sep + op.join('etc', 'flirtsch', 'measurecost1.sch')
         MNI152_T1_1mm_brain = ctx.get_standard("MNI152_T1_1mm_brain.nii.gz")
         MNI152_T1_1mm_brain_mask = ctx.get_standard("MNI152_T1_1mm_brain_mask.nii.gz")
 

bip/pipelines/IDPs_gen/IDP_all_align_to_T1.py

@@ -9,7 +9,6 @@
 # pylint: disable=C0103,E0602,C0114,C0115,C0116,R0913,R0914,R0915
 #
 
-import os
 import os.path as op
 import logging
 from fsl import wrappers

bip/pipelines/IDPs_gen/IDP_diff_TBSS.py

@@ -10,7 +10,6 @@
 # pylint: disable=W0613
 #
 
-import os
 import os.path as op
 import logging
 from pipe_tree import In, Out, Ref

bip/pipelines/IDPs_gen/IDP_diff_autoPtx.py

@@ -10,7 +10,6 @@
 # pylint: disable=W0613
 #
 
-import os
 import os.path as op
 import logging
 import nibabel as nib

bip/pipelines/IDPs_gen/IDP_diff_eddy_outliers.py

@@ -10,7 +10,6 @@
 # pylint: disable=W0613
 #
 
-import os
 import os.path as op
 import logging
 from pipe_tree import In, Out, Ref

bip/pipelines/IDPs_gen/IDP_func_TSNR.py

@@ -10,7 +10,6 @@
 # pylint: disable=W0613
 #
 
-import os
 import os.path as op
 import logging
 from fsl import wrappers

bip/pipelines/IDPs_gen/IDP_func_head_motion.py

@@ -10,7 +10,6 @@
 # pylint: disable=W0613
 #
 
-import os
 import os.path as op
 import json
 import logging

bip/pipelines/IDPs_gen/IDP_subject_COG_table.py

@@ -10,7 +10,6 @@
 # pylint: disable=W0613
 #
 
-import os
 import os.path as op
 import logging
 from pipe_tree import In, Out, Ref

bip/pipelines/IDPs_gen/IDP_subject_centre.py

@@ -9,7 +9,6 @@
 # pylint: disable=C0103,E0602,C0114,C0115,C0116,R0913,R0914,R0915
 #
 
-import os
 import os.path as op
 import json
 import logging

bip/pipelines/IDPs_gen/IDPs_generator.py

@@ -9,7 +9,6 @@
 # pylint: disable=C0103,E0602,C0114,C0115,C0116,R0913,R0914,R0915
 #
 
-import os
 import os.path as op
 import json
 import logging

bip/pipelines/dMRI_diff/diff_dtifit.py

@@ -10,7 +10,6 @@
 # pylint: disable=W0613
 #
 
-import os.path as op
 import logging
 from shutil import copyfile
 import numpy as np

bip/pipelines/fMRI_rest/rfMRI_netmats_fnc.py

@@ -10,7 +10,6 @@
 # pylint: disable=E1130
 #
 
-import os.path as op
 import numpy as np
 import nibabel as nib
 

bip/pipelines/struct_FS/FS_get_IDPs_fnc.py

@@ -139,7 +139,7 @@ def fix_aseg_data(data_dict, subjectDir):
     # For some reason, the VentricleChoroidVol is not caught by asegstats2table
     try:
         file_name = op.join(subjectDir, 'stats' , 'aseg.stats')
-        with open(, 'r', encoding="utf-8") as f:
+        with open(file_name, 'r', encoding="utf-8") as f:
             val=[x.split(',')[3].strip() for x in f.readlines() if 'VentricleChoroidVol' in x]
     except:
         val=["NaN"]

bip/pipelines/struct_T1/T1_QC_CNR_eyes.py

@@ -10,7 +10,6 @@
 # pylint: disable=W0613
 #
 
-import os
 import os.path as op
 import logging
 import nibabel as nib

bip/pipelines/struct_T1/T1_brain_extract.py

@@ -40,7 +40,7 @@ def run(ctx,
         T1_to_MNI_warp_coef:     Out,
         T1_to_MNI_warp_jac:      Out):
 
-    with redirect_logging(job_name(run), outdir=logs_dir),
+    with redirect_logging(job_name(run), outdir=logs_dir),\
          tempdir(op.join(tmp_dir, job_name(run))):
 
         T1_tmp_1                  = op.join(tmp_dir, 'T1_tmp_1.nii.gz')
@@ -67,7 +67,7 @@ def run(ctx,
         #Generate the actual affine from the orig_ud volume to the cut version
         #we haveand combine it to have an affine matrix from orig_ud to MNI
         wrappers.flirt(src=T1, ref=T1_orig_ud, omat=T1_to_T1_orig_ud_mat,
-               schedule = op.join(ctx.FSLDIR, 'etc', 'flirtsch', 'xyztrans.sch')
+              schedule = op.join(ctx.FSLDIR, 'etc', 'flirtsch', 'xyztrans.sch'))
         wrappers.invxfm(inmat=T1_to_T1_orig_ud_mat, omat=T1_orig_ud_to_T1_mat)
         wrappers.concatxfm(atob=T1_to_T1_orig_ud_mat,btoc=T1_orig_ud_to_std_mat,
                            atoc=T1_to_MNI_linear_mat)

bip/pipelines/struct_T1/T1_first.py

@@ -10,7 +10,7 @@
 # pylint: disable=W0613
 #
 
-import os.symlink
+import os
 import os.path as op
 import glob
 import logging