From c570ea0c0b3b76614fbce28164abe8b0dba79416 Mon Sep 17 00:00:00 2001
From: Fidel Alfaro Almagro <falmagro@fmrib.ox.ac.uk>
Date: Thu, 9 Mar 2023 12:56:32 +0000
Subject: [PATCH] Remove bugs that prevented the pipeline from running
---
bip/main.py | 81 +++++++++++--------
bip/pipelines/IDPs_gen/IDP_SWI_T2star.py | 1 -
bip/pipelines/IDPs_gen/IDP_T1_FIRST_vols.py | 1 -
bip/pipelines/IDPs_gen/IDP_T1_SIENAX.py | 1 -
bip/pipelines/IDPs_gen/IDP_T1_align_to_std.py | 3 +-
bip/pipelines/IDPs_gen/IDP_all_align_to_T1.py | 1 -
bip/pipelines/IDPs_gen/IDP_diff_TBSS.py | 1 -
bip/pipelines/IDPs_gen/IDP_diff_autoPtx.py | 1 -
.../IDPs_gen/IDP_diff_eddy_outliers.py | 1 -
bip/pipelines/IDPs_gen/IDP_func_TSNR.py | 1 -
.../IDPs_gen/IDP_func_head_motion.py | 1 -
.../IDPs_gen/IDP_subject_COG_table.py | 1 -
bip/pipelines/IDPs_gen/IDP_subject_centre.py | 1 -
bip/pipelines/IDPs_gen/IDPs_generator.py | 1 -
bip/pipelines/dMRI_diff/diff_dtifit.py | 1 -
bip/pipelines/fMRI_rest/rfMRI_netmats_fnc.py | 1 -
bip/pipelines/struct_FS/FS_get_IDPs_fnc.py | 2 +-
bip/pipelines/struct_T1/T1_QC_CNR_eyes.py | 1 -
bip/pipelines/struct_T1/T1_brain_extract.py | 4 +-
bip/pipelines/struct_T1/T1_first.py | 2 +-
bip/pipelines/struct_T1/T1_sienax.py | 4 -
.../struct_T2_FLAIR/T2_FLAIR_apply_bfc.py | 1 -
.../struct_T2_FLAIR/T2_FLAIR_bianca.py | 1 -
bip/pipelines/struct_swMRI/swMRI_proc.py | 1 -
24 files changed, 52 insertions(+), 62 deletions(-)
diff --git a/bip/main.py b/bip/main.py
index ab6ef49..b23e025 100755
--- a/bip/main.py
+++ b/bip/main.py
@@ -12,7 +12,6 @@ import sys
import logging
import argparse
import json
-from dataclasses import dataclass, field
from file_tree import FileTree
from pipe_tree import Pipeline, Ref
import bip
@@ -35,18 +34,17 @@ class Usage(Exception):
def __init__(self, msg):
self.msg = msg
-@dataclass
class Context:
- subject: str = ""
- BB_BIN_DIR: str = bip.__path__[0]
- FSLDIR: str = os.environ['FSLDIR']
- gdc: str = field(init=False)
- with open(self.get_data('dMRI/autoptx/struct_list.json'), 'r') as f:
- tract_struct = json.load(f)
+ def __init__(self, subject=""):
- def __post_init__(self):
- self.gdc = self.get_data('GDC/UKB.txt')
+ self.subject = subject
+ self.BB_BIN_DIR = bip.__path__[0]
+ self.FSLDIR = os.environ['FSLDIR']
+ self.gdc = self.get_data('GDC/UKB.txt')
+
+ with open(self.get_data('dMRI/autoptx/struct_list.json'), 'r') as f:
+ self.tract_struct = json.load(f)
@property
def MNI(self):
@@ -84,54 +82,67 @@ def parseArguments(ctx):
parser = MyParser(description='BioBank Pipeline Manager V. 2.0')
parser.add_argument("subjectFolder", help='Subject Folder', action="store")
parser.add_argument("-q", "--queue", help='Queue modifier (default: normal)',
- action="store", nargs="?", dest="queue", default="normal")
+ action="store", nargs="?", dest="queue", default="normal")
parser.add_argument("-n", "--normcheck", action="store_false", default=True,
- help='Do NOT check Normalisation in structural image (default if flag not used: Check normalisation)',
+ help='Do NOT check Normalisation in structural image '+\
+ '(default if flag not used: Check normalisation)',
dest="norm_check")
parser.add_argument("-P", "--namingPatterns", action="store", nargs="?",
default=ctx.get_data("config/naming_pattern_UKBB.json"),
- help='File with the naming patterns coming from dcm2niix (default: UKB naming patterns)',
+ help='File with the naming patterns coming from ' +\
+ 'dcm2niix (default: UKB naming patterns)',
dest="naming_patterns")
parser.add_argument("-c", "--coeff", action="store", nargs="?",
default=ctx.get_data("GDC/UKB.txt"),
- help='Coefficient file for the GDC (Gradient Distiortion Correction). \n' +
+ help='Coefficient file for the GDC (Gradient ' +\
+ 'Distiortion Correction). \n' +
' Options: \n' +
- ' none--> No GDC is performed' +
- ' <path to file> --> Uses the user-pecified gradients file' +
- ' --> If this option is not used, the default is the UKB',
+ ' none--> No GDC is performed' +
+ ' <path to file> --> Uses the user-pecified gradients file' +
+ ' --> If this option is not used, the default is the UKB',
dest="coeff")
- parser.add_argument("-C", "--coils_SWI", action="store", nargs="?", default=32,
- help='Number of coils for SWI data. Default: 32. 0 means "no separate coil data"',
+ parser.add_argument("-C", "--coils_SWI",action="store",nargs="?",default=32,
+ help='Number of coils for SWI data. Default: 32. 0 ' +\
+ 'means "no separate coil data"',
dest="coils_SWI")
- parser.add_argument("-E", "--echoes_SWI", action="store", nargs="?", default=2,
+ parser.add_argument("-E", "--echoes_SWI",action="store",nargs="?",default=2,
help='Number of echo times for SWI data (default: 2)',
dest="echoes_SWI")
parser.add_argument("-Q", "--basic_QC_file", action="store", nargs="?",
default=ctx.get_data("config/ideal_config_sizes_UKBB.json"),
- help='File with the ideal configuration of the files (default: UKB)',
+ help='File with the ideal configuration of the files +'\
+ '(default: UKB)',
dest="basic_QC_file")
- parser.add_argument("-p", "--complex_phase", action="store_true", default=False,
+ parser.add_argument("-p", "--complex_phase", action="store_true",
+ default=False,
help='SWI phase is complex (default: False)',
dest="complex_phase")
- parser.add_argument("-i", "--inverted_PED", action="store_true", default=False,
- help='Use if the dMRI Phase Encoding Direction is inverted (default if flag not used: False)',
+ parser.add_argument("-i", "--inverted_PED", action="store_true",
+ default=False,
+ help='Use if the dMRI Phase Encoding Direction is '+\
+ 'inverted (default if flag not used: False)',
dest="inverted_PED")
parser.add_argument("-b", "--b_value_shell", action="store", nargs="?",
default=1000,
- help='B value for the single shell data to be used (default: 1000)',
+ help='B value for the single shell data to be used '+\
+ '(default: 1000)',
dest="b_value_shell")
parser.add_argument("-B", "--B_files", action="store", nargs="?",
default="",
- help='Name of the directory with the bval and bvec files for dMRI data.\n' +
- 'If specified, the directory needs to have these 4 files: \n' +
+ help='Name of the directory with the bval and bvec '+\
+ 'files for dMRI data.\n' +
+ 'If specified, the directory needs to have these'+\
+ ' 4 files: \n' +
' - AP.bval\n ' +
' - AP.bvec\n ' +
' - PA.bval\n ' +
' - PA.bvec\n ' +
- 'If not set, the pipeline will use the dcm2niix generated files',
+ 'If not set, the pipeline will use the dcm2niix '+\
+ 'generated files',
dest="B_files")
- parser.add_argument("-S", "--num_shells", action="store", nargs="?", default=2,
- help='Number of different shells (B-values) for dMRI data.',
+ parser.add_argument("-S", "--num_shells",action="store",nargs="?",default=2,
+ help='Number of different shells (B-values) '+\
+ 'for dMRI data.',
dest="num_shells")
parser.add_argument("-X", "--exec_block", action="store", nargs="?", default=0,
help='Pipeline block(s) to execute: Option (default 0): \n' +
@@ -320,12 +331,12 @@ def main():
setup_logging(op.join(ctx.subject, 'logs', 'main.log'))
- tree = FileTree.read(ctx.get_data('FileTree'),
+ tree = FileTree.read(ctx.get_data('FileTree'),
subject=ctx.subject,
autoptx_tract=list(ctx.tract_struct.keys()),
netmats_dim=["25", "100"])
- # add submit parameters here that should
+ # add submit parameters here that should
# be applied to all jobs in the pipeline
pipe = Pipeline(default_submit=dict(logdir=Ref("logs_dir"),
export_vars=list(os.environ.keys())))
@@ -334,7 +345,7 @@ def main():
#ctx.save_context(tree.get('config_file'))
# This list will be filled with all desired outputs ... Later
- targets = []
+ #targets = []
struct_T1.add_to_pipeline (ctx, pipe, tree)
struct_T2_FLAIR.add_to_pipeline(ctx, pipe, tree)
@@ -349,8 +360,8 @@ def main():
# The skip-missing flag deals with cases where the subject is missing
# the data of some modalities. For more details, check:
- # https://open.win.ox.ac.uk/pages/ndcn0236/pipe-tree/tutorial.html#dealing-with-missing-data
- jobs = pipe.generate_jobs(tree).filter(targets, skip_missing=True)
+ # https://open.win.ox.ac.uk/pages/ndcn0236/pipe-tree/tutorial.html#dealing-with-missing-data
+ jobs = pipe.generate_jobs(tree).filter(None, skip_missing=True)
jobs.report()
diff --git a/bip/pipelines/IDPs_gen/IDP_SWI_T2star.py b/bip/pipelines/IDPs_gen/IDP_SWI_T2star.py
index 9105995..d030aa8 100755
--- a/bip/pipelines/IDPs_gen/IDP_SWI_T2star.py
+++ b/bip/pipelines/IDPs_gen/IDP_SWI_T2star.py
@@ -10,7 +10,6 @@
# pylint: disable=W0613
#
-import os
import os.path as op
import logging
from fsl import wrappers
diff --git a/bip/pipelines/IDPs_gen/IDP_T1_FIRST_vols.py b/bip/pipelines/IDPs_gen/IDP_T1_FIRST_vols.py
index 3aed0f4..7b270d5 100755
--- a/bip/pipelines/IDPs_gen/IDP_T1_FIRST_vols.py
+++ b/bip/pipelines/IDPs_gen/IDP_T1_FIRST_vols.py
@@ -10,7 +10,6 @@
# pylint: disable=W0613
#
-import os
import os.path as op
import logging
from fsl import wrappers
diff --git a/bip/pipelines/IDPs_gen/IDP_T1_SIENAX.py b/bip/pipelines/IDPs_gen/IDP_T1_SIENAX.py
index bf275eb..ec2b6e4 100755
--- a/bip/pipelines/IDPs_gen/IDP_T1_SIENAX.py
+++ b/bip/pipelines/IDPs_gen/IDP_T1_SIENAX.py
@@ -10,7 +10,6 @@
# pylint: disable=W0613
#
-import os
import os.path as op
import logging
from shutil import copyfile
diff --git a/bip/pipelines/IDPs_gen/IDP_T1_align_to_std.py b/bip/pipelines/IDPs_gen/IDP_T1_align_to_std.py
index c598f21..b5a51cd 100755
--- a/bip/pipelines/IDPs_gen/IDP_T1_align_to_std.py
+++ b/bip/pipelines/IDPs_gen/IDP_T1_align_to_std.py
@@ -9,6 +9,7 @@
# pylint: disable=C0103,E0602,C0114,C0115,C0116,R0913,R0914,R0915
#
+import os
import os.path as op
import logging
from fsl import wrappers
@@ -32,7 +33,7 @@ def run(ctx,
tmp_jac = op.join(tmp_dir, 'tmpjac.nii.gz')
tmp_mat = op.join(tmp_dir, 'tmp_mat.mat')
- MC=ctx.FSLDIR + '/etc/flirtschop.join(tmp_dir, 'op.join(tmp_dir, 'measurecost1.sch'
+ MC=ctx.FSLDIR + os.sep + op.join('etc', 'flirtsch', 'measurecost1.sch')
MNI152_T1_1mm_brain = ctx.get_standard("MNI152_T1_1mm_brain.nii.gz")
MNI152_T1_1mm_brain_mask = ctx.get_standard("MNI152_T1_1mm_brain_mask.nii.gz")
diff --git a/bip/pipelines/IDPs_gen/IDP_all_align_to_T1.py b/bip/pipelines/IDPs_gen/IDP_all_align_to_T1.py
index dbfac5e..f33de1d 100755
--- a/bip/pipelines/IDPs_gen/IDP_all_align_to_T1.py
+++ b/bip/pipelines/IDPs_gen/IDP_all_align_to_T1.py
@@ -9,7 +9,6 @@
# pylint: disable=C0103,E0602,C0114,C0115,C0116,R0913,R0914,R0915
#
-import os
import os.path as op
import logging
from fsl import wrappers
diff --git a/bip/pipelines/IDPs_gen/IDP_diff_TBSS.py b/bip/pipelines/IDPs_gen/IDP_diff_TBSS.py
index b12d1c2..342ca92 100755
--- a/bip/pipelines/IDPs_gen/IDP_diff_TBSS.py
+++ b/bip/pipelines/IDPs_gen/IDP_diff_TBSS.py
@@ -10,7 +10,6 @@
# pylint: disable=W0613
#
-import os
import os.path as op
import logging
from pipe_tree import In, Out, Ref
diff --git a/bip/pipelines/IDPs_gen/IDP_diff_autoPtx.py b/bip/pipelines/IDPs_gen/IDP_diff_autoPtx.py
index eb27510..3398805 100755
--- a/bip/pipelines/IDPs_gen/IDP_diff_autoPtx.py
+++ b/bip/pipelines/IDPs_gen/IDP_diff_autoPtx.py
@@ -10,7 +10,6 @@
# pylint: disable=W0613
#
-import os
import os.path as op
import logging
import nibabel as nib
diff --git a/bip/pipelines/IDPs_gen/IDP_diff_eddy_outliers.py b/bip/pipelines/IDPs_gen/IDP_diff_eddy_outliers.py
index a27c5d7..063c892 100755
--- a/bip/pipelines/IDPs_gen/IDP_diff_eddy_outliers.py
+++ b/bip/pipelines/IDPs_gen/IDP_diff_eddy_outliers.py
@@ -10,7 +10,6 @@
# pylint: disable=W0613
#
-import os
import os.path as op
import logging
from pipe_tree import In, Out, Ref
diff --git a/bip/pipelines/IDPs_gen/IDP_func_TSNR.py b/bip/pipelines/IDPs_gen/IDP_func_TSNR.py
index 9c68bab..047762b 100755
--- a/bip/pipelines/IDPs_gen/IDP_func_TSNR.py
+++ b/bip/pipelines/IDPs_gen/IDP_func_TSNR.py
@@ -10,7 +10,6 @@
# pylint: disable=W0613
#
-import os
import os.path as op
import logging
from fsl import wrappers
diff --git a/bip/pipelines/IDPs_gen/IDP_func_head_motion.py b/bip/pipelines/IDPs_gen/IDP_func_head_motion.py
index ec5c137..2685a07 100755
--- a/bip/pipelines/IDPs_gen/IDP_func_head_motion.py
+++ b/bip/pipelines/IDPs_gen/IDP_func_head_motion.py
@@ -10,7 +10,6 @@
# pylint: disable=W0613
#
-import os
import os.path as op
import json
import logging
diff --git a/bip/pipelines/IDPs_gen/IDP_subject_COG_table.py b/bip/pipelines/IDPs_gen/IDP_subject_COG_table.py
index ef09016..80cbb70 100755
--- a/bip/pipelines/IDPs_gen/IDP_subject_COG_table.py
+++ b/bip/pipelines/IDPs_gen/IDP_subject_COG_table.py
@@ -10,7 +10,6 @@
# pylint: disable=W0613
#
-import os
import os.path as op
import logging
from pipe_tree import In, Out, Ref
diff --git a/bip/pipelines/IDPs_gen/IDP_subject_centre.py b/bip/pipelines/IDPs_gen/IDP_subject_centre.py
index c189324..dbdcbde 100755
--- a/bip/pipelines/IDPs_gen/IDP_subject_centre.py
+++ b/bip/pipelines/IDPs_gen/IDP_subject_centre.py
@@ -9,7 +9,6 @@
# pylint: disable=C0103,E0602,C0114,C0115,C0116,R0913,R0914,R0915
#
-import os
import os.path as op
import json
import logging
diff --git a/bip/pipelines/IDPs_gen/IDPs_generator.py b/bip/pipelines/IDPs_gen/IDPs_generator.py
index a4ba815..cf4c32a 100755
--- a/bip/pipelines/IDPs_gen/IDPs_generator.py
+++ b/bip/pipelines/IDPs_gen/IDPs_generator.py
@@ -9,7 +9,6 @@
# pylint: disable=C0103,E0602,C0114,C0115,C0116,R0913,R0914,R0915
#
-import os
import os.path as op
import json
import logging
diff --git a/bip/pipelines/dMRI_diff/diff_dtifit.py b/bip/pipelines/dMRI_diff/diff_dtifit.py
index a6a900e..ea4cc46 100755
--- a/bip/pipelines/dMRI_diff/diff_dtifit.py
+++ b/bip/pipelines/dMRI_diff/diff_dtifit.py
@@ -10,7 +10,6 @@
# pylint: disable=W0613
#
-import os.path as op
import logging
from shutil import copyfile
import numpy as np
diff --git a/bip/pipelines/fMRI_rest/rfMRI_netmats_fnc.py b/bip/pipelines/fMRI_rest/rfMRI_netmats_fnc.py
index 106575f..398f032 100755
--- a/bip/pipelines/fMRI_rest/rfMRI_netmats_fnc.py
+++ b/bip/pipelines/fMRI_rest/rfMRI_netmats_fnc.py
@@ -10,7 +10,6 @@
# pylint: disable=E1130
#
-import os.path as op
import numpy as np
import nibabel as nib
diff --git a/bip/pipelines/struct_FS/FS_get_IDPs_fnc.py b/bip/pipelines/struct_FS/FS_get_IDPs_fnc.py
index 44fef49..273294f 100755
--- a/bip/pipelines/struct_FS/FS_get_IDPs_fnc.py
+++ b/bip/pipelines/struct_FS/FS_get_IDPs_fnc.py
@@ -139,7 +139,7 @@ def fix_aseg_data(data_dict, subjectDir):
# For some reason, the VentricleChoroidVol is not caught by asegstats2table
try:
file_name = op.join(subjectDir, 'stats' , 'aseg.stats')
- with open(, 'r', encoding="utf-8") as f:
+ with open(file_name, 'r', encoding="utf-8") as f:
val=[x.split(',')[3].strip() for x in f.readlines() if 'VentricleChoroidVol' in x]
except:
val=["NaN"]
diff --git a/bip/pipelines/struct_T1/T1_QC_CNR_eyes.py b/bip/pipelines/struct_T1/T1_QC_CNR_eyes.py
index 3637366..0b9f782 100755
--- a/bip/pipelines/struct_T1/T1_QC_CNR_eyes.py
+++ b/bip/pipelines/struct_T1/T1_QC_CNR_eyes.py
@@ -10,7 +10,6 @@
# pylint: disable=W0613
#
-import os
import os.path as op
import logging
import nibabel as nib
diff --git a/bip/pipelines/struct_T1/T1_brain_extract.py b/bip/pipelines/struct_T1/T1_brain_extract.py
index f4b1d03..c2aa51d 100755
--- a/bip/pipelines/struct_T1/T1_brain_extract.py
+++ b/bip/pipelines/struct_T1/T1_brain_extract.py
@@ -40,7 +40,7 @@ def run(ctx,
T1_to_MNI_warp_coef: Out,
T1_to_MNI_warp_jac: Out):
- with redirect_logging(job_name(run), outdir=logs_dir),
+ with redirect_logging(job_name(run), outdir=logs_dir),\
tempdir(op.join(tmp_dir, job_name(run))):
T1_tmp_1 = op.join(tmp_dir, 'T1_tmp_1.nii.gz')
@@ -67,7 +67,7 @@ def run(ctx,
#Generate the actual affine from the orig_ud volume to the cut version
#we haveand combine it to have an affine matrix from orig_ud to MNI
wrappers.flirt(src=T1, ref=T1_orig_ud, omat=T1_to_T1_orig_ud_mat,
- schedule = op.join(ctx.FSLDIR, 'etc', 'flirtsch', 'xyztrans.sch')
+ schedule = op.join(ctx.FSLDIR, 'etc', 'flirtsch', 'xyztrans.sch'))
wrappers.invxfm(inmat=T1_to_T1_orig_ud_mat, omat=T1_orig_ud_to_T1_mat)
wrappers.concatxfm(atob=T1_to_T1_orig_ud_mat,btoc=T1_orig_ud_to_std_mat,
atoc=T1_to_MNI_linear_mat)
diff --git a/bip/pipelines/struct_T1/T1_first.py b/bip/pipelines/struct_T1/T1_first.py
index 193e107..3a94f71 100755
--- a/bip/pipelines/struct_T1/T1_first.py
+++ b/bip/pipelines/struct_T1/T1_first.py
@@ -10,7 +10,7 @@
# pylint: disable=W0613
#
-import os.symlink
+import os
import os.path as op
import glob
import logging
diff --git a/bip/pipelines/struct_T1/T1_sienax.py b/bip/pipelines/struct_T1/T1_sienax.py
index eb22302..5a226bd 100755
--- a/bip/pipelines/struct_T1/T1_sienax.py
+++ b/bip/pipelines/struct_T1/T1_sienax.py
@@ -47,10 +47,6 @@ def run(ctx,
T1_tmp_mat_2 = op.join(tmp_dir, 'tmp_mat_2.mat')
T1_tmp_mat_3 = op.join(tmp_dir, 'tmp_mat_3.mat')
- FSLDIR = ctx.FSLDIR
- FSLDATA = FSLDIR + '/data'
- FSLMNI = FSLDATA + '/standard'
-
MNI = ctx.get_standard('MNI152_T1_1mm.nii.gz')
MNI_2mm_brain = ctx.get_standard('MNI152_T1_2mm_brain.nii.gz')
MNI_2mm_skull = ctx.get_standard('MNI152_T1_2mm_skull.nii.gz')
diff --git a/bip/pipelines/struct_T2_FLAIR/T2_FLAIR_apply_bfc.py b/bip/pipelines/struct_T2_FLAIR/T2_FLAIR_apply_bfc.py
index 3ed955a..e60003f 100755
--- a/bip/pipelines/struct_T2_FLAIR/T2_FLAIR_apply_bfc.py
+++ b/bip/pipelines/struct_T2_FLAIR/T2_FLAIR_apply_bfc.py
@@ -11,7 +11,6 @@
# pylint: disable=W0613
#
-import os
import os.path as op
import logging
from fsl import wrappers
diff --git a/bip/pipelines/struct_T2_FLAIR/T2_FLAIR_bianca.py b/bip/pipelines/struct_T2_FLAIR/T2_FLAIR_bianca.py
index 5d02a50..1cfbfec 100755
--- a/bip/pipelines/struct_T2_FLAIR/T2_FLAIR_bianca.py
+++ b/bip/pipelines/struct_T2_FLAIR/T2_FLAIR_bianca.py
@@ -9,7 +9,6 @@
# pylint: disable=W0613,C0301
#
-import os.path as op
import logging
from shutil import copyfile
from pipe_tree import In, Out, Ref
diff --git a/bip/pipelines/struct_swMRI/swMRI_proc.py b/bip/pipelines/struct_swMRI/swMRI_proc.py
index 08c5b20..7ff4aa5 100755
--- a/bip/pipelines/struct_swMRI/swMRI_proc.py
+++ b/bip/pipelines/struct_swMRI/swMRI_proc.py
@@ -10,7 +10,6 @@
# pylint: disable=W0613,R0912
#
-import os
import os.path as op
import logging
from shutil import copyfile
--
GitLab