Updating the code to use fsl-pipe instead of pipe-tree

bip/main.py

@@ -17,7 +17,7 @@ import logging
 import argparse
 import json
 from file_tree import FileTree
-from pipe_tree import Pipeline, Ref
+from fsl_pipe import Pipeline, Ref
 import bip
 from bip.utils.config import Config
 from bip.utils.log_utils import setup_logging
@@ -88,7 +88,11 @@ def parseArguments():
     # ARGUMENT PARSING CODE #
     #########################
     parser = MyParser(description='BioBank Pipeline Manager V. 2.0')
-    parser.add_argument("subjectFolder", help='Subject Folder', action="store")
+    parser.add_argument("subject", help='Subject. This can be either a ' +
+                                        'folder with the subject data or a ' +
+                                        'txt file with the list of folders ' +
+                                        'of the subjects to proces',
+                        action="store")
     parser.add_argument("-c", "--configDir",
                         help='Configuration folder: ' +
                              'A config.toml file must exist.',
@@ -101,56 +105,64 @@ def main():
     args = parseArguments()
 
     # Parse subject argument
-    subject = args.subjectFolder
-    subject = subject.strip()
-    if subject.endswith('/'):
-        subject = subject[0:len(subject)-1]
+    subj = args.subject
+    subj = subj.strip()
+    if subj.endswith('/'):
+        subj = subj[0:len(subj)-1]
 
-    if not op.isdir(subject):
-        sys.exit("\nError: The input folder for the subject does not exist.\n")
+    if not op.isdir(subj):
+        if op.isfile(subj):
+            with open(subj, 'r', encoding="utf-8") as f:
+                subjects = [x.strip() for x in f.readlines()]
+        else:
+            sys.exit("\nError: The folder for the subject does not exist.\n")
+
+    else:
+        subjects = [subj]
 
     configDir = args.configDir
 
-    ctx = Context(subject, cfgdir=configDir)
+    for subject in subjects:
+        ctx = Context(subject, cfgdir=configDir)
 
-    setup_logging(op.join(ctx.subject, 'logs', 'main.log'))
+        setup_logging(op.join(ctx.subject, 'logs', 'main.log'))
 
-    tree = FileTree.read(ctx.get_data('FileTree'),
-                         subject=ctx.subject,
-                         autoptx_tract=list(ctx.tract_struct.keys()),
-                         netmats_dim=["25", "100"])
+        tree = FileTree.read(ctx.get_data('FileTree'),
+                             subject=ctx.subject,
+                             autoptx_tract=list(ctx.tract_struct.keys()),
+                             netmats_dim=["25", "100"])
 
-    # add submit parameters here that should
-    # be applied to all jobs in the pipeline
-    pipe = Pipeline(default_submit={"logdir": Ref('logs_dir'),
-                                    "export_vars": list(os.environ.keys())})
+        # add submit parameters here that should
+        # be applied to all jobs in the pipeline
+        pipe = Pipeline(default_submit={"logdir": Ref('logs_dir'),
+                                        "export_vars": list(os.environ.keys())})
 
-    # pipe = Pipeline(default_submit=dict(logdir=Ref("logs_dir")))
-    # ctx.save_context(tree.get('config_file'))
+        # pipe = Pipeline(default_submit=dict(logdir=Ref("logs_dir")))
+        # ctx.save_context(tree.get('config_file'))
 
-    # This list will be filled with all desired outputs ... Later
-    # targets = []
+        # This list will be filled with all desired outputs ... Later
+        # targets = []
 
-    struct_T1.add_to_pipeline      (ctx, pipe, tree)
-    struct_T2_FLAIR.add_to_pipeline(ctx, pipe, tree)
-    struct_FS.add_to_pipeline      (ctx, pipe, tree)
-    struct_swMRI.add_to_pipeline   (ctx, pipe, tree)
-    struct_asl.add_to_pipeline     (ctx, pipe, tree)
-    dMRI_fieldmap.add_to_pipeline  (ctx, pipe, tree)
-    fMRI_task.add_to_pipeline      (ctx, pipe, tree)
-    fMRI_rest.add_to_pipeline      (ctx, pipe, tree)
-    dMRI_diff.add_to_pipeline      (ctx, pipe, tree)
-    IDPs_gen.add_to_pipeline       (ctx, pipe, tree)
+        struct_T1.add_to_pipeline      (ctx, pipe, tree)
+        struct_T2_FLAIR.add_to_pipeline(ctx, pipe, tree)
+        struct_FS.add_to_pipeline      (ctx, pipe, tree)
+        struct_swMRI.add_to_pipeline   (ctx, pipe, tree)
+        struct_asl.add_to_pipeline     (ctx, pipe, tree)
+        dMRI_fieldmap.add_to_pipeline  (ctx, pipe, tree)
+        fMRI_task.add_to_pipeline      (ctx, pipe, tree)
+        fMRI_rest.add_to_pipeline      (ctx, pipe, tree)
+        dMRI_diff.add_to_pipeline      (ctx, pipe, tree)
+        IDPs_gen.add_to_pipeline       (ctx, pipe, tree)
 
-    # The skip-missing flag deals with cases where the subject is missing
-    # the data of some modalities. For more details, check:
-    # https://open.win.ox.ac.uk/pages/ndcn0236/pipe-tree/tutorial.html#dealing-with-missing-data
-    jobs = pipe.generate_jobs(tree).filter(None, skip_missing=True)
+        # The skip-missing flag deals with cases where the subject is missing
+        # the data of some modalities. For more details, check:
+        # https://open.win.ox.ac.uk/pages/ndcn0236/pipe-tree/tutorial.html#dealing-with-missing-data
+        jobs = pipe.generate_jobs(tree).filter(None, skip_missing=True)
 
-    jobs.report()
+        jobs.report()
 
-    jobs.run("submit")
-    # jobs.run("local")
+        jobs.run("submit")
+        # jobs.run("local")
 
 if __name__ == "__main__":
     main()

bip/pipelines/IDPs_gen/IDP_SWI_T2star.py

@@ -12,7 +12,7 @@
 
 import os.path as op
 import logging
-from pipe_tree import In, Out, Ref
+from fsl_pipe import In, Out, Ref
 from fsl import wrappers
 from bip.utils.log_utils import redirect_logging, job_name
 

bip/pipelines/IDPs_gen/IDP_T1_FIRST_vols.py

@@ -12,7 +12,7 @@
 
 import os.path as op
 import logging
-from pipe_tree import In, Out, Ref
+from fsl_pipe import In, Out, Ref
 from fsl import wrappers
 from bip.utils.log_utils import redirect_logging, job_name
 

bip/pipelines/IDPs_gen/IDP_T1_GM_parcellation.py

@@ -12,7 +12,7 @@
 
 import os.path as op
 import logging
-from pipe_tree import In, Out, Ref
+from fsl_pipe import In, Out, Ref
 from fsl import wrappers
 from bip.utils.log_utils import redirect_logging, tempdir, job_name
 

bip/pipelines/IDPs_gen/IDP_T1_SIENAX.py

@@ -13,7 +13,7 @@
 import os.path as op
 import logging
 from shutil import copyfile
-from pipe_tree import In, Out, Ref
+from fsl_pipe import In, Out, Ref
 from bip.utils.log_utils import redirect_logging, job_name
 
 log = logging.getLogger(__name__)

bip/pipelines/IDPs_gen/IDP_T1_align_to_std.py

@@ -13,7 +13,7 @@
 import os
 import os.path as op
 import logging
-from pipe_tree import In, Out, Ref
+from fsl_pipe import In, Out, Ref
 from fsl import wrappers
 from bip.utils.log_utils import redirect_logging, tempdir, job_name
 

bip/pipelines/IDPs_gen/IDP_T1_noise_ratio.py

@@ -12,7 +12,7 @@
 
 import os.path as op
 import logging
-from pipe_tree import In, Out, Ref
+from fsl_pipe import In, Out, Ref
 from fsl import wrappers
 from bip.utils.log_utils import redirect_logging, tempdir, job_name
 

bip/pipelines/IDPs_gen/IDP_T2_FLAIR_WMH.py

@@ -12,7 +12,7 @@
 
 import logging
 from shutil import copyfile
-from pipe_tree import In, Out, Ref
+from fsl_pipe import In, Out, Ref
 from bip.utils.log_utils import redirect_logging, job_name
 
 log = logging.getLogger(__name__)

bip/pipelines/IDPs_gen/IDP_all_align_to_T1.py

@@ -12,7 +12,7 @@
 
 import os.path as op
 import logging
-from pipe_tree import In, Out, Ref
+from fsl_pipe import In, Out, Ref
 from fsl import wrappers
 from bip.utils.log_utils import redirect_logging, tempdir, job_name
 

bip/pipelines/IDPs_gen/IDP_diff_TBSS.py

@@ -12,7 +12,7 @@
 
 import os.path as op
 import logging
-from pipe_tree import In, Out, Ref
+from fsl_pipe import In, Out, Ref
 from bip.utils.log_utils import redirect_logging, job_name
 
 log = logging.getLogger(__name__)

bip/pipelines/IDPs_gen/IDP_diff_autoPtx.py

@@ -13,7 +13,7 @@
 import os.path as op
 import logging
 import nibabel as nib
-from pipe_tree import In, Out, Ref, Var
+from fsl_pipe import In, Out, Ref, Var
 from fsl import wrappers
 from bip.utils.log_utils import redirect_logging, tempdir, job_name
 

bip/pipelines/IDPs_gen/IDP_diff_eddy_outliers.py

@@ -12,7 +12,7 @@
 
 import os.path as op
 import logging
-from pipe_tree import In, Out, Ref
+from fsl_pipe import In, Out, Ref
 from bip.utils.log_utils import redirect_logging, job_name
 
 log = logging.getLogger(__name__)

bip/pipelines/IDPs_gen/IDP_func_TSNR.py

@@ -12,7 +12,7 @@
 
 import os.path as op
 import logging
-from pipe_tree import In, Out, Ref
+from fsl_pipe import In, Out, Ref
 from fsl import wrappers
 from bip.utils.log_utils import redirect_logging, tempdir, job_name
 

bip/pipelines/IDPs_gen/IDP_func_head_motion.py

@@ -13,7 +13,7 @@
 import os.path as op
 import json
 import logging
-from pipe_tree import In, Out, Ref
+from fsl_pipe import In, Out, Ref
 from bip.utils.log_utils import redirect_logging, job_name
 
 log = logging.getLogger(__name__)

bip/pipelines/IDPs_gen/IDP_func_task_activation.py

@@ -13,7 +13,7 @@ import os
 import os.path as op
 import shutil
 import logging
-from pipe_tree import In, Out, Ref
+from fsl_pipe import In, Out, Ref
 from fsl import wrappers
 from bip.utils.log_utils import redirect_logging, job_name
 

bip/pipelines/IDPs_gen/IDP_subject_COG_table.py

@@ -12,7 +12,7 @@
 
 import os.path as op
 import logging
-from pipe_tree import In, Out, Ref
+from fsl_pipe import In, Out, Ref
 from bip.utils.log_utils import redirect_logging, job_name
 
 log = logging.getLogger(__name__)

bip/pipelines/IDPs_gen/IDP_subject_ID.py

@@ -11,7 +11,7 @@
 #
 
 import logging
-from pipe_tree import In, Out, Ref
+from fsl_pipe import In, Out, Ref
 from bip.utils.log_utils import redirect_logging, job_name
 
 log = logging.getLogger(__name__)

bip/pipelines/IDPs_gen/IDP_subject_centre.py

@@ -12,7 +12,7 @@
 import os.path as op
 import json
 import logging
-from pipe_tree import In, Out, Ref
+from fsl_pipe import In, Out, Ref
 from bip.utils.log_utils import redirect_logging, job_name
 
 log = logging.getLogger(__name__)

bip/pipelines/IDPs_gen/IDPs_generator.py

@@ -12,7 +12,7 @@
 import os.path as op
 import json
 import logging
-from pipe_tree import In, Out, Ref
+from fsl_pipe import In, Out, Ref
 from bip.utils.log_utils import redirect_logging, job_name
 
 log = logging.getLogger(__name__)

bip/pipelines/QC_gen/T1/QC_T1_FIRST_vols.py

@@ -11,7 +11,7 @@
 #
 
 from shutil import copyfile
-from pipe_tree import In, Out
+from fsl_pipe import In, Out
 
 def run(ctx,
         IDP_T1_FIRST_vols: In,