Skip to content
GitLab
Commit 1e1a86bb authored by Andrei Roibu's avatar Andrei Roibu
Browse files

added mean subtraction back as an option

parent 9fb4dd1c
Hide whitespace changes
Inline Side-by-side
run.py
View file @ 1e1a86bb
......@@ -270,7 +270,7 @@ def evaluate_score(training_parameters, network_parameters, misc_parameters, eva
'rsfmri_mean_mask_path'],
dmri_mean_mask_path=mapping_evaluation_parameters[
'dmri_mean_mask_path'],
mean_reduction=mapping_evaluation_parameters['mean_reduction'],
mean_regression=mapping_evaluation_parameters['mean_regression'],
scaling_factors=mapping_evaluation_parameters['scaling_factors'],
regression_factors=mapping_evaluation_parameters['regression_factors'],
device=misc_parameters['device'],
......@@ -305,7 +305,8 @@ def evaluate_mapping(mapping_evaluation_parameters):
device = mapping_evaluation_parameters['device']
exit_on_error = mapping_evaluation_parameters['exit_on_error']
brain_mask_path = mapping_evaluation_parameters['brain_mask_path']
mean_reduction = mapping_evaluation_parameters['mean_reduction']
mean_regression = mapping_evaluation_parameters['mean_regression']
mean_subtraction = mapping_evaluation_parameters['mean_subtraction']
scaling_factors = mapping_evaluation_parameters['scaling_factors']
regression_factors = mapping_evaluation_parameters['regression_factors']
......@@ -317,7 +318,8 @@ def evaluate_mapping(mapping_evaluation_parameters):
brain_mask_path,
dmri_mean_mask_path,
rsfmri_mean_mask_path,
mean_reduction,
mean_regression,
mean_subtraction,
scaling_factors,
regression_factors,
device=device,
......
settings.ini
View file @ 1e1a86bb
......@@ -4,7 +4,7 @@ use_data_file = False
data_directory = "/well/win-biobank/projects/imaging/data/data3/subjectsAll/"
data_file = "/well/win-biobank/projects/imaging/data/data3/subjectsAll/subj_22k.txt"
k_fold = None
data_split_flag = True
data_split_flag = False
test_percentage = 5
subject_number = 12000
train_list = "datasets/train.txt"
......@@ -19,12 +19,13 @@ validation_target_file = "fMRI/rfMRI_25.dr/dr_stage2.nii.gz"
brain_mask_path = "utils/MNI152_T1_2mm_brain_mask.nii.gz"
rsfmri_mean_mask_path = "utils/mean_dr_stage2.nii.gz"
dmri_mean_mask_path = "utils/mean_tractsNormSummed_downsampled.nii.gz"
mean_reduction = True
mean_regression = False
mean_subtraction = True
[TRAINING]
experiment_name = "CU3D17-3"
pre_trained_path = "saved_models/CU3D17-3.pth.tar"
final_model_output_file = "CU3D17-3.pth.tar"
experiment_name = "VA2-1"
pre_trained_path = "saved_models/VA2-1.pth.tar"
final_model_output_file = "VA2-1.pth.tar"
training_batch_size = 5
validation_batch_size = 5
use_pre_trained = False
......@@ -32,9 +33,9 @@ learning_rate = 1e-5
optimizer_beta = (0.9, 0.999)
optimizer_epsilon = 1e-8
optimizer_weigth_decay = 1e-5
number_of_epochs = 200
number_of_epochs = 10
loss_log_period = 50
learning_rate_scheduler_step_size = 4
learning_rate_scheduler_step_size = 5
learning_rate_scheduler_gamma = 1e-1
use_last_checkpoint = False
......@@ -46,7 +47,7 @@ kernel_classification = 7
input_channels = 1
output_channels = 64
convolution_stride = 1
dropout = 0.2
dropout = 0
pool_kernel_size = 3
pool_stride = 2
up_mode = "upconv"
......
settings_evaluation.ini
View file @ 1e1a86bb
[MAPPING]
trained_model_path = "saved_models/test24.pth.tar"
prediction_output_path = "test24_predictions"
trained_model_path = "saved_models/VA2.pth.tar"
prediction_output_path = "VA2_predictions"
data_directory = "/well/win-biobank/projects/imaging/data/data3/subjectsAll/"
mapping_data_file = "dMRI/autoptx_preproc/tractsNormSummed.nii.gz"
data_list = "datasets/test.txt"
......@@ -10,6 +10,7 @@ dmri_mean_mask_path = "utils/mean_tractsNormSummed_downsampled.nii.gz"
mean_mask_path = "utils/mean_dr_stage2.nii.gz"
scaling_factors = "datasets/scaling_factors.pkl"
regression_factors = "datasets/regression_weights.pkl"
mean_reduction = True
mean_regression = False
mean_subtraction = True
device = 0
exit_on_error = True
utils/data_evaluation_utils.py
View file @ 1e1a86bb
......@@ -20,7 +20,6 @@ import numpy as np
import torch
import logging
import utils.data_utils as data_utils
import nibabel as nib
import matplotlib.pyplot as plt
import pandas as pd
from fsl.data.image import Image
......@@ -38,7 +37,7 @@ def evaluate_correlation(trained_model_path,
brain_mask_path,
rsfmri_mean_mask_path,
dmri_mean_mask_path,
mean_reduction,
mean_regression,
scaling_factors,
regression_factors,
device=0,
......@@ -58,7 +57,7 @@ def evaluate_correlation(trained_model_path,
brain_mask_path (str): Path to the MNI brain mask file
rsfmri_mean_mask_path (str): Path to the dualreg subject mean mask
dmri_mean_mask_path (str): Path to the dualreg subject mean mask
mean_reduction (bool): Flag indicating if the targets should be de-meaned using the mean_mask_path
mean_regression (bool): Flag indicating if the targets should be de-meaned using the mean_mask_path
scaling_factors (str): Path to the scaling factor file
regression_factors (str): Path to the linear regression weights file
device (str/int): Device type used for training (int - GPU id, str- CPU)
......@@ -123,7 +122,7 @@ def evaluate_correlation(trained_model_path,
file_path, subject, model,
device, cuda_available, brain_mask_path,
dmri_mean_mask_path, rsfmri_mean_mask_path, scaling_factors,
regression_factors, mean_reduction)
regression_factors, mean_regression)
for target_index, target_path in enumerate(target_paths):
target, target_demeaned = data_utils.load_and_preprocess_targets(
......@@ -172,7 +171,8 @@ def evaluate_mapping(trained_model_path,
brain_mask_path,
dmri_mean_mask_path,
rsfmri_mean_mask_path,
mean_reduction,
mean_regression,
mean_subtraction,
scaling_factors,
regression_factors,
device=0,
......@@ -191,7 +191,8 @@ def evaluate_mapping(trained_model_path,
brain_mask_path (str): Path to the MNI brain mask file
dmri_mean_mask_path (str): Path to the dualreg subject mean mask
rsfmri_mean_mask_path (str): Path to the summed tract mean mask
mean_reduction (bool): Flag indicating if the targets should be de-meaned using the mean_mask_path
mean_regression (bool): Flag indicating if the targets should be de-meaned by regression using the mean_mask_path
mean_subtraction (bool): Flag indicating if the targets should be de-meaned by subtraction using the mean_mask_path
scaling_factors (str): Path to the scaling factor file
regression_factors (str): Path to the linear regression weights file
device (str/int): Device type used for training (int - GPU id, str- CPU)
......@@ -251,7 +252,7 @@ def evaluate_mapping(trained_model_path,
file_path, subject, model,
device, cuda_available, brain_mask_path,
dmri_mean_mask_path, rsfmri_mean_mask_path, scaling_factors,
regression_factors, mean_reduction)
regression_factors, mean_regression, mean_subtraction)
# Generate New Header Affine
......@@ -269,7 +270,7 @@ def evaluate_mapping(trained_model_path,
output_nifti_image.save(output_nifti_path)
if mean_reduction == True:
if mean_regression == True:
output_complete_nifti_image = Image(
predicted_complete_volume, header=header, xform=xform)
......@@ -312,7 +313,8 @@ def _generate_volume_map(file_path,
rsfmri_mean_mask_path,
scaling_factors,
regression_factors,
mean_reduction=False):
mean_regression=False,
mean_subtraction=False):
"""rsfMRI Volume Generator
This function uses the trained model to generate a new volume
......@@ -328,18 +330,18 @@ def _generate_volume_map(file_path,
rsfmri_mean_mask_path (str): Path to the dualreg subject mean mask
scaling_factors (str): Path to the scaling factor file
regression_factors (str): Path to the linear regression weights file
mean_reduction (bool): Flag indicating if the targets should be de-meaned using the mean_mask_path
mean_regression (bool): Flag indicating if the targets should be de-meaned by regression using the mean_mask_path
mean_subtraction (bool): Flag indicating if the targets should be de-meaned by subtraction using the mean_mask_path
Returns
predicted_volume (np.array): Array containing the information regarding the generated volume
header (class): 'nibabel.nifti1.Nifti1Header' class object, containing volume metadata
"""
volume, header, xform = data_utils.load_and_preprocess_evaluation(
file_path)
volume, header, xform = data_utils.load_and_preprocess_evaluation(file_path)
volume = _regress_input(
volume, subject, dmri_mean_mask_path, regression_factors)
if mean_regression == True:
volume = _regress_input(volume, subject, dmri_mean_mask_path, regression_factors)
print('volume range:', np.min(volume), np.max(volume))
......@@ -367,11 +369,13 @@ def _generate_volume_map(file_path,
MNI152_T1_2mm_brain_mask = Image(brain_mask_path).data
if mean_reduction == True:
if mean_regression == True or mean_subtraction == True:
mean_mask = Image(rsfmri_mean_mask_path).data[:, :, :, 0]
weight = pd.read_pickle(regression_factors).loc[subject]['w_rsfMRI']
predicted_complete_volume = np.add(
output, np.multiply(weight, mean_mask))
if mean_regression == True:
weight = pd.read_pickle(regression_factors).loc[subject]['w_rsfMRI']
predicted_complete_volume = np.add(output, np.multiply(weight, mean_mask))
if mean_subtraction == True:
predicted_complete_volume = np.add(output, mean_mask)
print('predicted_complete_volume', np.min(
predicted_complete_volume), np.max(predicted_complete_volume))
......@@ -410,19 +414,19 @@ def _scale_input(volume, scaling_factors):
# min_value, max_value, _, _ = pickle.load(input_file)
# Steve Scaling
# min_value, max_value, _, _ = [0.0, 0.2, 0.0, 10.0]
min_value, max_value, _, _ = [0.0, 0.2, 0.0, 10.0]
# Andrei Scaling
min_value, max_value, _, _ = [-0.0539, 0.0969, -12.094, 14.6319]
# min_value, max_value, _, _ = [-0.0539, 0.0969, -12.094, 14.6319]
# Eliminating outliers
volume[volume > max_value] = max_value
volume[volume < min_value] = min_value
# Normalization to [0, 1]
# scaled_volume = np.divide(np.subtract(volume, min_value), np.subtract(max_value, min_value))
scaled_volume = np.divide(np.subtract(volume, min_value), np.subtract(max_value, min_value))
# Scaling between [-1, 1]
scaled_volume = np.add(-1.0, np.multiply(2.0, np.divide(np.subtract(volume, min_value), np.subtract(max_value, min_value))))
# scaled_volume = np.add(-1.0, np.multiply(2.0, np.divide(np.subtract(volume, min_value), np.subtract(max_value, min_value))))
return scaled_volume
......@@ -469,15 +473,15 @@ def _rescale_output(volume, scaling_factors):
# _, _, min_value, max_value = pickle.load(input_file)
# Steve Scaling
# _, _, min_value, max_value = [0.0, 0.2, 0.0, 10.0]
_, _, min_value, max_value = [0.0, 0.2, 0.0, 10.0]
# Andrei Scaling
_, _, min_value, max_value = [-0.0539, 0.0969, -12.094, 14.6319]
# _, _, min_value, max_value = [-0.0539, 0.0969, -12.094, 14.6319]
# Normalization to [0, 1]
# rescaled_volume = np.add(np.multiply(volume, np.subtract(max_value, min_value)), min_value)
rescaled_volume = np.add(np.multiply(volume, np.subtract(max_value, min_value)), min_value)
# Scaling between [-1, 1]
rescaled_volume = np.add(np.multiply(np.divide(np.add(volume, 1), 2), np.subtract(max_value, min_value)), min_value)
# rescaled_volume = np.add(np.multiply(np.divide(np.add(volume, 1), 2), np.subtract(max_value, min_value)), min_value)
return rescaled_volume
......
utils/data_utils.py
View file @ 1e1a86bb
......@@ -13,11 +13,9 @@ Usage:
"""
import os
import h5py
import pickle
from glob import glob
import numpy as np
import nibabel as nib
import torch
import torch.utils.data as data
import configparser
......@@ -328,7 +326,8 @@ class DataMapper(data.Dataset):
rsfMRI_mean_mask_path (str): Path to the dualreg mean mask
scaling_factors (str): Path to the file containing the scaling factors
regression_weights (str): Path to the file containing the regression_weights
mean_reduction (bool): Flag indicating if the targets should be de-meaned using the mean_mask_path
mean_subtraction (bool): Flag indicating if the targets should be de-meaned by subtraction using the mean_mask_path
mean_regression (bool): Flag indicating if the volumes should be de-meaned by regression using the mean_mask_path
Returns:
X_volume (torch.tensor): Tensor representation of the input data
......@@ -337,7 +336,7 @@ class DataMapper(data.Dataset):
"""
def __init__(self, filename, data_directory, data_file, output_targets, dMRI_mean_mask_path, rsfMRI_mean_mask_path, scaling_factors, regression_weights, mean_reduction=False):
def __init__(self, filename, data_directory, data_file, output_targets, dMRI_mean_mask_path, rsfMRI_mean_mask_path, scaling_factors, regression_weights, mean_subtraction=False, mean_regression=False):
# Initialize everything, and only store in memory the text data file.
# Memory usage limited by only storing the text string information, not the actual volumes.
# TODO: Currently, the timepoint in the fMRI data (y_volume) is hardcoded, only loading in the DMN. This needs to be updated in later iterations.
......@@ -349,7 +348,8 @@ class DataMapper(data.Dataset):
self._get_datasets()
self.dMRI_mean_mask_path = dMRI_mean_mask_path
self.rsfMRI_mean_mask_path = rsfMRI_mean_mask_path
self.mean_reduction = mean_reduction
self.mean_subtraction = mean_subtraction
self.mean_regression = mean_regression
self.scaling_factors = scaling_factors
self.regression_weights = regression_weights
......@@ -360,16 +360,15 @@ class DataMapper(data.Dataset):
X_path, y_path, subject_id = self.sample_pairs[index]
if self.mean_reduction == True:
X_volume = torch.from_numpy(self.scale_volume(self.linear_regress_mean(
X_path, subject_id, target_flag=False), target_flag=False))
y_volume = torch.from_numpy(self.scale_volume(self.linear_regress_mean(
y_path, subject_id, target_flag=True), target_flag=True))
if self.mean_regression == True:
X_volume = torch.from_numpy(self.scale_volume(self.linear_regress_mean(X_path, subject_id, target_flag=False), target_flag=False))
y_volume = torch.from_numpy(self.scale_volume(self.linear_regress_mean(y_path, subject_id, target_flag=True), target_flag=True))
elif self.mean_subtraction == True:
X_volume = torch.from_numpy(self.scale_volume(self.resample(X_path), target_flag=False))
y_volume = torch.from_numpy(self.scale_volume(self.subtract_mean(y_path), target_flag=True))
else:
X_volume = torch.from_numpy(self.scale_volume(
self.resample(X_path), target_flag=False))
y_volume = torch.from_numpy(self.scale_volume(
self.convert_to_numpy(y_path)[:, :, :, 0], target_flag=True))
X_volume = torch.from_numpy(self.scale_volume(self.resample(X_path), target_flag=False))
y_volume = torch.from_numpy(self.scale_volume(self.convert_to_numpy(y_path)[:, :, :, 0], target_flag=True))
return X_volume, y_volume
......@@ -441,6 +440,26 @@ class DataMapper(data.Dataset):
volume_numpy = self.read_data_files(path).data
return volume_numpy
def subtract_mean(self, path):
"""Mean Mask Substraction
Helper function which substracts the dualreg mean subject mask
Args:
mean_mask_path (str): Path to the dualreg subject mean mask
Returns:
subtracted_volume (np.array): Numpy array representation of the subtracted volume data
"""
dualreg_subject_mean = self.convert_to_numpy(self.rsfMRI_mean_mask_path)[:, :, :, 0]
volume = self.convert_to_numpy(path)[:, :, :, 0]
subtracted_volume = np.subtract(volume, dualreg_subject_mean)
return subtracted_volume
def linear_regress_mean(self, path, subject, target_flag):
"""Linear regressed mean subtraction
......@@ -490,10 +509,10 @@ class DataMapper(data.Dataset):
# min_input, max_input, min_target, max_target = pickle.load(input_file)
# # Steve Scaling
# min_input, max_input, min_target, max_target = [0.0, 0.2, 0.0, 10.0]
min_input, max_input, min_target, max_target = [0.0, 0.2, 0.0, 10.0]
# Andrei Scaling
min_input, max_input, min_target, max_target = [-0.0539, 0.0969, -12.094, 14.6319]
# min_input, max_input, min_target, max_target = [-0.0539, 0.0969, -12.094, 14.6319]
if target_flag == False:
min_value = min_input
......@@ -503,7 +522,7 @@ class DataMapper(data.Dataset):
max_value = max_target
# Set all negative elements to 0
# volume[volume < 0] = 0.0
volume[volume < 0] = 0.0
# Eliminating outliers
volume[volume > max_value] = max_value
......@@ -552,16 +571,17 @@ def get_datasets(data_parameters):
dMRI_mean_mask_path = data_parameters['dmri_mean_mask_path']
rsfMRI_mean_mask_path = data_parameters['rsfmri_mean_mask_path']
mean_reduction = data_parameters['mean_reduction']
mean_regression = data_parameters['mean_regression']
mean_subtraction = data_parameters['mean_subtraction']
scaling_factors = data_parameters['scaling_factors']
regression_weights = data_parameters['regression_weights']
return (
DataMapper(train_filename, data_directory, train_data_file, train_output_targets,
dMRI_mean_mask_path, rsfMRI_mean_mask_path, scaling_factors, regression_weights, mean_reduction),
dMRI_mean_mask_path, rsfMRI_mean_mask_path, scaling_factors, regression_weights, mean_subtraction, mean_regression),
DataMapper(validation_filename, data_directory, validation_data_file, validation_output_targets,
dMRI_mean_mask_path, rsfMRI_mean_mask_path, scaling_factors, regression_weights, mean_reduction)
dMRI_mean_mask_path, rsfMRI_mean_mask_path, scaling_factors, regression_weights, mean_subtraction, mean_regression)
)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment