Commit f214cf4c authored by Andrei-Claudiu Roibu's avatar Andrei-Claudiu Roibu 🖥
Browse files

added several forms of data scaling

parent f5c757b6
......@@ -217,27 +217,50 @@ def data_extremes(output_file_path, train_list, data_directory, train_inputs, tr
input_volume, _ = resampleToPixdims(Image(input_path), (2,2,2))
target_volume = np.subtract(Image(target_path).data[:,:,:,0], dualreg_subject_mean)
target_volume[target_volume < 0] = 0
# TODO: Currently, the volume of the fMRI data is hardcoded, only loading in the DMN. This needs to be updated in later iterations.
# if min_input == None:
# min_input = np.min(input_volume)
# elif min_input > np.min(input_volume):
# min_input = np.min(input_volume)
#
# if max_input == None:
# max_input = np.max(input_volume)
# elif max_input < np.max(input_volume):
# max_input = np.max(input_volume)
#
# if min_target == None:
# min_target = np.min(target_volume)
# elif min_target > np.min(target_volume):
# min_target = np.min(target_volume)
#
# if max_target == None:
# max_target = np.max(target_volume)
# elif max_target < np.max(target_volume):
# max_target = np.max(target_volume)
if min_input == None:
min_input = np.min(input_volume)
elif min_input > np.min(input_volume):
min_input = np.min(input_volume)
min_input = np.percentile(input_volume, 1)
elif min_input > np.percentile(input_volume, 1):
min_input = np.percentile(input_volume, 1)
if max_input == None:
max_input = np.max(input_volume)
elif max_input < np.max(input_volume):
max_input = np.max(input_volume)
max_input = np.percentile(input_volume, 99)
elif max_input < np.percentile(input_volume, 99):
max_input = np.percentile(input_volume, 99)
if min_target == None:
min_target = np.min(target_volume)
elif min_target > np.min(target_volume):
min_target = np.min(target_volume)
min_target = np.percentile(target_volume, 1)
elif min_target > np.percentile(target_volume, 1):
min_target = np.percentile(target_volume, 1)
if max_target == None:
max_target = np.max(target_volume)
elif max_target < np.max(target_volume):
max_target = np.max(target_volume)
max_target = np.percentile(target_volume, 99)
elif max_target < np.percentile(target_volume, 99):
max_target = np.percentile(target_volume, 99)
del input_volume, target_volume, input_path, target_path
......@@ -310,14 +333,12 @@ class DataMapper(data.Dataset):
X_path, y_path = self.sample_pairs[index]
X_volume = self.scale_volume(torch.from_numpy(self.resample(X_path)), self.scaling_factors, target_flag=False)
X_volume = torch.from_numpy(self.scale_volume(self.resample(X_path), self.scaling_factors, target_flag=False))
if self.mean_reduction == True:
y_volume = torch.from_numpy(
self.scale_volume(self.subtract_mean(y_path, self.mean_mask_path), self.scaling_factors, target_flag=True))
y_volume = torch.from_numpy(self.scale_volume(self.subtract_mean(y_path, self.mean_mask_path), self.scaling_factors, target_flag=True))
else:
y_volume = torch.from_numpy(
self.scale_volume(self.convert_to_numpy(y_path)[:, :, :, 0], self.scaling_factors, target_flag=True))
y_volume = torch.from_numpy(self.scale_volume(self.convert_to_numpy(y_path)[:, :, :, 0], self.scaling_factors, target_flag=True))
return X_volume, y_volume
......@@ -426,17 +447,30 @@ class DataMapper(data.Dataset):
with open(scaling_factors, 'rb') as input_file:
min_input, max_input, min_target, max_target = pickle.load(input_file)
# Steve Scaling
min_input, max_input, min_target, max_target = [0.0, 0.2, 0.0, 10.0]
if target_flag == False:
min_value = min_input
max_value = max_input
elif target_flag == True:
min_value = min_target
max_value = max_target
# Set all negative elements to 0
volume[volume < 0] = 0.0
# Eliminating outliers
volume[volume > max_value] = max_value
volume[volume < min_value] = min_value
# Normalization to [0, 1]
scaled_volume = np.divide(np.subtract(volume, min_value), np.subtract(max_value, min_value))
# Scaling between [-1, 1]
# scaled_volume = np.add(-1.0, np.multiply(2.0, np.divide(np.subtract(volume, min_value), np.subtract(max_value, min_value))))
# No scaling performed
# scaled_volume = volume
return scaled_volume
......@@ -526,56 +560,59 @@ def load_file_paths(data_directory, data_list, mapping_data_file, targets_direct
for volume in volumes_to_be_used]
else:
file_paths = [[os.path.join(data_directory, volume, mapping_data_file), os.path.join(
targets_directory, volume, )] for volume in volumes_to_be_used]
targets_directory, volume)] for volume in volumes_to_be_used]
return file_paths
def load_and_preprocess(file_paths, orientation):
"""Load & Preprocess
def load_and_preprocess_evaluation(file_path):
"""Load & Preprocessing before evaluation
This function is composed of two other function calls: one that calls a function loading the data, and another which preprocesses the data to the required format.
# TODO: Need to check if any more proprocessing would be required besides summing the tracts!
This function loads a nifty file and returns its volume and header information
Args:
file_paths (list): List containing the input data and target labelled output data
orientation (str): String detailing the current view (COR, SAG, AXL)
file_path (str): Path to the desired file
Returns:
volume (np.array): Array of training image data of data type dtype.
label_map (np.array): Array of labelled image data of data type dtype.
header (class): 'nibabel.nifti1.Nifti1Header' class object, containing image metadata
xform (np.array): Array of shape (4, 4), containing the adjusted voxel-to-world transformation for the spatial dimensions of the resampled data
Raises:
ValueError: "Orientation value is invalid. It must be either >>coronal<<, >>axial<< or >>sagital<< "
"""
volume, label_map, header = load(file_paths, orientation)
original_image = Image(file_path[0])
volume, xform = resampleToPixdims(original_image, (2, 2, 2))
header = Image(volume, header=original_image.header, xform=xform).header
return volume, label_map, header
return volume, header, xform
def load(file_path, orientation):
"""Load Data Function
def load_and_preprocess_targets(target_path, mean_mask_path):
"""Load & Preprocessing targets before evaluation
This function loads the required data files and extracts relevant information from it.
This function loads a nifty file and returns its volume, a de-meaned volume and header information
Args:
file_path (list): List containing the input data and target labelled output data
orientation (str): String detailing the current view (COR, SAG, AXL)
file_path (str): Path to the desired target file
mean_mask_path (str): Path to the dualreg subject mean mask
Returns:
volume (np.array): Array of training image data of data type dtype.
label_map (np.array): Array of labelled image data of data type dtype.
header (class): 'nibabel.nifti1.Nifti1Header' class object, containing image metadata
target (np.array): Array of training image data of data type dtype.
target_demeaned (np.array): Array of training data from which the group mean has been subtracted
Raises:
ValueError: "Orientation value is invalid. It must be either >>coronal<<, >>axial<< or >>sagital<< "
"""
nifty_volume, label_map = nib.load(file_path[0]), nib.load(file_path[1])
volume, label_map = nifty_volume.get_fdata(), label_map.ged_fdata()
target = Image(target_path[0]).data[:,:,:,0]
target_demeaned = np.subtract(target, Image(mean_mask_path).data[:,:,:,0])
# Do we need min-max normalization here? Will need to check when debuggint and testing
volume = (volume - np.min(volume)) / (np.max(volume) - np.min(volume))
return target, target_demeaned
volume, label_map = set_orientation(volume, label_map, orientation)
return volume, label_map, nifty_volume.header
# Deprecated Functions & Classes & Methods:
def set_orientation(volume, label_map, orientation):
......@@ -610,33 +647,6 @@ def set_orientation(volume, label_map, orientation):
"Orientation value is invalid. It must be either >>coronal<<, >>axial<< or >>sagital<< ")
def load_and_preprocess_evaluation(file_path):
"""Load & Preprocessing before evaluation
This function loads a nifty file and returns its volume and header information
Args:
file_path (str): Path to the desired file
Returns:
volume (np.array): Array of training image data of data type dtype.
header (class): 'nibabel.nifti1.Nifti1Header' class object, containing image metadata
xform (np.array): Array of shape (4, 4), containing the adjusted voxel-to-world transformation for the spatial dimensions of the resampled data
Raises:
ValueError: "Orientation value is invalid. It must be either >>coronal<<, >>axial<< or >>sagital<< "
"""
original_image = Image(file_path[0])
volume, xform = resampleToPixdims(original_image, (2, 2, 2))
header = Image(volume, header=original_image.header, xform=xform).header
return volume, header, xform
# Deprecated Functions & Classes & Methods:
def tract_sum_generator(folder_path):
"""Sums the tracts of different dMRI files
......@@ -810,3 +820,51 @@ def load_and_preprocess_evaluation2D(file_path, orientation, min_max=True):
"Orientation value is invalid. It must be either >>coronal<<, >>axial<< or >>sagital<< ")
return volume, header
def load_and_preprocess(file_paths, orientation):
"""Load & Preprocess
This function is composed of two other function calls: one that calls a function loading the data, and another which preprocesses the data to the required format.
# TODO: Need to check if any more proprocessing would be required besides summing the tracts!
Args:
file_paths (list): List containing the input data and target labelled output data
orientation (str): String detailing the current view (COR, SAG, AXL)
Returns:
volume (np.array): Array of training image data of data type dtype.
label_map (np.array): Array of labelled image data of data type dtype.
header (class): 'nibabel.nifti1.Nifti1Header' class object, containing image metadata
"""
volume, label_map, header = load(file_paths, orientation)
return volume, label_map, header
def load(file_path, orientation):
"""Load Data Function
This function loads the required data files and extracts relevant information from it.
Args:
file_path (list): List containing the input data and target labelled output data
orientation (str): String detailing the current view (COR, SAG, AXL)
Returns:
volume (np.array): Array of training image data of data type dtype.
label_map (np.array): Array of labelled image data of data type dtype.
header (class): 'nibabel.nifti1.Nifti1Header' class object, containing image metadata
"""
nifty_volume, label_map = nib.load(file_path[0]), nib.load(file_path[1])
volume, label_map = nifty_volume.get_fdata(), label_map.ged_fdata()
# Do we need min-max normalization here? Will need to check when debuggint and testing
volume = (volume - np.min(volume)) / (np.max(volume) - np.min(volume))
volume, label_map = set_orientation(volume, label_map, orientation)
return volume, label_map, nifty_volume.header
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment