added several forms of data scaling

utils/data_utils.py

@@ -217,27 +217,50 @@ def data_extremes(output_file_path, train_list, data_directory, train_inputs, tr
             input_volume, _ = resampleToPixdims(Image(input_path), (2,2,2))
             target_volume = np.subtract(Image(target_path).data[:,:,:,0], dualreg_subject_mean)
 
+            target_volume[target_volume < 0] = 0
+
             # TODO: Currently, the volume of the fMRI data is hardcoded, only loading in the DMN. This needs to be updated in later iterations.
 
+#            if min_input == None:
+#                min_input = np.min(input_volume)
+#            elif min_input > np.min(input_volume):
+#                min_input = np.min(input_volume)
+#
+#            if max_input == None:
+#                max_input = np.max(input_volume)
+#            elif max_input < np.max(input_volume):
+#                max_input = np.max(input_volume)
+#
+#            if min_target == None:
+#                min_target = np.min(target_volume)
+#            elif min_target > np.min(target_volume):
+#                min_target = np.min(target_volume)
+#
+#            if max_target == None:
+#                max_target = np.max(target_volume)
+#            elif max_target < np.max(target_volume):
+#                max_target = np.max(target_volume)
+
             if min_input == None:
-                min_input = np.min(input_volume)
-            elif min_input > np.min(input_volume):
-                min_input = np.min(input_volume)
+                min_input = np.percentile(input_volume, 1)
+            elif min_input > np.percentile(input_volume, 1):
+                min_input = np.percentile(input_volume, 1)
 
             if max_input == None:
-                max_input = np.max(input_volume)
-            elif max_input < np.max(input_volume):
-                max_input = np.max(input_volume)
+                max_input = np.percentile(input_volume, 99)
+            elif max_input < np.percentile(input_volume, 99):
+                max_input = np.percentile(input_volume, 99)
 
             if min_target == None:
-                min_target = np.min(target_volume)
-            elif min_target > np.min(target_volume):
-                min_target = np.min(target_volume)
+                min_target = np.percentile(target_volume, 1)
+            elif min_target > np.percentile(target_volume, 1):
+                min_target = np.percentile(target_volume, 1)
 
             if max_target == None:
-                max_target = np.max(target_volume)
-            elif max_target < np.max(target_volume):
-                max_target = np.max(target_volume)
+                max_target = np.percentile(target_volume, 99)
+            elif max_target < np.percentile(target_volume, 99):
+                max_target = np.percentile(target_volume, 99)
+
 
             del input_volume, target_volume, input_path, target_path
 
@@ -310,14 +333,12 @@ class DataMapper(data.Dataset):
 
         X_path, y_path = self.sample_pairs[index]
 
-        X_volume = self.scale_volume(torch.from_numpy(self.resample(X_path)), self.scaling_factors, target_flag=False)
+        X_volume = torch.from_numpy(self.scale_volume(self.resample(X_path), self.scaling_factors, target_flag=False))
 
         if self.mean_reduction == True:
-            y_volume = torch.from_numpy(
-                self.scale_volume(self.subtract_mean(y_path, self.mean_mask_path), self.scaling_factors, target_flag=True))
+            y_volume = torch.from_numpy(self.scale_volume(self.subtract_mean(y_path, self.mean_mask_path), self.scaling_factors, target_flag=True))
         else:
-            y_volume = torch.from_numpy(
-                self.scale_volume(self.convert_to_numpy(y_path)[:, :, :, 0], self.scaling_factors, target_flag=True))
+            y_volume = torch.from_numpy(self.scale_volume(self.convert_to_numpy(y_path)[:, :, :, 0], self.scaling_factors, target_flag=True))
 
         return X_volume, y_volume
 
@@ -426,6 +447,9 @@ class DataMapper(data.Dataset):
         with open(scaling_factors, 'rb') as input_file:
             min_input, max_input, min_target, max_target = pickle.load(input_file)
 
+        # Steve Scaling
+        min_input, max_input, min_target, max_target = [0.0, 0.2, 0.0, 10.0]
+
         if target_flag == False:
             min_value = min_input
             max_value = max_input
@@ -433,11 +457,21 @@ class DataMapper(data.Dataset):
             min_value = min_target
             max_value = max_target
 
+        # Set all negative elements to 0
+        volume[volume < 0] = 0.0        
+
+        # Eliminating outliers
+        volume[volume > max_value] = max_value
+        volume[volume < min_value] = min_value
+        
         # Normalization to [0, 1]
         scaled_volume = np.divide(np.subtract(volume, min_value), np.subtract(max_value, min_value)) 
         # Scaling between [-1, 1]
         # scaled_volume = np.add(-1.0, np.multiply(2.0, np.divide(np.subtract(volume, min_value), np.subtract(max_value, min_value))))
         
+        # No scaling performed
+        # scaled_volume = volume
+
         return scaled_volume
 
 
@@ -526,56 +560,59 @@ def load_file_paths(data_directory, data_list, mapping_data_file, targets_direct
                       for volume in volumes_to_be_used]
     else:
         file_paths = [[os.path.join(data_directory, volume, mapping_data_file), os.path.join(
-            targets_directory, volume, )] for volume in volumes_to_be_used]
+            targets_directory, volume)] for volume in volumes_to_be_used]
 
     return file_paths
 
 
-def load_and_preprocess(file_paths, orientation):
-    """Load & Preprocess
+def load_and_preprocess_evaluation(file_path):
+    """Load & Preprocessing before evaluation
 
-    This function is composed of two other function calls: one that calls a function loading the data, and another which preprocesses the data to the required format.
-    # TODO: Need to check if any more proprocessing would be required besides summing the tracts!
+    This function loads a nifty file and returns its volume and header information
 
     Args:
-        file_paths (list): List containing the input data and target labelled output data
-        orientation (str): String detailing the current view (COR, SAG, AXL)
+        file_path (str): Path to the desired file
 
     Returns:
         volume (np.array): Array of training image data of data type dtype.
-        label_map (np.array): Array of labelled image data of data type dtype.
         header (class): 'nibabel.nifti1.Nifti1Header' class object, containing image metadata
+        xform (np.array): Array of shape (4, 4), containing the adjusted voxel-to-world transformation for the spatial dimensions of the resampled data
+
+    Raises:
+        ValueError: "Orientation value is invalid. It must be either >>coronal<<, >>axial<< or >>sagital<< "
     """
 
-    volume, label_map, header = load(file_paths, orientation)
+    original_image = Image(file_path[0])
+    volume, xform = resampleToPixdims(original_image, (2, 2, 2))
+    header = Image(volume, header=original_image.header, xform=xform).header
 
-    return volume, label_map, header
+    return volume, header, xform
 
 
-def load(file_path, orientation):
-    """Load Data Function
+def load_and_preprocess_targets(target_path, mean_mask_path):
+    """Load & Preprocessing targets before evaluation
 
-    This function loads the required data files and extracts relevant information from it.
+    This function loads a nifty file and returns its volume, a de-meaned volume and header information
 
     Args:
-        file_path (list): List containing the input data and target labelled output data
-        orientation (str): String detailing the current view (COR, SAG, AXL)
+        file_path (str): Path to the desired target file
+        mean_mask_path (str): Path to the dualreg subject mean mask
 
     Returns:
-        volume (np.array): Array of training image data of data type dtype.
-        label_map (np.array): Array of labelled image data of data type dtype.
-        header (class): 'nibabel.nifti1.Nifti1Header' class object, containing image metadata
+        target (np.array): Array of training image data of data type dtype.
+        target_demeaned (np.array): Array of training data from which the group mean has been subtracted
+
+    Raises:
+        ValueError: "Orientation value is invalid. It must be either >>coronal<<, >>axial<< or >>sagital<< "
     """
 
-    nifty_volume, label_map = nib.load(file_path[0]), nib.load(file_path[1])
-    volume, label_map = nifty_volume.get_fdata(), label_map.ged_fdata()
+    target = Image(target_path[0]).data[:,:,:,0]
+    target_demeaned = np.subtract(target, Image(mean_mask_path).data[:,:,:,0])
 
-    # Do we need min-max normalization here? Will need to check when debuggint and testing
-    volume = (volume - np.min(volume)) / (np.max(volume) - np.min(volume))
+    return target, target_demeaned
 
-    volume, label_map = set_orientation(volume, label_map, orientation)
 
-    return volume, label_map, nifty_volume.header
+# Deprecated Functions & Classes & Methods:
 
 
 def set_orientation(volume, label_map, orientation):
@@ -610,33 +647,6 @@ def set_orientation(volume, label_map, orientation):
             "Orientation value is invalid. It must be either >>coronal<<, >>axial<< or >>sagital<< ")
 
 
-def load_and_preprocess_evaluation(file_path):
-    """Load & Preprocessing before evaluation
-
-    This function loads a nifty file and returns its volume and header information
-
-    Args:
-        file_path (str): Path to the desired file
-
-    Returns:
-        volume (np.array): Array of training image data of data type dtype.
-        header (class): 'nibabel.nifti1.Nifti1Header' class object, containing image metadata
-        xform (np.array): Array of shape (4, 4), containing the adjusted voxel-to-world transformation for the spatial dimensions of the resampled data
-
-    Raises:
-        ValueError: "Orientation value is invalid. It must be either >>coronal<<, >>axial<< or >>sagital<< "
-    """
-
-    original_image = Image(file_path[0])
-    volume, xform = resampleToPixdims(original_image, (2, 2, 2))
-    header = Image(volume, header=original_image.header, xform=xform).header
-
-    return volume, header, xform
-
-
-# Deprecated Functions & Classes & Methods:
-
-
 def tract_sum_generator(folder_path):
     """Sums the tracts of different dMRI files
 
@@ -810,3 +820,51 @@ def load_and_preprocess_evaluation2D(file_path, orientation, min_max=True):
             "Orientation value is invalid. It must be either >>coronal<<, >>axial<< or >>sagital<< ")
 
     return volume, header
+
+
+def load_and_preprocess(file_paths, orientation):
+    """Load & Preprocess
+
+    This function is composed of two other function calls: one that calls a function loading the data, and another which preprocesses the data to the required format.
+    # TODO: Need to check if any more proprocessing would be required besides summing the tracts!
+
+    Args:
+        file_paths (list): List containing the input data and target labelled output data
+        orientation (str): String detailing the current view (COR, SAG, AXL)
+
+    Returns:
+        volume (np.array): Array of training image data of data type dtype.
+        label_map (np.array): Array of labelled image data of data type dtype.
+        header (class): 'nibabel.nifti1.Nifti1Header' class object, containing image metadata
+    """
+
+    volume, label_map, header = load(file_paths, orientation)
+
+    return volume, label_map, header
+
+
+def load(file_path, orientation):
+    """Load Data Function
+
+    This function loads the required data files and extracts relevant information from it.
+
+    Args:
+        file_path (list): List containing the input data and target labelled output data
+        orientation (str): String detailing the current view (COR, SAG, AXL)
+
+    Returns:
+        volume (np.array): Array of training image data of data type dtype.
+        label_map (np.array): Array of labelled image data of data type dtype.
+        header (class): 'nibabel.nifti1.Nifti1Header' class object, containing image metadata
+    """
+
+    nifty_volume, label_map = nib.load(file_path[0]), nib.load(file_path[1])
+    volume, label_map = nifty_volume.get_fdata(), label_map.ged_fdata()
+
+    # Do we need min-max normalization here? Will need to check when debuggint and testing
+    volume = (volume - np.min(volume)) / (np.max(volume) - np.min(volume))
+
+    volume, label_map = set_orientation(volume, label_map, orientation)
+
+    return volume, label_map, nifty_volume.header
+