data_utils.py

"""Data Processing Functions

Description:

    This folder contains several functions which, either on their own or included in larger pieces of software, perform processing tasks on the data.

Usage:

    To use content from this folder, import the functions and instantiate them as you wish to use them:

        from utils.data_utils import function_name

"""

import os
import h5py
import pickle
from glob import glob
import numpy as np
import nibabel as nib
import torch
import torch.utils.data as data
import configparser
from fsl.data.image import Image
from fsl.utils.image.resample import resampleToPixdims
from sklearn.model_selection import KFold, train_test_split


def directory_reader(folder_location, subject_number=None, write_txt=False):
    """Produces a list of of data-tags which are accessible

    This function looks in a large data directory, and returns a list of sub-directories which are accessible.
    This is done as currently, not all UK Biobank Data is accessible due to privacy issues.

    Args:
        folder_location (str): A string containing the address of the required directory.
        write_txt (bool): Flag indicating if a .txt file should be created.
        suject_number (int): Number of subjects to be considered for a job. Useful when wanting to train on datasizes smaller than total datapoints available in a datafolder.
    Returns:
        subDirectoryList (list): A list of strings containing the available sub-directories. This is also printed out as a .txt file
    """
    if write_txt == True:
        out_file = open("files.txt", 'w')

    subDirectoryList = []

    number_of_subjects = 0

    if subject_number is None:
        subject_number = len(os.listdir(os.path.join(
            os.path.expanduser("~"), folder_location)))

    for directory in os.listdir(folder_location):
        if number_of_subjects < subject_number:
            if os.path.isdir(os.path.join(folder_location, directory)) and os.path.exists(os.path.join(folder_location, directory, "dMRI/autoptx_preproc/")) and os.path.exists(os.path.join(folder_location, directory, "fMRI/rfMRI_25.dr/")):
                filename = folder_location+directory
                if os.access(filename, os.R_OK):
                    string = directory
                    if write_txt == True:
                        out_file.write(string)
                        out_file.write("\n")
                    subDirectoryList.append(directory)
                    number_of_subjects += 1
        else:
            break

    return subDirectoryList


def data_file_reader(data_file_path, folder_location, subject_number=None):
    """Data File reader

    Args:
        data_file_path (str): Path to the file containing the data
        folder_location (str): A string containing the address of the required directory.
        subject_number (int): Number of subjects to be considered for a job. Useful when wanting to train on datasizes smaller than total datapoints available in a datafolder.

    Returns:
        subDirectoryList (list): A list of strings containing the available sub-directories
    """

    with open(data_file_path) as files:
        subDirectoryList = files.read().split('\n')
        subDirectoryList.remove('')

    for directory in subDirectoryList:
        if os.path.exists(os.path.join(folder_location, directory, "dMRI/autoptx_preproc/")) == False:
            if os.path.exists(os.path.join(folder_location, directory, "fMRI/rfMRI_25.dr/")) == False:
                subDirectoryList.remove(directory)

    if subject_number is not None:
        subDirectoryList = subDirectoryList[:subject_number]

    return subDirectoryList


def data_test_train_validation_split(data_folder_name, test_percentage, subject_number, data_directory, train_inputs, train_targets, mean_mask_path, data_file=None, K_fold=None):
    """Produces lists of train, test and validation data

    This function looks at the list of all available directories and returns three lists of dsub-directories.
    These lists are the lists required for training, testing and validation.

    Args:
        data_folder_name (str): The name of the folder where the string data is being output
        test_percentage (int): Percentage of data to be used for testing
        suject_number (int): Number of subjects to be considered for a job. Useful when wanting to train on datasizes smaller than total datapoints available in a datafolder.
        data_directory (str): A string containing the address of the required directory.
        train_inputs (str): Path to the training input files
        train_targets (str): Path to the training target files
        mean_mask_path (str): Path to the dualreg subject mean mask
        K_fold (int): Number of folds for splitting the training data
        data_file (str): Name of *.txt file containing a list of the required data

    """

    if data_file is not None:
        subDirectoryList = data_file_reader(
            data_file, data_directory, subject_number)
    else:
        subDirectoryList = directory_reader(data_directory, subject_number)

    subDirectoryList = np.array(subDirectoryList)
    create_folder(data_folder_name)

    train_data, test = train_test_split(
        subDirectoryList, test_size=test_percentage/100, random_state=42, shuffle=True)
    if os.path.exists(os.path.join(data_folder_name, 'test.txt')):
        os.remove(os.path.join(data_folder_name, 'test.txt'))
    np.savetxt(os.path.join(data_folder_name, 'test.txt'), test, fmt='%s')

    if K_fold is None:
        train, validation = train_test_split(
            train_data, test_size=int(len(test)), random_state=42, shuffle=True)

        if os.path.exists(os.path.join(data_folder_name, 'train.txt')):
            os.remove(os.path.join(data_folder_name, 'train.txt'))
        np.savetxt(os.path.join(data_folder_name,
                                'train.txt'), train, fmt='%s')

        if os.path.exists(os.path.join(data_folder_name, 'validation.txt')):
            os.remove(os.path.join(data_folder_name, 'validation.txt'))
        np.savetxt(os.path.join(data_folder_name, 'validation.txt'),
                   validation, fmt='%s')

        print('Generating training dataset min and max...')

        data_extremes(os.path.join(data_folder_name, 'scaling_factors.pkl'),
                      os.path.join(data_folder_name, 'train.txt'),
                      data_directory, train_inputs, train_targets, mean_mask_path)

        print('Global training dataset min and max values generated!')

    else:
        k_fold = KFold(n_splits=K_fold)
        k = 0
        for train_index, validation_index in k_fold.split(train_data):
            train, validation = train_data[train_index], train_data[validation_index]

            if os.path.exists(os.path.join(data_folder_name, 'train' + str(k+1) + '.txt')):
                os.remove(os.path.join(data_folder_name,
                                       'train' + str(k+1) + '.txt'))
            np.savetxt(os.path.join(data_folder_name, 'train' +
                                    str(k+1)+'.txt'), train, fmt='%s')

            if os.path.exists(os.path.join(data_folder_name, 'validation' + str(k+1) + '.txt')):
                os.remove(os.path.join(data_folder_name,
                                       'validation' + str(k+1) + '.txt'))
            np.savetxt(os.path.join(data_folder_name, 'validation' +
                                    str(k+1)+'.txt'), validation, fmt='%s')

            print(
                'Generating training dataset min and max for fold K = {}/{}...'.format(k+1, K_fold))

            data_extremes(os.path.join(data_folder_name, 'scaling_factors'+str(k+1)+'.pkl'),
                          os.path.join(data_folder_name,
                                       'train' + str(k+1) + '.txt'),
                          data_directory, train_inputs, train_targets, mean_mask_path)

            print(
                'Global training dataset min and max values for fold K = {}/{} generated!'.format(k+1, K_fold))

            k += 1


def data_extremes(output_file_path, train_list, data_directory, train_inputs, train_targets, mean_mask_path):
    """Calculates min-max for train dataset

    This function calculates the min and max of the entire training dataset.
    After, it saves the values to a pkl file.

    Args:
        output_file_path (str): Path to the output file
        train_list (str): Path to the file containing the list of training volumes
        data_directory (str): A string containing the address of the required directory.
        train_inputs (str): Path to the training input files
        train_targets (str): Path to the training target files
        mean_mask_path (str): Path to the dualreg subject mean mask
    """

    min_input = None
    max_input = None
    min_target = None
    max_target = None

    dualreg_subject_mean = Image(mean_mask_path).data[:, :, :, 0]

    with open(train_list) as volume_list:
        lines = volume_list.read().split('\n')

    for line in lines:
        if line == '':
            pass
        else:
            input_path = os.path.join(data_directory, line, train_inputs)
            target_path = os.path.join(data_directory, line, train_targets)

            input_volume, _ = resampleToPixdims(Image(input_path), (2,2,2))
            target_volume = np.subtract(Image(target_path).data[:,:,:,0], dualreg_subject_mean)

            target_volume[target_volume < 0] = 0

            # TODO: Currently, the volume of the fMRI data is hardcoded, only loading in the DMN. This needs to be updated in later iterations.

#            if min_input == None:
#                min_input = np.min(input_volume)
#            elif min_input > np.min(input_volume):
#                min_input = np.min(input_volume)
#
#            if max_input == None:
#                max_input = np.max(input_volume)
#            elif max_input < np.max(input_volume):
#                max_input = np.max(input_volume)
#
#            if min_target == None:
#                min_target = np.min(target_volume)
#            elif min_target > np.min(target_volume):
#                min_target = np.min(target_volume)
#
#            if max_target == None:
#                max_target = np.max(target_volume)
#            elif max_target < np.max(target_volume):
#                max_target = np.max(target_volume)

            if min_input == None:
                min_input = np.percentile(input_volume, 1)
            elif min_input > np.percentile(input_volume, 1):
                min_input = np.percentile(input_volume, 1)

            if max_input == None:
                max_input = np.percentile(input_volume, 99)
            elif max_input < np.percentile(input_volume, 99):
                max_input = np.percentile(input_volume, 99)

            if min_target == None:
                min_target = np.percentile(target_volume, 1)
            elif min_target > np.percentile(target_volume, 1):
                min_target = np.percentile(target_volume, 1)

            if max_target == None:
                max_target = np.percentile(target_volume, 99)
            elif max_target < np.percentile(target_volume, 99):
                max_target = np.percentile(target_volume, 99)


            del input_volume, target_volume, input_path, target_path

    if os.path.exists(output_file_path):
        os.remove(output_file_path)

    with open(output_file_path, 'wb') as output_file:
        pickle.dump([min_input, max_input, min_target,
                     max_target], output_file)


def update_shuffling_flag(file_name):
    """ Update shuffling flag

    Changes shuffling flag in settings to False once data has been shuffled

    Args:
        file_name (str): The settings file name
    """

    config = configparser.ConfigParser()
    config.read(file_name)
    config.set('DATA', 'data_split_flag', 'False')
    with open(file_name, 'w') as configfile:
        config.write(configfile)


class DataMapper(data.Dataset):
    """Data Mapper Class

    This class represents a generic parent class for mapping between keys and data samples.
    The class represents a subclass/child class of data.Dataset, inheriting its functionality.
    This class is composed of a __init__ constructor, a __getitem__(), supporting fetching a data sample for a given key, and __len__(), which returns the size of the dataset.
    This class also has several other helper functions.

    Args:
        filename (str): Path to file containing the relevant volume indicator numbers
        data_directory (str): Directory where the various subjects are stored.
        data_file (str): Intenal path for each subject to the relevant normalized summed dMRI tracts
        output_targets (str): Internal path for each subject to the relevant rsfMRI data
        mean_mask_path (str): Path to the dualreg subject mean mask
        scaling_factors (str): Path to the file containing the scaling factors
        mean_reduction (bool): Flag indicating if the targets should be de-meaned using the mean_mask_path

    Returns:
        X_volume (torch.tensor): Tensor representation of the input data
        y_volume (torch.tensor): Tensor representation of the output data
        int: lenght of the output

    """

    def __init__(self, filename, data_directory, data_file, output_targets, mean_mask_path, scaling_factors, mean_reduction=False):
        # Initialize everything, and only store in memory the text data file.
        # Memory usage limited by only storing the text string information, not the actual volumes.
        # TODO: Currently, the timepoint in the fMRI data (y_volume) is hardcoded, only loading in the DMN. This needs to be updated in later iterations.
        self.filename = filename
        self.data_directory = data_directory
        self.data_file = data_file
        self.output_targets = output_targets
        self.sample_pairs = []
        self._get_datasets()
        self.mean_mask_path = mean_mask_path
        self.mean_reduction = mean_reduction
        self.scaling_factors = scaling_factors

    def __len__(self):
        return len(self.sample_pairs)

    def __getitem__(self, index):

        X_path, y_path = self.sample_pairs[index]

        X_volume = torch.from_numpy(self.scale_volume(self.resample(X_path), self.scaling_factors, target_flag=False))

        if self.mean_reduction == True:
            y_volume = torch.from_numpy(self.scale_volume(self.subtract_mean(y_path, self.mean_mask_path), self.scaling_factors, target_flag=True))
        else:
            y_volume = torch.from_numpy(self.scale_volume(self.convert_to_numpy(y_path)[:, :, :, 0], self.scaling_factors, target_flag=True))

        return X_volume, y_volume

    def _get_datasets(self):
        """File path generator

        Helper function which reads all the various strings and generates the required paths.
        """

        with open(self.filename) as files:
            lines = files.read().split('\n')

        for line in lines:
            if line == '':
                pass
            else:
                X_path = os.path.join(
                    os.path.expanduser("~"), self.data_directory, line, self.data_file)
                y_path = os.path.join(
                    os.path.expanduser("~"), self.data_directory, line, self.output_targets)

                self.sample_pairs.append((X_path, y_path))

    def resample(self, path):
        """dMRI Resample

        Helper function downsampling the dMRI data from 1mm to 2mm.
        This is due to GPU memory / RAM limitations during training.
        The resampleToPixdims also carries out data smoothing.

        Args:
            path (str): Path to the relevant volume

        Returns:
            volume_resampled (np.array): Resampled volume
        """
        volume_resampled, _ = resampleToPixdims(
            self.read_data_files(path), (2, 2, 2))

        return volume_resampled

    def read_data_files(self, path):
        """Volume Reader

        Helper function reading the relvant volume. 

        Args:
            path (str): Path to the relevant volume

        Returns:
            volume_image (class): fsl.data.image.Image class
        """

        volume_image = Image(path)
        return volume_image

    def convert_to_numpy(self, path):
        """Numpy wrapper

        Helper function wrapping the conversaion of a volume to numpy

        Args:
            path (str): Path to the relevant volume.

        Returns
            volume_numpy (np.array): Numpy array representation of volume data.
        """

        volume_numpy = self.read_data_files(path).data
        return volume_numpy

    def subtract_mean(self, path, mean_mask_path):
        """Mean Mask Substraction

        Helper function which substracts the dualreg mean subject mask

        Args:
            mean_mask_path (str): Path to the dualreg subject mean mask

        Returns:
            subtracted_volume (np.array): Numpy array representation of the subtracted volume data
        """

        dualreg_subject_mean = self.convert_to_numpy(mean_mask_path)[:, :, :, 0]
  
        volume = self.convert_to_numpy(path)[:, :, :, 0]

        subtracted_volume = np.subtract(volume, dualreg_subject_mean)

        return subtracted_volume

    def scale_volume(self, volume, scaling_factors, target_flag=False):
        """Scaling function

        This function reads the scaling factors from the saved file and then scales the data.

        Args:
            volume (np.array): Unscalled volume
            scaling_factors (str): Path to the scaling factor file
            target_flag (bool): Flag signaling if the file is a target or an input

        Returns:
            scaled_volume (np.array): Scaled volume
        """

        with open(scaling_factors, 'rb') as input_file:
            min_input, max_input, min_target, max_target = pickle.load(input_file)

        # Steve Scaling
        min_input, max_input, min_target, max_target = [0.0, 0.2, 0.0, 10.0]

        if target_flag == False:
            min_value = min_input
            max_value = max_input
        elif target_flag == True:
            min_value = min_target
            max_value = max_target

        # Set all negative elements to 0
        volume[volume < 0] = 0.0        

        # Eliminating outliers
        volume[volume > max_value] = max_value
        volume[volume < min_value] = min_value
        
        # Normalization to [0, 1]
        scaled_volume = np.divide(np.subtract(volume, min_value), np.subtract(max_value, min_value)) 
        # Scaling between [-1, 1]
        # scaled_volume = np.add(-1.0, np.multiply(2.0, np.divide(np.subtract(volume, min_value), np.subtract(max_value, min_value))))
        
        # No scaling performed
        # scaled_volume = volume

        return scaled_volume


def get_datasets(data_parameters):
    """Data Loader Function.

    This function loads the various data file and returns the relevand mapped datasets.

    Args:
        data_parameters (dict): Dictionary containing relevant information for the datafiles.
        data_parameters = {
            data_directory: 'path/to/directory'
            train_data_file: 'training_data'
            train_output_targets: 'training_targets'
            train_list = 'train.txt'
            validation_list = 'validation.txt'
            validation_data_file: 'testing_data'
            validation_target_file: 'testing_targets'
        }

    Returns:
        touple: the relevant train and test datasets
    """

    train_filename = data_parameters['train_list']
    data_directory = data_parameters['data_directory']
    train_data_file = data_parameters['train_data_file']
    train_output_targets = data_parameters['train_output_targets']

    validation_filename = data_parameters['validation_list']
    validation_data_file = data_parameters['validation_data_file']
    validation_output_targets = data_parameters['validation_target_file']

    mean_mask_path = data_parameters['mean_mask_path']
    mean_reduction = data_parameters['mean_reduction']

    scaling_factors = data_parameters['scaling_factors']

    return (
        DataMapper(train_filename, data_directory,
                   train_data_file, train_output_targets, mean_mask_path, scaling_factors, mean_reduction),
        DataMapper(validation_filename, data_directory,
                   validation_data_file, validation_output_targets, mean_mask_path, scaling_factors, mean_reduction)
    )


def create_folder(path):
    """Folder Creator

    A function which creates a folder at a given path if one does not exist

    Args:
        path (str): destination to check for folder existance
    """

    if not os.path.exists(path):
        os.mkdir(path)


def load_file_paths(data_directory, data_list, mapping_data_file, targets_directory=None, target_file=None):
    """File Loader

    This function returns a list of combined file paths for the input data and labelled output data.

    Args:
        data_directory (str): Path to input data directory
        data_list (str): Path to a .txt file containing the input files for consideration
        mapping_data_file (str): Path to the input files
        targets_directory (str): Path to labelled data (Y-equivalent); None if during evaluation.

    Returns:
        file_paths (list): List containing the input data and target labelled output data

    Raises:
        ValueError: "Invalid data entry - check code and data entry format!"
    """

    if data_list:
        with open(data_list) as data_list_file:
            volumes_to_be_used = data_list_file.read().splitlines()
    else:
        volumes_to_be_used = [files for files in os.listdir(data_directory)]

    if targets_directory == None or target_file == None:
        file_paths = [[os.path.join(data_directory, volume, mapping_data_file)]
                      for volume in volumes_to_be_used]
    else:
        file_paths = [[os.path.join(data_directory, volume, mapping_data_file), os.path.join(
            targets_directory, volume)] for volume in volumes_to_be_used]

    return file_paths


def load_and_preprocess_evaluation(file_path):
    """Load & Preprocessing before evaluation

    This function loads a nifty file and returns its volume and header information

    Args:
        file_path (str): Path to the desired file

    Returns:
        volume (np.array): Array of training image data of data type dtype.
        header (class): 'nibabel.nifti1.Nifti1Header' class object, containing image metadata
        xform (np.array): Array of shape (4, 4), containing the adjusted voxel-to-world transformation for the spatial dimensions of the resampled data

    Raises:
        ValueError: "Orientation value is invalid. It must be either >>coronal<<, >>axial<< or >>sagital<< "
    """

    original_image = Image(file_path[0])
    volume, xform = resampleToPixdims(original_image, (2, 2, 2))
    header = Image(volume, header=original_image.header, xform=xform).header

    return volume, header, xform


def load_and_preprocess_targets(target_path, mean_mask_path):
    """Load & Preprocessing targets before evaluation

    This function loads a nifty file and returns its volume, a de-meaned volume and header information

    Args:
        file_path (str): Path to the desired target file
        mean_mask_path (str): Path to the dualreg subject mean mask

    Returns:
        target (np.array): Array of training image data of data type dtype.
        target_demeaned (np.array): Array of training data from which the group mean has been subtracted

    Raises:
        ValueError: "Orientation value is invalid. It must be either >>coronal<<, >>axial<< or >>sagital<< "
    """

    target = Image(target_path[0]).data[:,:,:,0]
    target_demeaned = np.subtract(target, Image(mean_mask_path).data[:,:,:,0])

    return target, target_demeaned


# Deprecated Functions & Classes & Methods:


def set_orientation(volume, label_map, orientation):
    """Load Data Orientation

    This function modifies the orientation of the input and output data depending on the required orientation.

    Args:
        volume (np.array): Array of training image data of data type dtype.
        label_map (np.array): Array of labelled image data of data type dtype.
        orientation (str): String detailing the current view (COR, SAG, AXL)

    Returns:
        volume (np.array): Array of training image data of data type dtype.
        label_map (np.array): Array of labelled image data of data type dtype.

    Raises:
        ValueError: Orientation value is invalid. It must be either >>coronal<<, >>axial<< or >>sagital<<
    """

    # TODO: will need to check if these alignments correpond with our data.
    # These alignments work for ADNI

    if orientation == "sagittal":
        return volume, label_map  # This is assumed to be the default orientation
    elif orientation == "axial":
        return volume.transpose((1, 2, 0)), label_map.transpose((1, 2, 0))
    elif orientation == "coronal":
        return volume.transpose((2, 0, 1)), label_map.transpose((2, 0, 1))
    else:
        raise ValueError(
            "Orientation value is invalid. It must be either >>coronal<<, >>axial<< or >>sagital<< ")


def tract_sum_generator(folder_path):
    """Sums the tracts of different dMRI files

    THIS FUNCTION IS NOT DEPRECATED: SummedTractMaps generated remotely

    When performing subject-specific probabilistic diffusion tractography using standard-space protocols, 27 tracts are created.
    This function loops through all the tracts, sums them and returns the summed tract map.
    This function also outputs the summed tract map as a Nifti (.nii.gz) file.

    Args:
        folder_location (str): A string containing the address of the required directory.
    """

    tractMapName = 'tracts/tractsNorm.nii.gz'

    subDirectoryList = directory_reader(folder_path)

    viableSubDirectories = len(subDirectoryList)
    counter = 0

    if not os.path.exists('/well/win/users/hsv459/functionmapper/datasets/dMRI'):
        if not os.path.exists('/well/win/users/hsv459/functionmapper/datasets'):
            os.mkdir('/well/win/users/hsv459/functionmapper/datasets')
        os.mkdir('/well/win/users/hsv459/functionmapper/datasets/dMRI')

    for subDirectory in subDirectoryList:
        tractedMapsPath = os.path.join(folder_location, str(
            subDirectory), 'dMRI/autoptx_preproc/tracts/')

        sum_flag = False  # This is a flat showing us if this is the first tracted to be summed

        print("Summing the tract number: {}/{}".format(counter, viableSubDirectories))

        for tract in os.listdir(tractedMapsPath):
            if os.path.isdir(os.path.join(tractedMapsPath, tract)):
                tractedMapPath = os.path.join(
                    tractedMapsPath, tract, tractMapName)
                tractedMapImg = nib.load(tractedMapPath)

                tractedMap = tractedMapImg.get_fdata()

                #  the affine array stores the relationship between voxel coordinates in the image data array and coordinates in the reference space

                tractedMapAffine = tractedMapImg.affine

                if sum_flag == False:
                    tractedMapSum = np.copy(tractedMap)
                else:
                    tractedMapSum = np.sum(tractedMapSum, tractedMap)

        tractedMapSumPath = '/well/win/users/hsv459/functionmapper/datasets/dMRI'
        tractsSumName = str(subDirectory) + ".nii.gz"
        tractedMapSumImg = nib.Nifti1Image(tractedMapSum, tractedMapAffine)
        nib.save(tractedMapSumImg, os.path.join(
            tractedMapSumPath, tractsSumName))

        counter += 1

    return None


class DataMapperHDF5(data.Dataset):
    """Data Mapper Class.

    THIS CLASS IS NOT DEPRECATED!

    This class represents a generic parent class for mapping between keys and data samples.
    The class represents a subclass/child class of data.Dataset, inheriting its functionality.
    This class is composed of a __init__ constructor, a __getitem__(), supporting fetching a data sample for a given key, and __len__(), which returns the size of the dataset.

    Args:
        X (HDF5 datafile): hierarchically organized input data
        y (HDF5 datafile): hierarchically organized output data 

    Returns:
        input_data (torch.tensor): Tensor representation of the input data
        label_data (torch.tensor): Tensor representation of the output data
        int: lenght of the output
    """

    def __init__(self, X, y):
        self.X = X
        self.y = y

    def __getitem__(self, index):
        input_data = torch.from_numpy(self.X[index])
        label_data = torch.from_numpy(self.y[index])
        return input_data, label_data

    def __len__(self):
        return len(self.y)


def get_datasetsHDF5(data_parameters):
    """Data Loader Function.

    THIS FUNCTION IS NOT DEPRECATED: Loader function rewritten. 

    This function loads the various data file and returns the relevand mapped datasets.

    Args:
        data_parameters (dict): Dictionary containing relevant information for the datafiles.
        data_parameters = {
            data_directory: 'path/to/directory'
            train_data_file: 'training_data'
            train_output_targets: 'training_targets'
            train_list = 'train.txt'
            validation_list = 'validation.txt'
            test_list = 'test.txt'
            test_data_file: 'testing_data'
            test_target_file: 'testing_targets'
        }

    Returns:
        touple: the relevant train and test datasets
    """

    training_data = h5py.File(os.path.join(
        data_parameters['data_directory'], data_parameters['training_data']), 'r')
    testing_data = h5py.File(os.path.join(
        data_parameters['data_directory'], data_parameters['testing_data']), 'r')

    training_labels = h5py.File(os.path.join(
        data_parameters['data_directory'], data_parameters['training_targets']), 'r')
    testing_labels = h5py.File(os.path.join(
        data_parameters['data_directory'], data_parameters['testing_targets']), 'r')

    return (
        DataMapperHDF5(training_data['data'][()],
                       training_labels['label'][()]),
        DataMapperHDF5(testing_data['data'][()], testing_labels['label'][()])
    )


def load_and_preprocess_evaluation2D(file_path, orientation, min_max=True):
    """Load & Preprocessing before evaluation

    This function loads a nifty file and returns its volume and header information

    Args:
        file_path (str): Path to the desired file
        orientation (str): String detailing the current view (COR, SAG, AXL)
        min_max (bool): Flag for inducing min-max normalization of the volume

    Returns:
        volume (np.array): Array of training image data of data type dtype.
        header (class): 'nibabel.nifti1.Nifti1Header' class object, containing image metadata

    Raises:
        ValueError: "Orientation value is invalid. It must be either >>coronal<<, >>axial<< or >>sagital<< "
    """

    nifty_volume = nib.load(file_path[0])

    volume = nifty_volume.get_fdata()
    header = nifty_volume.header

    if min_max:
        volume = (volume - np.min(volume)) / (np.max(volume) - np.min(volume))
    else:
        volume = np.round(volume)

    if orientation == "sagittal":
        return volume  # This is assumed to be the default orientation
    elif orientation == "axial":
        return volume.transpose((1, 2, 0))
    elif orientation == "coronal":
        return volume.transpose((2, 0, 1))
    else:
        raise ValueError(
            "Orientation value is invalid. It must be either >>coronal<<, >>axial<< or >>sagital<< ")

    return volume, header


def load_and_preprocess(file_paths, orientation):
    """Load & Preprocess

    This function is composed of two other function calls: one that calls a function loading the data, and another which preprocesses the data to the required format.
    # TODO: Need to check if any more proprocessing would be required besides summing the tracts!

    Args:
        file_paths (list): List containing the input data and target labelled output data
        orientation (str): String detailing the current view (COR, SAG, AXL)

    Returns:
        volume (np.array): Array of training image data of data type dtype.
        label_map (np.array): Array of labelled image data of data type dtype.
        header (class): 'nibabel.nifti1.Nifti1Header' class object, containing image metadata
    """

    volume, label_map, header = load(file_paths, orientation)

    return volume, label_map, header


def load(file_path, orientation):
    """Load Data Function

    This function loads the required data files and extracts relevant information from it.

    Args:
        file_path (list): List containing the input data and target labelled output data
        orientation (str): String detailing the current view (COR, SAG, AXL)

    Returns:
        volume (np.array): Array of training image data of data type dtype.
        label_map (np.array): Array of labelled image data of data type dtype.
        header (class): 'nibabel.nifti1.Nifti1Header' class object, containing image metadata
    """

    nifty_volume, label_map = nib.load(file_path[0]), nib.load(file_path[1])
    volume, label_map = nifty_volume.get_fdata(), label_map.ged_fdata()

    # Do we need min-max normalization here? Will need to check when debuggint and testing
    volume = (volume - np.min(volume)) / (np.max(volume) - np.min(volume))

    volume, label_map = set_orientation(volume, label_map, orientation)

    return volume, label_map, nifty_volume.header