Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Andrei-Claudiu Roibu
BrainMapper
Commits
1e1a86bb
Commit
1e1a86bb
authored
Jul 21, 2020
by
Andrei Roibu
Browse files
added mean subtraction back as an option
parent
9fb4dd1c
Changes
5
Hide whitespace changes
Inline
Side-by-side
run.py
View file @
1e1a86bb
...
...
@@ -270,7 +270,7 @@ def evaluate_score(training_parameters, network_parameters, misc_parameters, eva
'rsfmri_mean_mask_path'
],
dmri_mean_mask_path
=
mapping_evaluation_parameters
[
'dmri_mean_mask_path'
],
mean_re
duct
ion
=
mapping_evaluation_parameters
[
'mean_re
duct
ion'
],
mean_re
gress
ion
=
mapping_evaluation_parameters
[
'mean_re
gress
ion'
],
scaling_factors
=
mapping_evaluation_parameters
[
'scaling_factors'
],
regression_factors
=
mapping_evaluation_parameters
[
'regression_factors'
],
device
=
misc_parameters
[
'device'
],
...
...
@@ -305,7 +305,8 @@ def evaluate_mapping(mapping_evaluation_parameters):
device
=
mapping_evaluation_parameters
[
'device'
]
exit_on_error
=
mapping_evaluation_parameters
[
'exit_on_error'
]
brain_mask_path
=
mapping_evaluation_parameters
[
'brain_mask_path'
]
mean_reduction
=
mapping_evaluation_parameters
[
'mean_reduction'
]
mean_regression
=
mapping_evaluation_parameters
[
'mean_regression'
]
mean_subtraction
=
mapping_evaluation_parameters
[
'mean_subtraction'
]
scaling_factors
=
mapping_evaluation_parameters
[
'scaling_factors'
]
regression_factors
=
mapping_evaluation_parameters
[
'regression_factors'
]
...
...
@@ -317,7 +318,8 @@ def evaluate_mapping(mapping_evaluation_parameters):
brain_mask_path
,
dmri_mean_mask_path
,
rsfmri_mean_mask_path
,
mean_reduction
,
mean_regression
,
mean_subtraction
,
scaling_factors
,
regression_factors
,
device
=
device
,
...
...
settings.ini
View file @
1e1a86bb
...
...
@@ -4,7 +4,7 @@ use_data_file = False
data_directory
=
"/well/win-biobank/projects/imaging/data/data3/subjectsAll/"
data_file
=
"/well/win-biobank/projects/imaging/data/data3/subjectsAll/subj_22k.txt"
k_fold
=
None
data_split_flag
=
Tru
e
data_split_flag
=
Fals
e
test_percentage
=
5
subject_number
=
12000
train_list
=
"datasets/train.txt"
...
...
@@ -19,12 +19,13 @@ validation_target_file = "fMRI/rfMRI_25.dr/dr_stage2.nii.gz"
brain_mask_path
=
"utils/MNI152_T1_2mm_brain_mask.nii.gz"
rsfmri_mean_mask_path
=
"utils/mean_dr_stage2.nii.gz"
dmri_mean_mask_path
=
"utils/mean_tractsNormSummed_downsampled.nii.gz"
mean_reduction
=
True
mean_regression
=
False
mean_subtraction
=
True
[TRAINING]
experiment_name
=
"
CU3D17-3
"
pre_trained_path
=
"saved_models/
CU3D17-3
.pth.tar"
final_model_output_file
=
"
CU3D17-3
.pth.tar"
experiment_name
=
"
VA2-1
"
pre_trained_path
=
"saved_models/
VA2-1
.pth.tar"
final_model_output_file
=
"
VA2-1
.pth.tar"
training_batch_size
=
5
validation_batch_size
=
5
use_pre_trained
=
False
...
...
@@ -32,9 +33,9 @@ learning_rate = 1e-5
optimizer_beta
=
(0.9, 0.999)
optimizer_epsilon
=
1e-8
optimizer_weigth_decay
=
1e-5
number_of_epochs
=
20
0
number_of_epochs
=
1
0
loss_log_period
=
50
learning_rate_scheduler_step_size
=
4
learning_rate_scheduler_step_size
=
5
learning_rate_scheduler_gamma
=
1e-1
use_last_checkpoint
=
False
...
...
@@ -46,7 +47,7 @@ kernel_classification = 7
input_channels
=
1
output_channels
=
64
convolution_stride
=
1
dropout
=
0
.2
dropout
=
0
pool_kernel_size
=
3
pool_stride
=
2
up_mode
=
"upconv"
...
...
settings_evaluation.ini
View file @
1e1a86bb
[MAPPING]
trained_model_path
=
"saved_models/
test24
.pth.tar"
prediction_output_path
=
"
test24
_predictions"
trained_model_path
=
"saved_models/
VA2
.pth.tar"
prediction_output_path
=
"
VA2
_predictions"
data_directory
=
"/well/win-biobank/projects/imaging/data/data3/subjectsAll/"
mapping_data_file
=
"dMRI/autoptx_preproc/tractsNormSummed.nii.gz"
data_list
=
"datasets/test.txt"
...
...
@@ -10,6 +10,7 @@ dmri_mean_mask_path = "utils/mean_tractsNormSummed_downsampled.nii.gz"
mean_mask_path
=
"utils/mean_dr_stage2.nii.gz"
scaling_factors
=
"datasets/scaling_factors.pkl"
regression_factors
=
"datasets/regression_weights.pkl"
mean_reduction
=
True
mean_regression
=
False
mean_subtraction
=
True
device
=
0
exit_on_error
=
True
utils/data_evaluation_utils.py
View file @
1e1a86bb
...
...
@@ -20,7 +20,6 @@ import numpy as np
import
torch
import
logging
import
utils.data_utils
as
data_utils
import
nibabel
as
nib
import
matplotlib.pyplot
as
plt
import
pandas
as
pd
from
fsl.data.image
import
Image
...
...
@@ -38,7 +37,7 @@ def evaluate_correlation(trained_model_path,
brain_mask_path
,
rsfmri_mean_mask_path
,
dmri_mean_mask_path
,
mean_re
duct
ion
,
mean_re
gress
ion
,
scaling_factors
,
regression_factors
,
device
=
0
,
...
...
@@ -58,7 +57,7 @@ def evaluate_correlation(trained_model_path,
brain_mask_path (str): Path to the MNI brain mask file
rsfmri_mean_mask_path (str): Path to the dualreg subject mean mask
dmri_mean_mask_path (str): Path to the dualreg subject mean mask
mean_re
duct
ion (bool): Flag indicating if the targets should be de-meaned using the mean_mask_path
mean_re
gress
ion (bool): Flag indicating if the targets should be de-meaned using the mean_mask_path
scaling_factors (str): Path to the scaling factor file
regression_factors (str): Path to the linear regression weights file
device (str/int): Device type used for training (int - GPU id, str- CPU)
...
...
@@ -123,7 +122,7 @@ def evaluate_correlation(trained_model_path,
file_path
,
subject
,
model
,
device
,
cuda_available
,
brain_mask_path
,
dmri_mean_mask_path
,
rsfmri_mean_mask_path
,
scaling_factors
,
regression_factors
,
mean_re
duct
ion
)
regression_factors
,
mean_re
gress
ion
)
for
target_index
,
target_path
in
enumerate
(
target_paths
):
target
,
target_demeaned
=
data_utils
.
load_and_preprocess_targets
(
...
...
@@ -172,7 +171,8 @@ def evaluate_mapping(trained_model_path,
brain_mask_path
,
dmri_mean_mask_path
,
rsfmri_mean_mask_path
,
mean_reduction
,
mean_regression
,
mean_subtraction
,
scaling_factors
,
regression_factors
,
device
=
0
,
...
...
@@ -191,7 +191,8 @@ def evaluate_mapping(trained_model_path,
brain_mask_path (str): Path to the MNI brain mask file
dmri_mean_mask_path (str): Path to the dualreg subject mean mask
rsfmri_mean_mask_path (str): Path to the summed tract mean mask
mean_reduction (bool): Flag indicating if the targets should be de-meaned using the mean_mask_path
mean_regression (bool): Flag indicating if the targets should be de-meaned by regression using the mean_mask_path
mean_subtraction (bool): Flag indicating if the targets should be de-meaned by subtraction using the mean_mask_path
scaling_factors (str): Path to the scaling factor file
regression_factors (str): Path to the linear regression weights file
device (str/int): Device type used for training (int - GPU id, str- CPU)
...
...
@@ -251,7 +252,7 @@ def evaluate_mapping(trained_model_path,
file_path
,
subject
,
model
,
device
,
cuda_available
,
brain_mask_path
,
dmri_mean_mask_path
,
rsfmri_mean_mask_path
,
scaling_factors
,
regression_factors
,
mean_re
du
ction
)
regression_factors
,
mean_re
gression
,
mean_subtra
ction
)
# Generate New Header Affine
...
...
@@ -269,7 +270,7 @@ def evaluate_mapping(trained_model_path,
output_nifti_image
.
save
(
output_nifti_path
)
if
mean_re
duct
ion
==
True
:
if
mean_re
gress
ion
==
True
:
output_complete_nifti_image
=
Image
(
predicted_complete_volume
,
header
=
header
,
xform
=
xform
)
...
...
@@ -312,7 +313,8 @@ def _generate_volume_map(file_path,
rsfmri_mean_mask_path
,
scaling_factors
,
regression_factors
,
mean_reduction
=
False
):
mean_regression
=
False
,
mean_subtraction
=
False
):
"""rsfMRI Volume Generator
This function uses the trained model to generate a new volume
...
...
@@ -328,18 +330,18 @@ def _generate_volume_map(file_path,
rsfmri_mean_mask_path (str): Path to the dualreg subject mean mask
scaling_factors (str): Path to the scaling factor file
regression_factors (str): Path to the linear regression weights file
mean_reduction (bool): Flag indicating if the targets should be de-meaned using the mean_mask_path
mean_regression (bool): Flag indicating if the targets should be de-meaned by regression using the mean_mask_path
mean_subtraction (bool): Flag indicating if the targets should be de-meaned by subtraction using the mean_mask_path
Returns
predicted_volume (np.array): Array containing the information regarding the generated volume
header (class): 'nibabel.nifti1.Nifti1Header' class object, containing volume metadata
"""
volume
,
header
,
xform
=
data_utils
.
load_and_preprocess_evaluation
(
file_path
)
volume
,
header
,
xform
=
data_utils
.
load_and_preprocess_evaluation
(
file_path
)
volume
=
_regress_input
(
volume
,
subject
,
dmri_mean_mask_path
,
regression_factors
)
if
mean_regression
==
True
:
volume
=
_regress_input
(
volume
,
subject
,
dmri_mean_mask_path
,
regression_factors
)
print
(
'volume range:'
,
np
.
min
(
volume
),
np
.
max
(
volume
))
...
...
@@ -367,11 +369,13 @@ def _generate_volume_map(file_path,
MNI152_T1_2mm_brain_mask
=
Image
(
brain_mask_path
).
data
if
mean_re
du
ction
==
True
:
if
mean_re
gression
==
True
or
mean_subtra
ction
==
True
:
mean_mask
=
Image
(
rsfmri_mean_mask_path
).
data
[:,
:,
:,
0
]
weight
=
pd
.
read_pickle
(
regression_factors
).
loc
[
subject
][
'w_rsfMRI'
]
predicted_complete_volume
=
np
.
add
(
output
,
np
.
multiply
(
weight
,
mean_mask
))
if
mean_regression
==
True
:
weight
=
pd
.
read_pickle
(
regression_factors
).
loc
[
subject
][
'w_rsfMRI'
]
predicted_complete_volume
=
np
.
add
(
output
,
np
.
multiply
(
weight
,
mean_mask
))
if
mean_subtraction
==
True
:
predicted_complete_volume
=
np
.
add
(
output
,
mean_mask
)
print
(
'predicted_complete_volume'
,
np
.
min
(
predicted_complete_volume
),
np
.
max
(
predicted_complete_volume
))
...
...
@@ -410,19 +414,19 @@ def _scale_input(volume, scaling_factors):
# min_value, max_value, _, _ = pickle.load(input_file)
# Steve Scaling
#
min_value, max_value, _, _ = [0.0, 0.2, 0.0, 10.0]
min_value
,
max_value
,
_
,
_
=
[
0.0
,
0.2
,
0.0
,
10.0
]
# Andrei Scaling
min_value
,
max_value
,
_
,
_
=
[
-
0.0539
,
0.0969
,
-
12.094
,
14.6319
]
#
min_value, max_value, _, _ = [-0.0539, 0.0969, -12.094, 14.6319]
# Eliminating outliers
volume
[
volume
>
max_value
]
=
max_value
volume
[
volume
<
min_value
]
=
min_value
# Normalization to [0, 1]
#
scaled_volume = np.divide(np.subtract(volume, min_value), np.subtract(max_value, min_value))
scaled_volume
=
np
.
divide
(
np
.
subtract
(
volume
,
min_value
),
np
.
subtract
(
max_value
,
min_value
))
# Scaling between [-1, 1]
scaled_volume
=
np
.
add
(
-
1.0
,
np
.
multiply
(
2.0
,
np
.
divide
(
np
.
subtract
(
volume
,
min_value
),
np
.
subtract
(
max_value
,
min_value
))))
#
scaled_volume = np.add(-1.0, np.multiply(2.0, np.divide(np.subtract(volume, min_value), np.subtract(max_value, min_value))))
return
scaled_volume
...
...
@@ -469,15 +473,15 @@ def _rescale_output(volume, scaling_factors):
# _, _, min_value, max_value = pickle.load(input_file)
# Steve Scaling
#
_, _, min_value, max_value = [0.0, 0.2, 0.0, 10.0]
_
,
_
,
min_value
,
max_value
=
[
0.0
,
0.2
,
0.0
,
10.0
]
# Andrei Scaling
_
,
_
,
min_value
,
max_value
=
[
-
0.0539
,
0.0969
,
-
12.094
,
14.6319
]
#
_, _, min_value, max_value = [-0.0539, 0.0969, -12.094, 14.6319]
# Normalization to [0, 1]
#
rescaled_volume = np.add(np.multiply(volume, np.subtract(max_value, min_value)), min_value)
rescaled_volume
=
np
.
add
(
np
.
multiply
(
volume
,
np
.
subtract
(
max_value
,
min_value
)),
min_value
)
# Scaling between [-1, 1]
rescaled_volume
=
np
.
add
(
np
.
multiply
(
np
.
divide
(
np
.
add
(
volume
,
1
),
2
),
np
.
subtract
(
max_value
,
min_value
)),
min_value
)
#
rescaled_volume = np.add(np.multiply(np.divide(np.add(volume, 1), 2), np.subtract(max_value, min_value)), min_value)
return
rescaled_volume
...
...
utils/data_utils.py
View file @
1e1a86bb
...
...
@@ -13,11 +13,9 @@ Usage:
"""
import
os
import
h5py
import
pickle
from
glob
import
glob
import
numpy
as
np
import
nibabel
as
nib
import
torch
import
torch.utils.data
as
data
import
configparser
...
...
@@ -328,7 +326,8 @@ class DataMapper(data.Dataset):
rsfMRI_mean_mask_path (str): Path to the dualreg mean mask
scaling_factors (str): Path to the file containing the scaling factors
regression_weights (str): Path to the file containing the regression_weights
mean_reduction (bool): Flag indicating if the targets should be de-meaned using the mean_mask_path
mean_subtraction (bool): Flag indicating if the targets should be de-meaned by subtraction using the mean_mask_path
mean_regression (bool): Flag indicating if the volumes should be de-meaned by regression using the mean_mask_path
Returns:
X_volume (torch.tensor): Tensor representation of the input data
...
...
@@ -337,7 +336,7 @@ class DataMapper(data.Dataset):
"""
def
__init__
(
self
,
filename
,
data_directory
,
data_file
,
output_targets
,
dMRI_mean_mask_path
,
rsfMRI_mean_mask_path
,
scaling_factors
,
regression_weights
,
mean_
reduct
ion
=
False
):
def
__init__
(
self
,
filename
,
data_directory
,
data_file
,
output_targets
,
dMRI_mean_mask_path
,
rsfMRI_mean_mask_path
,
scaling_factors
,
regression_weights
,
mean_
subtraction
=
False
,
mean_regress
ion
=
False
):
# Initialize everything, and only store in memory the text data file.
# Memory usage limited by only storing the text string information, not the actual volumes.
# TODO: Currently, the timepoint in the fMRI data (y_volume) is hardcoded, only loading in the DMN. This needs to be updated in later iterations.
...
...
@@ -349,7 +348,8 @@ class DataMapper(data.Dataset):
self
.
_get_datasets
()
self
.
dMRI_mean_mask_path
=
dMRI_mean_mask_path
self
.
rsfMRI_mean_mask_path
=
rsfMRI_mean_mask_path
self
.
mean_reduction
=
mean_reduction
self
.
mean_subtraction
=
mean_subtraction
self
.
mean_regression
=
mean_regression
self
.
scaling_factors
=
scaling_factors
self
.
regression_weights
=
regression_weights
...
...
@@ -360,16 +360,15 @@ class DataMapper(data.Dataset):
X_path
,
y_path
,
subject_id
=
self
.
sample_pairs
[
index
]
if
self
.
mean_reduction
==
True
:
X_volume
=
torch
.
from_numpy
(
self
.
scale_volume
(
self
.
linear_regress_mean
(
X_path
,
subject_id
,
target_flag
=
False
),
target_flag
=
False
))
y_volume
=
torch
.
from_numpy
(
self
.
scale_volume
(
self
.
linear_regress_mean
(
y_path
,
subject_id
,
target_flag
=
True
),
target_flag
=
True
))
if
self
.
mean_regression
==
True
:
X_volume
=
torch
.
from_numpy
(
self
.
scale_volume
(
self
.
linear_regress_mean
(
X_path
,
subject_id
,
target_flag
=
False
),
target_flag
=
False
))
y_volume
=
torch
.
from_numpy
(
self
.
scale_volume
(
self
.
linear_regress_mean
(
y_path
,
subject_id
,
target_flag
=
True
),
target_flag
=
True
))
elif
self
.
mean_subtraction
==
True
:
X_volume
=
torch
.
from_numpy
(
self
.
scale_volume
(
self
.
resample
(
X_path
),
target_flag
=
False
))
y_volume
=
torch
.
from_numpy
(
self
.
scale_volume
(
self
.
subtract_mean
(
y_path
),
target_flag
=
True
))
else
:
X_volume
=
torch
.
from_numpy
(
self
.
scale_volume
(
self
.
resample
(
X_path
),
target_flag
=
False
))
y_volume
=
torch
.
from_numpy
(
self
.
scale_volume
(
self
.
convert_to_numpy
(
y_path
)[:,
:,
:,
0
],
target_flag
=
True
))
X_volume
=
torch
.
from_numpy
(
self
.
scale_volume
(
self
.
resample
(
X_path
),
target_flag
=
False
))
y_volume
=
torch
.
from_numpy
(
self
.
scale_volume
(
self
.
convert_to_numpy
(
y_path
)[:,
:,
:,
0
],
target_flag
=
True
))
return
X_volume
,
y_volume
...
...
@@ -441,6 +440,26 @@ class DataMapper(data.Dataset):
volume_numpy
=
self
.
read_data_files
(
path
).
data
return
volume_numpy
def
subtract_mean
(
self
,
path
):
"""Mean Mask Substraction
Helper function which substracts the dualreg mean subject mask
Args:
mean_mask_path (str): Path to the dualreg subject mean mask
Returns:
subtracted_volume (np.array): Numpy array representation of the subtracted volume data
"""
dualreg_subject_mean
=
self
.
convert_to_numpy
(
self
.
rsfMRI_mean_mask_path
)[:,
:,
:,
0
]
volume
=
self
.
convert_to_numpy
(
path
)[:,
:,
:,
0
]
subtracted_volume
=
np
.
subtract
(
volume
,
dualreg_subject_mean
)
return
subtracted_volume
def
linear_regress_mean
(
self
,
path
,
subject
,
target_flag
):
"""Linear regressed mean subtraction
...
...
@@ -490,10 +509,10 @@ class DataMapper(data.Dataset):
# min_input, max_input, min_target, max_target = pickle.load(input_file)
# # Steve Scaling
#
min_input, max_input, min_target, max_target = [0.0, 0.2, 0.0, 10.0]
min_input
,
max_input
,
min_target
,
max_target
=
[
0.0
,
0.2
,
0.0
,
10.0
]
# Andrei Scaling
min_input
,
max_input
,
min_target
,
max_target
=
[
-
0.0539
,
0.0969
,
-
12.094
,
14.6319
]
#
min_input, max_input, min_target, max_target = [-0.0539, 0.0969, -12.094, 14.6319]
if
target_flag
==
False
:
min_value
=
min_input
...
...
@@ -503,7 +522,7 @@ class DataMapper(data.Dataset):
max_value
=
max_target
# Set all negative elements to 0
#
volume[volume < 0] = 0.0
volume
[
volume
<
0
]
=
0.0
# Eliminating outliers
volume
[
volume
>
max_value
]
=
max_value
...
...
@@ -552,16 +571,17 @@ def get_datasets(data_parameters):
dMRI_mean_mask_path
=
data_parameters
[
'dmri_mean_mask_path'
]
rsfMRI_mean_mask_path
=
data_parameters
[
'rsfmri_mean_mask_path'
]
mean_reduction
=
data_parameters
[
'mean_reduction'
]
mean_regression
=
data_parameters
[
'mean_regression'
]
mean_subtraction
=
data_parameters
[
'mean_subtraction'
]
scaling_factors
=
data_parameters
[
'scaling_factors'
]
regression_weights
=
data_parameters
[
'regression_weights'
]
return
(
DataMapper
(
train_filename
,
data_directory
,
train_data_file
,
train_output_targets
,
dMRI_mean_mask_path
,
rsfMRI_mean_mask_path
,
scaling_factors
,
regression_weights
,
mean_
reduct
ion
),
dMRI_mean_mask_path
,
rsfMRI_mean_mask_path
,
scaling_factors
,
regression_weights
,
mean_
subtraction
,
mean_regress
ion
),
DataMapper
(
validation_filename
,
data_directory
,
validation_data_file
,
validation_output_targets
,
dMRI_mean_mask_path
,
rsfMRI_mean_mask_path
,
scaling_factors
,
regression_weights
,
mean_
reduct
ion
)
dMRI_mean_mask_path
,
rsfMRI_mean_mask_path
,
scaling_factors
,
regression_weights
,
mean_
subtraction
,
mean_regress
ion
)
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment