RF: Don't use nibabel Nifti1Image.get_fdata, as it causes RAM blow-out. Store

image data in its native type instead. Use 64 bit when comparing arrays, as
we can't remember why we decided 128 bit was needed (it probably isn't).

pyfeeds/evaluate.py

@@ -290,6 +290,8 @@ def compareAllFiles(pyf, test, outputDir, benchmarkDir, logFile=None):
         # the test fails for this file.
         try:
 
+            log.debug('Comparing %s <-> %s...', testfile, benchmark)
+
             for rname, rfunc in routines:
 
                 # if this is a file group evaluation
@@ -396,9 +398,8 @@ def evalHeader(pyf, testfile, benchmark, alldims=True):
     Returns 0 if they all match, 1 otherwise.
     """
 
-    img1 = pyf.imageCache[testfile]
-    img2 = pyf.imageCache[benchmark]
-
+    img1   = pyf.imageCache[testfile][0]
+    img2   = pyf.imageCache[benchmark][0]
     hdr1   = img1.header
     hdr2   = img2.header
     fields = ['dim',       'pixdim',     'intent_code',
@@ -438,12 +439,8 @@ def evalImage(pyf, testfile, benchmark):
 
     The :func:`cmpArrays` function does the calculation.
     """
-
-    img1  = pyf.imageCache[testfile]
-    img2  = pyf.imageCache[benchmark]
-    data1 = img1.get_fdata()
-    data2 = img2.get_fdata()
-
+    data1  = pyf.imageCache[testfile][1]
+    data2  = pyf.imageCache[benchmark][1]
     return cmpArrays(data1, data2)
 
 
@@ -480,11 +477,8 @@ def evalVectorImage(pyf, testfile, benchmark):
     if evalImage(pyf, testfile, benchmark) == 0.0:
         return 0
     pion2 = np.pi / 2
-    img1  = pyf.imageCache[testfile]
-    img2  = pyf.imageCache[benchmark]
-
-    data1 = img1.get_fdata().reshape(-1, 3).T
-    data2 = img2.get_fdata().reshape(-1, 3).T
+    data1  = pyf.imageCache[testfile][ 1].reshape(-1, 3).T
+    data2  = pyf.imageCache[benchmark][1].reshape(-1, 3).T
 
     # Calculate the length of each vector,
     # discard vectors of length 0, and
@@ -567,10 +561,10 @@ def evalPolarCoordinateImageGroup(pyf, testfiles, benchmarks):
         raise ValueError('Wrong number of files: {} <-> {}'
                          ''.format(testfiles, benchmarks))
 
-    testtheta  = pyf.imageCache[testtheta[ 0]].get_fdata()
-    testphi    = pyf.imageCache[testphi[   0]].get_fdata()
-    benchtheta = pyf.imageCache[benchtheta[0]].get_fdata()
-    benchphi   = pyf.imageCache[benchphi[  0]].get_fdata()
+    testtheta  = pyf.imageCache[testtheta[ 0]][1]
+    testphi    = pyf.imageCache[testphi[   0]][1]
+    benchtheta = pyf.imageCache[benchtheta[0]][1]
+    benchphi   = pyf.imageCache[benchphi[  0]][1]
 
     if any((testphi   .shape != testtheta.shape,
             benchtheta.shape != testtheta.shape,
@@ -603,8 +597,8 @@ def evalGiftiVertices(pyf, testfile, benchmark):
     function.
     """
 
-    surf1 = pyf.imageCache[testfile]
-    surf2 = pyf.imageCache[benchmark]
+    surf1 = pyf.imageCache[testfile][1]
+    surf2 = pyf.imageCache[benchmark][1]
 
     # NIFTI_INTENT_POINTSET == 1008
     verts1 = [d for d in surf1.darrays if d.intent == 1008][0].data
@@ -620,15 +614,8 @@ def cmpArrays(arr1, arr2):
     by the combined data range of the two arrays.
     """
 
-    # December 2021: numpy on macOS/M1
-    # doesn't support float128. Hopefully
-    # can be removed in the near future.
-    if hasattr(np, 'float128'):
-        arr1 = np.asarray(arr1, dtype=np.float128)
-        arr2 = np.asarray(arr2, dtype=np.float128)
-    else:
-        arr1 = np.asarray(arr1, dtype=np.float64)
-        arr2 = np.asarray(arr2, dtype=np.float64)
+    arr1 = np.asarray(arr1, dtype=np.float64)
+    arr2 = np.asarray(arr2, dtype=np.float64)
 
     # Non-finite values (nan/inf)
     # must match in both arrays.

pyfeeds/imagecache.py

@@ -37,7 +37,7 @@ def size(img):
     return kb / 1048576.0
 
 
-class ImageCache(object):
+class ImageCache:
     """The ``ImageCache`` is just a cache for loading ``nibabel`` images (and
     other data types).  Its purpose is to avoid re-loading the same image
     multiple times - once an image has been loaded through the cache, it will
@@ -48,11 +48,11 @@ class ImageCache(object):
         cache = ImageCache()
 
         # the image is loaded on first access
-        img = cache['myimage.nii.gz']
+        img, data = cache['myimage.nii.gz']
 
         # on subsequent accesses, the image
         # is returned from the cache
-        img = cache['myimage.nii.gz']
+        img, data = cache['myimage.nii.gz']
     """
 
     def __init__(self, maxsize=32768):
@@ -71,7 +71,9 @@ class ImageCache(object):
 
     def __getitem__(self, imagefile):
         """Return the ``nibabel`` image corresponding to the given
-        ``imagefile``, loading it if necessary.
+        ``imagefile``, loading it if necessary. A tuple is returned,
+        containing the ``nibabel`` image, and a ``numpy``array with
+        the image data.
         """
 
         imagefile = op.realpath(op.abspath(imagefile))
@@ -86,8 +88,9 @@ class ImageCache(object):
             return image
 
         image           = nib.load(imagefile)
-        self[imagefile] = image
-        return image
+        data            = np.asanyarray(image.dataobj)
+        self[imagefile] = image, data
+        return image, data
 
 
     def __setitem__(self, imagefile, image):
@@ -95,15 +98,21 @@ class ImageCache(object):
         storing derived, in-memory-only images.
         """
 
+        # assignments through __getitem__ will
+        # be an (image, ndarray) tuple
+        if isinstance(image, nib.Nifti1Image):
+            data = np.asanyarray(image.dataobj)
+        else:
+            image, data = image
+
         # imagefile may not be a valid
         # file system path, but we transform
         # it in this way because all keys
         # passed to __getitem__ get
         # transformed in the same way.
-
         imagefile                    = op.realpath(op.abspath(imagefile))
         imagesize                    = size(image)
-        self.__images[    imagefile] = image
+        self.__images[    imagefile] = image, data
         self.__imagesizes[imagefile] = imagesize
 
         newsize = self.__currentsize + imagesize