Commit 48984e01 authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

Merge branch 'rf/imglob-perf' into 'master'

MNT: Improve performance of im* scripts

See merge request fsl/fslpy!310
parents f3d34686 418c0b69
...@@ -2,8 +2,8 @@ This document contains the ``fslpy`` release history in reverse chronological ...@@ -2,8 +2,8 @@ This document contains the ``fslpy`` release history in reverse chronological
order. order.
3.7.0 (Under development) 3.7.0 (Friday 20th August 2021)
------------------------- -------------------------------
Added Added
...@@ -17,6 +17,10 @@ Added ...@@ -17,6 +17,10 @@ Added
Changed Changed
^^^^^^^ ^^^^^^^
* Performance of the :mod:`.imglob`, :mod:`.imln`, :mod:`imtest`, :mod:`.imrm`
and :mod:`.remove_ext` scripts has been improved, by re-organising them to
avoid unnecessary and expensive imports such as ``numpy``.
* The default behaviour of the :func:`fsl.utils.run.run` function (and hence * The default behaviour of the :func:`fsl.utils.run.run` function (and hence
that of all :mod:`fsl.wrappers` functions) has been changed so that the that of all :mod:`fsl.wrappers` functions) has been changed so that the
standard output and error of the called command is now forwarded to the standard output and error of the called command is now forwarded to the
......
...@@ -10,14 +10,9 @@ NIFTI/ANALYZE image files. ...@@ -10,14 +10,9 @@ NIFTI/ANALYZE image files.
import sys import sys
import warnings import glob
import fsl.utils.path as fslpath import fsl.utils.path as fslpath
# See atlasq.py for explanation
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=FutureWarning)
import fsl.data.image as fslimage
usage = """ usage = """
Usage: imglob [-extension/extensions] <list of names> Usage: imglob [-extension/extensions] <list of names>
...@@ -25,8 +20,17 @@ Usage: imglob [-extension/extensions] <list of names> ...@@ -25,8 +20,17 @@ Usage: imglob [-extension/extensions] <list of names>
-extensions for image list with full extensions -extensions for image list with full extensions
""".strip() """.strip()
exts = fslimage.ALLOWED_EXTENSIONS
groups = fslimage.FILE_GROUPS # The lists below are defined in the
# fsl.data.image class, but are duplicated
# here for performance (to avoid import of
# nibabel/numpy/etc).
exts = ['.nii.gz', '.nii', '.img', '.hdr', '.img.gz', '.hdr.gz']
"""List of supported image file extensions. """
groups = [('.hdr', '.img'), ('.hdr.gz', '.img.gz')]
"""List of known image file groups (image/header file pairs). """
def imglob(paths, output=None): def imglob(paths, output=None):
...@@ -59,12 +63,27 @@ def imglob(paths, output=None): ...@@ -59,12 +63,27 @@ def imglob(paths, output=None):
imgfiles = [] imgfiles = []
# Expand any wildcard paths if provided.
# Depending on the way that imglob is
# invoked, this may not get done by the
# calling shell.
expanded = []
for path in paths:
if any(c in path for c in '*?[]'):
expanded.extend(glob.glob(path))
else:
expanded.append(path)
paths = expanded
# Build a list of all image files (both # Build a list of all image files (both
# hdr and img and otherwise) that match # hdr and img and otherwise) that match
for path in paths: for path in paths:
try: try:
path = fslimage.removeExt(path) path = fslpath.removeExt(path, allowedExts=exts)
imgfiles.extend(fslimage.addExt(path, unambiguous=False)) imgfiles.extend(fslpath.addExt(path,
allowedExts=exts,
unambiguous=False))
except fslpath.PathError: except fslpath.PathError:
continue continue
......
...@@ -17,19 +17,22 @@ to NIFTI image files. ...@@ -17,19 +17,22 @@ to NIFTI image files.
import os.path as op import os.path as op
import os import os
import sys import sys
import warnings
import fsl.utils.path as fslpath import fsl.utils.path as fslpath
# See atlasq.py for explanation # The lists below are defined in the
with warnings.catch_warnings(): # fsl.data.image class, but are duplicated
warnings.filterwarnings("ignore", category=FutureWarning) # here for performance (to avoid import of
import fsl.data.image as fslimage # nibabel/numpy/etc).
exts = ['.nii.gz', '.nii',
'.img', '.hdr',
'.img.gz', '.hdr.gz',
'.mnc', '.mnc.gz']
"""List of file extensions that are supported by ``imtest``.
"""
ALLOWED_EXTENSIONS = fslimage.ALLOWED_EXTENSIONS + ['.mnc', '.mnc.gz'] groups = [('.hdr', '.img'), ('.hdr.gz', '.img.gz')]
"""List of file extensions that are supported by ``imln``. """ """List of known image file groups (image/header file pairs). """
usage = """ usage = """
...@@ -50,8 +53,8 @@ def main(argv=None): ...@@ -50,8 +53,8 @@ def main(argv=None):
return 1 return 1
target, linkbase = argv target, linkbase = argv
target = fslpath.removeExt(target, ALLOWED_EXTENSIONS) target = fslpath.removeExt(target, exts)
linkbase = fslpath.removeExt(linkbase, ALLOWED_EXTENSIONS) linkbase = fslpath.removeExt(linkbase, exts)
# Target must exist, so we can # Target must exist, so we can
# infer the correct extension(s). # infer the correct extension(s).
...@@ -59,8 +62,8 @@ def main(argv=None): ...@@ -59,8 +62,8 @@ def main(argv=None):
# (e.g. a.img without a.hdr). # (e.g. a.img without a.hdr).
try: try:
targets = fslpath.getFileGroup(target, targets = fslpath.getFileGroup(target,
allowedExts=ALLOWED_EXTENSIONS, allowedExts=exts,
fileGroups=fslimage.FILE_GROUPS, fileGroups=groups,
unambiguous=True) unambiguous=True)
except Exception as e: except Exception as e:
print(f'Error: {e}') print(f'Error: {e}')
...@@ -70,7 +73,7 @@ def main(argv=None): ...@@ -70,7 +73,7 @@ def main(argv=None):
if not op.exists(target): if not op.exists(target):
continue continue
ext = fslpath.getExt(target, ALLOWED_EXTENSIONS) ext = fslpath.getExt(target, exts)
link = f'{linkbase}{ext}' link = f'{linkbase}{ext}'
try: try:
......
...@@ -13,22 +13,22 @@ import itertools as it ...@@ -13,22 +13,22 @@ import itertools as it
import os.path as op import os.path as op
import os import os
import sys import sys
import warnings
import fsl.utils.path as fslpath import fsl.utils.path as fslpath
# See atlasq.py for explanation
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=FutureWarning)
import fsl.data.image as fslimage
usage = """Usage: imrm <list of image names to remove> usage = """Usage: imrm <list of image names to remove>
NB: filenames can be basenames or not NB: filenames can be basenames or not
""".strip() """.strip()
ALLOWED_EXTENSIONS = fslimage.ALLOWED_EXTENSIONS + ['.mnc', '.mnc.gz'] # This list is defined in the
# fsl.data.image class, but are duplicated
# here for performance (to avoid import of
# nibabel/numpy/etc).
exts = ['.nii.gz', '.nii',
'.img', '.hdr',
'.img.gz', '.hdr.gz',
'.mnc', '.mnc.gz']
"""List of file extensions that are removed by ``imrm``. """ """List of file extensions that are removed by ``imrm``. """
...@@ -42,9 +42,9 @@ def main(argv=None): ...@@ -42,9 +42,9 @@ def main(argv=None):
print(usage) print(usage)
return 1 return 1
prefixes = [fslpath.removeExt(p, ALLOWED_EXTENSIONS) for p in argv] prefixes = [fslpath.removeExt(p, exts) for p in argv]
for prefix, ext in it.product(prefixes, ALLOWED_EXTENSIONS): for prefix, ext in it.product(prefixes, exts):
path = f'{prefix}{ext}' path = f'{prefix}{ext}'
......
...@@ -11,18 +11,22 @@ not, without having to know the file suffix (.nii, .nii.gz, etc). ...@@ -11,18 +11,22 @@ not, without having to know the file suffix (.nii, .nii.gz, etc).
import os.path as op import os.path as op
import sys import sys
import warnings
import fsl.utils.path as fslpath import fsl.utils.path as fslpath
# See atlasq.py for explanation
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=FutureWarning)
import fsl.data.image as fslimage
# The lists below are defined in the
# fsl.data.image class, but are duplicated
# here for performance (to avoid import of
# nibabel/numpy/etc).
exts = ['.nii.gz', '.nii',
'.img', '.hdr',
'.img.gz', '.hdr.gz',
'.mnc', '.mnc.gz']
"""List of file extensions that are supported by ``imtest``.
"""
ALLOWED_EXTENSIONS = fslimage.ALLOWED_EXTENSIONS + ['.mnc', '.mnc.gz'] groups = [('.hdr', '.img'), ('.hdr.gz', '.img.gz')]
"""List of file extensions that are supported by ``imln``. """ """List of known image file groups (image/header file pairs). """
def main(argv=None): def main(argv=None):
...@@ -38,7 +42,7 @@ def main(argv=None): ...@@ -38,7 +42,7 @@ def main(argv=None):
print('0') print('0')
return 0 return 0
path = fslpath.removeExt(argv[0], ALLOWED_EXTENSIONS) path = fslpath.removeExt(argv[0], exts)
path = op.realpath(path) path = op.realpath(path)
# getFileGroup will raise an error # getFileGroup will raise an error
...@@ -47,8 +51,8 @@ def main(argv=None): ...@@ -47,8 +51,8 @@ def main(argv=None):
# image) does not exist # image) does not exist
try: try:
fslpath.getFileGroup(path, fslpath.getFileGroup(path,
allowedExts=ALLOWED_EXTENSIONS, allowedExts=exts,
fileGroups=fslimage.FILE_GROUPS, fileGroups=groups,
unambiguous=True) unambiguous=True)
print('1') print('1')
except fslpath.PathError: except fslpath.PathError:
......
...@@ -6,22 +6,22 @@ ...@@ -6,22 +6,22 @@
# #
import sys import sys
import warnings
import fsl.utils.path as fslpath import fsl.utils.path as fslpath
# See atlasq.py for explanation
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=FutureWarning)
import fsl.data.image as fslimage
usage = """Usage: remove_ext <list of image paths to remove extension from> usage = """Usage: remove_ext <list of image paths to remove extension from>
""".strip() """.strip()
ALLOWED_EXTENSIONS = fslimage.ALLOWED_EXTENSIONS + ['.mnc', '.mnc.gz'] # This list is defined in the
# fsl.data.image class, but are duplicated
# here for performance (to avoid import of
# nibabel/numpy/etc).
exts = ['.nii.gz', '.nii',
'.img', '.hdr',
'.img.gz', '.hdr.gz',
'.mnc', '.mnc.gz']
"""List of file extensions that are removed by ``remove_ext``. """ """List of file extensions that are removed by ``remove_ext``. """
...@@ -40,7 +40,7 @@ def main(argv=None): ...@@ -40,7 +40,7 @@ def main(argv=None):
removed = [] removed = []
for path in argv: for path in argv:
removed.append(fslpath.removeExt(path, ALLOWED_EXTENSIONS)) removed.append(fslpath.removeExt(path, exts))
print(' '.join(removed)) print(' '.join(removed))
......
...@@ -37,8 +37,6 @@ import re ...@@ -37,8 +37,6 @@ import re
from typing import Sequence, Tuple, Union from typing import Sequence, Tuple, Union
from fsl.utils.platform import platform
PathLike = Union[str, pathlib.Path] PathLike = Union[str, pathlib.Path]
...@@ -596,6 +594,7 @@ def winpath(path): ...@@ -596,6 +594,7 @@ def winpath(path):
This requires WSL2 which supports the ``\\wsl$\\`` network path. This requires WSL2 which supports the ``\\wsl$\\`` network path.
wslpath is assumed to be an absolute path. wslpath is assumed to be an absolute path.
""" """
from fsl.utils.platform import platform # pylint: disable=import-outside-toplevel # noqa: E501
if not platform.fslwsl: if not platform.fslwsl:
return path return path
else: else:
......
...@@ -111,26 +111,15 @@ class Platform(notifier.Notifier): ...@@ -111,26 +111,15 @@ class Platform(notifier.Notifier):
self.WX_MAC_CARBON = WX_MAC_CARBON self.WX_MAC_CARBON = WX_MAC_CARBON
self.WX_GTK = WX_GTK self.WX_GTK = WX_GTK
self.__inSSHSession = False # initialise fsldir - see fsldir.setter
self.__inVNCSession = False self.fsldir = self.fsldir
# These are all initialised on first access
self.__glVersion = None self.__glVersion = None
self.__glRenderer = None self.__glRenderer = None
self.__glIsSoftware = None self.__glIsSoftware = None
self.__fslVersion = None self.__fslVersion = None
self.__canHaveGui = None
# initialise fsldir - see fsldir.setter
self.fsldir = self.fsldir
# Determine if a display is available. We do
# this once at init (instead of on-demand in
# the canHaveGui method) because calling the
# IsDisplayAvailable function will cause the
# application to steal focus under OSX!
try:
import wx
self.__canHaveGui = wx.App.IsDisplayAvailable()
except ImportError:
self.__canHaveGui = False
# If one of the SSH_/VNC environment # If one of the SSH_/VNC environment
# variables is set, then we're probably # variables is set, then we're probably
...@@ -177,7 +166,7 @@ class Platform(notifier.Notifier): ...@@ -177,7 +166,7 @@ class Platform(notifier.Notifier):
the event loop is called periodically, and so is not always running. the event loop is called periodically, and so is not always running.
""" """
try: try:
import wx import wx # pylint: disable=import-outside-toplevel
app = wx.GetApp() app = wx.GetApp()
# TODO Previously this conditional # TODO Previously this conditional
...@@ -216,6 +205,17 @@ class Platform(notifier.Notifier): ...@@ -216,6 +205,17 @@ class Platform(notifier.Notifier):
'Equivalent functionality is available in fsleyes-widgets.') 'Equivalent functionality is available in fsleyes-widgets.')
def canHaveGui(self): def canHaveGui(self):
"""``True`` if it is possible to create a GUI, ``False`` otherwise. """ """``True`` if it is possible to create a GUI, ``False`` otherwise. """
# Determine if a display is available. Note that
# calling the IsDisplayAvailable function will
# cause the application to steal focus under OSX!
if self.__canHaveGui is None:
try:
import wx # pylint: disable=import-outside-toplevel
self.__canHaveGui = wx.App.IsDisplayAvailable()
except ImportError:
self.__canHaveGui = False
return self.__canHaveGui return self.__canHaveGui
...@@ -261,14 +261,14 @@ class Platform(notifier.Notifier): ...@@ -261,14 +261,14 @@ class Platform(notifier.Notifier):
if not self.canHaveGui: if not self.canHaveGui:
return WX_UNKNOWN return WX_UNKNOWN
import wx import wx # pylint: disable=import-outside-toplevel
pi = [t.lower() for t in wx.PlatformInfo] pi = [t.lower() for t in wx.PlatformInfo]
if any(['cocoa' in p for p in pi]): plat = WX_MAC_COCOA if any('cocoa' in p for p in pi): plat = WX_MAC_COCOA
elif any(['carbon' in p for p in pi]): plat = WX_MAC_CARBON elif any('carbon' in p for p in pi): plat = WX_MAC_CARBON
elif any(['gtk' in p for p in pi]): plat = WX_GTK elif any('gtk' in p for p in pi): plat = WX_GTK
else: plat = WX_UNKNOWN else: plat = WX_UNKNOWN
if plat is WX_UNKNOWN: if plat is WX_UNKNOWN:
log.warning('Could not determine wx platform from ' log.warning('Could not determine wx platform from '
...@@ -290,7 +290,7 @@ class Platform(notifier.Notifier): ...@@ -290,7 +290,7 @@ class Platform(notifier.Notifier):
if not self.canHaveGui: if not self.canHaveGui:
return WX_UNKNOWN return WX_UNKNOWN
import wx import wx # pylint: disable=import-outside-toplevel
pi = [t.lower() for t in wx.PlatformInfo] pi = [t.lower() for t in wx.PlatformInfo]
isPhoenix = False isPhoenix = False
...@@ -323,7 +323,9 @@ class Platform(notifier.Notifier): ...@@ -323,7 +323,9 @@ class Platform(notifier.Notifier):
@property @property
def fslwsl(self): def fslwsl(self):
"""Boolean flag indicating whether FSL is installed in Windows Subsystem for Linux """ """Boolean flag indicating whether FSL is installed in Windows
Subsystem for Linux
"""
return self.fsldir is not None and self.fsldir.startswith("\\\\wsl$") return self.fsldir is not None and self.fsldir.startswith("\\\\wsl$")
...@@ -352,8 +354,9 @@ class Platform(notifier.Notifier): ...@@ -352,8 +354,9 @@ class Platform(notifier.Notifier):
if op.exists(versionFile): if op.exists(versionFile):
with open(versionFile, 'rt') as f: with open(versionFile, 'rt') as f:
# split string at colon for new hash style versions # split string at colon for new hash style versions
# first object in list is the non-hashed version string (e.g. 6.0.2) # first object in list is the non-hashed version string
# if no ":hash:" then standard FSL version string is still returned # (e.g. 6.0.2) if no ":hash:" then standard FSL version
# string is still returned
self.__fslVersion = f.read().strip().split(":")[0] self.__fslVersion = f.read().strip().split(":")[0]
self.notify(value=value) self.notify(value=value)
......
...@@ -47,7 +47,7 @@ import re ...@@ -47,7 +47,7 @@ import re
import string import string
__version__ = '3.7.0.dev0' __version__ = '3.8.0.dev0'
"""Current version number, as a string. """ """Current version number, as a string. """
......
...@@ -100,6 +100,9 @@ def test_imglob_shouldPass2(): ...@@ -100,6 +100,9 @@ def test_imglob_shouldPass2():
('file1.hdr file1.img file2.nii', 'file1 file2', 'primary', 'file1.hdr file2.nii'), ('file1.hdr file1.img file2.nii', 'file1 file2', 'primary', 'file1.hdr file2.nii'),
('file1.hdr file1.img file2.nii', 'file1 file2', 'all', 'file1.hdr file1.img file2.nii'), ('file1.hdr file1.img file2.nii', 'file1 file2', 'all', 'file1.hdr file1.img file2.nii'),
# muiltiple files, given wildcard
('file1.nii file2.nii', 'file*', 'prefix', 'file1 file2'),
# no file # no file
('file.nii', 'bag', 'prefix', ''), ('file.nii', 'bag', 'prefix', ''),
('file.nii', 'bag', 'primary', ''), ('file.nii', 'bag', 'primary', ''),
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment