Commit 399b03b7 authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

Merge branch 'enh/3d-depth-picking' into 'master'

ENH: Depth picking for volume overlays

See merge request fsl/fsleyes/fsleyes!290
parents eeca288e 5ca0fcec
......@@ -13,14 +13,22 @@ chronological order.
-------------------------
Added
^^^^^
* The 3D view now allows the display location to be set to the corresponding
location under the mouse on a volume overlay, by shift+clicking (!290).
Changed
^^^^^^^
* Small improvementsto the *File* |right_arrow| *Add from XNAT* dialog.
* The *Sample along line* tool now supports 2D and multi-channel (e.g. RGB)
images (currently plotting the mean intensity across channels for the
latter).
* Small improvementsto the *File* |right_arrow| *Add from XNAT* dialog (!291).
Fixed
......
......@@ -122,12 +122,16 @@ MOV skipTest, 1;
MOV finalColour, startingColour;
# Set depth.w > 0 if depth has
# not been set, < 0 otherwise.
# If depth has already been set,
# we want to pass through its
# initial value.
SUB depth.w, 0, finalColour.a;
# Set depth.w < 0 if depth has already been
# set in a previous pass, > 0 otherwise. If
# depth has already been set, we want to pass
# through that value, rather than calculating
# it again. Initial value for a depth texture
# is 1 (and depth textures are clamped to 0
# [closer to screen], 1 [farther from screen]),
# so if it is < 1, we assume it has already
# been set.
SUB depth.w, depth.x, 1;
CMP depth.w, depth.w, -1, 1;
......@@ -209,7 +213,10 @@ MIN skipTest.x, skipTest.x, tempVar.x;
# Check whether the accumulated
# colour alpha is already high enough
# colour alpha is already high enough.
# If it is, we don't consider the
# values from any more samples on the
# ray.
SLT tempVar.x, finalColour.a, 0.95;
MAD tempVar.x, tempVar.x, 2.0, -1;
MIN skipTest.x, skipTest.x, tempVar.x;
......@@ -250,32 +257,55 @@ MUL colour.rgb, colour, colour.a;
SUB tempVar.x, 1, finalColour.a;
MAD tempColour, tempVar.x, colour, finalColour;
# Figure out if we should consider this
# sample for depth calculation:
# - skipTest.x tells us whether this sample
# should be discarded
# - depth.w tells us whether we already have
# a depth value
# - {{ param_screenSize }}.z tells us whether
# we are modulating alpha by voxel intensity
# (depth is calculated differently depending
# on whether we are or not)
#
# If modulating alpha by voxel intensity, we
# take the depth of the first sample on the ray
# with intensity >= 0.1 (voxel values are
# normalised to [0, 1] w.r.t. current display
# range).
#
# If not blending by voxel intensity, we
# take the depth of the first sample.
#
# We accumulate all of these conditionals into
# tempVar.x, such that it is == 0 if we should
# take the depth value from this sample, < 0
# otherwise.
SLT tempVar.x, skipTest.x, 0; # set x if should skip
SLT tempVar.y, depth.w, 0; # set y if already have depth
SUB tempVar.z, voxValue.x, {{ param_negCmap }}.y; # set z if below threshold
SLT tempVar.z, voxValue.x, 0.1;
ADD tempVar.z, tempVar.z, {{ param_screenSize }}.z; # and blending by intensity
SGE tempVar.z, tempVar.z, 2;
ADD tempVar.x, tempVar.x, tempVar.y;
ADD tempVar.x, tempVar.x, tempVar.z;
MUL tempVar.x, tempVar.x, -1;
# Calculate the screen depth of
# the current ray position. If we
# have just taken our first sample,
# save the depth, as we will use
# it for the final fragment depth.
# Figure out if this is the first
# sample taken by testing whether
# skipTest.x > 0, and depth.w > 0
# (the latter is true if the
# startingColour.a == 0).
ADD depth.y, depth.w, skipTest.x;
MUL depth.y, depth.y, -1;
DP4 depth.z, {{ param_tex2ScreenXform }}, texCoord;
CMP depth.x, depth.y, depth.z, depth.x;
# Calculate the screen depth of the
# current ray position, storing it
# in tempvar.z
DP4 tempVar.z, {{ param_tex2ScreenXform }}, texCoord;
# Save the depth value to depth.x, based on
# conditional tests above.
CMP depth.x, tempVar.x, depth.x, tempVar.z;
# If we have just set the depth,
# set depth.w to < 0 so that we
# don't overwrite the depth on a
# subsequent sample.
SUB depth.y, depth.w, skipTest.x;
MAD depth.y, depth.y, -1, 0.5;
CMP depth.w, depth.y, 1, -1;
CMP depth.w, tempVar.x, depth.w, -1;
# Only update the accumulated colour
......@@ -294,9 +324,25 @@ CMP finalColour, skipTest.x, finalColour, tempColour;
ADD texCoord.xyz, texCoord, {{ param_rayStep }};
ADD clipTexCoord.xyz, clipTexCoord, {{ param_rayStep }};
{% endfor %}
# set tempVar.w to -1 if this is the final
# pass, and we still don't have a depth
# value, 0 otherwise.
SGE tempVar.x, {{ param_settings }}.z, 1;
SGE tempVar.y, depth.w, 1;
ADD tempVar.x, tempVar.x, tempVar.y;
SGE tempVar.x, tempVar.x, 2;
MUL tempVar.w, tempVar.x, -1;
# Calculate the depth value corresponding
# to the very first sample in this ray.
ADD tempVar.xyz, {{ param_rayStep }}, {{ varying_texCoord }};
DP4 tempVar.z, {{ param_tex2ScreenXform }}, tempVar;
# Use it if we don't already have a depth
CMP depth.x, tempVar.w, tempVar.z, depth.x;
# If startingColour.a == 0 and
# finalColour.a == 0,
......
......@@ -344,7 +344,8 @@ void main(void) {
vec3 texCoord = fragTexCoord;
vec4 colour = vec4(0);
vec4 finalColour = vec4(0);
vec4 depth = vec4(0);
float depth = 0;
float firstDepth = 0;
int nsamples = 0;
float voxValue;
int clipIdx;
......@@ -385,47 +386,67 @@ void main(void) {
* voxel was not clipped and was
* not NaN.
*/
if (sample_volume(texCoord, vec3(0, 0, 0), vec3(0, 0, 0), voxValue, colour)) {
if (lighting) {
colour.rgb = volume_lighting(texCoord,
imageTexture,
lightPos,
colour.rgb,
numClipPlanes,
clipPlanes,
clipMode);
}
if (!sample_volume(texCoord, vec3(0, 0, 0), vec3(0, 0, 0), voxValue, colour)) {
continue;
}
if (nsamples == 1) {
firstDepth = (tex2ScreenXform * vec4(texCoord, 1.0)).z;
}
if (lighting) {
colour.rgb = volume_lighting(texCoord,
imageTexture,
lightPos,
colour.rgb,
numClipPlanes,
clipPlanes,
clipMode);
}
/*
* weight the sample opacity by the voxel intensity
* (normalised w.r.t. the current display range)
*/
if (blendByIntensity) {
colour.a = 1 - pow(1 - clamp(voxValue, 0, 1), 1 - blendFactor);
/*
* weight the sample opacity by the voxel intensity
* (normalised w.r.t. the current display range)
* When alpha is being modulated by voxel intensity,
* we set the fragment depth to the location of the
* first sample with intensity greater than 0.1
* (voxValue is normalised to [0, 1] w.r.t. the
* current display range). The 0.1 threshold is
* completely arbitrary, but seems to work well.
*/
if (blendByIntensity) {
colour.a = 1 - pow(1 - clamp(voxValue, 0, 1), 1 - blendFactor);
}
/* Or just weight by blend factor */
else {
colour.a = 1 - blendFactor;
if (depth == 0 && (voxValue - texZero) >= 0.1) {
depth = (tex2ScreenXform * vec4(texCoord, 1.0)).z;
}
colour.rgb *= colour.a;
finalColour += (1 - finalColour.a) * colour;
nsamples += 1;
}
/* Or just weight by blend factor */
else {
colour.a = 1 - blendFactor;
/*
* If this is the first sample on the ray,
* set the fragment depth to its location
* When alpha is not modulated by voxel intensity,
* we simply set the fragment depth to the
* position of the first sample on the ray.
*/
if (nsamples == 1) {
depth = tex2ScreenXform * vec4(texCoord, 1.0);
depth = firstDepth;
}
}
colour.rgb *= colour.a;
finalColour += (1 - finalColour.a) * colour;
nsamples += 1;
}
if (nsamples > 0) {
if (depth == 0) {
depth = firstDepth;
}
finalColour.a *= alpha;
gl_FragDepth = depth.z;
gl_FragDepth = depth;
gl_FragColor = finalColour;
}
else {
......
......@@ -261,9 +261,19 @@ class Volume3DOpts(object):
#
# The projection matrix puts depth into
# [-1, 1], but we want it in [0, 1]
zscale = affine.scaleOffsetXform([1, 1, 0.5], [0, 0, 0.5])
zscale = affine.scaleOffsetXform([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
xform = affine.concat(zscale, proj, xform)
# The Scene3DViewProfile needs to know the image
# texture to screen transformation so it can
# transform screen locations into image
# coordinates. So we construct an appropriate
# transform and cache it in the overlay list
# whenever it is recalculated.
scr2disp = affine.concat(t2dmat, affine.invert(xform))
self.overlayList.setData(
self.overlay, 'screen2DisplayXform_{}'.format(id(self)), scr2disp)
return rayStep, xform
......
......@@ -256,6 +256,14 @@ def draw3D(self, xform=None, bbox=None):
with glroutines.enabled((gl.GL_VERTEX_ARRAY)), \
glroutines.disabled((gl.GL_BLEND)):
# The depth value for a fragment will
# not necessary be set at the same
# time that the fragment colour is
# set, so we need to use <= for depth
# testing so that unset depth values
# do not cause depth clipping.
gl.glDepthFunc(gl.GL_LEQUAL)
for i in range(outerLoop):
settings = list(settings)
......@@ -280,6 +288,8 @@ def draw3D(self, xform=None, bbox=None):
dest, src = src, dest
gl.glDepthFunc(gl.GL_LESS)
shader.unloadAtts()
shader.unload()
......
......@@ -231,6 +231,7 @@ def draw3D(self, xform=None, bbox=None):
:arg bbox: An optional bounding box.
"""
ovl = self.overlay
opts = self.opts
canvas = self.canvas
copts = canvas.opts
......@@ -240,9 +241,9 @@ def draw3D(self, xform=None, bbox=None):
rayStep , texform = opts.calculateRayCastSettings(xform, proj)
rayStep = affine.transformNormal(
rayStep, self.imageTexture.texCoordXform(self.overlay.shape))
rayStep, self.imageTexture.texCoordXform(ovl.shape))
texform = affine.concat(
texform, self.imageTexture.invTexCoordXform(self.overlay.shape))
texform, self.imageTexture.invTexCoordXform(ovl.shape))
# If lighting is enabled, we specify the light
# position in image texture coordinates, to make
......
......@@ -731,8 +731,8 @@ class GLVolume(glimageobject.GLImageObject):
fslgl.glvolume_funcs.draw3D(self, *args, **kwargs)
# Apply smoothing if needed. If smoothing
# is enabled, the final final render will
# be in renderTexture2
# is enabled, the final render will be in
# renderTexture2
if opts.smoothing > 0:
self.smoothFilter.set(offsets=[1.0 / sw, 1.0 / sh])
self.smoothFilter.osApply(self.renderTexture1,
......@@ -759,7 +759,7 @@ class GLVolume(glimageobject.GLImageObject):
# contains the final render. Otherwise,
# rt2 contains the final render, but rt1
# contains the depth information. So we
# need to # temporarily replace rt2.depth
# need to temporarily replace rt2.depth
# with rt1.depth.
if opts.smoothing > 0:
src = self.renderTexture2
......
......@@ -11,8 +11,9 @@
import logging
import wx
import numpy as np
import wx
import numpy as np
import OpenGL.GL as gl
import fsleyes_props as props
import fsl.transform.affine as affine
......@@ -153,8 +154,8 @@ class Scene3DViewProfile(profiles.Profile):
self.__rotateMousePos = mousePos
canvas.opts.rotation = affine.concat(rot,
self.__lastRot,
self.__baseXform)
self.__lastRot,
self.__baseXform)
def _rotateModeLeftMouseUp(self, ev, canvas, mousePos, canvasPos):
......@@ -228,30 +229,70 @@ class Scene3DViewProfile(profiles.Profile):
Updates the :attr:`DisplayContext.location` property.
"""
from fsl.data.mesh import Mesh
from fsl.data.mesh import Mesh
from fsl.data.image import Image
displayCtx = self.displayCtx
ovl = displayCtx.getSelectedOverlay()
displayCtx = self.displayCtx
overlayList = self.overlayList
ovl = displayCtx.getSelectedOverlay()
if ovl is None:
return
opts = self.displayCtx.getOpts(ovl)
# The canvasPos is located on the near clipping
# plane (see Scene3DCanvas.canvasToWorld).
# We also need the corresponding point on the
# far clipping plane.
farPos = canvas.canvasToWorld(mousePos[0], mousePos[1], near=False)
# For non-mesh overlays, we select a point which
# is in between the near/far clipping planes.
if not isinstance(ovl, Mesh):
# For image overlays, we transform screen
# coordinates into display coordinates, via
# a texture to screen coord affine, which
# is cached by the glvolume draw functions.
if isinstance(ovl, Image):
screen2Display = overlayList.getData(
ovl, 'screen2DisplayXform_{}'.format(id(opts)), None)
posDir = farPos - canvasPos
dist = affine.veclength(posDir)
posDir = affine.normalise(posDir)
midPos = canvasPos + 0.5 * dist * posDir
if screen2Display is None:
return
self.displayCtx.location.xyz = midPos
# Retrieve the depth for the current
# fragment. Images are drawn to an off-screen
# texture (see GLVolume.draw3d), so we can get
# the depth from there.
globj = canvas.getGLObject(ovl)
tex = globj.renderTexture1.depthTexture
with tex.bound():
# There's no function to read part of
# a texture in GL < 4.5, so we have
# to read the entire depth testure.
buf = gl.glGetTexImage(gl.GL_TEXTURE_2D,
0,
tex.baseFormat,
tex.textureType,
None)
# Get the mouse coords, and transform
# them into normalised device coordinates
# (NDCs, in the range [0, 1] - see
# Volume3DOpts.calculateRayCastSettings),
x, y = mousePos
w, h = canvas.GetSize()
# The depth texure is stored as uint32,
# but represents floating point values in
# the range [0, 1].
buf = np.frombuffer(buf, dtype=tex.dtype).reshape(h, w)
z = buf[y, x] / 4294967295.0
x = x / w
y = y / h
# Transform NDCs into display coordinates
xyz = affine.transform([x, y, z], screen2Display)
self.displayCtx.location.xyz = xyz
else:
opts = self.displayCtx.getOpts(ovl)
......
......@@ -60,8 +60,9 @@ cli_tests = """
-dl 3d -cp 25 0 0 -cp 50 90 45 -cp 50 90 -45 -m intersection
-dl 3d -cp 75 0 0 -cp 50 90 45 -cp 50 90 -45 -m union
-dl 3d -cp 75 0 0 -cp 50 90 45 -cp 50 90 -45 -m complement
"""
cli_tests_2d = """
# 2D images
-dl -rot -45 -30 0 {{roi('3d.nii.gz', (0, 17, 0, 14, 6, 7))}} {{roi('3d.nii.gz', (0, 17, 6, 7, 0, 14))}} {{roi('3d.nii.gz', (8, 9, 0, 14, 0, 14))}}
-dl -rot 125 -45 0 {{roi('3d.nii.gz', (0, 17, 0, 14, 6, 7))}} {{roi('3d.nii.gz', (0, 17, 6, 7, 0, 14))}} {{roi('3d.nii.gz', (8, 9, 0, 14, 0, 14))}}
......@@ -92,6 +93,21 @@ def test_overlay_volume_3d():
threshold=40)
# Blending multiple volumes in GL14 doesn't quite work,
# because of limitations in the fragment shader w.r.t.
# calculating fragment depth.
@pytest.mark.skipif('not haveGL21()')
def test_overlay_volume_3d_2d():
extras = {
'roi' : roi
}
run_cli_tests('test_overlay_volume_3d',
cli_tests_2d,
extras=extras,
scene='3d',
threshold=40)
@pytest.mark.skipif('not haveGL21()')
def test_overlay_volume_3d_lighting():
run_cli_tests('test_overlay_volume_3d_lighting',
......
......@@ -2,6 +2,7 @@
.. |alt_key| unicode:: U+2325
.. |command_key| unicode:: U+2318
.. |control_key| unicode:: U+2303
.. |shift_key| unicode:: U+21E7
.. |reset_zoom_icon| image:: images/reset_zoom_icon.png
.. |gear_icon| image:: images/gear_icon.png
......@@ -33,6 +34,9 @@ You can interact with the 3D view in the following ways:
- Hold down the |command_key| or |control_key| and spin the mouse wheel to
zoom in and out.
- Hold down the |shift_key| and click to change the display location,
relative to the currently selected image or surface.
To reset the view, click on the |reset_zoom_icon| button on the toolbar.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment