improve transparency & outlines

- proper per-group transparency with wboit off
- fixed outlines with transparent backgound

CHANGELOG.md

@@ -8,10 +8,12 @@ Note that since we don't clearly distinguish between a public and private interf
 
 - Fix case handling of ``struct_conf`` mmCIF enumeration field (#425)
 - Add support for outlines around transparent objects
+- Improve per-group transparency when wboit is switched off
 - Fix ``allowTransparentBackfaces`` for per-group transparency
 - Fix ``FormatRegistry.isApplicable`` returning true for unregistered formats
 - Fix: handle building of ``GridLookup3D`` with zero cell size
 - Fix ``ignoreLight`` for direct-volume rendering with webgl1
+- Fix (non-black) outlines when using transparent background
 
 ## [v3.7.0] - 2022-04-13
 

src/mol-canvas3d/passes/draw.ts

@@ -142,9 +142,9 @@ export class DrawPass {
         if (PostprocessingPass.isEnabled(postprocessingProps)) {
             if (PostprocessingPass.isOutlineEnabled(postprocessingProps)) {
                 this.depthTarget.bind();
-                renderer.clear(false);
+                renderer.clearDepth(true);
                 if (scene.getOpacityAverage() < 1) {
-                    renderer.renderDepthWboit(scene.primitives, camera, null);
+                    renderer.renderDepthTransparent(scene.primitives, camera, this.depthTexturePrimitives);
                 }
             }
 
@@ -183,7 +183,7 @@ export class DrawPass {
             // extensions.depthTexture is unsupported (i.e. depthTarget is set)
             if (this.depthTargetPrimitives) {
                 this.depthTargetPrimitives.bind();
-                renderer.clear(false);
+                renderer.clearDepth(true);
                 renderer.renderDepthOpaque(scene.primitives, camera, null);
                 this.colorTarget.bind();
             }
@@ -197,9 +197,9 @@ export class DrawPass {
 
                 if (PostprocessingPass.isOutlineEnabled(postprocessingProps)) {
                     this.depthTarget.bind();
-                    renderer.clear(false);
+                    renderer.clearDepth(true);
                     if (scene.getOpacityAverage() < 1) {
-                        renderer.renderDepthTransparent(scene.primitives, camera, null);
+                        renderer.renderDepthTransparent(scene.primitives, camera, this.depthTexturePrimitives);
                     }
                 }
 

src/mol-gl/renderable/schema.ts

@@ -1,5 +1,5 @@
 /**
- * Copyright (c) 2018-2021 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2018-2022 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
@@ -153,7 +153,7 @@ export const GlobalUniformSchema = {
 
     uXrayEdgeFalloff: UniformSpec('f'),
 
-    uRenderWboit: UniformSpec('b'),
+    uRenderMask: UniformSpec('i'),
     uMarkingDepthTest: UniformSpec('b'),
     uMarkingType: UniformSpec('i'),
     uPickType: UniformSpec('i'),

src/mol-gl/renderer.ts

@@ -54,14 +54,13 @@ interface Renderer {
     readonly props: Readonly<RendererProps>
 
     clear: (toBackgroundColor: boolean, ignoreTransparentBackground?: boolean) => void
-    clearDepth: () => void
+    clearDepth: (packed?: boolean) => void
     update: (camera: ICamera) => void
 
     renderPick: (group: Scene.Group, camera: ICamera, variant: 'pick' | 'depth', depthTexture: Texture | null, pickType: PickType) => void
     renderDepth: (group: Scene.Group, camera: ICamera, depthTexture: Texture | null) => void
     renderDepthOpaque: (group: Scene.Group, camera: ICamera, depthTexture: Texture | null) => void
     renderDepthTransparent: (group: Scene.Group, camera: ICamera, depthTexture: Texture | null) => void
-    renderDepthWboit: (group: Scene.Group, camera: ICamera, depthTexture: Texture | null) => void
     renderMarkingDepth: (group: Scene.Group, camera: ICamera, depthTexture: Texture | null) => void
     renderMarkingMask: (group: Scene.Group, camera: ICamera, depthTexture: Texture | null) => void
     renderBlended: (group: Scene.Group, camera: ICamera, depthTexture: Texture | null) => void
@@ -143,6 +142,12 @@ namespace Renderer {
         BlendedBack = 2
     }
 
+    const enum Mask {
+        All = 0,
+        Opaque = 1,
+        Transparent = 2,
+    }
+
     export function create(ctx: WebGLContext, props: Partial<RendererProps> = {}): Renderer {
         const { gl, state, stats } = ctx;
         const p = PD.merge(RendererParams, PD.getDefaultValues(RendererParams), props);
@@ -155,6 +160,8 @@ namespace Renderer {
         let transparentBackground = false;
 
         const emptyDepthTexture = ctx.resources.texture('image-uint8', 'rgba', 'ubyte', 'nearest');
+        emptyDepthTexture.define(1, 1);
+        emptyDepthTexture.load({ array: new Uint8Array([255, 255, 255, 255]), width: 1, height: 1 });
         const sharedTexturesList: Textures = [
             ['tDepth', emptyDepthTexture]
         ];
@@ -199,7 +206,7 @@ namespace Renderer {
             uFogFar: ValueCell.create(10000),
             uFogColor: ValueCell.create(bgColor),
 
-            uRenderWboit: ValueCell.create(false),
+            uRenderMask: ValueCell.create(0),
             uMarkingDepthTest: ValueCell.create(false),
             uPickType: ValueCell.create(PickType.None),
             uMarkingType: ValueCell.create(MarkingType.None),
@@ -329,7 +336,7 @@ namespace Renderer {
             ValueCell.updateIfChanged(globalUniforms.uTransparentBackground, transparentBackground);
         };
 
-        const updateInternal = (group: Scene.Group, camera: ICamera, depthTexture: Texture | null, renderWboit: boolean, markingDepthTest: boolean) => {
+        const updateInternal = (group: Scene.Group, camera: ICamera, depthTexture: Texture | null, renderMask: Mask, markingDepthTest: boolean) => {
             arrayMapUpsert(sharedTexturesList, 'tDepth', depthTexture || emptyDepthTexture);
 
             ValueCell.update(globalUniforms.uModel, group.view);
@@ -338,7 +345,7 @@ namespace Renderer {
             ValueCell.update(globalUniforms.uModelViewProjection, Mat4.mul(modelViewProjection, modelView, camera.projection));
             ValueCell.update(globalUniforms.uInvModelViewProjection, Mat4.invert(invModelViewProjection, modelViewProjection));
 
-            ValueCell.updateIfChanged(globalUniforms.uRenderWboit, renderWboit);
+            ValueCell.updateIfChanged(globalUniforms.uRenderMask, renderMask);
             ValueCell.updateIfChanged(globalUniforms.uMarkingDepthTest, markingDepthTest);
 
             state.enable(gl.SCISSOR_TEST);
@@ -357,7 +364,7 @@ namespace Renderer {
             state.enable(gl.DEPTH_TEST);
             state.depthMask(true);
 
-            updateInternal(group, camera, depthTexture, false, false);
+            updateInternal(group, camera, depthTexture, Mask.All, false);
             ValueCell.updateIfChanged(globalUniforms.uPickType, pickType);
 
             const { renderables } = group;
@@ -373,7 +380,7 @@ namespace Renderer {
             state.enable(gl.DEPTH_TEST);
             state.depthMask(true);
 
-            updateInternal(group, camera, depthTexture, false, false);
+            updateInternal(group, camera, depthTexture, Mask.All, false);
 
             const { renderables } = group;
             for (let i = 0, il = renderables.length; i < il; ++i) {
@@ -386,12 +393,14 @@ namespace Renderer {
             state.enable(gl.DEPTH_TEST);
             state.depthMask(true);
 
-            updateInternal(group, camera, depthTexture, false, false);
+            updateInternal(group, camera, depthTexture, Mask.Opaque, false);
 
             const { renderables } = group;
             for (let i = 0, il = renderables.length; i < il; ++i) {
                 const r = renderables[i];
-                if (r.state.opaque) renderObject(r, 'depth', Flag.None);
+                if (r.state.opaque && r.values.transparencyAverage.ref.value !== 1 && !r.values.dXrayShaded?.ref.value) {
+                    renderObject(r, 'depth', Flag.None);
+                }
             }
         };
 
@@ -400,27 +409,12 @@ namespace Renderer {
             state.enable(gl.DEPTH_TEST);
             state.depthMask(true);
 
-            updateInternal(group, camera, depthTexture, false, false);
-
-            const { renderables } = group;
-            for (let i = 0, il = renderables.length; i < il; ++i) {
-                const r = renderables[i];
-                if (!r.state.opaque) renderObject(r, 'depth', Flag.None);
-            }
-        };
-
-        const renderDepthWboit = (group: Scene.Group, camera: ICamera, depthTexture: Texture | null) => {
-            state.disable(gl.BLEND);
-            state.enable(gl.DEPTH_TEST);
-            state.depthMask(true);
-
-            updateInternal(group, camera, depthTexture, true, false);
+            updateInternal(group, camera, depthTexture, Mask.Transparent, false);
 
             const { renderables } = group;
             for (let i = 0, il = renderables.length; i < il; ++i) {
                 const r = renderables[i];
-
-                if (!r.state.opaque || r.values.transparencyAverage.ref.value > 0) {
+                if (!r.state.opaque || r.values.transparencyAverage.ref.value > 0 || r.values.dXrayShaded?.ref.value) {
                     renderObject(r, 'depth', Flag.None);
                 }
             }
@@ -431,7 +425,7 @@ namespace Renderer {
             state.enable(gl.DEPTH_TEST);
             state.depthMask(true);
 
-            updateInternal(group, camera, depthTexture, false, false);
+            updateInternal(group, camera, depthTexture, Mask.All, false);
             ValueCell.updateIfChanged(globalUniforms.uMarkingType, MarkingType.Depth);
 
             const { renderables } = group;
@@ -449,7 +443,7 @@ namespace Renderer {
             state.enable(gl.DEPTH_TEST);
             state.depthMask(true);
 
-            updateInternal(group, camera, depthTexture, false, !!depthTexture);
+            updateInternal(group, camera, depthTexture, Mask.All, !!depthTexture);
             ValueCell.updateIfChanged(globalUniforms.uMarkingType, MarkingType.Mask);
 
             const { renderables } = group;
@@ -472,7 +466,7 @@ namespace Renderer {
             state.enable(gl.DEPTH_TEST);
             state.depthMask(true);
 
-            updateInternal(group, camera, depthTexture, false, false);
+            updateInternal(group, camera, depthTexture, Mask.Opaque, false);
 
             const { renderables } = group;
             for (let i = 0, il = renderables.length; i < il; ++i) {
@@ -488,7 +482,7 @@ namespace Renderer {
         const renderBlendedTransparent = (group: Scene.Group, camera: ICamera, depthTexture: Texture | null) => {
             state.enable(gl.DEPTH_TEST);
 
-            updateInternal(group, camera, depthTexture, false, false);
+            updateInternal(group, camera, depthTexture, Mask.Transparent, false);
 
             const { renderables } = group;
 
@@ -510,7 +504,7 @@ namespace Renderer {
             state.depthMask(false);
             for (let i = 0, il = renderables.length; i < il; ++i) {
                 const r = renderables[i];
-                if (!r.state.opaque && !r.state.writeDepth) {
+                if ((!r.state.opaque && !r.state.writeDepth) || r.values.transparencyAverage.ref.value > 0) {
                     if (r.values.uDoubleSided?.ref.value) {
                         // render frontfaces and backfaces separately to avoid artefacts
                         if (r.values.dTransparentBackfaces?.ref.value !== 'opaque') {
@@ -528,7 +522,7 @@ namespace Renderer {
             state.blendFunc(gl.ONE, gl.ONE_MINUS_SRC_ALPHA);
             state.enable(gl.BLEND);
 
-            updateInternal(group, camera, depthTexture, false, false);
+            updateInternal(group, camera, depthTexture, Mask.Transparent, false);
 
             const { renderables } = group;
             for (let i = 0, il = renderables.length; i < il; ++i) {
@@ -544,7 +538,7 @@ namespace Renderer {
             state.enable(gl.DEPTH_TEST);
             state.depthMask(true);
 
-            updateInternal(group, camera, depthTexture, false, false);
+            updateInternal(group, camera, depthTexture, Mask.Opaque, false);
 
             const { renderables } = group;
             for (let i = 0, il = renderables.length; i < il; ++i) {
@@ -560,7 +554,7 @@ namespace Renderer {
         };
 
         const renderWboitTransparent = (group: Scene.Group, camera: ICamera, depthTexture: Texture | null) => {
-            updateInternal(group, camera, depthTexture, true, false);
+            updateInternal(group, camera, depthTexture, Mask.Transparent, false);
 
             const { renderables } = group;
             for (let i = 0, il = renderables.length; i < il; ++i) {
@@ -591,11 +585,18 @@ namespace Renderer {
                 }
                 gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
             },
-            clearDepth: () => {
+            clearDepth: (packed = false) => {
                 state.enable(gl.SCISSOR_TEST);
-                state.enable(gl.DEPTH_TEST);
-                state.depthMask(true);
-                gl.clear(gl.DEPTH_BUFFER_BIT);
+
+                if (packed) {
+                    state.colorMask(true, true, true, true);
+                    state.clearColor(1, 1, 1, 1);
+                    gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
+                } else {
+                    state.enable(gl.DEPTH_TEST);
+                    state.depthMask(true);
+                    gl.clear(gl.DEPTH_BUFFER_BIT);
+                }
             },
             update,
 
@@ -603,7 +604,6 @@ namespace Renderer {
             renderDepth,
             renderDepthOpaque,
             renderDepthTransparent,
-            renderDepthWboit,
             renderMarkingDepth,
             renderMarkingMask,
             renderBlended,

src/mol-gl/shader/chunks/assign-material-color.glsl.ts

@@ -31,14 +31,34 @@ export const assign_material_color = `
 #elif defined(dRenderVariant_pick)
     vec4 material = vColor;
 #elif defined(dRenderVariant_depth)
-    if (uRenderWboit && gl_FragColor.a == 1.0) {
+    if (fragmentDepth > getDepth(gl_FragCoord.xy / uDrawingBufferSize)) {
         discard;
     }
-    #ifdef enabledFragDepth
-        vec4 material = packDepthToRGBA(gl_FragDepthEXT);
+
+    #ifndef dXrayShaded
+        #if defined(dTransparency)
+            float dta = 1.0 - vTransparency;
+            if (vTransparency < 0.2) dta = 1.0; // hard cutoff looks better
+
+            if (uRenderMask == MaskTransparent && uAlpha * dta == 1.0) {
+                discard;
+            } else if (uRenderMask == MaskOpaque && uAlpha * dta < 1.0) {
+                discard;
+            }
+        #else
+            if (uRenderMask == MaskTransparent && uAlpha == 1.0) {
+                discard;
+            } else if (uRenderMask == MaskOpaque && uAlpha < 1.0) {
+                discard;
+            }
+        #endif
     #else
-        vec4 material = packDepthToRGBA(gl_FragCoord.z);
+        if (uRenderMask == MaskOpaque) {
+            discard;
+        }
     #endif
+
+    vec4 material = packDepthToRGBA(fragmentDepth);
 #elif defined(dRenderVariant_marking)
     vec4 material;
     if(uMarkingType == 1) {
@@ -65,55 +85,41 @@ export const assign_material_color = `
     }
 #endif
 
-// apply screendoor transparency
-#if defined(dTransparency)
+// apply per-group transparency
+#if defined(dTransparency) && (defined(dRenderVariant_pick) || defined(dRenderVariant_color))
     float ta = 1.0 - vTransparency;
-    #if defined(dRenderVariant_colorWboit)
-        if (vTransparency < 0.2) ta = 1.0; // hard cutoff looks better with wboit
-    #endif
+    if (vTransparency < 0.2) ta = 1.0; // hard cutoff looks better
 
     #if defined(dRenderVariant_pick)
         if (ta < uPickingAlphaThreshold)
             discard; // ignore so the element below can be picked
-    #else
+    #elif defined(dRenderVariant_color)
+        material.a *= ta;
+
         #if defined(dRenderVariant_colorBlended)
             #if defined(dTransparentBackfaces_off)
-                if (interior && ta < 1.0) discard;
-            #endif
-
-            float at = 0.0;
-
-            // shift by view-offset during multi-sample rendering to allow for blending
-            vec2 coord = gl_FragCoord.xy + uViewOffset * 0.25;
-
-            const mat4 thresholdMatrix = mat4(
-                1.0 / 17.0,  9.0 / 17.0,  3.0 / 17.0, 11.0 / 17.0,
-                13.0 / 17.0,  5.0 / 17.0, 15.0 / 17.0,  7.0 / 17.0,
-                4.0 / 17.0, 12.0 / 17.0,  2.0 / 17.0, 10.0 / 17.0,
-                16.0 / 17.0,  8.0 / 17.0, 14.0 / 17.0,  6.0 / 17.0
-            );
-            int ci = int(intMod(coord.x, 4.0));
-            int ri = int(intMod(coord.y, 4.0));
-            #if __VERSION__ == 100
-                vec4 i = vec4(float(ci * 4 + ri));
-                vec4 v = thresholdMatrix[0] * vec4(equal(i, vec4(0.0, 1.0, 2.0, 3.0))) +
-                    thresholdMatrix[1] * vec4(equal(i, vec4(4.0, 5.0, 6.0, 7.0))) +
-                    thresholdMatrix[2] * vec4(equal(i, vec4(8.0, 9.0, 10.0, 11.0))) +
-                    thresholdMatrix[3] * vec4(equal(i, vec4(12.0, 13.0, 14.0, 15.0)));
-                at = v.x + v.y + v.z + v.w;
-            #else
-                at = thresholdMatrix[ci][ri];
-            #endif
-
-            if (ta < 0.99 && (ta < 0.01 || ta < at)) {
-                #if defined(dTransparentBackfaces_opaque)
-                    if (!interior) discard;
-                #else
+                if ((uRenderMask == MaskOpaque && material.a < 1.0) ||
+                    (uRenderMask == MaskTransparent && material.a == 1.0) ||
+                    (interior && material.a < 1.0)
+                ) {
                     discard;
-                #endif
-            }
-        #elif defined(dRenderVariant_colorWboit)
-            material.a *= ta;
+                }
+            #elif defined(dTransparentBackfaces_on)
+                if ((uRenderMask == MaskOpaque && material.a < 1.0) ||
+                    (uRenderMask == MaskTransparent && material.a == 1.0)
+                ) {
+                    discard;
+                }
+            #elif defined(dTransparentBackfaces_opaque)
+                if (interior) {
+                    material.a = 1.0;
+                } else if (
+                    (uRenderMask == MaskOpaque && material.a < 1.0) ||
+                    (uRenderMask == MaskTransparent && material.a == 1.0)
+                ) {
+                    discard;
+                }
+            #endif
         #endif
     #endif
 #endif

src/mol-gl/shader/chunks/common-frag-params.glsl.ts

@@ -64,7 +64,7 @@ uniform float uXrayEdgeFalloff;
 
 uniform mat4 uProjection;
 
-uniform bool uRenderWboit;
+uniform int uRenderMask;
 uniform bool uMarkingDepthTest;
 
 uniform sampler2D tDepth;

src/mol-gl/shader/chunks/common.glsl.ts

@@ -17,6 +17,10 @@ export const common = `
     #define dColorType_varying
 #endif
 
+#define MaskAll 0
+#define MaskOpaque 1
+#define MaskTransparent 2
+
 //
 
 #define PI 3.14159265

src/mol-gl/shader/chunks/wboit-write.glsl.ts

@@ -7,11 +7,11 @@
 
 export const wboit_write = `
 #if defined(dRenderVariant_colorWboit)
-    if (!uRenderWboit) {
+    if (uRenderMask == MaskOpaque) {
         if (preFogAlpha < 1.0) {
             discard;
         }
-    } else if (uRenderWboit) {
+    } else if (uRenderMask == MaskTransparent) {
         // the 'fragmentDepth > 0.99' check is to handle precision issues with packed depth
         if (preFogAlpha != 1.0 && (fragmentDepth < getDepth(gl_FragCoord.xy / uDrawingBufferSize) || fragmentDepth > 0.99)) {
             #ifdef dTransparentBackfaces_off

src/mol-gl/shader/direct-volume.frag.ts

@@ -72,7 +72,7 @@ uniform float uPickingAlphaThreshold;
 uniform bool uTransparentBackground;
 uniform float uXrayEdgeFalloff;
 
-uniform bool uRenderWboit;
+uniform int uRenderMask;
 
 uniform float uNear;
 uniform float uFar;
@@ -141,12 +141,7 @@ float transferFunction(float value) {
 
 float getDepth(const in vec2 coords) {
     #ifdef depthTextureSupport
-        if (!uRenderWboit) {
-            // in case of opaque volumes (and depth texture support)
-            return texture2D(tDepth, coords).r;
-        } else {
-            return unpackRGBAToDepth(texture2D(tDepth, coords));
-        }
+        return texture2D(tDepth, coords).r;
     #else
         return unpackRGBAToDepth(texture2D(tDepth, coords));
     #endif

src/mol-gl/shader/postprocessing.frag.ts

@@ -57,11 +57,11 @@ bool isBackground(const in float depth) {
     return depth == 1.0;
 }
 
-float getOutline(const in vec2 coords, out float closestTexel) {
+float getOutline(const in vec2 coords, const in float opaqueDepth, out float closestTexel) {
     float backgroundViewZ = uFar + 3.0 * uMaxPossibleViewZDiff;
     vec2 invTexSize = 1.0 / uTexSize;
 
-    float selfDepth = min(getDepthOpaque(coords), getDepthTransparent(coords));
+    float selfDepth = min(opaqueDepth, getDepthTransparent(coords));
     float selfViewZ = isBackground(selfDepth) ? backgroundViewZ : getViewZ(selfDepth);
 
     float outline = 1.0;
@@ -77,14 +77,15 @@ float getOutline(const in vec2 coords, out float closestTexel) {
             vec4 sampleOutlineCombined = texture2D(tOutlines, sampleCoords);
             float sampleOutline = sampleOutlineCombined.r;
             float sampleOutlineDepth = unpackRGToUnitInterval(sampleOutlineCombined.gb);
+            float sampleOutlineViewZ = isBackground(sampleOutlineDepth) ? backgroundViewZ : getViewZ(sampleOutlineDepth);
 
-            if (sampleOutline == 0.0 && sampleOutlineDepth < closestTexel && abs(selfViewZ - sampleOutlineDepth) > uMaxPossibleViewZDiff) {
+            if (sampleOutline == 0.0 && sampleOutlineDepth < closestTexel && abs(selfViewZ - sampleOutlineViewZ) > uMaxPossibleViewZDiff) {
                 outline = 0.0;
                 closestTexel = sampleOutlineDepth;
             }
         }
     }
-    return outline;
+    return closestTexel < opaqueDepth ? outline : 1.0;
 }
 
 float getSsao(vec2 coords) {
@@ -104,11 +105,11 @@ void main(void) {
 
     float viewDist;
     float fogFactor;
+    float opaqueDepth = getDepthOpaque(coords);
 
     #ifdef dOcclusionEnable
-        float depth = getDepthOpaque(coords);
-        if (!isBackground(depth)) {
-            viewDist = abs(getViewZ(depth));
+        if (!isBackground(opaqueDepth)) {
+            viewDist = abs(getViewZ(opaqueDepth));
             fogFactor = smoothstep(uFogNear, uFogFar, viewDist);
             float occlusionFactor = getSsao(coords + uOcclusionOffset);
             if (!uTransparentBackground) {
@@ -122,16 +123,15 @@ void main(void) {
     // outline needs to be handled after occlusion to keep them clean
     #ifdef dOutlineEnable
         float closestTexel;
-        float outline = getOutline(coords, closestTexel);
-
+        float outline = getOutline(coords, opaqueDepth, closestTexel);
         if (outline == 0.0) {
-            color.rgb = mix(uOutlineColor, color.rgb, outline);
             viewDist = abs(getViewZ(closestTexel));
             fogFactor = smoothstep(uFogNear, uFogFar, viewDist);
             if (!uTransparentBackground) {
-                color.rgb = mix(color.rgb, uFogColor, fogFactor);
+                color.rgb = mix(uOutlineColor, uFogColor, fogFactor);
             } else {
                 color.a = 1.0 - fogFactor;
+                color.rgb = mix(uOutlineColor, color.rgb, fogFactor);
             }
         }
     #endif