Merge pull request #932 from molstar/sharpening

Sharpening

CHANGELOG.md

@@ -22,6 +22,10 @@ Note that since we don't clearly distinguish between a public and private interf
 - Add `blockIndex` parameter to TrajectoryFromMmCif
 - Fix bounding sphere calculation for "element-like" visuals
 - Fix RCSB PDB validation report URL
+- Add sharpening postprocessing option
+- Take pixel-ratio into account for outline scale
+- Gracefully handle missing HTMLImageElement
+- Fix pixel-ratio changes not applied to all render passes
 
 ## [v3.39.0] - 2023-09-02
 

src/extensions/backgrounds/index.ts

@@ -1,5 +1,5 @@
 /**
- * Copyright (c) 2022 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2022-2023 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
@@ -28,6 +28,12 @@ export const Backgrounds = PluginBehavior.create<{ }>({
     ctor: class extends PluginBehavior.Handler<{ }> {
         register(): void {
             this.ctx.config.set(PluginConfig.Background.Styles, [
+                [{
+                    variant: {
+                        name: 'off',
+                        params: {}
+                    }
+                }, 'Off'],
                 [{
                     variant: {
                         name: 'radialGradient',

src/mol-canvas3d/passes/background.ts

@@ -372,6 +372,12 @@ function getSkyboxTexture(ctx: WebGLContext, assetManager: AssetManager, faces:
     const cubeAssets = getCubeAssets(assetManager, faces);
     const cubeFaces = getCubeFaces(assetManager, cubeAssets);
     const assets = [cubeAssets.nx, cubeAssets.ny, cubeAssets.nz, cubeAssets.px, cubeAssets.py, cubeAssets.pz];
+    if (typeof HTMLImageElement === 'undefined') {
+        console.error(`Missing "HTMLImageElement" required for background skybox`);
+        onload?.(true);
+        return { texture: createNullTexture(), assets };
+    }
+
     const texture = ctx.resources.cubeTexture(cubeFaces, true, onload);
     return { texture, assets };
 }
@@ -393,6 +399,15 @@ function areImageTexturePropsEqual(sourceA: ImageProps['source'], sourceB: Image
 }
 
 function getImageTexture(ctx: WebGLContext, assetManager: AssetManager, source: ImageProps['source'], onload?: (errored?: boolean) => void): { texture: Texture, asset: Asset } {
+    const asset = source.name === 'url'
+        ? Asset.getUrlAsset(assetManager, source.params)
+        : source.params!;
+    if (typeof HTMLImageElement === 'undefined') {
+        console.error(`Missing "HTMLImageElement" required for background image`);
+        onload?.(true);
+        return { texture: createNullTexture(), asset };
+    }
+
     const texture = ctx.resources.texture('image-uint8', 'rgba', 'ubyte', 'linear');
     const img = new Image();
     img.onload = () => {
@@ -405,9 +420,7 @@ function getImageTexture(ctx: WebGLContext, assetManager: AssetManager, source:
     img.onerror = () => {
         onload?.(true);
     };
-    const asset = source.name === 'url'
-        ? Asset.getUrlAsset(assetManager, source.params)
-        : source.params!;
+
     assetManager.resolve(asset, 'binary').run().then(a => {
         const blob = new Blob([a.data]);
         img.src = URL.createObjectURL(blob);

src/mol-canvas3d/passes/cas.ts

@@ -0,0 +1,120 @@
+/**
+ * Copyright (c) 2023 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
+ */
+
+import { QuadSchema, QuadValues } from '../../mol-gl/compute/util';
+import { ComputeRenderable, createComputeRenderable } from '../../mol-gl/renderable';
+import { DefineSpec, TextureSpec, UniformSpec, Values } from '../../mol-gl/renderable/schema';
+import { ShaderCode } from '../../mol-gl/shader-code';
+import { WebGLContext } from '../../mol-gl/webgl/context';
+import { createComputeRenderItem } from '../../mol-gl/webgl/render-item';
+import { Texture } from '../../mol-gl/webgl/texture';
+import { Vec2 } from '../../mol-math/linear-algebra';
+import { ValueCell } from '../../mol-util';
+import { ParamDefinition as PD } from '../../mol-util/param-definition';
+import { quad_vert } from '../../mol-gl/shader/quad.vert';
+import { Viewport } from '../camera/util';
+import { RenderTarget } from '../../mol-gl/webgl/render-target';
+import { isTimingMode } from '../../mol-util/debug';
+import { cas_frag } from '../../mol-gl/shader/cas.frag';
+
+export const CasParams = {
+    sharpness: PD.Numeric(0.5, { min: 0, max: 1, step: 0.05 }),
+    denoise: PD.Boolean(true),
+};
+export type CasProps = PD.Values<typeof CasParams>
+
+export class CasPass {
+    private readonly renderable: CasRenderable;
+
+    constructor(private webgl: WebGLContext, input: Texture) {
+        this.renderable = getCasRenderable(webgl, input);
+    }
+
+    private updateState(viewport: Viewport) {
+        const { gl, state } = this.webgl;
+
+        state.enable(gl.SCISSOR_TEST);
+        state.disable(gl.BLEND);
+        state.disable(gl.DEPTH_TEST);
+        state.depthMask(false);
+
+        const { x, y, width, height } = viewport;
+        state.viewport(x, y, width, height);
+        state.scissor(x, y, width, height);
+
+        state.clearColor(0, 0, 0, 1);
+        gl.clear(gl.COLOR_BUFFER_BIT);
+    }
+
+    setSize(width: number, height: number) {
+        ValueCell.update(this.renderable.values.uTexSizeInv, Vec2.set(this.renderable.values.uTexSizeInv.ref.value, 1 / width, 1 / height));
+    }
+
+    update(input: Texture, props: CasProps) {
+        const { values } = this.renderable;
+        const { sharpness, denoise } = props;
+
+        let needsUpdate = false;
+
+        if (values.tColor.ref.value !== input) {
+            ValueCell.update(this.renderable.values.tColor, input);
+            needsUpdate = true;
+        }
+
+        ValueCell.updateIfChanged(values.uSharpness, 2 - 2 * Math.pow(sharpness, 0.25));
+
+        if (values.dDenoise.ref.value !== denoise) needsUpdate = true;
+        ValueCell.updateIfChanged(values.dDenoise, denoise);
+
+        if (needsUpdate) {
+            this.renderable.update();
+        }
+    }
+
+    render(viewport: Viewport, target: RenderTarget | undefined) {
+        if (isTimingMode) this.webgl.timer.mark('CasPass.render');
+        if (target) {
+            target.bind();
+        } else {
+            this.webgl.unbindFramebuffer();
+        }
+        this.updateState(viewport);
+        this.renderable.render();
+        if (isTimingMode) this.webgl.timer.markEnd('CasPass.render');
+    }
+}
+
+//
+
+const CasSchema = {
+    ...QuadSchema,
+    tColor: TextureSpec('texture', 'rgba', 'ubyte', 'linear'),
+    uTexSizeInv: UniformSpec('v2'),
+
+    uSharpness: UniformSpec('f'),
+    dDenoise: DefineSpec('boolean'),
+};
+const CasShaderCode = ShaderCode('cas', quad_vert, cas_frag);
+type CasRenderable = ComputeRenderable<Values<typeof CasSchema>>
+
+function getCasRenderable(ctx: WebGLContext, colorTexture: Texture): CasRenderable {
+    const width = colorTexture.getWidth();
+    const height = colorTexture.getHeight();
+
+    const values: Values<typeof CasSchema> = {
+        ...QuadValues,
+        tColor: ValueCell.create(colorTexture),
+        uTexSizeInv: ValueCell.create(Vec2.create(1 / width, 1 / height)),
+
+        uSharpness: ValueCell.create(0.5),
+        dDenoise: ValueCell.create(true),
+    };
+
+    const schema = { ...CasSchema };
+    const renderItem = createComputeRenderItem(ctx, 'triangles', CasShaderCode, schema, values);
+
+    return createComputeRenderable(renderItem, values);
+}

src/mol-canvas3d/passes/draw.ts

@@ -115,19 +115,19 @@ export class DrawPass {
 
             ValueCell.update(this.copyFboTarget.values.uTexSize, Vec2.set(this.copyFboTarget.values.uTexSize.ref.value, width, height));
             ValueCell.update(this.copyFboPostprocessing.values.uTexSize, Vec2.set(this.copyFboPostprocessing.values.uTexSize.ref.value, width, height));
+        }
 
-            if (this.wboit?.supported) {
-                this.wboit.setSize(width, height);
-            }
-
-            if (this.dpoit?.supported) {
-                this.dpoit.setSize(width, height);
-            }
+        if (this.wboit?.supported) {
+            this.wboit.setSize(width, height);
+        }
 
-            this.marking.setSize(width, height);
-            this.postprocessing.setSize(width, height);
-            this.antialiasing.setSize(width, height);
+        if (this.dpoit?.supported) {
+            this.dpoit.setSize(width, height);
         }
+
+        this.marking.setSize(width, height);
+        this.postprocessing.setSize(width, height);
+        this.antialiasing.setSize(width, height);
     }
 
     private _renderDpoit(renderer: Renderer, camera: ICamera, scene: Scene, iterations: number, transparentBackground: boolean, postprocessingProps: PostprocessingProps) {

src/mol-canvas3d/passes/postprocessing.ts

@@ -33,6 +33,7 @@ import { BackgroundParams, BackgroundPass } from './background';
 import { AssetManager } from '../../mol-util/assets';
 import { Light } from '../../mol-gl/renderer';
 import { shadows_frag } from '../../mol-gl/shader/shadows.frag';
+import { CasParams, CasPass } from './cas';
 
 const OutlinesSchema = {
     ...QuadSchema,
@@ -399,6 +400,10 @@ export const PostprocessingParams = {
         smaa: PD.Group(SmaaParams),
         off: PD.Group({})
     }, { options: [['fxaa', 'FXAA'], ['smaa', 'SMAA'], ['off', 'Off']], description: 'Smooth pixel edges' }),
+    sharpening: PD.MappedStatic('off', {
+        on: PD.Group(CasParams),
+        off: PD.Group({})
+    }, { cycle: true, description: 'Contrast Adaptive Sharpening' }),
     background: PD.Group(BackgroundParams, { isFlat: true }),
 };
 
@@ -762,8 +767,8 @@ export class PostprocessingPass {
 
         if (props.outline.name === 'on') {
             const transparentOutline = props.outline.params.includeTransparent ?? true;
-            const outlineScale = props.outline.params.scale - 1;
-            const outlineThreshold = 50 * props.outline.params.threshold;
+            const outlineScale = Math.max(1, Math.round(props.outline.params.scale * this.webgl.pixelRatio)) - 1;
+            const outlineThreshold = 50 * props.outline.params.threshold * this.webgl.pixelRatio;
 
             ValueCell.updateIfChanged(this.outlinesRenderable.values.uNear, camera.near);
             ValueCell.updateIfChanged(this.outlinesRenderable.values.uFar, camera.far);
@@ -953,8 +958,11 @@ export class AntialiasingPass {
     }
 
     readonly target: RenderTarget;
+    private readonly internalTarget: RenderTarget;
+
     private readonly fxaa: FxaaPass;
     private readonly smaa: SmaaPass;
+    private readonly cas: CasPass;
 
     constructor(webgl: WebGLContext, private drawPass: DrawPass) {
         const { colorTarget } = drawPass;
@@ -962,8 +970,11 @@ export class AntialiasingPass {
         const height = colorTarget.getHeight();
 
         this.target = webgl.createRenderTarget(width, height, false);
+        this.internalTarget = webgl.createRenderTarget(width, height, false);
+
         this.fxaa = new FxaaPass(webgl, this.target.texture);
         this.smaa = new SmaaPass(webgl, this.target.texture);
+        this.cas = new CasPass(webgl, this.target.texture);
     }
 
     setSize(width: number, height: number) {
@@ -972,41 +983,69 @@ export class AntialiasingPass {
 
         if (width !== w || height !== h) {
             this.target.setSize(width, height);
+            this.internalTarget.setSize(width, height);
             this.fxaa.setSize(width, height);
             if (this.smaa.supported) this.smaa.setSize(width, height);
+            this.cas.setSize(width, height);
         }
     }
 
-    private _renderFxaa(camera: ICamera, toDrawingBuffer: boolean, props: PostprocessingProps) {
+    private _renderFxaa(camera: ICamera, target: RenderTarget | undefined, props: PostprocessingProps) {
         if (props.antialiasing.name !== 'fxaa') return;
 
         const input = PostprocessingPass.isEnabled(props)
             ? this.drawPass.postprocessing.target.texture
             : this.drawPass.colorTarget.texture;
         this.fxaa.update(input, props.antialiasing.params);
-        this.fxaa.render(camera.viewport, toDrawingBuffer ? undefined : this.target);
+        this.fxaa.render(camera.viewport, target);
     }
 
-    private _renderSmaa(camera: ICamera, toDrawingBuffer: boolean, props: PostprocessingProps) {
+    private _renderSmaa(camera: ICamera, target: RenderTarget | undefined, props: PostprocessingProps) {
         if (props.antialiasing.name !== 'smaa') return;
 
         const input = PostprocessingPass.isEnabled(props)
             ? this.drawPass.postprocessing.target.texture
             : this.drawPass.colorTarget.texture;
         this.smaa.update(input, props.antialiasing.params);
-        this.smaa.render(camera.viewport, toDrawingBuffer ? undefined : this.target);
+        this.smaa.render(camera.viewport, target);
     }
 
-    render(camera: ICamera, toDrawingBuffer: boolean, props: PostprocessingProps) {
-        if (props.antialiasing.name === 'off') return;
-
+    private _renderAntialiasing(camera: ICamera, target: RenderTarget | undefined, props: PostprocessingProps) {
         if (props.antialiasing.name === 'fxaa') {
-            this._renderFxaa(camera, toDrawingBuffer, props);
+            this._renderFxaa(camera, target, props);
         } else if (props.antialiasing.name === 'smaa') {
-            if (!this.smaa.supported) {
-                throw new Error('SMAA not supported, missing "HTMLImageElement"');
-            }
-            this._renderSmaa(camera, toDrawingBuffer, props);
+            this._renderSmaa(camera, target, props);
+        }
+    }
+
+    private _renderCas(camera: ICamera, target: RenderTarget | undefined, props: PostprocessingProps) {
+        if (props.sharpening.name !== 'on') return;
+
+        const input = props.antialiasing.name !== 'off'
+            ? this.internalTarget.texture
+            : PostprocessingPass.isEnabled(props)
+                ? this.drawPass.postprocessing.target.texture
+                : this.drawPass.colorTarget.texture;
+        this.cas.update(input, props.sharpening.params);
+        this.cas.render(camera.viewport, target);
+    }
+
+    render(camera: ICamera, toDrawingBuffer: boolean, props: PostprocessingProps) {
+        if (props.antialiasing.name === 'off' && props.sharpening.name === 'off') return;
+
+        if (props.antialiasing.name === 'smaa' && !this.smaa.supported) {
+            console.error('SMAA not supported, missing "HTMLImageElement"');
+            return;
+        }
+
+        const target = toDrawingBuffer ? undefined : this.target;
+        if (props.sharpening.name === 'off') {
+            this._renderAntialiasing(camera, target, props);
+        } else if (props.antialiasing.name === 'off') {
+            this._renderCas(camera, target, props);
+        } else {
+            this._renderAntialiasing(camera, this.internalTarget, props);
+            this._renderCas(camera, target, props);
         }
     }
 }

src/mol-gl/shader/cas.frag.ts

@@ -0,0 +1,145 @@
+export const cas_frag = `
+precision mediump float;
+precision mediump sampler2D;
+
+uniform sampler2D tColor;
+uniform vec2 uTexSizeInv;
+
+uniform float uSharpness;
+
+// adapted from https://www.shadertoy.com/view/stXSWB
+
+/*
+* FidelityFX Super Resolution scales up a low resolution
+* image, while adding fine detail.
+*
+* MIT Open License
+*
+* https://gpuopen.com/fsr
+*
+* Left: FSR processed
+* Right: Original texture, bilinear interpolation
+*
+* Mouse at top: Sharpness 0 stops (maximum)
+* Mouse at bottom: Sharpness 2 stops (minimum)
+*
+* It works in two passes-
+*   EASU upsamples the image with a clamped Lanczos kernel.
+*   RCAS sharpens the image at the target resolution.
+*
+* I needed to make a few changes to improve readability and
+* WebGL compatibility in an algorithm I don't fully understand.
+* Expect bugs.
+*
+* Shader not currently running for WebGL1 targets (eg. mobile Safari)
+*
+* There is kind of no point to using FSR in Shadertoy, as it renders buffers
+* at full target resolution. But this might be useful for WebGL based demos
+* running smaller-than-target render buffers.
+*
+* For sharpening with a full resolution render buffer,
+* FidelityFX CAS is a better option.
+* https://www.shadertoy.com/view/ftsXzM
+*
+* For readability and compatibility, these optimisations have been removed:
+*   * Fast approximate inverse and inversesqrt
+*   * textureGather fetches (not WebGL compatible)
+*   * Multiplying by reciprocal instead of division
+*
+* Apologies to AMD for the numerous slowdowns and errors I have introduced.
+*
+*/
+
+/***** RCAS *****/
+#define FSR_RCAS_LIMIT (0.25-(1.0/16.0))
+
+// Input callback prototypes that need to be implemented by calling shader
+vec4 FsrRcasLoadF(vec2 p);
+//------------------------------------------------------------------------------------------------------------------------------
+void FsrRcasCon(
+    out float con,
+    // The scale is {0.0 := maximum, to N>0, where N is the number of stops (halving) of the reduction of sharpness}.
+    float sharpness
+) {
+    // Transform from stops to linear value.
+    con = exp2(-sharpness);
+}
+
+vec3 FsrRcasF(
+    vec2 ip, // Integer pixel position in output.
+    float con
+) {
+    // Constant generated by RcasSetup().
+    // Algorithm uses minimal 3x3 pixel neighborhood.
+    //    b
+    //  d e f
+    //    h
+    vec2 sp = vec2(ip);
+    vec3 b = FsrRcasLoadF(sp + vec2( 0,-1)).rgb;
+    vec3 d = FsrRcasLoadF(sp + vec2(-1, 0)).rgb;
+    vec3 e = FsrRcasLoadF(sp).rgb;
+    vec3 f = FsrRcasLoadF(sp + vec2( 1, 0)).rgb;
+    vec3 h = FsrRcasLoadF(sp + vec2( 0, 1)).rgb;
+
+    // Luma times 2.
+    float bL = b.g + .5 * (b.b + b.r);
+    float dL = d.g + .5 * (d.b + d.r);
+    float eL = e.g + .5 * (e.b + e.r);
+    float fL = f.g + .5 * (f.b + f.r);
+    float hL = h.g + .5 * (h.b + h.r);
+
+    // Noise detection.
+    #ifdef dDenoise
+        float nz = .25 * (bL + dL + fL + hL) - eL;
+        nz=clamp(
+            abs(nz)
+            /(
+                max(max(bL,dL),max(eL,max(fL,hL)))
+                -min(min(bL,dL),min(eL,min(fL,hL)))
+            ),
+            0., 1.
+        );
+        nz=1.-.5*nz;
+    #endif
+
+    // Min and max of ring.
+    vec3 mn4 = min(b, min(f, h));
+    vec3 mx4 = max(b, max(f, h));
+
+    // Immediate constants for peak range.
+    vec2 peakC = vec2(1., -4.);
+
+    // Limiters, these need to be high precision RCPs.
+    vec3 hitMin = mn4 / (4. * mx4);
+    vec3 hitMax = (peakC.x - mx4) / (4.* mn4 + peakC.y);
+    vec3 lobeRGB = max(-hitMin, hitMax);
+    float lobe = max(
+        -FSR_RCAS_LIMIT,
+        min(max(lobeRGB.r, max(lobeRGB.g, lobeRGB.b)), 0.)
+    )*con;
+
+    // Apply noise removal.
+    #ifdef dDenoise
+        lobe *= nz;
+    #endif
+
+    // Resolve, which needs the medium precision rcp approximation to avoid visible tonality changes.
+    return (lobe * (b + d + h + f) + e) / (4. * lobe + 1.);
+}
+
+
+vec4 FsrRcasLoadF(vec2 p) {
+    return texture2D(tColor, p * uTexSizeInv);
+}
+
+void main() {
+    // Set up constants
+    float con;
+    FsrRcasCon(con, uSharpness);
+
+    // Perform RCAS pass
+    vec3 col = FsrRcasF(gl_FragCoord.xy, con);
+
+    gl_FragColor = vec4(col, FsrRcasLoadF(gl_FragCoord.xy).a);
+}
+`;