handle fog in outline shader

src/mol-canvas3d/canvas3d.ts

@@ -132,7 +132,7 @@ namespace Canvas3D {
 
         const drawPass = new DrawPass(webgl, renderer, scene, debugHelper)
         const pickPass = new PickPass(webgl, renderer, scene, 0.5)
-        const postprocessing = new PostprocessingPass(webgl, drawPass, p.postprocessing)
+        const postprocessing = new PostprocessingPass(webgl, camera, drawPass, p.postprocessing)
         const multiSample = new MultiSamplePass(webgl, camera, drawPass, postprocessing, p.multiSample)
 
         let isUpdating = false

src/mol-canvas3d/passes/postprocessing.ts

@@ -12,10 +12,11 @@ import { Texture } from 'mol-gl/webgl/texture';
 import { ValueCell } from 'mol-util';
 import { createComputeRenderItem } from 'mol-gl/webgl/render-item';
 import { createComputeRenderable, ComputeRenderable } from 'mol-gl/renderable';
-import { Vec2 } from 'mol-math/linear-algebra';
+import { Vec2, Vec3 } from 'mol-math/linear-algebra';
 import { ParamDefinition as PD } from 'mol-util/param-definition';
 import { createRenderTarget, RenderTarget } from 'mol-gl/webgl/render-target';
 import { DrawPass } from './draw';
+import { Camera } from 'mol-canvas3d/camera';
 
 const PostprocessingSchema = {
     ...QuadSchema,
@@ -23,6 +24,14 @@ const PostprocessingSchema = {
     tDepth: TextureSpec('texture', 'rgba', 'ubyte', 'nearest'),
     uTexSize: UniformSpec('v2'),
 
+    dUseFog: DefineSpec('boolean'),
+    dOrthographic: DefineSpec('number'),
+    uNear: UniformSpec('f'),
+    uFar: UniformSpec('f'),
+    uFogNear: UniformSpec('f'),
+    uFogFar: UniformSpec('f'),
+    uFogColor: UniformSpec('v3'),
+
     dOcclusionEnable: DefineSpec('boolean'),
     dOcclusionKernelSize: DefineSpec('number'),
     uOcclusionBias: UniformSpec('f'),
@@ -44,6 +53,8 @@ export const PostprocessingParams = {
     outlineEnable: PD.Boolean(false),
     outlineScale: PD.Numeric(1, { min: 0, max: 10, step: 1 }),
     outlineThreshold: PD.Numeric(0.8, { min: 0, max: 1, step: 0.01 }),
+
+    useFog: PD.Boolean(true),
 }
 export type PostprocessingProps = PD.Values<typeof PostprocessingParams>
 
@@ -57,6 +68,14 @@ function getPostprocessingRenderable(ctx: WebGLContext, colorTexture: Texture, d
         tDepth: ValueCell.create(depthTexture),
         uTexSize: ValueCell.create(Vec2.create(colorTexture.width, colorTexture.height)),
 
+        dUseFog: ValueCell.create(p.useFog),
+        dOrthographic: ValueCell.create(0),
+        uNear: ValueCell.create(1),
+        uFar: ValueCell.create(10000),
+        uFogNear: ValueCell.create(10000),
+        uFogFar: ValueCell.create(10000),
+        uFogColor: ValueCell.create(Vec3.create(1, 1, 1)),
+
         dOcclusionEnable: ValueCell.create(p.occlusionEnable),
         dOcclusionKernelSize: ValueCell.create(p.occlusionKernelSize),
         uOcclusionBias: ValueCell.create(p.occlusionBias),
@@ -84,7 +103,7 @@ export class PostprocessingPass {
     props: PostprocessingProps
     renderable: PostprocessingRenderable
 
-    constructor(private webgl: WebGLContext, drawPass: DrawPass, props: Partial<PostprocessingProps>) {
+    constructor(private webgl: WebGLContext, private camera: Camera, drawPass: DrawPass, props: Partial<PostprocessingProps>) {
         const { gl } = webgl
         this.target = createRenderTarget(webgl, gl.drawingBufferWidth, gl.drawingBufferHeight)
         this.props = { ...PD.getDefaultValues(PostprocessingParams), ...props }
@@ -132,10 +151,21 @@ export class PostprocessingPass {
             ValueCell.update(this.renderable.values.uOutlineThreshold, props.outlineThreshold)
         }
 
+        if (props.useFog !== undefined) {
+            this.props.useFog = props.useFog
+            ValueCell.update(this.renderable.values.dUseFog, props.useFog)
+        }
+
         this.renderable.update()
     }
 
     render(toDrawingBuffer: boolean) {
+        ValueCell.update(this.renderable.values.uFar, this.camera.state.far)
+        ValueCell.update(this.renderable.values.uNear, this.camera.state.near)
+        ValueCell.update(this.renderable.values.uFogFar, this.camera.state.fogFar)
+        ValueCell.update(this.renderable.values.uFogNear, this.camera.state.fogNear)
+        ValueCell.update(this.renderable.values.dOrthographic, this.camera.state.mode === 'orthographic' ? 1 : 0)
+
         const { gl, state } = this.webgl
         if (toDrawingBuffer) {
             this.webgl.unbindFramebuffer()

src/mol-gl/shader/postprocessing.frag.ts

@@ -7,6 +7,12 @@ uniform sampler2D tColor;
 uniform sampler2D tDepth;
 uniform vec2 uTexSize;
 
+uniform float uNear;
+uniform float uFar;
+uniform float uFogNear;
+uniform float uFogFar;
+uniform vec3 uFogColor;
+
 uniform float uOcclusionBias;
 uniform float uOcclusionRadius;
 
@@ -14,10 +20,11 @@ uniform float uOutlineScale;
 uniform float uOutlineThreshold;
 
 const float noiseAmount = 0.0002;
+const vec4 occlusionColor = vec4(0.0, 0.0, 0.0, 1.0);
 
 #include common
 
-float noise(vec2 coords) {
+float noise(const in vec2 coords) {
 	float a = 12.9898;
 	float b = 78.233;
 	float c = 43758.5453;
@@ -27,7 +34,23 @@ float noise(vec2 coords) {
 	return fract(sin(sn) * c);
 }
 
-float getDepth(in vec2 coords) {
+float perspectiveDepthToViewZ(const in float invClipZ, const in float near, const in float far) {
+	return (near * far) / ((far - near) * invClipZ - far);
+}
+
+float orthographicDepthToViewZ(const in float linearClipZ, const in float near, const in float far) {
+	return linearClipZ * (near - far) - near;
+}
+
+float getViewZ(const in float depth) {
+	#if dOrthographic == 1
+		return orthographicDepthToViewZ(depth, uNear, uFar);
+	#else
+		return perspectiveDepthToViewZ(depth, uNear, uFar);
+	#endif
+}
+
+float getDepth(const in vec2 coords) {
 	#ifdef dPackedDepth
 		return unpackRGBAToDepth(texture2D(tDepth, coords));
 	#else
@@ -35,7 +58,7 @@ float getDepth(in vec2 coords) {
 	#endif
 }
 
-float calcSSAO(in vec2 coords, in float depth) {
+float calcSSAO(const in vec2 coords, const in float depth) {
 	float occlusionFactor = 0.0;
 
 	for (int i = -dOcclusionKernelSize; i <= dOcclusionKernelSize; i++) {
@@ -50,7 +73,7 @@ float calcSSAO(in vec2 coords, in float depth) {
 	return occlusionFactor / float((2 * dOcclusionKernelSize + 1) * (2 * dOcclusionKernelSize + 1));
 }
 
-float calcEdgeDepth(in vec2 coords) {
+vec2 calcEdgeDepth(const in vec2 coords) {
     vec2 invTexSize = 1.0 / uTexSize;
     float halfScaleFloor = floor(uOutlineScale * 0.5);
     float halfScaleCeil = ceil(uOutlineScale * 0.5);
@@ -68,25 +91,36 @@ float calcEdgeDepth(in vec2 coords) {
     float depthFiniteDifference0 = depth1 - depth0;
     float depthFiniteDifference1 = depth3 - depth2;
 
-    return sqrt(pow(depthFiniteDifference0, 2.0) + pow(depthFiniteDifference1, 2.0)) * 100.0;
+    return vec2(
+		sqrt(pow(depthFiniteDifference0, 2.0) + pow(depthFiniteDifference1, 2.0)) * 100.0,
+		min(depth0, min(depth1, min(depth2, depth3)))
+	);
 }
 
 void main(void) {
 	vec2 coords = gl_FragCoord.xy / uTexSize;
 	vec4 color = texture2D(tColor, coords);
 
+	#ifdef dOutlineEnable
+		vec2 edgeDepth = calcEdgeDepth(coords);
+		float edgeFlag = step(edgeDepth.x, uOutlineThreshold);
+    	color.rgb *= edgeFlag;
+		#ifdef dUseFog
+			float viewDist = abs(getViewZ(edgeDepth.y));
+			float fogFactor = smoothstep(uFogNear, uFogFar, viewDist) * (1.0 - edgeFlag);
+			color.rgb = mix(color.rgb, uFogColor, fogFactor);
+		#endif
+	#endif
+
+	// occlusion needs to be handled after outline to darken them properly
 	#ifdef dOcclusionEnable
 		float depth = getDepth(coords);
 		if (depth != 1.0) {
 			float occlusionFactor = calcSSAO(coords, depth);
-			color = mix(color, vec4(0.0, 0.0, 0.0, 1.0), uOcclusionBias * occlusionFactor);
+			color = mix(color, occlusionColor, uOcclusionBias * occlusionFactor);
 		}
 	#endif
 
-	#ifdef dOutlineEnable
-    	color.rgb *= (step(calcEdgeDepth(coords), uOutlineThreshold));
-	#endif
-
 	gl_FragColor = color;
 }
 `