picking improvements

- get 3d position from depth
- option to render an object only in color pass

src/mol-canvas3d/canvas3d.ts

@@ -27,7 +27,7 @@ import { Canvas3dInteractionHelper } from './helper/interaction-events';
 import { PostprocessingParams, PostprocessingPass } from './passes/postprocessing';
 import { MultiSampleParams, MultiSamplePass } from './passes/multi-sample';
 import { DrawPass } from './passes/draw';
-import { PickPass } from './passes/pick';
+import { PickData, PickPass } from './passes/pick';
 import { ImagePass, ImageProps } from './passes/image';
 import { Sphere3D } from '../mol-math/geometry';
 import { isDebugMode } from '../mol-util/debug';
@@ -100,7 +100,7 @@ interface Canvas3D {
     requestDraw(force?: boolean): void
     animate(): void
     pause(): void
-    identify(x: number, y: number): PickingId | undefined
+    identify(x: number, y: number): PickData | undefined
     mark(loci: Representation.Loci, action: MarkerAction): void
     getLoci(pickingId: PickingId | undefined): Representation.Loci
 
@@ -132,9 +132,9 @@ const cancelAnimationFrame = typeof window !== 'undefined'
     : (handle: number) => clearImmediate(handle as unknown as NodeJS.Immediate);
 
 namespace Canvas3D {
-    export interface HoverEvent { current: Representation.Loci, buttons: ButtonsType, button: ButtonsType.Flag, modifiers: ModifiersKeys }
+    export interface HoverEvent { current: Representation.Loci, buttons: ButtonsType, button: ButtonsType.Flag, modifiers: ModifiersKeys, page?: Vec2, position?: Vec3 }
     export interface DragEvent { current: Representation.Loci, buttons: ButtonsType, button: ButtonsType.Flag, modifiers: ModifiersKeys, pageStart: Vec2, pageEnd: Vec2 }
-    export interface ClickEvent { current: Representation.Loci, buttons: ButtonsType, button: ButtonsType.Flag, modifiers: ModifiersKeys }
+    export interface ClickEvent { current: Representation.Loci, buttons: ButtonsType, button: ButtonsType.Flag, modifiers: ModifiersKeys, position?: Vec3 }
 
     export function fromCanvas(canvas: HTMLCanvasElement, props: PartialCanvas3DProps = {}, attribs: Partial<{ antialias: boolean, pixelScale: number }> = {}) {
         const gl = getGLContext(canvas, {
@@ -346,7 +346,7 @@ namespace Canvas3D {
             animationFrameHandle = 0;
         }
 
-        function identify(x: number, y: number): PickingId | undefined {
+        function identify(x: number, y: number): PickData | undefined {
             return webgl.isContextLost ? undefined : pickPass.identify(x, y);
         }
 

src/mol-canvas3d/helper/bounding-sphere-helper.ts

@@ -160,5 +160,5 @@ const instanceMaterialId = getNextMaterialId();
 
 function createBoundingSphereRenderObject(mesh: Mesh, color: Color, materialId: number, transform?: TransformData) {
     const values = Mesh.Utils.createValuesSimple(mesh, { alpha: 0.1, doubleSided: false }, color, 1, transform);
-    return createRenderObject('mesh', values, { visible: true, alphaFactor: 1, pickable: false, opaque: false, writeDepth: false }, materialId);
+    return createRenderObject('mesh', values, { visible: true, alphaFactor: 1, pickable: false, colorOnly: false, opaque: false, writeDepth: false }, materialId);
 }

src/mol-canvas3d/helper/interaction-events.ts

@@ -9,7 +9,7 @@ import { PickingId } from '../../mol-geo/geometry/picking';
 import { Representation } from '../../mol-repr/representation';
 import InputObserver, { ModifiersKeys, ButtonsType } from '../../mol-util/input/input-observer';
 import { RxEventHelper } from '../../mol-util/rx-event-helper';
-import { Vec2 } from '../../mol-math/linear-algebra';
+import { Vec2, Vec3 } from '../../mol-math/linear-algebra';
 import { Camera } from '../camera';
 
 type Canvas3D = import('../canvas3d').Canvas3D
@@ -34,6 +34,7 @@ export class Canvas3dInteractionHelper {
     private endY = -1;
 
     private id: PickingId | undefined = void 0;
+    private position: Vec3 | undefined = void 0;
 
     private currentIdentifyT = 0;
     private isInteracting = false;
@@ -61,14 +62,16 @@ export class Canvas3dInteractionHelper {
         }
 
         if (xyChanged) {
-            this.id = this.canvasIdentify(this.endX, this.endY);
+            const pickData = this.canvasIdentify(this.endX, this.endY);
+            this.id = pickData?.id;
+            this.position = pickData?.position;
             this.startX = this.endX;
             this.startY = this.endY;
         }
 
         if (e === InputEvent.Click) {
             const loci = this.getLoci(this.id);
-            this.events.click.next({ current: loci, buttons: this.buttons, button: this.button, modifiers: this.modifiers });
+            this.events.click.next({ current: loci, buttons: this.buttons, button: this.button, modifiers: this.modifiers, position: this.position });
             this.prevLoci = loci;
             return;
         }
@@ -78,11 +81,8 @@ export class Canvas3dInteractionHelper {
         }
 
         const loci = this.getLoci(this.id);
-        // only broadcast the latest hover
-        if (!Representation.Loci.areEqual(this.prevLoci, loci)) {
-            this.events.hover.next({ current: loci, buttons: this.buttons, button: this.button, modifiers: this.modifiers });
-            this.prevLoci = loci;
-        }
+        this.events.hover.next({ current: loci, buttons: this.buttons, button: this.button, modifiers: this.modifiers, page: Vec2.create(this.endX, this.endY), position: this.position });
+        this.prevLoci = loci;
     }
 
     tick(t: number) {
@@ -129,9 +129,9 @@ export class Canvas3dInteractionHelper {
     }
 
     private modify(modifiers: ModifiersKeys) {
-        if (Representation.Loci.isEmpty(this.prevLoci) || ModifiersKeys.areEqual(modifiers, this.modifiers)) return;
+        if (ModifiersKeys.areEqual(modifiers, this.modifiers)) return;
         this.modifiers = modifiers;
-        this.events.hover.next({ current: this.prevLoci, buttons: this.buttons, button: this.button, modifiers: this.modifiers });
+        this.events.hover.next({ current: this.prevLoci, buttons: this.buttons, button: this.button, modifiers: this.modifiers, page: Vec2.create(this.endX, this.endY), position: this.position });
     }
 
     private outsideViewport(x: number, y: number) {

src/mol-canvas3d/passes/pick.ts

@@ -10,26 +10,32 @@ import Scene from '../../mol-gl/scene';
 import { WebGLContext } from '../../mol-gl/webgl/context';
 import { GraphicsRenderVariant } from '../../mol-gl/webgl/render-item';
 import { RenderTarget } from '../../mol-gl/webgl/render-target';
-import { decodeFloatRGB } from '../../mol-util/float-packing';
+import { Vec3 } from '../../mol-math/linear-algebra';
+import { decodeFloatRGB, unpackRGBAToDepth } from '../../mol-util/float-packing';
 import { Camera, ICamera } from '../camera';
 import { StereoCamera } from '../camera/stereo';
+import { cameraUnproject } from '../camera/util';
 import { HandleHelper } from '../helper/handle-helper';
 import { DrawPass } from './draw';
 
 const NullId = Math.pow(2, 24) - 2;
 
+export type PickData = { id: PickingId, position: Vec3 }
+
 export class PickPass {
     pickDirty = true
 
     objectPickTarget: RenderTarget
     instancePickTarget: RenderTarget
     groupPickTarget: RenderTarget
+    depthPickTarget: RenderTarget
 
     isStereo = false
 
     private objectBuffer: Uint8Array
     private instanceBuffer: Uint8Array
     private groupBuffer: Uint8Array
+    private depthBuffer: Uint8Array
 
     private pickScale: number
     private pickWidth: number
@@ -43,6 +49,7 @@ export class PickPass {
         this.objectPickTarget = webgl.createRenderTarget(this.pickWidth, this.pickHeight);
         this.instancePickTarget = webgl.createRenderTarget(this.pickWidth, this.pickHeight);
         this.groupPickTarget = webgl.createRenderTarget(this.pickWidth, this.pickHeight);
+        this.depthPickTarget = webgl.createRenderTarget(this.pickWidth, this.pickHeight);
 
         this.setupBuffers();
     }
@@ -53,6 +60,7 @@ export class PickPass {
             this.objectBuffer = new Uint8Array(bufferSize);
             this.instanceBuffer = new Uint8Array(bufferSize);
             this.groupBuffer = new Uint8Array(bufferSize);
+            this.depthBuffer = new Uint8Array(bufferSize);
         }
     }
 
@@ -68,6 +76,7 @@ export class PickPass {
             this.objectPickTarget.setSize(this.pickWidth, this.pickHeight);
             this.instancePickTarget.setSize(this.pickWidth, this.pickHeight);
             this.groupPickTarget.setSize(this.pickWidth, this.pickHeight);
+            this.depthPickTarget.setSize(this.pickWidth, this.pickHeight);
 
             this.setupBuffers();
         }
@@ -107,6 +116,9 @@ export class PickPass {
         this.groupPickTarget.bind();
         this.renderVariant('pickGroup');
 
+        this.depthPickTarget.bind();
+        this.renderVariant('depth');
+
         this.pickDirty = false;
     }
 
@@ -121,14 +133,27 @@ export class PickPass {
 
         this.groupPickTarget.bind();
         webgl.readPixels(0, 0, this.pickWidth, this.pickHeight, this.groupBuffer);
+
+        this.depthPickTarget.bind();
+        webgl.readPixels(0, 0, this.pickWidth, this.pickHeight, this.depthBuffer);
+    }
+
+    private getBufferIdx(x: number, y: number): number {
+        return (y * this.pickWidth + x) * 4;
+    }
+
+    private getDepth(x: number, y: number): number {
+        const idx = this.getBufferIdx(x, y);
+        const b = this.depthBuffer;
+        return unpackRGBAToDepth(b[idx], b[idx + 1], b[idx + 2], b[idx + 3]);
     }
 
     private getId(x: number, y: number, buffer: Uint8Array) {
-        const idx = (y * this.pickWidth + x) * 4;
+        const idx = this.getBufferIdx(x, y);
         return decodeFloatRGB(buffer[idx], buffer[idx + 1], buffer[idx + 2]);
     }
 
-    identify(x: number, y: number): PickingId | undefined {
+    identify(x: number, y: number): PickData | undefined {
         const { webgl, pickScale, camera: { viewport } } = this;
         if (webgl.isContextLost) return;
 
@@ -168,7 +193,12 @@ export class PickPass {
         const groupId = this.getId(xp, yp, this.groupBuffer);
         // console.log('groupId', groupId);
         if (groupId === -1 || groupId === NullId) return;
-        // console.log({ objectId, instanceId, groupId });
-        return { objectId, instanceId, groupId };
+
+        const z = this.getDepth(xp, yp);
+        const position = Vec3.create(x, gl.drawingBufferHeight - y, z);
+        cameraUnproject(position, position, viewport, this.camera.inverseProjectionView);
+
+        // console.log({ { objectId, instanceId, groupId }, position} );
+        return { id: { objectId, instanceId, groupId }, position };
     }
 }

src/mol-geo/geometry/base.ts

@@ -78,6 +78,7 @@ export namespace BaseGeometry {
             visible: true,
             alphaFactor: 1,
             pickable: true,
+            colorOnly: false,
             opaque,
             writeDepth: opaque,
         };

src/mol-geo/geometry/picking.ts

@@ -15,8 +15,3 @@ export namespace PickingId {
         return a.objectId === b.objectId && a.instanceId === b.instanceId && a.groupId === b.groupId;
     }
 }
-
-export interface PickingInfo {
-    label: string
-    data?: any
-}

src/mol-gl/_spec/renderer.spec.ts

@@ -92,6 +92,7 @@ function createPoints() {
         visible: true,
         alphaFactor: 1,
         pickable: true,
+        colorOnly: false,
         opaque: true,
         writeDepth: true
     };

src/mol-gl/renderable.ts

@@ -17,6 +17,7 @@ export type RenderableState = {
     visible: boolean
     alphaFactor: number
     pickable: boolean
+    colorOnly: boolean
     opaque: boolean
     writeDepth: boolean,
 }

src/mol-gl/renderer.ts

@@ -368,7 +368,9 @@ namespace Renderer {
                 }
             } else { // picking & depth
                 for (let i = 0, il = renderables.length; i < il; ++i) {
-                    renderObject(renderables[i], variant, depthTexture);
+                    if (!renderables[i].state.colorOnly) {
+                        renderObject(renderables[i], variant, depthTexture);
+                    }
                 }
             }
 

src/mol-plugin-state/manager/interactivity.ts

@@ -15,7 +15,7 @@ import { shallowEqual } from '../../mol-util/object';
 import { ParamDefinition as PD } from '../../mol-util/param-definition';
 import { StatefulPluginComponent } from '../component';
 import { StructureSelectionManager } from './structure/selection';
-import { Vec2 } from '../../mol-math/linear-algebra';
+import { Vec2, Vec3 } from '../../mol-math/linear-algebra';
 
 export { InteractivityManager };
 
@@ -70,9 +70,9 @@ namespace InteractivityManager {
     export type Params = typeof Params
     export type Props = PD.Values<Params>
 
-    export interface HoverEvent { current: Representation.Loci, buttons: ButtonsType, button: ButtonsType.Flag, modifiers: ModifiersKeys }
+    export interface HoverEvent { current: Representation.Loci, buttons: ButtonsType, button: ButtonsType.Flag, modifiers: ModifiersKeys, page?: Vec2, position?: Vec3 }
     export interface DragEvent { current: Representation.Loci, buttons: ButtonsType, button: ButtonsType.Flag, modifiers: ModifiersKeys, pageStart: Vec2, pageEnd: Vec2 }
-    export interface ClickEvent { current: Representation.Loci, buttons: ButtonsType, button: ButtonsType.Flag, modifiers: ModifiersKeys }
+    export interface ClickEvent { current: Representation.Loci, buttons: ButtonsType, button: ButtonsType.Flag, modifiers: ModifiersKeys, position?: Vec3 }
 
     export type LociMarkProvider = (loci: Representation.Loci, action: MarkerAction) => void
 

src/mol-repr/representation.ts

@@ -174,6 +174,8 @@ namespace Representation {
         alphaFactor: number
         /** Controls if the representation's renderobjects are pickable or not */
         pickable: boolean
+        /** Controls if the representation's renderobjects is rendered in color pass (i.e., not pick and depth) or not */
+        colorOnly: boolean
         /** Overpaint applied to the representation's renderobjects */
         overpaint: Overpaint
         /** Per group transparency applied to the representation's renderobjects */
@@ -188,12 +190,13 @@ namespace Representation {
         markerActions: MarkerActions
     }
     export function createState(): State {
-        return { visible: true, alphaFactor: 1, pickable: true, syncManually: false, transform: Mat4.identity(), overpaint: Overpaint.Empty, transparency: Transparency.Empty, clipping: Clipping.Empty, markerActions: MarkerActions.All };
+        return { visible: true, alphaFactor: 1, pickable: true, colorOnly: false, syncManually: false, transform: Mat4.identity(), overpaint: Overpaint.Empty, transparency: Transparency.Empty, clipping: Clipping.Empty, markerActions: MarkerActions.All };
     }
     export function updateState(state: State, update: Partial<State>) {
         if (update.visible !== undefined) state.visible = update.visible;
         if (update.alphaFactor !== undefined) state.alphaFactor = update.alphaFactor;
         if (update.pickable !== undefined) state.pickable = update.pickable;
+        if (update.colorOnly !== undefined) state.colorOnly = update.colorOnly;
         if (update.overpaint !== undefined) state.overpaint = update.overpaint;
         if (update.transparency !== undefined) state.transparency = update.transparency;
         if (update.clipping !== undefined) state.clipping = update.clipping;
@@ -363,6 +366,7 @@ namespace Representation {
                 if (state.visible !== undefined) Visual.setVisibility(renderObject, state.visible);
                 if (state.alphaFactor !== undefined) Visual.setAlphaFactor(renderObject, state.alphaFactor);
                 if (state.pickable !== undefined) Visual.setPickable(renderObject, state.pickable);
+                if (state.colorOnly !== undefined) Visual.setColorOnly(renderObject, state.colorOnly);
                 if (state.overpaint !== undefined) {
                     // TODO
                 }

src/mol-repr/shape/representation.ts

@@ -203,6 +203,7 @@ export function ShapeRepresentation<D, G extends Geometry, P extends Geometry.Pa
                 if (state.visible !== undefined) Visual.setVisibility(_renderObject, state.visible);
                 if (state.alphaFactor !== undefined) Visual.setAlphaFactor(_renderObject, state.alphaFactor);
                 if (state.pickable !== undefined) Visual.setPickable(_renderObject, state.pickable);
+                if (state.colorOnly !== undefined) Visual.setColorOnly(_renderObject, state.colorOnly);
                 if (state.overpaint !== undefined) {
                     Visual.setOverpaint(_renderObject, state.overpaint, lociApply, true);
                 }

src/mol-repr/structure/complex-visual.ts

@@ -217,6 +217,9 @@ export function ComplexVisual<G extends Geometry, P extends StructureParams & Ge
         setPickable(pickable: boolean) {
             Visual.setPickable(renderObject, pickable);
         },
+        setColorOnly(colorOnly: boolean) {
+            Visual.setColorOnly(renderObject, colorOnly);
+        },
         setTransform(matrix?: Mat4, instanceMatrices?: Float32Array | null) {
             Visual.setTransform(renderObject, matrix, instanceMatrices);
         },

src/mol-repr/structure/units-visual.ts

@@ -274,6 +274,9 @@ export function UnitsVisual<G extends Geometry, P extends StructureParams & Geom
         setPickable(pickable: boolean) {
             Visual.setPickable(renderObject, pickable);
         },
+        setColorOnly(colorOnly: boolean) {
+            Visual.setColorOnly(renderObject, colorOnly);
+        },
         setTransform(matrix?: Mat4, instanceMatrices?: Float32Array | null) {
             Visual.setTransform(renderObject, matrix, instanceMatrices);
         },

src/mol-repr/visual.ts

@@ -41,6 +41,7 @@ interface Visual<D, P extends PD.Params> {
     setVisibility: (visible: boolean) => void
     setAlphaFactor: (alphaFactor: number) => void
     setPickable: (pickable: boolean) => void
+    setColorOnly: (colorOnly: boolean) => void
     setTransform: (matrix?: Mat4, instanceMatrices?: Float32Array | null) => void
     setOverpaint: (overpaint: Overpaint) => void
     setTransparency: (transparency: Transparency) => void
@@ -62,6 +63,10 @@ namespace Visual {
         if (renderObject) renderObject.state.pickable = pickable;
     }
 
+    export function setColorOnly(renderObject: GraphicsRenderObject | undefined, colorOnly: boolean) {
+        if (renderObject) renderObject.state.colorOnly = colorOnly;
+    }
+
     export function mark(renderObject: GraphicsRenderObject | undefined, loci: Loci, action: MarkerAction, lociApply: LociApply) {
         if (!renderObject) return false;
 

src/mol-repr/volume/representation.ts

@@ -190,6 +190,9 @@ export function VolumeVisual<G extends Geometry, P extends VolumeParams & Geomet
         setPickable(pickable: boolean) {
             Visual.setPickable(renderObject, pickable);
         },
+        setColorOnly(colorOnly: boolean) {
+            Visual.setColorOnly(renderObject, colorOnly);
+        },
         setTransform(matrix?: Mat4, instanceMatrices?: Float32Array | null) {
             Visual.setTransform(renderObject, matrix, instanceMatrices);
         },

src/tests/browser/render-shape.ts

@@ -41,7 +41,7 @@ let prevReprLoci = Representation.Loci.Empty;
 const canvas3d = Canvas3D.fromCanvas(canvas);
 canvas3d.animate();
 canvas3d.input.move.subscribe(({x, y}) => {
-    const pickingId = canvas3d.identify(x, y);
+    const pickingId = canvas3d.identify(x, y)?.id;
     let label = '';
     if (pickingId) {
         const reprLoci = canvas3d.getLoci(pickingId);

src/tests/browser/render-structure.ts

@@ -51,7 +51,7 @@ parent.appendChild(info);
 
 let prevReprLoci = Representation.Loci.Empty;
 canvas3d.input.move.pipe(throttleTime(100)).subscribe(({x, y}) => {
-    const pickingId = canvas3d.identify(x, y);
+    const pickingId = canvas3d.identify(x, y)?.id;
     let label = '';
     if (pickingId) {
         const reprLoci = canvas3d.getLoci(pickingId);