wip, picking

src/mol-app/ui/visualization/image-canvas.tsx

@@ -65,10 +65,9 @@ export class ImageCanvas extends React.Component<{ imageData: ImageData, aspectR
             style={{
                 width: this.state.width + 6,
                 height: this.state.height + 6,
-                position: 'absolute',
+                margin: 10,
+                display: 'inline-block',
                 border: '3px white solid',
-                bottom: 10,
-                left: 10,
             }}
         >
             <canvas

src/mol-app/ui/visualization/viewport.tsx

@@ -94,11 +94,18 @@ export const Logo = () =>
     </div>
 
 
-export class Viewport extends View<ViewportController, {}, { noWebGl?: boolean, showLogo?: boolean, imageData?: ImageData, aspectRatio: number }> {
+type ViewportState = {
+    noWebGl: boolean,
+    showLogo: boolean,
+    aspectRatio: number,
+    images: { [k: string]: ImageData }
+}
+
+export class Viewport extends View<ViewportController, ViewportState, { noWebGl?: boolean, showLogo?: boolean, aspectRatio: number }> {
     private container: HTMLDivElement | null = null;
     private canvas: HTMLCanvasElement | null = null;
     private defaultBg = { r: 1, g: 1, b: 1 }
-    state = { noWebGl: false, showLogo: true, imageData: undefined, aspectRatio: 1 };
+    state: ViewportState = { noWebGl: false, showLogo: true, images: {}, aspectRatio: 1 };
 
     componentDidMount() {
         if (!this.canvas || !this.container || !this.controller.context.initStage(this.canvas, this.container)) {
@@ -117,7 +124,13 @@ export class Viewport extends View<ViewportController, {}, { noWebGl?: boolean,
         viewer.didDraw.subscribe(() => {
             // this.setState({ imageData: viewer.getImageData() })
             viewer.pick()
-            this.setState({ imageData: viewer.getPickImageData() })
+            this.setState({
+                images: {
+                    'object': viewer.getImageData('pickObject'),
+                    'instance': viewer.getImageData('pickInstance'),
+                    'element': viewer.getImageData('pickElement')
+                }
+            })
         })
 
         if (this.container) {
@@ -143,14 +156,6 @@ export class Viewport extends View<ViewportController, {}, { noWebGl?: boolean,
     render() {
         if (this.state.noWebGl) return this.renderMissing();
 
-        // const imageData = new ImageData(256, 128)
-
-        let image: JSX.Element | undefined
-        const imageData = this.state.imageData
-        if (imageData) {
-            image = <ImageCanvas imageData={imageData} aspectRatio={this.state.aspectRatio} maxWidth={256} maxHeight={256} />
-        }
-
         const color = this.controller.latestState.clearColor! || this.defaultBg;
         return <div className='molstar-viewport' style={{ backgroundColor: `rgb(${255 * color.r}, ${255 * color.g}, ${255 * color.b})` }}>
             <div ref={elm => this.container = elm} className='molstar-viewport-container'>
@@ -158,7 +163,18 @@ export class Viewport extends View<ViewportController, {}, { noWebGl?: boolean,
             </div>
             {this.state.showLogo ? <Logo /> : void 0}
             <ViewportControls controller={this.controller} />
-            {image}
+            <div
+                style={{
+                    position: 'absolute',
+                    bottom: 10,
+                    left: 10,
+                }}
+            >
+                {Object.keys(this.state.images).map(k => {
+                    const imageData = this.state.images[k]
+                    return <ImageCanvas key={k} imageData={imageData} aspectRatio={this.state.aspectRatio} maxWidth={256} maxHeight={256} />
+                })}
+            </div>
         </div>;
     }
 }

src/mol-gl/renderable.ts

@@ -5,7 +5,8 @@
  */
 
 import { Program } from './webgl/program';
-import { RenderableValues } from './renderable/schema';
+import { RenderableValues, Values, RenderableSchema } from './renderable/schema';
+import { RenderVariant, RenderItem } from './webgl/render-item';
 
 export type RenderableState = {
     visible: boolean
@@ -13,16 +14,26 @@ export type RenderableState = {
 }
 
 export interface Renderable<T extends RenderableValues> {
-    draw: () => void
-    pick: () => void
-    values: T
-    state: RenderableState
-    name: string
-    drawProgram: Program
-    pickProgram: Program
+    readonly values: T
+    readonly state: RenderableState
+
+    render: (variant: RenderVariant) => void
+    getProgram: (variant: RenderVariant) => Program
     update: () => void
     dispose: () => void
 }
 
+export function createRenderable<T extends Values<RenderableSchema>>(renderItem: RenderItem, values: T, state: RenderableState): Renderable<T> {
+    return {
+        get values () { return values },
+        get state () { return state },
+
+        render: (variant: RenderVariant) => renderItem.render(variant),
+        getProgram: (variant: RenderVariant) => renderItem.getProgram(variant),
+        update: () => renderItem.update(),
+        dispose: () => renderItem.destroy()
+    }
+}
+
 export { PointRenderable, PointSchema, PointValues } from './renderable/point'
 export { MeshRenderable, MeshSchema, MeshValues } from './renderable/mesh'

src/mol-gl/renderable/mesh.ts

@@ -4,7 +4,7 @@
  * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
-import { Renderable, RenderableState } from '../renderable'
+import { Renderable, RenderableState, createRenderable } from '../renderable'
 import { Context } from '../webgl/context';
 import { createRenderItem } from '../webgl/render-item';
 import { GlobalUniformSchema, BaseSchema, AttributeSpec, ElementsSpec, DefineSpec, Values, InternalSchema } from '../renderable/schema';
@@ -30,23 +30,5 @@ export function MeshRenderable(ctx: Context, id: number, values: MeshValues, sta
     const schaderCode = MeshShaderCode
     const renderItem = createRenderItem(ctx, 'triangles', schaderCode, schema, { ...values, ...internalValues })
 
-    return {
-        draw: () => {
-            renderItem.draw()
-        },
-        pick: () => {
-            renderItem.pick()
-        },
-        get values () { return values },
-        get state () { return state },
-        name: 'mesh',
-        get drawProgram () { return renderItem.drawProgram },
-        get pickProgram () { return renderItem.pickProgram },
-        update: () => {
-            renderItem.update()
-        },
-        dispose: () => {
-            renderItem.destroy()
-        }
-    }
+    return createRenderable(renderItem, values, state)
 }

src/mol-gl/renderable/point.ts

@@ -4,7 +4,7 @@
  * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
-import { Renderable, RenderableState } from '../renderable'
+import { Renderable, RenderableState, createRenderable } from '../renderable'
 import { Context } from '../webgl/context';
 import { createRenderItem } from '../webgl/render-item';
 import { GlobalUniformSchema, BaseSchema, AttributeSpec, UniformSpec, DefineSpec, Values, InternalSchema } from '../renderable/schema';
@@ -29,23 +29,5 @@ export function PointRenderable(ctx: Context, id: number, values: PointValues, s
     const schaderCode = PointShaderCode
     const renderItem = createRenderItem(ctx, 'points', schaderCode, schema, { ...values, ...internalValues })
 
-    return {
-        draw: () => {
-            renderItem.draw()
-        },
-        pick: () => {
-            renderItem.pick()
-        },
-        get values () { return values },
-        get state () { return state },
-        name: 'point',
-        get drawProgram () { return renderItem.drawProgram },
-        get pickProgram () { return renderItem.pickProgram },
-        update: () => {
-            renderItem.update()
-        },
-        dispose: () => {
-            renderItem.destroy()
-        }
-    }
+    return createRenderable(renderItem, values, state)
 }

src/mol-gl/renderer.ts

@@ -15,17 +15,21 @@ import { Renderable } from './renderable';
 import { Color } from 'mol-util/color';
 import { ValueCell } from 'mol-util';
 import { RenderableValues, GlobalUniformValues } from './renderable/schema';
+import { RenderVariant } from './webgl/render-item';
 
 export interface RendererStats {
     programCount: number
     shaderCount: number
+
     bufferCount: number
+    framebufferCount: number
+    renderbufferCount: number
     textureCount: number
     vaoCount: number
 }
 
 interface Renderer {
-    render: (scene: Scene, pick: boolean) => void
+    render: (scene: Scene, variant: RenderVariant) => void
 
     setViewport: (viewport: Viewport) => void
     setClearColor: (color: Color) => void
@@ -77,8 +81,8 @@ namespace Renderer {
         }
 
         let currentProgramId = -1
-        const renderObject = (r: Renderable<RenderableValues>, pick: boolean) => {
-            const program = pick ? r.pickProgram : r.drawProgram
+        const renderObject = (r: Renderable<RenderableValues>, variant: RenderVariant) => {
+            const program = r.getProgram(variant)
             if (r.state.visible) {
                 if (currentProgramId !== program.id) {
                     program.use()
@@ -101,15 +105,11 @@ namespace Renderer {
 
                 gl.depthMask(r.state.depthMask)
 
-                if (pick) {
-                    r.pick()
-                } else {
-                    r.draw()
-                }
+                r.render(variant)
             }
         }
 
-        const render = (scene: Scene, pick: boolean) => {
+        const render = (scene: Scene, variant: RenderVariant) => {
             ValueCell.update(globalUniforms.uView, camera.view)
             ValueCell.update(globalUniforms.uProjection, camera.projection)
 
@@ -120,11 +120,11 @@ namespace Renderer {
 
             gl.disable(gl.BLEND)
             gl.enable(gl.DEPTH_TEST)
-            scene.eachOpaque((r) => renderObject(r, pick))
+            scene.eachOpaque((r) => renderObject(r, variant))
 
             gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA)
             gl.enable(gl.BLEND)
-            scene.eachTransparent((r) => renderObject(r, pick))
+            scene.eachTransparent((r) => renderObject(r, variant))
 
             gl.finish()
         }
@@ -147,10 +147,14 @@ namespace Renderer {
             },
 
             get stats(): RendererStats {
+                console.log(ctx)
                 return {
                     programCount: ctx.programCache.count,
                     shaderCount: ctx.shaderCache.count,
+
                     bufferCount: ctx.bufferCount,
+                    framebufferCount: ctx.framebufferCount,
+                    renderbufferCount: ctx.renderbufferCount,
                     textureCount: ctx.textureCount,
                     vaoCount: ctx.vaoCount,
                 }

src/mol-gl/shader/mesh.frag

@@ -41,7 +41,6 @@ void main() {
     #if defined(dColorType_objectPicking) || defined(dColorType_instancePicking) || defined(dColorType_elementPicking)
         // gl_FragColor = vec4(material.r, material.g, material.a, 1.0);
         gl_FragColor = material;
-        gl_FragColor.a = 1.0;
     #else
         // determine surface to light direction
         // vec4 viewLightPosition = view * vec4(lightPosition, 1.0);

src/mol-gl/shader/utils/encode-id-rgba.glsl

@@ -3,7 +3,7 @@
 #define MAX_ID 16777216.0
 
 vec4 encodeIdRGBA( const in float v ) {
-	return encodeFloatRGBA(1.0 - ((v + 1.0) / MAX_ID));
+	return encodeFloatRGBA(((v + 1.0) / MAX_ID));
 }
 
 #pragma glslify: export(encodeIdRGBA)

src/mol-gl/webgl/context.ts

@@ -71,8 +71,8 @@ export interface Context {
     textureCount: number
     vaoCount: number
 
-    readPixels: (x: number, y: number, width: number, height: number, buffer: Uint8Array) => void
     unbindFramebuffer: () => void
+    readPixels: (x: number, y: number, width: number, height: number, buffer: Uint8Array) => void
     destroy: () => void
 }
 
@@ -110,6 +110,7 @@ export function createContext(gl: WebGLRenderingContext): Context {
         textureCount: 0,
         vaoCount: 0,
 
+        unbindFramebuffer: () => unbindFramebuffer(gl),
         readPixels: (x: number, y: number, width: number, height: number, buffer: Uint8Array) => {
             if (gl.checkFramebufferStatus(gl.FRAMEBUFFER) === gl.FRAMEBUFFER_COMPLETE) {
                 gl.readPixels(x, y, width, height, gl.RGBA, gl.UNSIGNED_BYTE, buffer)
@@ -117,7 +118,7 @@ export function createContext(gl: WebGLRenderingContext): Context {
                 console.error('Reading pixels failed. Framebuffer not complete.')
             }
         },
-        unbindFramebuffer: () => unbindFramebuffer(gl),
+
         destroy: () => {
             unbindResources(gl)
             programCache.dispose()

src/mol-gl/webgl/framebuffer.ts

@@ -29,10 +29,7 @@ export function createFramebuffer (ctx: Context): Framebuffer {
     return {
         id: getNextFramebufferId(),
 
-        bind: () => {
-            gl.bindFramebuffer(gl.FRAMEBUFFER, _framebuffer)
-        },
-
+        bind: () => gl.bindFramebuffer(gl.FRAMEBUFFER, _framebuffer),
         destroy: () => {
             if (destroyed) return
             gl.deleteFramebuffer(_framebuffer)

src/mol-gl/webgl/render-item.ts

@@ -9,10 +9,11 @@ import { createTextures } from './texture';
 import { Context } from './context';
 import { ShaderCode, addShaderDefines } from '../shader-code';
 import { Program } from './program';
-import { RenderableSchema, RenderableValues, AttributeSpec, getValueVersions, splitValues } from '../renderable/schema';
+import { RenderableSchema, RenderableValues, AttributeSpec, getValueVersions, splitValues, Values } from '../renderable/schema';
 import { idFactory } from 'mol-util/id-factory';
 import { deleteVertexArray, createVertexArray } from './vertex-array';
 import { ValueCell } from 'mol-util';
+import { ReferenceItem } from 'mol-util/reference-cache';
 
 const getNextRenderItemId = idFactory()
 
@@ -33,15 +34,24 @@ export function getDrawMode(ctx: Context, drawMode: DrawMode) {
 
 export interface RenderItem {
     readonly id: number
-    readonly drawProgram: Program
-    readonly pickProgram: Program
+    getProgram: (variant: RenderVariant) => Program
 
+    render: (variant: RenderVariant) => void
     update: () => void
-    draw: () => void
-    pick: () => void
     destroy: () => void
 }
 
+const RenderVariantDefines = {
+    'draw': {},
+    'pickObject': { dColorType: ValueCell.create('objectPicking') },
+    'pickInstance': { dColorType: ValueCell.create('instancePicking') },
+    'pickElement': { dColorType: ValueCell.create('elementPicking') }
+}
+export type RenderVariant = keyof typeof RenderVariantDefines
+
+type ProgramVariants = { [k: string]: ReferenceItem<Program> }
+type VertexArrayVariants = { [k: string]: WebGLVertexArrayObjectOES | undefined }
+
 export function createRenderItem(ctx: Context, drawMode: DrawMode, shaderCode: ShaderCode, schema: RenderableSchema, values: RenderableValues): RenderItem {
     const id = getNextRenderItemId()
     const { programCache } = ctx
@@ -51,13 +61,14 @@ export function createRenderItem(ctx: Context, drawMode: DrawMode, shaderCode: S
     const versions = getValueVersions(values)
 
     const glDrawMode = getDrawMode(ctx, drawMode)
-    let drawProgram = programCache.get(ctx, {
-        shaderCode: addShaderDefines(defineValues, shaderCode),
-        schema
-    })
-    let pickProgram = programCache.get(ctx, {
-        shaderCode: addShaderDefines({ ...defineValues, dColorType: ValueCell.create('elementPicking') }, shaderCode),
-        schema
+
+    const programs: ProgramVariants = {}
+    Object.keys(RenderVariantDefines).forEach(k => {
+        const variantDefineValues: Values<RenderableSchema> = (RenderVariantDefines as any)[k]
+        programs[k] = programCache.get(ctx, {
+            shaderCode: addShaderDefines({ ...defineValues, ...variantDefineValues }, shaderCode),
+            schema
+        })
     })
 
     const textures = createTextures(ctx, schema, textureValues)
@@ -69,40 +80,36 @@ export function createRenderItem(ctx: Context, drawMode: DrawMode, shaderCode: S
         elementsBuffer = createElementsBuffer(ctx, elements.ref.value)
     }
 
-    let drawVertexArray: WebGLVertexArrayObjectOES | undefined = createVertexArray(ctx, drawProgram.value, attributeBuffers, elementsBuffer)
-    let pickVertexArray: WebGLVertexArrayObjectOES | undefined = createVertexArray(ctx, pickProgram.value, attributeBuffers, elementsBuffer)
+    const vertexArrays: VertexArrayVariants = {}
+    Object.keys(RenderVariantDefines).forEach(k => {
+        vertexArrays[k] = createVertexArray(ctx, programs[k].value, attributeBuffers, elementsBuffer)
+    })
 
     let drawCount = values.drawCount.ref.value
     let instanceCount = values.instanceCount.ref.value
 
     let destroyed = false
 
-    function render(program: Program, vertexArray: WebGLVertexArrayObjectOES | undefined) {
-        program.setUniforms(uniformValues)
-        if (oesVertexArrayObject && vertexArray) {
-            oesVertexArrayObject.bindVertexArrayOES(vertexArray)
-        } else {
-            program.bindAttributes(attributeBuffers)
-            if (elementsBuffer) elementsBuffer.bind()
-        }
-        program.bindTextures(textures)
-        if (elementsBuffer) {
-            angleInstancedArrays.drawElementsInstancedANGLE(glDrawMode, drawCount, elementsBuffer._dataType, 0, instanceCount);
-        } else {
-            angleInstancedArrays.drawArraysInstancedANGLE(glDrawMode, 0, drawCount, instanceCount)
-        }
-    }
-
     return {
         id,
-        get drawProgram () { return drawProgram.value },
-        get pickProgram () { return pickProgram.value },
-
-        draw: () => {
-            render(drawProgram.value, drawVertexArray)
-        },
-        pick: () => {
-            render(pickProgram.value, pickVertexArray)
+        getProgram: (variant: RenderVariant) => programs[variant].value,
+
+        render: (variant: RenderVariant) => {
+            const program = programs[variant].value
+            const vertexArray = vertexArrays[variant]
+            program.setUniforms(uniformValues)
+            if (oesVertexArrayObject && vertexArray) {
+                oesVertexArrayObject.bindVertexArrayOES(vertexArray)
+            } else {
+                program.bindAttributes(attributeBuffers)
+                if (elementsBuffer) elementsBuffer.bind()
+            }
+            program.bindTextures(textures)
+            if (elementsBuffer) {
+                angleInstancedArrays.drawElementsInstancedANGLE(glDrawMode, drawCount, elementsBuffer._dataType, 0, instanceCount);
+            } else {
+                angleInstancedArrays.drawArraysInstancedANGLE(glDrawMode, 0, drawCount, instanceCount)
+            }
         },
         update: () => {
             let defineChange = false
@@ -116,17 +123,14 @@ export function createRenderItem(ctx: Context, drawMode: DrawMode, shaderCode: S
             })
 
             if (defineChange) {
-                console.log('some defines changed, need to rebuild program')
-                drawProgram.free()
-                drawProgram = programCache.get(ctx, {
-                    shaderCode: addShaderDefines(defineValues, shaderCode),
-                    schema
-                })
-
-                pickProgram.free()
-                pickProgram = programCache.get(ctx, {
-                    shaderCode: addShaderDefines({ ...defineValues, dColorType: ValueCell.create('elementPicking') }, shaderCode),
-                    schema
+                console.log('some defines changed, need to rebuild programs')
+                Object.keys(RenderVariantDefines).forEach(k => {
+                    const variantDefineValues: Values<RenderableSchema> = (RenderVariantDefines as any)[k]
+                    programs[k].free()
+                    programs[k] = programCache.get(ctx, {
+                        shaderCode: addShaderDefines({ ...defineValues, ...variantDefineValues }, shaderCode),
+                        schema
+                    })
                 })
             }
 
@@ -175,11 +179,10 @@ export function createRenderItem(ctx: Context, drawMode: DrawMode, shaderCode: S
             }
 
             if (defineChange || bufferChange) {
-                console.log('program/defines or buffers changed, rebuild vao')
-                deleteVertexArray(ctx, drawVertexArray)
-                drawVertexArray = createVertexArray(ctx, drawProgram.value, attributeBuffers, elementsBuffer)
-                deleteVertexArray(ctx, pickVertexArray)
-                pickVertexArray = createVertexArray(ctx, drawProgram.value, attributeBuffers, elementsBuffer)
+                console.log('program/defines or buffers changed, rebuild vaos')
+                Object.keys(RenderVariantDefines).forEach(k => {
+                    vertexArrays[k] = createVertexArray(ctx, programs[k].value, attributeBuffers, elementsBuffer)
+                })
             }
 
             Object.keys(textureValues).forEach(k => {
@@ -193,13 +196,13 @@ export function createRenderItem(ctx: Context, drawMode: DrawMode, shaderCode: S
         },
         destroy: () => {
             if (!destroyed) {
-                drawProgram.free()
-                pickProgram.free()
+                Object.keys(RenderVariantDefines).forEach(k => {
+                    programs[k].free()
+                    deleteVertexArray(ctx, vertexArrays[k])
+                })
                 Object.keys(textures).forEach(k => textures[k].destroy())
                 Object.keys(attributeBuffers).forEach(k => attributeBuffers[k].destroy())
                 if (elementsBuffer) elementsBuffer.destroy()
-                deleteVertexArray(ctx, drawVertexArray)
-                deleteVertexArray(ctx, pickVertexArray)
                 destroyed = true
             }
         }

src/mol-gl/webgl/render-target.ts

@@ -8,7 +8,7 @@ import { Context, createImageData } from './context'
 import { idFactory } from 'mol-util/id-factory';
 import { createTexture } from './texture';
 import { createFramebuffer } from './framebuffer';
-// import { createRenderbuffer } from './renderbuffer';
+import { createRenderbuffer } from './renderbuffer';
 
 const getNextRenderTargetId = idFactory()
 
@@ -34,16 +34,12 @@ export function createRenderTarget (ctx: Context, _width: number, _height: numbe
     targetTexture.load(image)
 
     const framebuffer = createFramebuffer(ctx)
-    framebuffer.bind()
 
     // attach the texture as the first color attachment
-    gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, targetTexture.texture, 0);
+    targetTexture.attachFramebuffer(framebuffer, 'color0')
 
-    // const depthRenderbuffer = createRenderbuffer(ctx)
-    // depthRenderbuffer.bind()
-    // // make a depth buffer and the same size as the targetTexture
-    // gl.renderbufferStorage(gl.RENDERBUFFER, gl.DEPTH_COMPONENT16, targetTexture.width, targetTexture.height);
-    // gl.framebufferRenderbuffer(gl.FRAMEBUFFER, gl.DEPTH_ATTACHMENT, gl.RENDERBUFFER, depthRenderbuffer);
+    // make a depth renderbuffer of the same size as the targetTexture
+    const depthRenderbuffer = createRenderbuffer(ctx, 'depth16', 'depth', _width, _height)
 
     let destroyed = false
 
@@ -61,6 +57,8 @@ export function createRenderTarget (ctx: Context, _width: number, _height: numbe
             image.width = _width
             image.height = _height
             targetTexture.load(image)
+
+            depthRenderbuffer.setSize(_width, _height)
         },
         getImageData: () => {
             framebuffer.bind()
@@ -71,7 +69,7 @@ export function createRenderTarget (ctx: Context, _width: number, _height: numbe
             if (destroyed) return
             targetTexture.destroy()
             framebuffer.destroy()
-            // depthRenderbuffer.destroy()
+            depthRenderbuffer.destroy()
             destroyed = true
         }
     }

src/mol-gl/webgl/renderbuffer.ts

@@ -9,28 +9,63 @@ import { idFactory } from 'mol-util/id-factory';
 
 const getNextRenderbufferId = idFactory()
 
+export type RenderbufferFormat = 'depth16' | 'stencil8' | 'rgba4' | 'depth-stencil'
+export type RenderbufferAttachment = 'depth' | 'stencil' | 'depth-stencil' | 'color0'
+
+export function getFormat(ctx: Context, format: RenderbufferFormat) {
+    const { gl } = ctx
+    switch (format) {
+        case 'depth16': return gl.DEPTH_COMPONENT16
+        case 'stencil8': return gl.STENCIL_INDEX8
+        case 'rgba4': return gl.RGBA4
+        case 'depth-stencil': return gl.DEPTH_STENCIL
+    }
+}
+
+export function getAttachment(ctx: Context, attachment: RenderbufferAttachment) {
+    const { gl } = ctx
+    switch (attachment) {
+        case 'depth': return gl.DEPTH_ATTACHMENT
+        case 'stencil': return gl.STENCIL_ATTACHMENT
+        case 'depth-stencil': return gl.DEPTH_STENCIL_ATTACHMENT
+        case 'color0': return gl.COLOR_ATTACHMENT0
+    }
+}
+
 export interface Renderbuffer {
     readonly id: number
 
     bind: () => void
+    setSize: (width: number, height: number) => void
+
     destroy: () => void
 }
 
-export function createRenderbuffer (ctx: Context): Renderbuffer {
+export function createRenderbuffer (ctx: Context, format: RenderbufferFormat, attachment: RenderbufferAttachment, _width: number, _height: number): Renderbuffer {
     const { gl } = ctx
     const _renderbuffer = gl.createRenderbuffer()
     if (_renderbuffer === null) {
         throw new Error('Could not create WebGL renderbuffer')
     }
 
+    const bind = () => gl.bindRenderbuffer(gl.RENDERBUFFER, _renderbuffer)
+    const _format = getFormat(ctx, format)
+    const _attachment = getAttachment(ctx, attachment)
+
+    bind()
+    gl.renderbufferStorage(gl.RENDERBUFFER, _format, _width, _height)
+    gl.framebufferRenderbuffer(gl.FRAMEBUFFER, _attachment, gl.RENDERBUFFER, _renderbuffer)
+
     let destroyed = false
     ctx.renderbufferCount += 1
 
     return {
         id: getNextRenderbufferId(),
 
-        bind: () => {
-            gl.bindRenderbuffer(gl.RENDERBUFFER, _renderbuffer)
+        bind,
+        setSize: (_width: number, _height: number) => {
+            bind()
+            gl.renderbufferStorage(gl.RENDERBUFFER, _format, _width, _height)
         },
 
         destroy: () => {

src/mol-gl/webgl/texture.ts

@@ -9,11 +9,13 @@ import { TextureImage } from '../renderable/util';
 import { ValueCell } from 'mol-util';
 import { RenderableSchema } from '../renderable/schema';
 import { idFactory } from 'mol-util/id-factory';
+import { Framebuffer } from './framebuffer';
 
 const getNextTextureId = idFactory()
 
 export type TextureFormat = 'rgb' | 'rgba'
 export type TextureType = 'ubyte' | 'uint'
+export type TextureAttachment = 'depth' | 'stencil' | 'color0'
 
 export function getFormat(ctx: Context, format: TextureFormat) {
     const { gl } = ctx
@@ -31,9 +33,17 @@ export function getType(ctx: Context, type: TextureType) {
     }
 }
 
+export function getAttachment(ctx: Context, attachment: TextureAttachment) {
+    const { gl } = ctx
+    switch (attachment) {
+        case 'depth': return gl.DEPTH_ATTACHMENT
+        case 'stencil': return gl.STENCIL_ATTACHMENT
+        case 'color0': return gl.COLOR_ATTACHMENT0
+    }
+}
+
 export interface Texture {
     readonly id: number
-    readonly texture: WebGLTexture
     readonly format: number
     readonly type: number
 
@@ -43,6 +53,7 @@ export interface Texture {
     load: (image: TextureImage) => void
     bind: (id: TextureId) => void
     unbind: (id: TextureId) => void
+    attachFramebuffer: (framebuffer: Framebuffer, attachment: TextureAttachment) => void
     destroy: () => void
 }
 
@@ -72,7 +83,6 @@ export function createTexture(ctx: Context, _format: TextureFormat, _type: Textu
 
     return {
         id,
-        texture,
         format,
         type,
 
@@ -102,6 +112,10 @@ export function createTexture(ctx: Context, _format: TextureFormat, _type: Textu
             gl.activeTexture(gl.TEXTURE0 + id)
             gl.bindTexture(gl.TEXTURE_2D, null)
         },
+        attachFramebuffer: (framebuffer: Framebuffer, attachment: TextureAttachment) => {
+            framebuffer.bind()
+            gl.framebufferTexture2D(gl.FRAMEBUFFER, getAttachment(ctx, attachment), gl.TEXTURE_2D, texture, 0)
+        },
         destroy: () => {
             if (destroyed) return
             gl.deleteTexture(texture)

src/mol-math/linear-algebra/3d/vec4.ts

@@ -140,6 +140,10 @@ namespace Vec4 {
         out[3] = m[3] * x + m[7] * y + m[11] * z + m[15] * w;
         return out;
     }
+
+    export function dot(a: Vec4, b: Vec4) {
+        return a[0] * b[0] + a[1] * b[1] + a[2] * b[2] + a[3] * b[3];
+    }
 }
 
 export default Vec4

src/mol-util/input/input-observer.ts

@@ -93,6 +93,13 @@ export type ClickInput = {
     pageY: number,
 } & BaseInput
 
+export type MoveInput = {
+    x: number,
+    y: number,
+    pageX: number,
+    pageY: number,
+} & BaseInput
+
 export type PinchInput = {
     delta: number,
     distance: number,
@@ -124,6 +131,7 @@ interface InputObserver {
     wheel: Subject<WheelInput>,
     pinch: Subject<PinchInput>,
     click: Subject<ClickInput>,
+    move: Subject<MoveInput>,
     resize: Subject<ResizeInput>,
 
     dispose: () => void
@@ -153,6 +161,7 @@ namespace InputObserver {
 
         const drag = new Subject<DragInput>()
         const click = new Subject<ClickInput>()
+        const move = new Subject<MoveInput>()
         const wheel = new Subject<WheelInput>()
         const pinch = new Subject<PinchInput>()
         const resize = new Subject<ResizeInput>()
@@ -169,6 +178,7 @@ namespace InputObserver {
             wheel,
             pinch,
             click,
+            move,
             resize,
 
             dispose
@@ -343,13 +353,15 @@ namespace InputObserver {
 
         function onPointerMove (ev: PointerEvent) {
             eventOffset(pointerEnd, ev)
+            const { pageX, pageY } = ev
+            const [ x, y ] = pointerEnd
+            move.next({ x, y, pageX, pageY, buttons, modifiers })
+
             if (dragging === DraggingState.Stopped) return
 
             Vec2.div(pointerDelta, Vec2.sub(pointerDelta, pointerEnd, pointerStart), getClientSize(rectSize))
 
             const isStart = dragging === DraggingState.Started
-            const { pageX, pageY } = ev
-            const [ x, y ] = pointerEnd
             const [ dx, dy ] = pointerDelta
             drag.next({ x, y, dx, dy, pageX, pageY, buttons, modifiers, isStart })
 

src/mol-view/viewer.ts

@@ -6,7 +6,7 @@
 
 import { BehaviorSubject } from 'rxjs';
 
-import { Vec3, Mat4, EPSILON } from 'mol-math/linear-algebra'
+import { Vec3, Mat4, EPSILON, Vec4 } from 'mol-math/linear-algebra'
 import InputObserver from 'mol-util/input/input-observer'
 import * as SetUtils from 'mol-util/set'
 import Renderer, { RendererStats } from 'mol-gl/renderer'
@@ -20,6 +20,7 @@ import { createContext } from 'mol-gl/webgl/context';
 import { Representation } from 'mol-geo/representation';
 import { createRenderTarget } from 'mol-gl/webgl/render-target';
 import Scene from 'mol-gl/scene';
+import { RenderVariant } from 'mol-gl/webgl/render-item';
 
 interface Viewer {
     center: (p: Vec3) => void
@@ -36,6 +37,7 @@ interface Viewer {
     requestDraw: () => void
     animate: () => void
     pick: () => void
+    identify: (x: number, y: number) => void
 
     reprCount: BehaviorSubject<number>
     didDraw: BehaviorSubject<number>
@@ -43,8 +45,7 @@ interface Viewer {
     handleResize: () => void
     resetCamera: () => void
     downloadScreenshot: () => void
-    getImageData: () => ImageData
-    getPickImageData: () => ImageData
+    getImageData: (variant: RenderVariant) => ImageData
 
     input: InputObserver
     stats: RendererStats
@@ -72,6 +73,7 @@ namespace Viewer {
 
         const input = InputObserver.create(canvas)
         input.resize.subscribe(handleResize)
+        input.move.subscribe(({x, y}) => identify(x, y))
 
         const camera = PerspectiveCamera.create({
             near: 0.1,
@@ -97,19 +99,23 @@ namespace Viewer {
         const scene = Scene.create(ctx)
         const renderer = Renderer.create(ctx, camera)
 
-        const rtScale = 1 / 4
-        const renderTarget = createRenderTarget(ctx, Math.round(canvas.width * rtScale), Math.round(canvas.height * rtScale))
+        const pickScale = 1 // 1 / 4
+        const pickWidth = Math.round(canvas.width * pickScale)
+        const pickHeight = Math.round(canvas.height * pickScale)
+        const objectPickTarget = createRenderTarget(ctx, pickWidth, pickHeight)
+        const instancePickTarget = createRenderTarget(ctx, pickWidth, pickHeight)
+        const elementPickTarget = createRenderTarget(ctx, pickWidth, pickHeight)
 
         let drawPending = false
         const prevProjectionView = Mat4.zero()
 
-        function render(pick: boolean, force?: boolean) {
+        function render(variant: RenderVariant, force?: boolean) {
             let didRender = false
             controls.update()
             camera.update()
             if (force || !Mat4.areEqual(camera.projectionView, prevProjectionView, EPSILON.Value)) {
                 Mat4.copy(prevProjectionView, camera.projectionView)
-                renderer.render(scene, pick)
+                renderer.render(scene, variant)
                 didRender = true
             }
             return didRender
@@ -119,7 +125,7 @@ namespace Viewer {
             ctx.unbindFramebuffer()
             const viewport = { x: 0, y: 0, width: canvas.width, height: canvas.height }
             renderer.setViewport(viewport)
-            if (render(false, force)) {
+            if (render('draw', force)) {
                 didDraw.next(performance.now() - startTime)
             }
             drawPending = false
@@ -136,6 +142,31 @@ namespace Viewer {
             window.requestAnimationFrame(() => animate())
         }
 
+        const decodeFactors = Vec4.create(1, 1/255, 1/65025, 1/16581375)
+        function decodeFloatRGBA(rgba: Vec4) {
+            return Vec4.dot(rgba, decodeFactors);
+        }
+
+        function identify (x: number, y: number) {
+            y = canvas.height - y // flip y
+            const xp = Math.round(x * pickScale)
+            const yp = Math.round(y * pickScale)
+            console.log('position', x, y, xp, yp)
+
+            const buffer = new Uint8Array(4)
+            elementPickTarget.bind()
+            ctx.readPixels(xp, yp, 1, 1, buffer)
+            console.log('identify', buffer[0], buffer[1], buffer[2], buffer[3])
+            const v = Vec4.create(buffer[0], buffer[1], buffer[2], buffer[3])
+            const d = decodeFloatRGBA(v)
+            console.log(d)
+            console.log(d * 16777216)
+
+            ctx.unbindFramebuffer()
+            ctx.readPixels(x, y, 1, 1, buffer)
+            console.log('color', buffer[0], buffer[1], buffer[2], buffer[3])
+        }
+
         handleResize()
 
         return {
@@ -184,9 +215,16 @@ namespace Viewer {
             requestDraw,
             animate,
             pick: () => {
-                renderTarget.bind()
-                render(true, true)
+                objectPickTarget.bind()
+                render('pickObject', true)
+
+                instancePickTarget.bind()
+                render('pickInstance', true)
+
+                elementPickTarget.bind()
+                render('pickElement', true)
             },
+            identify,
 
             handleResize,
             resetCamera: () => {
@@ -195,11 +233,13 @@ namespace Viewer {
             downloadScreenshot: () => {
                 // TODO
             },
-            getImageData: () => {
-                return renderer.getImageData()
-            },
-            getPickImageData: () => {
-                return renderTarget.getImageData()
+            getImageData: (variant: RenderVariant) => {
+                switch (variant) {
+                    case 'draw': return renderer.getImageData()
+                    case 'pickObject': return objectPickTarget.getImageData()
+                    case 'pickInstance': return instancePickTarget.getImageData()
+                    case 'pickElement': return elementPickTarget.getImageData()
+                }
             },
             reprCount,
             didDraw,
@@ -224,7 +264,12 @@ namespace Viewer {
             renderer.setViewport(viewport)
             Viewport.copy(camera.viewport, viewport)
             Viewport.copy(controls.viewport, viewport)
-            renderTarget.setSize(Math.round(canvas.width * rtScale), Math.round(canvas.height * rtScale))
+
+            const pickWidth = Math.round(canvas.width * pickScale)
+            const pickHeight = Math.round(canvas.height * pickScale)
+            objectPickTarget.setSize(pickWidth, pickHeight)
+            instancePickTarget.setSize(pickWidth, pickHeight)
+            elementPickTarget.setSize(pickWidth, pickHeight)
         }
     }
 }