image geometry

src/mol-geo/geometry/geometry.ts

@@ -21,8 +21,9 @@ import { TransformData } from './transform-data';
 import { Theme } from '../../mol-theme/theme';
 import { RenderObjectValues } from '../../mol-gl/render-object';
 import { TextureMesh } from './texture-mesh/texture-mesh';
+import { Image } from './image/image';
 
-export type GeometryKind = 'mesh' | 'points' | 'spheres' | 'text' | 'lines' | 'direct-volume' | 'texture-mesh'
+export type GeometryKind = 'mesh' | 'points' | 'spheres' | 'text' | 'lines' | 'direct-volume' | 'image' | 'texture-mesh'
 
 export type Geometry<T extends GeometryKind = GeometryKind> =
     T extends 'mesh' ? Mesh :
@@ -31,7 +32,8 @@ export type Geometry<T extends GeometryKind = GeometryKind> =
                 T extends 'text' ? Text :
                     T extends 'lines' ? Lines :
                         T extends 'direct-volume' ? DirectVolume :
-                            T extends 'texture-mesh' ? TextureMesh : never
+                            T extends 'image' ? Image :
+                                T extends 'texture-mesh' ? TextureMesh : never
 
 type GeometryParams<T extends GeometryKind> =
     T extends 'mesh' ? Mesh.Params :
@@ -40,7 +42,8 @@ type GeometryParams<T extends GeometryKind> =
                 T extends 'text' ? Text.Params :
                     T extends 'lines' ? Lines.Params :
                         T extends 'direct-volume' ? DirectVolume.Params :
-                            T extends 'texture-mesh' ? TextureMesh.Params : never
+                            T extends 'image' ? Image.Params :
+                                T extends 'texture-mesh' ? TextureMesh.Params : never
 
 export interface GeometryUtils<G extends Geometry, P extends PD.Params = GeometryParams<G['kind']>, V = RenderObjectValues<G['kind']>> {
     Params: P
@@ -64,6 +67,7 @@ export namespace Geometry {
             case 'text': return geometry.charCount * 2 * 3;
             case 'lines': return geometry.lineCount * 2 * 3;
             case 'direct-volume': return 12 * 3;
+            case 'image': return 2 * 3;
             case 'texture-mesh': return geometry.vertexCount;
         }
     }
@@ -78,6 +82,8 @@ export namespace Geometry {
                 return getDrawCount(geometry) === 0 ? 0 : (arrayMax(geometry.groupBuffer.ref.value) + 1);
             case 'direct-volume':
                 return 1;
+            case 'image':
+                return arrayMax(geometry.groupTexture.ref.value.array) + 1;
             case 'texture-mesh':
                 return geometry.groupCount;
         }
@@ -92,6 +98,7 @@ export namespace Geometry {
             case 'text': return Text.Utils as any;
             case 'lines': return Lines.Utils as any;
             case 'direct-volume': return DirectVolume.Utils as any;
+            case 'image': return Image.Utils as any;
             case 'texture-mesh': return TextureMesh.Utils as any;
         }
     }

src/mol-geo/geometry/image/image.ts

@@ -0,0 +1,211 @@
+/**
+ * Copyright (c) 2020 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
+ */
+
+import { hashFnv32a } from '../../../mol-data/util';
+import { LocationIterator } from '../../../mol-geo/util/location-iterator';
+import { RenderableState } from '../../../mol-gl/renderable';
+import { calculateInvariantBoundingSphere, calculateTransformBoundingSphere, TextureImage } from '../../../mol-gl/renderable/util';
+import { Sphere3D } from '../../../mol-math/geometry';
+import { Vec2 } from '../../../mol-math/linear-algebra';
+import { Theme } from '../../../mol-theme/theme';
+import { ValueCell } from '../../../mol-util';
+import { Color } from '../../../mol-util/color';
+import { ParamDefinition as PD } from '../../../mol-util/param-definition';
+import { BaseGeometry } from '../base';
+import { createColors } from '../color-data';
+import { GeometryUtils } from '../geometry';
+import { createMarkers } from '../marker-data';
+import { createEmptyOverpaint } from '../overpaint-data';
+import { TransformData } from '../transform-data';
+import { createEmptyTransparency } from '../transparency-data';
+import { ImageValues } from '../../../mol-gl/renderable/image';
+import { fillSerial } from '../../../mol-util/array';
+
+const QuadIndices = new Uint32Array([
+    0, 1, 2,
+    1, 3, 2
+]);
+
+const QuadUvs = new Float32Array([
+    0, 1,
+    0, 0,
+    1, 1,
+    1, 0
+]);
+
+export const InterpolationTypes = {
+    'nearest': 'Nearest',
+    'catmulrom': 'Catmulrom (Cubic)',
+    'mitchell': 'Mitchell (Cubic)',
+    'bspline': 'B-Spline (Cubic)'
+};
+export type InterpolationTypes = keyof typeof InterpolationTypes;
+export const InterpolationTypeNames = Object.keys(InterpolationTypes) as InterpolationTypes[];
+
+export { Image };
+
+interface Image {
+    readonly kind: 'image',
+
+    readonly imageTexture: ValueCell<TextureImage<Float32Array>>,
+    readonly imageTextureDim: ValueCell<Vec2>,
+    readonly cornerBuffer: ValueCell<Float32Array>,
+    readonly groupTexture: ValueCell<TextureImage<Float32Array>>,
+
+    /** Bounding sphere of the image */
+    boundingSphere: Sphere3D
+}
+
+namespace Image {
+    export function create(imageTexture: TextureImage<Float32Array>, corners: Float32Array, groupTexture: TextureImage<Float32Array>, image?: Image): Image {
+        return image ?
+            update(imageTexture, corners, groupTexture, image) :
+            fromData(imageTexture, corners, groupTexture);
+    }
+
+    function hashCode(image: Image) {
+        return hashFnv32a([
+            image.cornerBuffer.ref.version
+        ]);
+    }
+
+    function fromData(imageTexture: TextureImage<Float32Array>, corners: Float32Array, groupTexture: TextureImage<Float32Array>): Image {
+        const boundingSphere = Sphere3D();
+        let currentHash = -1;
+
+        const width = imageTexture.width;
+        const height = imageTexture.height;
+
+        const image = {
+            kind: 'image' as const,
+            imageTexture: ValueCell.create(imageTexture),
+            imageTextureDim: ValueCell.create(Vec2.create(width, height)),
+            cornerBuffer: ValueCell.create(corners),
+            groupTexture: ValueCell.create(groupTexture),
+            get boundingSphere() {
+                const newHash = hashCode(image);
+                if (newHash !== currentHash) {
+                    const b = getBoundingSphere(image.cornerBuffer.ref.value);
+                    Sphere3D.copy(boundingSphere, b);
+                    currentHash = newHash;
+                }
+                return boundingSphere;
+            },
+        };
+        return image;
+    }
+
+    function update(imageTexture: TextureImage<Uint8Array | Float32Array>, corners: Float32Array, groupTexture: TextureImage<Float32Array>, image: Image): Image {
+
+        const width = imageTexture.width;
+        const height = imageTexture.height;
+
+        ValueCell.update(image.imageTexture, imageTexture);
+        ValueCell.update(image.imageTextureDim, Vec2.set(image.imageTextureDim.ref.value, width, height));
+        ValueCell.update(image.cornerBuffer, corners);
+        ValueCell.update(image.groupTexture, groupTexture);
+        return image;
+    }
+
+    export function createEmpty(image?: Image): Image {
+        return {} as Image; // TODO
+    }
+
+    export const Params = {
+        ...BaseGeometry.Params,
+        interpolation: PD.Select('bspline', PD.objectToOptions(InterpolationTypes)),
+    };
+    export type Params = typeof Params
+
+    export const Utils: GeometryUtils<Image, Params> = {
+        Params,
+        createEmpty,
+        createValues,
+        createValuesSimple,
+        updateValues,
+        updateBoundingSphere,
+        createRenderableState,
+        updateRenderableState
+    };
+
+    function createValues(image: Image, transform: TransformData, locationIt: LocationIterator, theme: Theme, props: PD.Values<Params>): ImageValues {
+
+        const { instanceCount, groupCount } = locationIt;
+        const color = createColors(locationIt, theme.color);
+        const marker = createMarkers(instanceCount * groupCount);
+        const overpaint = createEmptyOverpaint();
+        const transparency = createEmptyTransparency();
+
+        const counts = { drawCount: QuadIndices.length, groupCount, instanceCount };
+
+        const invariantBoundingSphere = Sphere3D.clone(image.boundingSphere);
+        const boundingSphere = calculateTransformBoundingSphere(invariantBoundingSphere, transform.aTransform.ref.value, instanceCount);
+
+        return {
+            ...color,
+            ...marker,
+            ...overpaint,
+            ...transparency,
+            ...transform,
+            ...BaseGeometry.createValues(props, counts),
+
+            aPosition: image.cornerBuffer,
+            aUv: ValueCell.create(QuadUvs),
+            elements: ValueCell.create(QuadIndices),
+
+            // aGroup is used as a vertex index here, group id is in tGroupTex
+            aGroup: ValueCell.create(fillSerial(new Float32Array(4))),
+            boundingSphere: ValueCell.create(boundingSphere),
+            invariantBoundingSphere: ValueCell.create(invariantBoundingSphere),
+
+            dInterpolation: ValueCell.create(props.interpolation),
+
+            uImageTexDim: image.imageTextureDim,
+            tImageTex: image.imageTexture,
+            tGroupTex: image.groupTexture,
+        };
+    }
+
+    function createValuesSimple(image: Image, props: Partial<PD.Values<Params>>, colorValue: Color, sizeValue: number, transform?: TransformData) {
+        const s = BaseGeometry.createSimple(colorValue, sizeValue, transform);
+        const p = { ...PD.getDefaultValues(Params), ...props };
+        return createValues(image, s.transform, s.locationIterator, s.theme, p);
+    }
+
+    function updateValues(values: ImageValues, props: PD.Values<Params>) {
+        ValueCell.updateIfChanged(values.uAlpha, props.alpha);
+        ValueCell.updateIfChanged(values.dInterpolation, props.interpolation);
+    }
+
+    function updateBoundingSphere(values: ImageValues, image: Image) {
+        const invariantBoundingSphere = Sphere3D.clone(image.boundingSphere);
+        const boundingSphere = calculateTransformBoundingSphere(invariantBoundingSphere, values.aTransform.ref.value, values.instanceCount.ref.value);
+
+        if (!Sphere3D.equals(boundingSphere, values.boundingSphere.ref.value)) {
+            ValueCell.update(values.boundingSphere, boundingSphere);
+        }
+        if (!Sphere3D.equals(invariantBoundingSphere, values.invariantBoundingSphere.ref.value)) {
+            ValueCell.update(values.invariantBoundingSphere, invariantBoundingSphere);
+        }
+    }
+
+    function createRenderableState(props: PD.Values<Params>): RenderableState {
+        const state = BaseGeometry.createRenderableState(props);
+        state.opaque = false;
+        return state;
+    }
+
+    function updateRenderableState(state: RenderableState, props: PD.Values<Params>) {
+        BaseGeometry.updateRenderableState(state, props);
+        state.opaque = false;
+    }
+}
+
+//
+
+function getBoundingSphere(corners: Float32Array) {
+    return calculateInvariantBoundingSphere(corners, corners.length / 3, 1);
+}

src/mol-gl/render-object.ts

@@ -14,6 +14,7 @@ import { LinesValues, LinesRenderable } from './renderable/lines';
 import { SpheresValues, SpheresRenderable } from './renderable/spheres';
 import { TextValues, TextRenderable } from './renderable/text';
 import { TextureMeshValues, TextureMeshRenderable } from './renderable/texture-mesh';
+import { ImageValues, ImageRenderable } from './renderable/image';
 
 const getNextId = idFactory(0, 0x7FFFFFFF);
 
@@ -27,7 +28,7 @@ export interface GraphicsRenderObject<T extends RenderObjectType = RenderObjectT
     readonly materialId: number
 }
 
-export type RenderObjectType = 'mesh' | 'points' | 'spheres' | 'text' | 'lines' | 'direct-volume' | 'texture-mesh'
+export type RenderObjectType = 'mesh' | 'points' | 'spheres' | 'text' | 'lines' | 'direct-volume' | 'image' | 'texture-mesh'
 
 export type RenderObjectValues<T extends RenderObjectType> =
     T extends 'mesh' ? MeshValues :
@@ -36,7 +37,8 @@ export type RenderObjectValues<T extends RenderObjectType> =
                 T extends 'text' ? TextValues :
                     T extends 'lines' ? LinesValues :
                         T extends 'direct-volume' ? DirectVolumeValues :
-                            T extends 'texture-mesh' ? TextureMeshValues : never
+                            T extends 'image' ? ImageValues :
+                                T extends 'texture-mesh' ? TextureMeshValues : never
 
 //
 
@@ -52,6 +54,7 @@ export function createRenderable<T extends RenderObjectType>(ctx: WebGLContext,
         case 'text': return TextRenderable(ctx, o.id, o.values as TextValues, o.state, o.materialId);
         case 'lines': return LinesRenderable(ctx, o.id, o.values as LinesValues, o.state, o.materialId);
         case 'direct-volume': return DirectVolumeRenderable(ctx, o.id, o.values as DirectVolumeValues, o.state, o.materialId);
+        case 'image': return ImageRenderable(ctx, o.id, o.values as ImageValues, o.state, o.materialId);
         case 'texture-mesh': return TextureMeshRenderable(ctx, o.id, o.values as TextureMeshValues, o.state, o.materialId);
     }
     throw new Error('unsupported type');

src/mol-gl/renderable/image.ts

@@ -0,0 +1,41 @@
+/**
+ * Copyright (c) 2020 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
+ */
+
+import { Renderable, RenderableState, createRenderable } from '../renderable';
+import { WebGLContext } from '../webgl/context';
+import { createGraphicsRenderItem } from '../webgl/render-item';
+import { AttributeSpec, Values, GlobalUniformSchema, InternalSchema, TextureSpec, ElementsSpec, DefineSpec, InternalValues, BaseSchema, UniformSpec } from './schema';
+import { ImageShaderCode } from '../shader-code';
+import { ValueCell } from '../../mol-util';
+import { InterpolationTypeNames } from '../../mol-geo/geometry/image/image';
+
+export const ImageSchema = {
+    ...BaseSchema,
+
+    aPosition: AttributeSpec('float32', 3, 0),
+    aUv: AttributeSpec('float32', 2, 0),
+
+    elements: ElementsSpec('uint32'),
+
+    uImageTexDim: UniformSpec('v2'),
+    tImageTex: TextureSpec('image-float32', 'rgba', 'float', 'nearest'),
+    tGroupTex: TextureSpec('image-float32', 'alpha', 'float', 'nearest'),
+
+    dInterpolation: DefineSpec('string', InterpolationTypeNames),
+};
+export type ImageSchema = typeof ImageSchema
+export type ImageValues = Values<ImageSchema>
+
+export function ImageRenderable(ctx: WebGLContext, id: number, values: ImageValues, state: RenderableState, materialId: number): Renderable<ImageValues> {
+    const schema = { ...GlobalUniformSchema, ...InternalSchema, ...ImageSchema };
+    const internalValues: InternalValues = {
+        uObjectId: ValueCell.create(id),
+        uPickable: ValueCell.create(state.pickable ? 1 : 0),
+    };
+    const shaderCode = ImageShaderCode;
+    const renderItem = createGraphicsRenderItem(ctx, 'triangles', shaderCode, schema, { ...values, ...internalValues }, materialId);
+    return createRenderable(renderItem, values, state);
+}

src/mol-gl/shader-code.ts

@@ -126,6 +126,10 @@ import direct_volume_vert from './shader/direct-volume.vert';
 import direct_volume_frag from './shader/direct-volume.frag';
 export const DirectVolumeShaderCode = ShaderCode('direct-volume', direct_volume_vert, direct_volume_frag, { fragDepth: true });
 
+import image_vert from './shader/image.vert';
+import image_frag from './shader/image.frag';
+export const ImageShaderCode = ShaderCode('image', image_vert, image_frag, { fragDepth: true });
+
 //
 
 export type ShaderDefines = {

src/mol-gl/shader/image.frag.ts

@@ -0,0 +1,127 @@
+/**
+ * Copyright (c) 2020 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
+ */
+export default `
+precision highp float;
+precision highp int;
+
+#include common
+#include common_frag_params
+
+uniform vec2 uImageTexDim;
+uniform sampler2D tImageTex;
+uniform sampler2D tGroupTex;
+
+varying vec2 vUv;
+varying float vInstance;
+
+#if defined(dInterpolation_catmulrom) || defined(dInterpolation_mitchell) || defined(dInterpolation_bspline)
+    #define dInterpolation_cubic
+#endif
+
+#if defined(dInterpolation_cubic)
+    #if defined(dInterpolation_catmulrom) || defined(dInterpolation_mitchell)
+        #if defined(dInterpolation_catmulrom)
+            const float B = 0.0;
+            const float C = 0.5;
+        #elif defined(dInterpolation_mitchell)
+            const float B = 0.333;
+            const float C = 0.333;
+        #endif
+
+        float cubicFilter( float x ){
+            float f = x;
+            if( f < 0.0 ){
+                f = -f;
+            }
+            if( f < 1.0 ){
+                return ( ( 12.0 - 9.0 * B - 6.0 * C ) * ( f * f * f ) +
+                    ( -18.0 + 12.0 * B + 6.0 *C ) * ( f * f ) +
+                    ( 6.0 - 2.0 * B ) ) / 6.0;
+            }else if( f >= 1.0 && f < 2.0 ){
+                return ( ( -B - 6.0 * C ) * ( f * f * f )
+                    + ( 6.0 * B + 30.0 * C ) * ( f *f ) +
+                    ( - ( 12.0 * B ) - 48.0 * C  ) * f +
+                    8.0 * B + 24.0 * C ) / 6.0;
+            }else{
+                return 0.0;
+            }
+        }
+    #elif defined(dInterpolation_bspline)
+        float cubicFilter(float x) {
+            float f = x;
+            if (f < 0.0) {
+                f = -f;
+            }
+            if (f >= 0.0 && f <= 1.0){
+                return (2.0 / 3.0) + (0.5) * (f * f * f) - (f * f);
+            } else if (f > 1.0 && f <= 2.0) {
+                return 1.0 / 6.0 * pow((2.0 - f), 3.0);
+            }
+            return 1.0;
+        }
+    #endif
+
+    vec4 biCubic(sampler2D tex, vec2 texCoord) {
+        vec2 texelSize = 1.0 / uImageTexDim;
+        texCoord -= texelSize / 2.0;
+        vec4 nSum = vec4(0.0);
+        float nDenom = 0.0;
+        vec2 cell = fract(texCoord * uImageTexDim);
+        for (float m = -1.0; m <= 2.0; ++m) {
+            for (float n = -1.0; n <= 2.0; ++n) {
+                vec4 vecData = texture2D(tex, texCoord + texelSize * vec2(m, n));
+                float c = cubicFilter(m - cell.x) * cubicFilter(-n + cell.y);
+                nSum += vecData * c;
+                nDenom += c;
+            }
+        }
+        return nSum / nDenom;
+    }
+#endif
+
+void main() {
+    #if defined(dInterpolation_cubic)
+        vec4 imageData = biCubic(tImageTex, vUv);
+    #else
+        vec4 imageData = texture2D(tImageTex, vUv);
+    #endif
+
+    #if defined(dRenderVariant_pick)
+        if (imageData.a < 0.3)
+            discard;
+
+        if (uPickable == 1) {
+            #if defined(dRenderVariant_pickObject)
+                gl_FragColor = vec4(encodeFloatRGB(float(uObjectId)), 1.0);
+            #elif defined(dRenderVariant_pickInstance)
+                gl_FragColor = vec4(encodeFloatRGB(vInstance), 1.0);
+            #elif defined(dRenderVariant_pickGroup)
+                float group = texture2D(tGroupTex, vUv).a;
+                gl_FragColor = vec4(encodeFloatRGB(group), 1.0);
+            #endif
+        } else {
+            gl_FragColor = vec4(0.0, 0.0, 0.0, 1.0); // set to empty picking id
+        }
+    #elif defined(dRenderVariant_depth)
+        if (imageData.a < 0.01)
+            discard;
+
+        #ifdef enabledFragDepth
+            gl_FragColor = packDepthToRGBA(gl_FragDepthEXT);
+        #else
+            gl_FragColor = packDepthToRGBA(gl_FragCoord.z);
+        #endif
+    #elif defined(dRenderVariant_color)
+        if (imageData.a < 0.01)
+            discard;
+
+        gl_FragColor = imageData;
+        gl_FragColor.a *= uAlpha;
+
+        #include apply_fog
+    #endif
+}
+`;

src/mol-gl/shader/image.vert.ts

@@ -0,0 +1,27 @@
+/**
+ * Copyright (c) 2020 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
+ */
+export default `
+precision highp float;
+precision highp int;
+
+#include common
+#include common_vert_params
+
+attribute vec3 aPosition;
+attribute vec2 aUv;
+attribute mat4 aTransform;
+attribute float aInstance;
+
+varying vec2 vUv;
+varying float vInstance;
+
+void main() {
+    #include assign_position
+
+    vUv = aUv;
+    vInstance = aInstance;
+}
+`;