geo export support smoothed overpaint/transparency

src/extensions/geo-export/glb-exporter.ts

@@ -2,6 +2,7 @@
  * Copyright (c) 2021 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * @author Sukolsak Sakshuwong <sukolsak@stanford.edu>
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
 import { BaseValues } from '../../mol-gl/renderable/schema';
@@ -30,6 +31,12 @@ const FLOAT = 5126;
 const ARRAY_BUFFER = 34962;
 const ELEMENT_ARRAY_BUFFER = 34963;
 
+const GLTF_MAGIC_BYTE = 0x46546C67;
+const JSON_CHUNK_TYPE = 0x4E4F534A;
+const BIN_CHUNK_TYPE = 0x004E4942;
+const JSON_PAD_CHAR = 0x20;
+const BIN_PAD_CHAR = 0x00;
+
 export type GlbData = {
     glb: Uint8Array
 }
@@ -126,23 +133,16 @@ export class GlbExporter extends MeshExporter<GlbData> {
         };
     }
 
-    private addColorBuffer(values: BaseValues, groups: Float32Array | Uint8Array, vertexCount: number, instanceIndex: number, isGeoTexture: boolean, interpolatedColors: Uint8Array | undefined) {
-        const groupCount = values.uGroupCount.ref.value;
+    private addColorBuffer(values: BaseValues, groups: Float32Array | Uint8Array, vertexCount: number, instanceIndex: number, isGeoTexture: boolean, interpolatedColors: Uint8Array | undefined, interpolatedOverpaint: Uint8Array | undefined, interpolatedTransparency: Uint8Array | undefined) {
         const uAlpha = values.uAlpha.ref.value;
-        const dTransparency = values.dTransparency.ref.value;
-        const tTransparency = values.tTransparency.ref.value;
 
         const colorArray = new Uint8Array(vertexCount * 4);
 
         for (let i = 0; i < vertexCount; ++i) {
-            let color = GlbExporter.getColor(values, groups, vertexCount, instanceIndex, isGeoTexture, interpolatedColors, i);
+            let color = GlbExporter.getColor(values, groups, vertexCount, instanceIndex, isGeoTexture, interpolatedColors, interpolatedOverpaint, i);
 
-            let alpha = uAlpha;
-            if (dTransparency) {
-                const group = isGeoTexture ? GlbExporter.getGroup(groups, i) : groups[i];
-                const transparency = tTransparency.array[instanceIndex * groupCount + group] / 255;
-                alpha *= 1 - transparency;
-            }
+            const transparency = GlbExporter.getTransparency(values, groups, vertexCount, instanceIndex, isGeoTexture, interpolatedTransparency, i);
+            const alpha = uAlpha * (1 - transparency);
 
             color = Color.sRGBToLinear(color);
             Color.toArray(color, colorArray, i * 4);
@@ -163,6 +163,8 @@ export class GlbExporter extends MeshExporter<GlbData> {
         const t = Mat4();
 
         const colorType = values.dColorType.ref.value;
+        const overpaintType = values.dOverpaintType.ref.value;
+        const transparencyType = values.dTransparencyType.ref.value;
         const dTransparency = values.dTransparency.ref.value;
         const aTransform = values.aTransform.ref.value;
         const instanceCount = values.uInstanceCount.ref.value;
@@ -173,6 +175,18 @@ export class GlbExporter extends MeshExporter<GlbData> {
             interpolatedColors = GlbExporter.getInterpolatedColors(mesh!.vertices, mesh!.vertexCount, values, stride, colorType, webgl!);
         }
 
+        let interpolatedOverpaint: Uint8Array | undefined;
+        if (overpaintType === 'volumeInstance') {
+            const stride = isGeoTexture ? 4 : 3;
+            interpolatedOverpaint = GlbExporter.getInterpolatedOverpaint(mesh!.vertices, mesh!.vertexCount, values, stride, overpaintType, webgl!);
+        }
+
+        let interpolatedTransparency: Uint8Array | undefined;
+        if (transparencyType === 'volumeInstance') {
+            const stride = isGeoTexture ? 4 : 3;
+            interpolatedTransparency = GlbExporter.getInterpolatedTransparency(mesh!.vertices, mesh!.vertexCount, values, stride, transparencyType, webgl!);
+        }
+
         // instancing
         const sameGeometryBuffers = mesh !== undefined;
         const sameColorBuffer = sameGeometryBuffers && colorType !== 'instance' && !colorType.endsWith('Instance') && !dTransparency;
@@ -201,7 +215,7 @@ export class GlbExporter extends MeshExporter<GlbData> {
 
                 // create a color buffer if needed
                 if (instanceIndex === 0 || !sameColorBuffer) {
-                    colorAccessorIndex = this.addColorBuffer(values, groups, vertexCount, instanceIndex, isGeoTexture, interpolatedColors);
+                    colorAccessorIndex = this.addColorBuffer(values, groups, vertexCount, instanceIndex, isGeoTexture, interpolatedColors, interpolatedOverpaint, interpolatedTransparency);
                 }
 
                 // glTF mesh
@@ -279,13 +293,13 @@ export class GlbExporter extends MeshExporter<GlbData> {
         const jsonBuffer = new Uint8Array(jsonString.length);
         asciiWrite(jsonBuffer, jsonString);
 
-        const [jsonChunk, jsonChunkLength] = createChunk(0x4E4F534A, [jsonBuffer.buffer], jsonBuffer.length, 0x20);
-        const [binaryChunk, binaryChunkLength] = createChunk(0x004E4942, this.binaryBuffer, binaryBufferLength, 0x00);
+        const [jsonChunk, jsonChunkLength] = createChunk(JSON_CHUNK_TYPE, [jsonBuffer.buffer], jsonBuffer.length, JSON_PAD_CHAR);
+        const [binaryChunk, binaryChunkLength] = createChunk(BIN_CHUNK_TYPE, this.binaryBuffer, binaryBufferLength, BIN_PAD_CHAR);
 
         const glbBufferLength = 12 + jsonChunkLength + binaryChunkLength;
         const header = new ArrayBuffer(12);
         const headerDataView = new DataView(header);
-        headerDataView.setUint32(0, 0x46546C67, true); // magic number "glTF"
+        headerDataView.setUint32(0, GLTF_MAGIC_BYTE, true); // magic number "glTF"
         headerDataView.setUint32(4, 2, true); // version
         headerDataView.setUint32(8, glbBufferLength, true); // length
         const glbBuffer = [header, ...jsonChunk, ...binaryChunk];

src/extensions/geo-export/mesh-exporter.ts

@@ -2,6 +2,7 @@
  * Copyright (c) 2021 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * @author Sukolsak Sakshuwong <sukolsak@stanford.edu>
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
 import { sort, arraySwap } from '../../mol-data/util';
@@ -26,7 +27,7 @@ import { RuntimeContext } from '../../mol-task';
 import { Color } from '../../mol-util/color/color';
 import { decodeFloatRGB } from '../../mol-util/float-packing';
 import { RenderObjectExporter, RenderObjectExportData } from './render-object-exporter';
-import { readTexture } from '../../mol-gl/compute/util';
+import { readAlphaTexture, readTexture } from '../../mol-gl/compute/util';
 
 const GeoExportName = 'geo-export';
 
@@ -103,6 +104,31 @@ export abstract class MeshExporter<D extends RenderObjectExportData> implements
         return interpolated.array;
     }
 
+    protected static getInterpolatedOverpaint(vertices: Float32Array, vertexCount: number, values: BaseValues, stride: 3 | 4, colorType: 'volumeInstance', webgl: WebGLContext) {
+        const overpaintGridTransform = values.uOverpaintGridTransform.ref.value;
+        const overpaintGridDim = values.uOverpaintGridDim.ref.value;
+        const overpaintTexDim = values.uOverpaintTexDim.ref.value;
+        const aTransform = values.aTransform.ref.value;
+        const instanceCount = values.uInstanceCount.ref.value;
+
+        const overpaintGrid = readTexture(webgl, values.tOverpaintGrid.ref.value).array;
+        const interpolated = getTrilinearlyInterpolated({ vertexCount, instanceCount, transformBuffer: aTransform, positionBuffer: vertices, colorType, grid: overpaintGrid, gridDim: overpaintGridDim, gridTexDim: overpaintTexDim, gridTransform: overpaintGridTransform, vertexStride: stride, colorStride: 4, outputStride: 4 });
+        return interpolated.array;
+    }
+
+    protected static getInterpolatedTransparency(vertices: Float32Array, vertexCount: number, values: BaseValues, stride: 3 | 4, colorType: 'volumeInstance', webgl: WebGLContext) {
+        const transparencyGridTransform = values.uTransparencyGridTransform.ref.value;
+        const transparencyGridDim = values.uTransparencyGridDim.ref.value;
+        const transparencyTexDim = values.uTransparencyTexDim.ref.value;
+        const aTransform = values.aTransform.ref.value;
+        const instanceCount = values.uInstanceCount.ref.value;
+
+        const transparencyGrid = readAlphaTexture(webgl, values.tTransparencyGrid.ref.value).array;
+        const interpolated = getTrilinearlyInterpolated({ vertexCount, instanceCount, transformBuffer: aTransform, positionBuffer: vertices, colorType, grid: transparencyGrid, gridDim: transparencyGridDim, gridTexDim: transparencyTexDim, gridTransform: transparencyGridTransform, vertexStride: stride, colorStride: 4, outputStride: 1, itemOffset: 3 });
+
+        return interpolated.array;
+    }
+
     protected static quantizeColors(colorArray: Uint8Array, vertexCount: number) {
         if (vertexCount <= 1024) return;
         const rgb = Vec3();
@@ -184,11 +210,12 @@ export abstract class MeshExporter<D extends RenderObjectExportData> implements
         }
     }
 
-    protected static getColor(values: BaseValues, groups: Float32Array | Uint8Array, vertexCount: number, instanceIndex: number, isGeoTexture: boolean, interpolatedColors: Uint8Array | undefined, vertexIndex: number): Color {
+    protected static getColor(values: BaseValues, groups: Float32Array | Uint8Array, vertexCount: number, instanceIndex: number, isGeoTexture: boolean, interpolatedColors: Uint8Array | undefined, interpolatedOverpaint: Uint8Array | undefined, vertexIndex: number): Color {
         const groupCount = values.uGroupCount.ref.value;
         const colorType = values.dColorType.ref.value;
         const uColor = values.uColor.ref.value;
         const tColor = values.tColor.ref.value.array;
+        const overpaintType = values.dOverpaintType.ref.value;
         const dOverpaint = values.dOverpaint.ref.value;
         const tOverpaint = values.tOverpaint.ref.value.array;
 
@@ -226,15 +253,69 @@ export abstract class MeshExporter<D extends RenderObjectExportData> implements
         }
 
         if (dOverpaint) {
-            const group = isGeoTexture ? MeshExporter.getGroup(groups, vertexIndex) : groups[vertexIndex];
-            const overpaintColor = Color.fromArray(tOverpaint, (instanceIndex * groupCount + group) * 4);
-            const overpaintAlpha = tOverpaint[(instanceIndex * groupCount + group) * 4 + 3] / 255;
+            let overpaintColor: Color;
+            let overpaintAlpha: number;
+            switch (overpaintType) {
+                case 'groupInstance': {
+                    const group = isGeoTexture ? MeshExporter.getGroup(groups, vertexIndex) : groups[vertexIndex];
+                    const idx = (instanceIndex * groupCount + group) * 4;
+                    overpaintColor = Color.fromArray(tOverpaint, idx);
+                    overpaintAlpha = tOverpaint[idx + 3] / 255;
+                    break;
+                }
+                case 'vertexInstance': {
+                    const idx = (instanceIndex * vertexCount + vertexIndex) * 4;
+                    overpaintColor = Color.fromArray(tOverpaint, idx);
+                    overpaintAlpha = tOverpaint[idx + 3] / 255;
+                    break;
+                }
+                case 'volumeInstance': {
+                    const idx = (instanceIndex * vertexCount + vertexIndex) * 4;
+                    overpaintColor = Color.fromArray(interpolatedOverpaint!, idx);
+                    overpaintAlpha = interpolatedOverpaint![idx + 3] / 255;
+                    break;
+                }
+                default: throw new Error('Unsupported overpaint type.');
+            }
+            // interpolate twice to avoid darkening due to empty overpaint
+            overpaintColor = Color.interpolate(color, overpaintColor, overpaintAlpha);
             color = Color.interpolate(color, overpaintColor, overpaintAlpha);
         }
 
         return color;
     }
 
+    protected static getTransparency(values: BaseValues, groups: Float32Array | Uint8Array, vertexCount: number, instanceIndex: number, isGeoTexture: boolean, interpolatedTransparency: Uint8Array | undefined, vertexIndex: number): number {
+        const groupCount = values.uGroupCount.ref.value;
+        const dTransparency = values.dTransparency.ref.value;
+        const tTransparency = values.tTransparency.ref.value.array;
+        const transparencyType = values.dTransparencyType.ref.value;
+
+        let transparency: number = 0;
+        if (dTransparency) {
+            switch (transparencyType) {
+                case 'groupInstance': {
+                    const group = isGeoTexture ? MeshExporter.getGroup(groups, vertexIndex) : groups[vertexIndex];
+                    const idx = (instanceIndex * groupCount + group);
+                    transparency = tTransparency[idx] / 255;
+                    break;
+                }
+                case 'vertexInstance': {
+                    const idx = (instanceIndex * vertexCount + vertexIndex);
+                    transparency = tTransparency[idx] / 255;
+                    break;
+                }
+                case 'volumeInstance': {
+                    const idx = (instanceIndex * vertexCount + vertexIndex);
+                    transparency = interpolatedTransparency![idx] / 255;
+                    break;
+                }
+                default: throw new Error('Unsupported transparency type.');
+            }
+        }
+        return transparency;
+    }
+
     protected abstract addMeshWithColors(input: AddMeshInput): void;
 
     private async addMesh(values: MeshValues, webgl: WebGLContext, ctx: RuntimeContext) {

src/extensions/geo-export/obj-exporter.ts

@@ -2,6 +2,7 @@
  * Copyright (c) 2021 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * @author Sukolsak Sakshuwong <sukolsak@stanford.edu>
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
 import { asciiWrite } from '../../mol-io/common/ascii';
@@ -47,7 +48,7 @@ export class ObjExporter extends MeshExporter<ObjData> {
         StringBuilder.newline(this.obj);
         if (!this.materialSet.has(material)) {
             this.materialSet.add(material);
-            const [r, g, b] = Color.toRgbNormalized(color);
+            const [r, g, b] = Color.toRgbNormalized(color).map(v => Math.round(v * 1000) / 1000);
             const mtl = this.mtl;
             StringBuilder.writeSafe(mtl, `newmtl ${material}\n`);
             StringBuilder.writeSafe(mtl, 'illum 2\n'); // illumination model
@@ -77,18 +78,27 @@ export class ObjExporter extends MeshExporter<ObjData> {
         const tmpV = Vec3();
         const stride = isGeoTexture ? 4 : 3;
 
-        const groupCount = values.uGroupCount.ref.value;
         const colorType = values.dColorType.ref.value;
+        const overpaintType = values.dOverpaintType.ref.value;
+        const transparencyType = values.dTransparencyType.ref.value;
         const uAlpha = values.uAlpha.ref.value;
-        const dTransparency = values.dTransparency.ref.value;
-        const tTransparency = values.tTransparency.ref.value;
         const aTransform = values.aTransform.ref.value;
         const instanceCount = values.uInstanceCount.ref.value;
 
         let interpolatedColors: Uint8Array | undefined;
         if (colorType === 'volume' || colorType === 'volumeInstance') {
             interpolatedColors = ObjExporter.getInterpolatedColors(mesh!.vertices, mesh!.vertexCount, values, stride, colorType, webgl!);
-            ObjExporter.quantizeColors(interpolatedColors, mesh!.vertexCount);
+        }
+
+        let interpolatedOverpaint: Uint8Array | undefined;
+        if (overpaintType === 'volumeInstance') {
+            interpolatedOverpaint = ObjExporter.getInterpolatedOverpaint(mesh!.vertices, mesh!.vertexCount, values, stride, overpaintType, webgl!);
+        }
+
+        let interpolatedTransparency: Uint8Array | undefined;
+        if (transparencyType === 'volumeInstance') {
+            const stride = isGeoTexture ? 4 : 3;
+            interpolatedTransparency = ObjExporter.getInterpolatedTransparency(mesh!.vertices, mesh!.vertexCount, values, stride, transparencyType, webgl!);
         }
 
         await ctx.update({ isIndeterminate: false, current: 0, max: instanceCount });
@@ -126,17 +136,21 @@ export class ObjExporter extends MeshExporter<ObjData> {
                 StringBuilder.newline(obj);
             }
 
-            // face
+            // color
+            const quantizedColors = new Uint8Array(drawCount * 3);
             for (let i = 0; i < drawCount; i += 3) {
                 const v = isGeoTexture ? i : indices![i];
-                const color = ObjExporter.getColor(values, groups, vertexCount, instanceIndex, isGeoTexture, interpolatedColors, v);
-
-                let alpha = uAlpha;
-                if (dTransparency) {
-                    const group = isGeoTexture ? ObjExporter.getGroup(groups, i) : groups[indices![i]];
-                    const transparency = tTransparency.array[instanceIndex * groupCount + group] / 255;
-                    alpha *= 1 - transparency;
-                }
+                const color = ObjExporter.getColor(values, groups, vertexCount, instanceIndex, isGeoTexture, interpolatedColors, interpolatedOverpaint, v);
+                Color.toArray(color, quantizedColors, i);
+            }
+            ObjExporter.quantizeColors(quantizedColors, mesh!.vertexCount);
+
+            // face
+            for (let i = 0; i < drawCount; i += 3) {
+                const color = Color.fromArray(quantizedColors, i);
+
+                const transparency = ObjExporter.getTransparency(values, groups, vertexCount, instanceIndex, isGeoTexture, interpolatedTransparency, i);
+                const alpha = Math.round(uAlpha * (1 - transparency) * 10) / 10; // quantized
 
                 this.updateMaterial(color, alpha);
 

src/extensions/geo-export/usdz-exporter.ts

@@ -2,6 +2,7 @@
  * Copyright (c) 2021 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * @author Sukolsak Sakshuwong <sukolsak@stanford.edu>
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
 import { Style } from '../../mol-gl/renderer';
@@ -42,7 +43,7 @@ export class UsdzExporter extends MeshExporter<UsdzData> {
         const materialKey = UsdzExporter.getMaterialKey(color, alpha);
         if (this.materialSet.has(materialKey)) return;
         this.materialSet.add(materialKey);
-        const [r, g, b] = Color.toRgbNormalized(color);
+        const [r, g, b] = Color.toRgbNormalized(color).map(v => Math.round(v * 1000) / 1000);
         this.materials.push(`
 def Material "material${materialKey}"
 {
@@ -68,18 +69,28 @@ def Material "material${materialKey}"
         const tmpV = Vec3();
         const stride = isGeoTexture ? 4 : 3;
 
-        const groupCount = values.uGroupCount.ref.value;
         const colorType = values.dColorType.ref.value;
+        const overpaintType = values.dOverpaintType.ref.value;
+        const transparencyType = values.dTransparencyType.ref.value;
         const uAlpha = values.uAlpha.ref.value;
-        const dTransparency = values.dTransparency.ref.value;
-        const tTransparency = values.tTransparency.ref.value;
         const aTransform = values.aTransform.ref.value;
         const instanceCount = values.uInstanceCount.ref.value;
 
         let interpolatedColors: Uint8Array | undefined;
         if (colorType === 'volume' || colorType === 'volumeInstance') {
             interpolatedColors = UsdzExporter.getInterpolatedColors(mesh!.vertices, mesh!.vertexCount, values, stride, colorType, webgl!);
-            UsdzExporter.quantizeColors(interpolatedColors, mesh!.vertexCount);
+        }
+
+        let interpolatedOverpaint: Uint8Array | undefined;
+        if (overpaintType === 'volumeInstance') {
+            const stride = isGeoTexture ? 4 : 3;
+            interpolatedOverpaint = UsdzExporter.getInterpolatedOverpaint(mesh!.vertices, mesh!.vertexCount, values, stride, overpaintType, webgl!);
+        }
+
+        let interpolatedTransparency: Uint8Array | undefined;
+        if (transparencyType === 'volumeInstance') {
+            const stride = isGeoTexture ? 4 : 3;
+            interpolatedTransparency = UsdzExporter.getInterpolatedTransparency(mesh!.vertices, mesh!.vertexCount, values, stride, transparencyType, webgl!);
         }
 
         await ctx.update({ isIndeterminate: false, current: 0, max: instanceCount });
@@ -129,17 +140,21 @@ def Material "material${materialKey}"
             }
 
             // color
-            const faceIndicesByMaterial = new Map<number, number[]>();
+            const quantizedColors = new Uint8Array(drawCount * 3);
             for (let i = 0; i < drawCount; i += 3) {
                 const v = isGeoTexture ? i : indices![i];
-                const color = UsdzExporter.getColor(values, groups, vertexCount, instanceIndex, isGeoTexture, interpolatedColors, v);
+                const color = UsdzExporter.getColor(values, groups, vertexCount, instanceIndex, isGeoTexture, interpolatedColors, interpolatedOverpaint, v);
+                Color.toArray(color, quantizedColors, i);
+            }
+            UsdzExporter.quantizeColors(quantizedColors, mesh!.vertexCount);
 
-                let alpha = uAlpha;
-                if (dTransparency) {
-                    const group = isGeoTexture ? UsdzExporter.getGroup(groups, i) : groups[indices![i]];
-                    const transparency = tTransparency.array[instanceIndex * groupCount + group] / 255;
-                    alpha *= 1 - transparency;
-                }
+            // material
+            const faceIndicesByMaterial = new Map<number, number[]>();
+            for (let i = 0; i < drawCount; i += 3) {
+                const color = Color.fromArray(quantizedColors, i);
+
+                const transparency = UsdzExporter.getTransparency(values, groups, vertexCount, instanceIndex, isGeoTexture, interpolatedTransparency, i);
+                const alpha = Math.round(uAlpha * (1 - transparency) * 10) / 10; // quantized
 
                 this.addMaterial(color, alpha);