mp4 export prototype

src/apps/viewer/index.ts

@@ -34,6 +34,7 @@ import { StateObjectSelector } from '../../mol-state';
 import { PluginStateObject } from '../../mol-plugin-state/objects';
 import { StateTransforms } from '../../mol-plugin-state/transforms';
 import { createVolumeRepresentationParams } from '../../mol-plugin-state/helpers/volume-representation-params';
+import { Mp4EncoderTestUI } from '../../extensions/mp4-export/ui';
 
 require('mol-plugin-ui/skin/light.scss');
 
@@ -134,6 +135,8 @@ export class Viewer {
             : elementOrId;
         if (!element) throw new Error(`Could not get element with id '${elementOrId}'`);
         this.plugin = createPlugin(element, spec);
+
+        this.plugin.customStructureControls.set('mp4-encoder', Mp4EncoderTestUI as any);
     }
 
     setRemoteSnapshot(id: string) {

src/extensions/mp4-export/encoder.ts

@@ -0,0 +1,67 @@
+import * as HME from 'h264-mp4-encoder';
+import { canvasToBlob } from '../../mol-canvas3d/util';
+import { PluginContext } from '../../mol-plugin/context';
+
+export class Mp4Encoder {
+
+    createImagePass() {
+        const pass = this.plugin.canvas3d!.getImagePass({
+            transparentBackground: false,
+            cameraHelper: { axes: { name: 'off', params: {} } },
+            multiSample: { mode: 'off', sampleLevel: 2 }, // { mode: 'on', sampleLevel: 2 },
+            postprocessing: this.plugin.canvas3d!.props.postprocessing
+        });
+        return pass;
+    }
+
+    sleep() {
+        return new Promise(res => setTimeout(res, 16.6));
+    }
+
+    async generate() {
+        const w = 1024, h = 768;
+
+        const encoder = await HME.createH264MP4Encoder();
+        encoder.width = w;
+        encoder.height = h;
+        encoder.initialize();
+
+        console.log('creating image pass');
+        const pass = this.createImagePass();
+
+        const canvas = document.createElement('canvas');
+        canvas.width = w;
+        canvas.height = h;
+        const canvasCtx = canvas.getContext('2d')!;
+
+        const imageData: Uint8ClampedArray[] = [];
+        const N = 60;
+        console.log({ w, h });
+        for (let i = 0; i < N; i++) {
+            const image = pass.getImageData(w, h);
+            if (i === 0) canvasCtx.putImageData(image, 0, 0);
+
+            imageData.push(image.data);
+            console.log(`frame ${i + 1}/${N}`);
+            await this.sleep();
+        }
+
+        let ii = 0;
+        for (const f of imageData) {
+            encoder.addFrameRgba(f);
+            console.log(`added ${++ii}/${N}`);
+        }
+
+        console.log('finalizing');
+        encoder.finalize();
+        console.log('finalized');
+        const uint8Array = encoder.FS.readFile(encoder.outputFilename);
+        console.log('encoded');
+        encoder.delete();
+
+        return { movie: uint8Array, image: await canvasToBlob(canvas, 'png') };
+    }
+
+    constructor(private plugin: PluginContext) {
+    }
+}

src/extensions/mp4-export/ui.tsx

@@ -0,0 +1,36 @@
+import React from 'react';
+import { CollapsableControls, CollapsableState } from '../../mol-plugin-ui/base';
+import { Button } from '../../mol-plugin-ui/controls/common';
+import { download } from '../../mol-util/download';
+import { Mp4Encoder } from './encoder';
+
+interface State {
+    data?: { movie: Uint8Array, image: Blob };
+}
+
+export class Mp4EncoderTestUI extends CollapsableControls<{}, State> {
+    protected defaultState(): State & CollapsableState  {
+        return {
+            header: 'Export MP4',
+            isCollapsed: false,
+            brand: { accent: 'cyan' }
+        };
+    }
+    protected renderControls(): JSX.Element | null {
+        return <>
+            <Button onClick={() => this.generate()}>Generate</Button>
+            {this.state.data && <Button onClick={() => this.save()}>Save</Button>}
+        </>;
+    }
+
+    save() {
+        download(new Blob([this.state.data!.movie]), 'test.mp4');
+        // download(this.state.data!.image, 'test.png');
+    }
+
+    async generate() {
+        const encoder = new Mp4Encoder(this.plugin);
+        const data = await encoder.generate();
+        this.setState({ data });
+    }
+}

src/mol-canvas3d/passes/image.ts

@@ -28,8 +28,8 @@ export const ImageParams = {
 export type ImageProps = PD.Values<typeof ImageParams>
 
 export class ImagePass {
-    private _width = 1024
-    private _height = 768
+    private _width = 0
+    private _height = 0
     private _camera = new Camera()
 
     readonly props: ImageProps
@@ -59,7 +59,7 @@ export class ImagePass {
             handle: helper.handle,
         };
 
-        this.setSize(this._width, this._height);
+        this.setSize(1024, 768);
     }
 
     setSize(width: number, height: number) {