screenshot perf improvements

src/mol-plugin-ui/skin/base/components/controls.scss

@@ -391,16 +391,20 @@
     text-align: center;
     padding: $control-spacing;
 
-    img {
+    canvas {
+        @include user-select(none);
+
         max-height: 180px;
         max-width: 100%;
         display: 'block';
+    }
 
+    canvas.msp-transparent-screenshot {
         background-color: $default-background;
-    background-image: linear-gradient(45deg, lightgrey 25%, transparent 25%, transparent 75%, lightgrey 75%, lightgrey),
-    linear-gradient(45deg, lightgrey 25%, transparent 25%, transparent 75%, lightgrey 75%, lightgrey);
-    background-size: 30px 30px;
-    background-position: 0 0, 15px 15px;
+        background-image: linear-gradient(45deg, lightgrey 25%, transparent 25%, transparent 75%, lightgrey 75%, lightgrey),
+        linear-gradient(45deg, lightgrey 25%, transparent 25%, transparent 75%, lightgrey 75%, lightgrey);
+        background-size: 30px 30px;
+        background-position: 0 0, 15px 15px;
     }
 
     > span {

src/mol-plugin-ui/viewport/screenshot.tsx

@@ -6,17 +6,16 @@
  */
 
 import * as React from 'react';
-import { ParamDefinition as PD } from '../../mol-util/param-definition';
 import { ParameterControls } from '../controls/parameters';
 import { PluginUIComponent } from '../base';
 import { debounceTime } from 'rxjs/operators';
 import { Subject } from 'rxjs';
-import { ViewportScreenshotHelper } from '../../mol-plugin/util/viewport-screenshot';
+import { ViewportScreenshotHelper, ViewportScreenshotHelperParams } from '../../mol-plugin/util/viewport-screenshot';
 import { Button, ExpandGroup } from '../controls/common';
 import { CameraHelperProps } from '../../mol-canvas3d/helper/camera-helper';
 import { PluginCommands } from '../../mol-plugin/commands';
 import { StateExportImportControls, LocalStateSnapshotParams } from '../state/snapshots';
-import { CopySvg, GetAppSvg, LaunchSvg } from '../controls/icons';
+import { CopySvg, GetAppSvg } from '../controls/icons';
 
 interface ImageControlsState {
     showPreview: boolean
@@ -30,18 +29,36 @@ interface ImageControlsState {
 export class DownloadScreenshotControls extends PluginUIComponent<{ close: () => void }, ImageControlsState> {
     state: ImageControlsState = {
         showPreview: true,
-        isDisabled: false,
-        resolution: this.plugin.helpers.viewportScreenshot?.currentResolution,
-        transparent: this.plugin.helpers.viewportScreenshot?.transparent,
-        axes: this.plugin.helpers.viewportScreenshot?.axes
+        isDisabled: false
     } as ImageControlsState
 
-    private imgRef = React.createRef<HTMLImageElement>()
+    private canvasRef = React.createRef<HTMLCanvasElement>()
     private updateQueue = new Subject();
 
-    private preview = async () => {
-        if (!this.imgRef.current) return;
-        this.imgRef.current!.src = await this.plugin.helpers.viewportScreenshot!.imageData();
+    private preview = () => {
+        if (!this.state.showPreview || !this.canvasRef.current || this.state.isDisabled) return;
+
+        const { canvas, width, height } = this.plugin.helpers.viewportScreenshot?.getPreview()!;
+        const ctx = this.canvasRef.current.getContext('2d');
+        if (!ctx) return;
+
+        const a0 = width / height;
+
+        const target = this.canvasRef.current;
+        const w = this.canvasRef.current.clientWidth;
+        const h = this.canvasRef.current.clientHeight;
+        target.width = w;
+        target.height = h;
+        const a1 = w / h;
+
+        ctx.clearRect(0, 0, w, h);
+        if (a0 <= a1) {
+            const t = h * a0;
+            ctx.drawImage(canvas, Math.round((w - t) / 2), 0, Math.round(t), h);
+        } else {
+            const t = w / a0;
+            ctx.drawImage(canvas, 0, Math.round((h - t) / 2), w, Math.round(t));
+        }
     }
 
     private download = () => {
@@ -51,7 +68,6 @@ export class DownloadScreenshotControls extends PluginUIComponent<{ close: () =>
 
     private copy = async () => {
         await this.plugin.helpers.viewportScreenshot?.copyToClipboard();
-        this.props.close();
         PluginCommands.Toast.Show(this.plugin, {
             message: 'Copied to clipboard.',
             title: 'Screenshot',
@@ -59,30 +75,16 @@ export class DownloadScreenshotControls extends PluginUIComponent<{ close: () =>
         });
     }
 
-    private handlePreview() {
-        if (this.state.showPreview) {
-            this.preview();
-        }
-    }
-
-    componentDidUpdate() {
-        this.updateQueue.next();
-    }
-
     componentDidMount() {
         if (!this.plugin.canvas3d) return;
 
-        this.subscribe(debounceTime(250)(this.updateQueue), () => this.handlePreview());
+        this.subscribe(this.updateQueue.pipe(debounceTime(33)), () => this.preview());
 
         this.subscribe(this.plugin.events.canvas3d.settingsUpdated, () => {
-            this.plugin.helpers.viewportScreenshot!.imagePass.setProps({
-                multiSample: { mode: 'on', sampleLevel: 2 },
-                postprocessing: this.plugin.canvas3d?.props.postprocessing
-            });
             this.updateQueue.next();
         });
 
-        this.subscribe(debounceTime(250)(this.plugin.canvas3d.didDraw), () => {
+        this.subscribe(this.plugin.canvas3d.didDraw.pipe(debounceTime(150)), () => {
             if (this.state.isDisabled) return;
             this.updateQueue.next();
         });
@@ -92,20 +94,16 @@ export class DownloadScreenshotControls extends PluginUIComponent<{ close: () =>
             if (!v) this.updateQueue.next();
         });
 
-        this.handlePreview();
+        this.subscribe(this.plugin.helpers.viewportScreenshot!.behaviors.values, () => {
+            this.forceUpdate();
+            this.updateQueue.next();
+        });
+
+        this.preview();
     }
 
-    private setProps = (p: { param: PD.Base<any>, name: string, value: any }) => {
-        if (p.name === 'resolution') {
-            this.plugin.helpers.viewportScreenshot!.currentResolution = p.value;
-            this.setState({ resolution: p.value });
-        } else if (p.name === 'transparent') {
-            this.plugin.helpers.viewportScreenshot!.transparent = p.value;
-            this.setState({ transparent: p.value });
-        } else if (p.name === 'axes') {
-            this.plugin.helpers.viewportScreenshot!.axes = p.value;
-            this.setState({ axes: p.value });
-        }
+    private setValues = (p: ViewportScreenshotHelperParams) => {
+        this.plugin.helpers.viewportScreenshot!.behaviors.values.next(p);
     }
 
     downloadToFileJson = () => {
@@ -121,17 +119,22 @@ export class DownloadScreenshotControls extends PluginUIComponent<{ close: () =>
         PluginCommands.State.Snapshots.OpenFile(this.plugin, { file: e.target.files![0] });
     }
 
+    onCanvasClick = (e: React.MouseEvent) => {
+        e.preventDefault();
+    };
+
     render() {
+        const values = this.plugin.helpers.viewportScreenshot!.values;
+
         return <div>
             <div className='msp-image-preview'>
-                <img ref={this.imgRef} /><br />
-                <span>Right-click the image to Copy.</span>
+                <canvas ref={this.canvasRef} onClick={this.onCanvasClick} onContextMenu={this.onCanvasClick} style={{ width: '100%', height: '180px' }} className={values.transparent ? 'msp-transparent-screenshot' : void 0}></canvas><br />
             </div>
             <div className='msp-flex-row'>
-                {!(navigator.clipboard as any).write && <Button icon={CopySvg} onClick={this.copy} disabled={this.state.isDisabled}>Copy</Button>}
+                {!!(navigator.clipboard as any).write && <Button icon={CopySvg} onClick={this.copy} disabled={this.state.isDisabled}>Copy</Button>}
                 <Button icon={GetAppSvg} onClick={this.download} disabled={this.state.isDisabled}>Download</Button>
             </div>
-            <ParameterControls params={this.plugin.helpers.viewportScreenshot!.params} values={this.plugin.helpers.viewportScreenshot!.values} onChange={this.setProps} isDisabled={this.state.isDisabled} />
+            <ParameterControls params={this.plugin.helpers.viewportScreenshot!.params} values={values} onChangeValues={this.setValues} isDisabled={this.state.isDisabled} />
             <ExpandGroup header='State'>
                 <StateExportImportControls onAction={this.props.close} />
                 <ExpandGroup header='Save Options' initiallyExpanded={false} noOffset>

src/mol-plugin/util/viewport-screenshot.ts

@@ -5,26 +5,28 @@
  * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
-import { PluginContext } from '../context';
+import { CameraHelperParams } from '../../mol-canvas3d/helper/camera-helper';
 import { ImagePass } from '../../mol-canvas3d/passes/image';
-import { StateSelection } from '../../mol-state';
-import { PluginStateObject } from '../../mol-plugin-state/objects';
-import { Task, RuntimeContext } from '../../mol-task';
 import { canvasToBlob } from '../../mol-canvas3d/util';
+import { PluginComponent } from '../../mol-plugin-state/component';
+import { PluginStateObject } from '../../mol-plugin-state/objects';
+import { StateSelection } from '../../mol-state';
+import { RuntimeContext, Task } from '../../mol-task';
 import { download } from '../../mol-util/download';
 import { ParamDefinition as PD } from '../../mol-util/param-definition';
-import { SyncRuntimeContext } from '../../mol-task/execution/synchronous';
-import { CameraHelperParams, CameraHelperProps } from '../../mol-canvas3d/helper/camera-helper';
 import { SetUtils } from '../../mol-util/set';
+import { PluginContext } from '../context';
 
-export { ViewportScreenshotHelper };
+export { ViewportScreenshotHelper, ViewportScreenshotHelperParams };
 
 namespace ViewportScreenshotHelper {
     export type ResolutionSettings = PD.Values<ReturnType<ViewportScreenshotHelper['createParams']>>['resolution']
     export type ResolutionTypes = ResolutionSettings['name']
 }
 
-class ViewportScreenshotHelper {
+type ViewportScreenshotHelperParams = PD.Values<ReturnType<ViewportScreenshotHelper['createParams']>>
+
+class ViewportScreenshotHelper extends PluginComponent {
     private createParams() {
         const max = Math.min(this.plugin.canvas3d ? this.plugin.canvas3d.webgl.maxRenderbufferSize : 4096, 4096);
         return {
@@ -56,12 +58,16 @@ class ViewportScreenshotHelper {
         return this._params = this.createParams();
     }
 
+    readonly behaviors = {
+        values: this.ev.behavior<ViewportScreenshotHelperParams>({
+            transparent: this.params.transparent.defaultValue,
+            axes: { name: 'off', params: {} },
+            resolution: this.params.resolution.defaultValue
+        })
+    };
+
     get values() {
-        return {
-            transparent: this.transparent,
-            axes: this.axes,
-            resolution: this.currentResolution
-        };
+        return this.behaviors.values.value;
     }
 
     private getCanvasSize() {
@@ -71,36 +77,38 @@ class ViewportScreenshotHelper {
         };
     }
 
-    transparent = this.params.transparent.defaultValue
-    axes: CameraHelperProps['axes'] = { name: 'off', params: {} }
-    currentResolution = this.params.resolution.defaultValue
-
     private getSize() {
-        switch (this.currentResolution.name ) {
+        const values = this.values;
+        switch (values.resolution.name ) {
             case 'viewport': return this.getCanvasSize();
             case 'hd': return { width: 1280, height: 720 };
             case 'full-hd': return { width: 1920, height: 1080 };
             case 'ultra-hd': return { width: 3840, height: 2160 };
-            default: return { width: this.currentResolution.params.width, height: this.currentResolution.params.height };
+            default: return { width: values.resolution.params.width, height: values.resolution.params.height };
         }
     }
 
-    private _imagePass: ImagePass;
-
-    get imagePass() {
-        if (this._imagePass) return this._imagePass;
-
+    private createPass(mutlisample: boolean) {
         const c = this.plugin.canvas3d!;
-        this._imagePass = c.getImagePass({
-            transparentBackground: this.transparent,
-            cameraHelper: { axes: this.axes },
+        return this.plugin.canvas3d!.getImagePass({
+            transparentBackground: this.values.transparent,
+            cameraHelper: { axes: this.values.axes },
             multiSample: {
-                mode: 'on',
+                mode: mutlisample ? 'on' : 'off',
                 sampleLevel: c.webgl.extensions.colorBufferFloat ? 4 : 2
             },
             postprocessing: c.props.postprocessing
         });
-        return this._imagePass;
+    }
+
+    private _previewPass: ImagePass;
+    private get previewPass() {
+        return this._previewPass || (this._previewPass = this.createPass(false));
+    }
+
+    private _imagePass: ImagePass;
+    get imagePass() {
+        return this._imagePass || (this._imagePass = this.createPass(true));
     }
 
     getFilename() {
@@ -116,27 +124,41 @@ class ViewportScreenshotHelper {
         return canvas;
     }();
 
-    // private preview = () => {
-    //     const { width, height } = this.getSize()
-    //     if (width <= 0 || height <= 0) return
-
-    //     let w: number, h: number
-    //     const aH = maxHeightUi / height
-    //     const aW = maxWidthUi / width
-    //     if (aH < aW) {
-    //         h = Math.round(Math.min(maxHeightUi, height))
-    //         w = Math.round(width * (h / height))
-    //     } else {
-    //         w = Math.round(Math.min(maxWidthUi, width))
-    //         h = Math.round(height * (w / width))
-    //     }
-    //     setCanvasSize(this.canvas, w, h)
-    //     const pixelRatio = this.plugin.canvas3d?.webgl.pixelRatio || 1
-    //     const pw = Math.round(w * pixelRatio)
-    //     const ph = Math.round(h * pixelRatio)
-    //     const imageData = this.imagePass.getImageData(pw, ph)
-    //     this.canvasContext.putImageData(imageData, 0, 0)
-    // }
+    private previewCanvas = function () {
+        const canvas = document.createElement('canvas');
+        return canvas;
+    }();
+
+    getPreview(maxDim = 640) {
+        const { width, height } = this.getSize();
+        if (width <= 0 || height <= 0) return;
+
+        const f = width / height;
+
+        let w = 0, h = 0;
+        if (f > 1) {
+            w = maxDim;
+            h = Math.round(maxDim / f);
+        } else {
+            h = maxDim;
+            w = Math.round(maxDim * f);
+        }
+
+        this.previewPass.setProps({
+            cameraHelper: { axes: this.values.axes },
+            transparentBackground: this.values.transparent,
+            // TODO: optimize because this creates a copy of a large object!
+            postprocessing: this.plugin.canvas3d!.props.postprocessing
+        });
+        const imageData = this.previewPass.getImageData(w, h);
+        const canvas = this.previewCanvas;
+        canvas.width = imageData.width;
+        canvas.height = imageData.height;
+        const canvasCtx = canvas.getContext('2d');
+        if (!canvasCtx) throw new Error('Could not create canvas 2d context');
+        canvasCtx.putImageData(imageData, 0, 0);
+        return { canvas, width: w, height: h };
+    }
 
     private async draw(ctx: RuntimeContext) {
         const { width, height } = this.getSize();
@@ -144,9 +166,10 @@ class ViewportScreenshotHelper {
 
         await ctx.update('Rendering image...');
         this.imagePass.setProps({
-            cameraHelper: { axes: this.axes },
-            transparentBackground: this.transparent,
-            postprocessing: this.plugin.canvas3d!.props.postprocessing // TODO this line should not be required, updating should work by listening to this.plugin.events.canvas3d.settingsUpdated
+            cameraHelper: { axes: this.values.axes },
+            transparentBackground: this.values.transparent,
+            // TODO: optimize because this creates a copy of a large object!
+            postprocessing: this.plugin.canvas3d!.props.postprocessing
         });
         const imageData = this.imagePass.getImageData(width, height);
 
@@ -160,7 +183,7 @@ class ViewportScreenshotHelper {
         return;
     }
 
-    async copyToClipboard() {
+    private copyToClipboardTask() {
         const cb = navigator.clipboard as any;
 
         if (!cb.write) {
@@ -168,41 +191,37 @@ class ViewportScreenshotHelper {
             return;
         }
 
-        const blob = await canvasToBlob(this.canvas, 'png');
-        const item = new ClipboardItem({ 'image/png': blob });
-        cb.write([item]);
+        return Task.create('Copy Image', async ctx => {
+            await this.draw(ctx);
+            await ctx.update('Downloading image...');
+            const blob = await canvasToBlob(this.canvas, 'png');
+            const item = new ClipboardItem({ 'image/png': blob });
+            cb.write([item]);
+            this.plugin.log.message('Image copied to clipboard.');
+        });
+    }
 
-        this.plugin.log.message('Image copied to clipboard.');
+    copyToClipboard() {
+        const task = this.copyToClipboardTask();
+        if (!task) return;
+        return this.plugin.runTask(task);
     }
 
     private downloadTask() {
         return Task.create('Download Image', async ctx => {
-            this.draw(ctx);
+            await this.draw(ctx);
             await ctx.update('Downloading image...');
             const blob = await canvasToBlob(this.canvas, 'png');
             download(blob, this.getFilename());
         });
     }
 
-    downloadCurrent() {
-        return this.plugin.runTask(Task.create('Download Image', async ctx => {
-            await ctx.update('Downloading image...');
-            const blob = await canvasToBlob(this.canvas, 'png');
-            download(blob, this.getFilename());
-        }));
-    }
-
-    async imageData() {
-        await this.draw(SyncRuntimeContext);
-        return this.canvas.toDataURL();
-    }
-
     download() {
         this.plugin.runTask(this.downloadTask());
     }
 
     constructor(private plugin: PluginContext) {
-
+        super();
     }
 }