add zip file support to zenodo extension

CHANGELOG.md

@@ -8,7 +8,7 @@ Note that since we don't clearly distinguish between a public and private interf
 
 - Fix handling of mmcif with empty ``label_*`` fields
 - Add LoadTrajectory action
-- Add Zenodo import extension (load structures, trajectories, and volumes)
+- Add Zenodo import extension (load structures, trajectories, volumes, and zip files)
 - Fix loading of some compressed files within sessions
 
 ## [v3.3.1] - 2022-02-27

src/extensions/zenodo/ui.tsx

@@ -4,6 +4,7 @@
  * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
+import { DownloadFile } from '../../mol-plugin-state/actions/file';
 import { DownloadStructure, LoadTrajectory } from '../../mol-plugin-state/actions/structure';
 import { DownloadDensity } from '../../mol-plugin-state/actions/volume';
 import { TrajectoryFormatCategory } from '../../mol-plugin-state/formats/trajectory';
@@ -51,7 +52,7 @@ interface State {
 }
 
 const ZenodoImportParams = {
-    record: PD.Text('847637', { description: 'Zenodo ID.' })
+    record: PD.Text('438727', { description: 'Zenodo ID.' })
 };
 
 function createImportParams(files: ZenodoFile[], plugin: PluginContext) {
@@ -59,6 +60,7 @@ function createImportParams(files: ZenodoFile[], plugin: PluginContext) {
     const topologyOpts: [string, string][] = [];
     const coordinatesOpts: [string, string][] = [];
     const volumeOpts: [string, string][] = [];
+    const compressedOpts: [string, string][] = [];
 
     const structureExts = new Map<string, { format: string, isBinary: boolean }>();
     const volumeExts = new Map<string, { format: string, isBinary: boolean }>();
@@ -84,6 +86,8 @@ function createImportParams(files: ZenodoFile[], plugin: PluginContext) {
             topologyOpts.push([`${file.links.self}|${file.type}|false`, file.key]);
         } else if (file.type === 'xtc' || file.type === 'dcd') {
             coordinatesOpts.push([`${file.links.self}|${file.type}|true`, file.key]);
+        } else if (file.type === 'zip') {
+            compressedOpts.push([`${file.links.self}|${file.type}|true`, file.key]);
         }
     }
 
@@ -108,6 +112,11 @@ function createImportParams(files: ZenodoFile[], plugin: PluginContext) {
         params.volume = PD.Select(volumeOpts[0][0], volumeOpts);
     }
 
+    if (compressedOpts.length) {
+        if (!defaultType) defaultType = 'compressed';
+        params.compressed = PD.Select(compressedOpts[0][0], compressedOpts);
+    }
+
     return {
         type: PD.MappedStatic(defaultType, Object.keys(params).length ? params : { '': PD.EmptyGroup() })
     };
@@ -210,6 +219,15 @@ export class ZenodoImportUI extends CollapsableControls<{}, State> {
                         }
                     }
                 }));
+            } else if (t.name === 'compressed') {
+                const [url, format, isBinary] = t.params.split('|');
+
+                await this.plugin.runTask(this.plugin.state.data.applyAction(DownloadFile, {
+                    url,
+                    format: format as any,
+                    isBinary: isBinary === 'true',
+                    visuals: true
+                }));
             }
         } catch (e) {
             console.error(e);
@@ -240,7 +258,7 @@ export class ZenodoImportUI extends CollapsableControls<{}, State> {
     private renderRecordInfo(record: ZenodoRecord) {
         return <div style={{ marginBottom: 10 }}>
             <div className='msp-help-text'>
-                <div>{`${record.metadata.title} (${record.id})`}</div>
+                <div>Record {`${record.id}`}: <i>{`${record.metadata.title}`}</i></div>
             </div>
             <Button onClick={this.clearRecord} style={{ marginTop: 1 }} disabled={this.state.busy}>
                 Clear

src/mol-plugin-state/actions/file.ts

@@ -1,5 +1,5 @@
 /**
- * Copyright (c) 2019-2020 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2019-2022 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
@@ -13,6 +13,27 @@ import { ParamDefinition as PD } from '../../mol-util/param-definition';
 import { unzip } from '../../mol-util/zip/zip';
 import { PluginStateObject } from '../objects';
 
+async function processFile(file: Asset.File, plugin: PluginContext, format: string, visuals: boolean) {
+    const info = getFileInfo(file.file!);
+    const isBinary = plugin.dataFormats.binaryExtensions.has(info.ext);
+    const { data } = await plugin.builders.data.readFile({ file, isBinary });
+    const provider = format === 'auto'
+        ? plugin.dataFormats.auto(info, data.cell?.obj!)
+        : plugin.dataFormats.get(format);
+
+    if (!provider) {
+        plugin.log.warn(`OpenFiles: could not find data provider for '${info.name}.${info.ext}'`);
+        await plugin.state.data.build().delete(data).commit();
+        return;
+    }
+
+    // need to await so that the enclosing Task finishes after the update is done.
+    const parsed = await provider.parse(plugin, data);
+    if (visuals) {
+        await provider.visuals?.(plugin, parsed);
+    }
+};
+
 export const OpenFiles = StateAction.build({
     display: { name: 'Open Files', description: 'Load one or more files and optionally create default visuals' },
     from: PluginStateObject.Root,
@@ -36,36 +57,19 @@ export const OpenFiles = StateAction.build({
             return;
         }
 
-        const processFile = async (file: Asset.File) => {
-            const info = getFileInfo(file.file!);
-            const isBinary = plugin.dataFormats.binaryExtensions.has(info.ext);
-            const { data } = await plugin.builders.data.readFile({ file, isBinary });
-            const provider = params.format.name === 'auto'
-                ? plugin.dataFormats.auto(info, data.cell?.obj!)
-                : plugin.dataFormats.get(params.format.params);
-
-            if (!provider) {
-                plugin.log.warn(`OpenFiles: could not find data provider for '${info.name}.${info.ext}'`);
-                return;
-            }
-
-            // need to await so that the enclosing Task finishes after the update is done.
-            const parsed = await provider.parse(plugin, data);
-            if (params.visuals) {
-                await provider.visuals?.(plugin, parsed);
-            }
-        };
-
         for (const file of params.files) {
             try {
                 if (file.file && file.name.toLowerCase().endsWith('.zip')) {
                     const zippedFiles = await unzip(taskCtx, await file.file.arrayBuffer());
                     for (const [fn, filedata] of Object.entries(zippedFiles)) {
-                        const asset = Asset.File(new File([filedata as Uint8Array], fn));
-                        await processFile(asset);
+                        if (!(filedata instanceof Uint8Array) || filedata.length === 0) continue;
+
+                        const asset = Asset.File(new File([filedata], fn));
+                        await processFile(asset, plugin, 'auto', params.visuals);
                     }
                 } else {
-                    await processFile(file);
+                    const format = params.format.name === 'auto' ? 'auto' : params.format.params;
+                    await processFile(file, plugin, format, params.visuals);
                 }
             } catch (e) {
                 console.error(e);
@@ -79,7 +83,7 @@ export const DownloadFile = StateAction.build({
     display: { name: 'Download File', description: 'Load one or more file from an URL' },
     from: PluginStateObject.Root,
     params: (a, ctx: PluginContext) => {
-        const { options } = ctx.dataFormats;
+        const options = [...ctx.dataFormats.options, ['zip', 'Zip'] as const];
         return {
             url: PD.Url(''),
             format: PD.Select(options[0][0], options),
@@ -92,16 +96,30 @@ export const DownloadFile = StateAction.build({
 
     await state.transaction(async () => {
         try {
-            const provider = plugin.dataFormats.get(params.format);
-            if (!provider) {
-                plugin.log.warn(`DownloadFile: could not find data provider for '${params.format}'`);
-                return;
-            }
+            if (params.format === 'zip') {
+                // TODO: add ReadZipFile transformer so this can be saved as a simple state snaphot,
+                //       would need support for extracting individual files from zip
+                const data = await plugin.builders.data.download({ url: params.url, isBinary: true });
+                const zippedFiles = await unzip(taskCtx, (data.obj?.data as Uint8Array).buffer);
+                for (const [fn, filedata] of Object.entries(zippedFiles)) {
+                    if (!(filedata instanceof Uint8Array) || filedata.length === 0) continue;
+
+                    const asset = Asset.File(new File([filedata], fn));
 
-            const data = await plugin.builders.data.download({ url: params.url, isBinary: params.isBinary });
-            const parsed = await provider.parse(plugin, data);
-            if (params.visuals) {
-                await provider.visuals?.(plugin, parsed);
+                    await processFile(asset, plugin, 'auto', params.visuals);
+                }
+            } else {
+                const provider = plugin.dataFormats.get(params.format);
+                if (!provider) {
+                    plugin.log.warn(`DownloadFile: could not find data provider for '${params.format}'`);
+                    return;
+                }
+
+                const data = await plugin.builders.data.download({ url: params.url, isBinary: params.isBinary });
+                const parsed = await provider.parse(plugin, data);
+                if (params.visuals) {
+                    await provider.visuals?.(plugin, parsed);
+                }
             }
         } catch (e) {
             console.error(e);

src/mol-plugin-state/actions/structure.ts

@@ -385,6 +385,7 @@ export const LoadTrajectory = StateAction.build({
 
             if (!provider) {
                 ctx.log.warn(`LoadTrajectory: could not find data provider for '${info.name}.${info.ext}'`);
+                await ctx.state.data.build().delete(data).commit();
                 return;
             }