refactored AssetManager

- support assets in OpenFile and Download transforms

src/extensions/cellpack/model.ts

@@ -28,6 +28,7 @@ import { createModels } from '../../mol-model-formats/structure/basic/parser';
 import { CellpackPackingPreset, CellpackMembranePreset } from './preset';
 import { AjaxTask } from '../../mol-util/data-source';
 import { CellPackInfoProvider } from './property';
+import { Asset } from '../../mol-util/assets';
 
 function getCellPackModelUrl(fileName: string, baseUrl: string) {
     return `${baseUrl}/results/${fileName}`;
@@ -392,7 +393,7 @@ async function loadMembrane(name: string, plugin: PluginContext, runtime: Runtim
     let b = state.build().toRoot();
     if (fname in ingredientFiles) {
         const file = ingredientFiles[fname];
-        b = b.apply(StateTransforms.Data.ReadFile, { file, isBinary: true, label: file.name }, { state: { isGhost: true } });
+        b = b.apply(StateTransforms.Data.ReadFile, { file: Asset.File(file), isBinary: true, label: file.name }, { state: { isGhost: true } });
     } else {
         const url = `${params.baseUrl}/membranes/${name}.bcif`;
         b = b.apply(StateTransforms.Data.Download, { url, isBinary: true, label: name }, { state: { isGhost: true } });

src/extensions/rcsb/validation-report/prop.ts

@@ -111,8 +111,9 @@ namespace ValidationReport {
     }
 
     export async function open(ctx: CustomProperty.Context, model: Model, props: FileSourceProps): Promise<ValidationReport> {
-        if (props.input === null) throw new Error('No file given');
-        const xml = await readFromFile(props.input, 'xml').runInContext(ctx.runtime);
+        // TODO: this should use the asset manager and release the file "somehow"
+        if (!(props.input?.file instanceof File)) throw new Error('No file given');
+        const xml = await readFromFile(props.input.file, 'xml').runInContext(ctx.runtime);
         return fromXml(xml, model);
     }
 

src/mol-plugin-state/actions/file.ts

@@ -10,6 +10,7 @@ import { Task } from '../../mol-task';
 import { getFileInfo } from '../../mol-util/file-info';
 import { ParamDefinition as PD } from '../../mol-util/param-definition';
 import { PluginStateObject } from '../objects';
+import { Asset } from '../../mol-util/assets';
 
 export const OpenFiles = StateAction.build({
     display: { name: 'Open Files', description: 'Load one or more files and optionally create default visuals' },
@@ -35,7 +36,7 @@ export const OpenFiles = StateAction.build({
                 const file = params.files[i];
                 const info = getFileInfo(file);
                 const isBinary = plugin.dataFormats.binaryExtensions.has(info.ext);
-                const { data } = await plugin.builders.data.readFile({ file, isBinary });
+                const { data } = await plugin.builders.data.readFile({ file: Asset.File(file), isBinary });
                 const provider = params.format === 'auto'
                     ? plugin.dataFormats.auto(info, data.cell?.obj!)
                     : plugin.dataFormats.get(params.format);

src/mol-plugin-state/builder/data.ts

@@ -31,7 +31,7 @@ export class DataBuilder {
 
     async readFile(params: StateTransformer.Params<ReadFile>, options?: Partial<StateTransform.Options>) {
         const data = await this.dataState.build().toRoot().apply(ReadFile, params, options).commit({ revertOnError: true });
-        const fileInfo = getFileInfo(params.file || '');
+        const fileInfo = getFileInfo(params.file?.file || '');
         return { data: data, fileInfo };
     }
 

src/mol-plugin-state/transforms/data.ts

@@ -5,22 +5,33 @@
  * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
-import { PluginStateTransform } from '../objects';
-import { PluginStateObject as SO } from '../objects';
-import { Task } from '../../mol-task';
-import { CIF } from '../../mol-io/reader/cif';
-import { PluginContext } from '../../mol-plugin/context';
-import { ParamDefinition as PD } from '../../mol-util/param-definition';
-import { StateTransformer, StateObject } from '../../mol-state';
-import { readFromFile, ajaxGetMany } from '../../mol-util/data-source';
+import { isTypedArray } from '../../mol-data/db/column-helpers';
 import * as CCP4 from '../../mol-io/reader/ccp4/parser';
+import { CIF } from '../../mol-io/reader/cif';
 import * as DSN6 from '../../mol-io/reader/dsn6/parser';
 import * as PLY from '../../mol-io/reader/ply/parser';
 import { parsePsf } from '../../mol-io/reader/psf/parser';
-import { isTypedArray } from '../../mol-data/db/column-helpers';
-import { AssetManager } from '../../mol-plugin/util/asset-manager';
+import { PluginContext } from '../../mol-plugin/context';
+import { StateObject, StateTransformer } from '../../mol-state';
+import { Task } from '../../mol-task';
+import { ajaxGetMany } from '../../mol-util/data-source';
+import { ParamDefinition as PD } from '../../mol-util/param-definition';
+import { PluginStateObject as SO, PluginStateTransform } from '../objects';
+import { Asset } from '../../mol-util/assets';
 
 export { Download };
+export { DownloadBlob };
+export { RawData };
+export { ReadFile };
+export { ParseBlob };
+export { ParseCif };
+export { ParsePsf };
+export { ParsePly };
+export { ParseCcp4 };
+export { ParseDsn6 };
+export { ImportString };
+export { ImportJson };
+export { ParseJson };
 type Download = typeof Download
 const Download = PluginStateTransform.BuiltIn({
     name: 'download',
@@ -34,14 +45,19 @@ const Download = PluginStateTransform.BuiltIn({
         body: PD.Optional(PD.Text(''))
     }
 })({
-    apply({ params: p }, globalCtx: PluginContext) {
+    apply({ params: p }, plugin: PluginContext) {
         return Task.create('Download', async ctx => {
-            const data = await globalCtx.fetch({ url: p.url, type: p.isBinary ? 'binary' : 'string', body: p.body }).runInContext(ctx);
+            const data = await plugin.managers.asset.resolve(Asset.Url(p.url, { body: p.body }), p.isBinary ? 'binary' : 'string').runInContext(ctx);
             return p.isBinary
                 ? new SO.Data.Binary(data as Uint8Array, { label: p.label ? p.label : p.url })
                 : new SO.Data.String(data as string, { label: p.label ? p.label : p.url });
         });
     },
+    dispose({ params: p }, plugin: PluginContext) {
+        if (p) {
+            plugin.managers.asset.release(Asset.Url(p.url, { body: p.body }));
+        }
+    },
     update({ oldParams, newParams, b }) {
         if (oldParams.url !== newParams.url || oldParams.isBinary !== newParams.isBinary) return StateTransformer.UpdateResult.Recreate;
         if (oldParams.label !== newParams.label) {
@@ -52,7 +68,6 @@ const Download = PluginStateTransform.BuiltIn({
     }
 });
 
-export { DownloadBlob };
 type DownloadBlob = typeof DownloadBlob
 const DownloadBlob = PluginStateTransform.BuiltIn({
     name: 'download-blob',
@@ -99,7 +114,6 @@ const DownloadBlob = PluginStateTransform.BuiltIn({
     // }
 });
 
-export { RawData };
 type RawData = typeof RawData
 const RawData = PluginStateTransform.BuiltIn({
     name: 'raw-data',
@@ -131,7 +145,6 @@ const RawData = PluginStateTransform.BuiltIn({
     }
 });
 
-export { ReadFile };
 type ReadFile = typeof ReadFile
 const ReadFile = PluginStateTransform.BuiltIn({
     name: 'read-file',
@@ -151,27 +164,19 @@ const ReadFile = PluginStateTransform.BuiltIn({
                 return StateObject.Null;
             }
 
-            let file: File;
-            if (AssetManager.isItem(p.file)) {
-                if (!plugin.managers.asset.has(p.file.id)) {
-                    throw new Error(`No asset found for '${p.file.name}'`);
-                }
-                file = plugin.managers.asset.get(p.file.id)!;
-                // will be added again below with new id
-                plugin.managers.asset.remove(p.file.id);
-            } else {
-                file = p.file;
-            }
-
-            const data = await readFromFile(file, p.isBinary ? 'binary' : 'string').runInContext(ctx);
+            const data = await plugin.managers.asset.resolve(p.file, p.isBinary ? 'binary' : 'string').runInContext(ctx);
             const o = p.isBinary
                 ? new SO.Data.Binary(data as Uint8Array, { label: p.label ? p.label : p.file.name })
                 : new SO.Data.String(data as string, { label: p.label ? p.label : p.file.name });
 
-            plugin.managers.asset.set(o.id, file);
             return o;
         });
     },
+    dispose({ params }, plugin: PluginContext) {
+        if (params?.file) {
+            plugin.managers.asset.release(params.file);
+        }
+    },
     update({ oldParams, newParams, b }) {
         if (oldParams.label !== newParams.label) {
             (b.label as string) = newParams.label || oldParams.file?.name || '';
@@ -182,7 +187,6 @@ const ReadFile = PluginStateTransform.BuiltIn({
     isSerializable: () => ({ isSerializable: false, reason: 'Cannot serialize user loaded files.' })
 });
 
-export { ParseBlob };
 type ParseBlob = typeof ParseBlob
 const ParseBlob = PluginStateTransform.BuiltIn({
     name: 'parse-blob',
@@ -226,7 +230,6 @@ const ParseBlob = PluginStateTransform.BuiltIn({
     // }
 });
 
-export { ParseCif };
 type ParseCif = typeof ParseCif
 const ParseCif = PluginStateTransform.BuiltIn({
     name: 'parse-cif',
@@ -243,7 +246,6 @@ const ParseCif = PluginStateTransform.BuiltIn({
     }
 });
 
-export { ParsePsf };
 type ParsePsf = typeof ParsePsf
 const ParsePsf = PluginStateTransform.BuiltIn({
     name: 'parse-psf',
@@ -260,7 +262,6 @@ const ParsePsf = PluginStateTransform.BuiltIn({
     }
 });
 
-export { ParsePly };
 type ParsePly = typeof ParsePly
 const ParsePly = PluginStateTransform.BuiltIn({
     name: 'parse-ply',
@@ -277,7 +278,6 @@ const ParsePly = PluginStateTransform.BuiltIn({
     }
 });
 
-export { ParseCcp4 };
 type ParseCcp4 = typeof ParseCcp4
 const ParseCcp4 = PluginStateTransform.BuiltIn({
     name: 'parse-ccp4',
@@ -294,7 +294,6 @@ const ParseCcp4 = PluginStateTransform.BuiltIn({
     }
 });
 
-export { ParseDsn6 };
 type ParseDsn6 = typeof ParseDsn6
 const ParseDsn6 = PluginStateTransform.BuiltIn({
     name: 'parse-dsn6',
@@ -311,7 +310,6 @@ const ParseDsn6 = PluginStateTransform.BuiltIn({
     }
 });
 
-export { ImportString };
 type ImportString = typeof ImportString
 const ImportString = PluginStateTransform.BuiltIn({
     name: 'import-string',
@@ -337,7 +335,6 @@ const ImportString = PluginStateTransform.BuiltIn({
     isSerializable: () => ({ isSerializable: false, reason: 'Cannot serialize user imported strings.' })
 });
 
-export { ImportJson };
 type ImportJson = typeof ImportJson
 const ImportJson = PluginStateTransform.BuiltIn({
     name: 'import-json',
@@ -363,7 +360,6 @@ const ImportJson = PluginStateTransform.BuiltIn({
     isSerializable: () => ({ isSerializable: false, reason: 'Cannot serialize user imported JSON.' })
 });
 
-export { ParseJson };
 type ParseJson = typeof ParseJson
 const ParseJson = PluginStateTransform.BuiltIn({
     name: 'parse-json',

src/mol-plugin-ui/controls/parameters.tsx

@@ -25,6 +25,7 @@ import { Icon } from './icons';
 import { legendFor } from './legend';
 import LineGraphComponent from './line-graph/line-graph-component';
 import { Slider, Slider2 } from './slider';
+import { Asset } from '../../mol-util/assets';
 
 export type ParameterControlsCategoryFilter = string | null | (string | null)[]
 
@@ -787,7 +788,7 @@ export class Mat4Control extends React.PureComponent<ParamProps<PD.Mat4>, { isEx
 
 export class FileControl extends React.PureComponent<ParamProps<PD.FileParam>> {
     change(value: File) {
-        this.props.onChange({ name: this.props.name, param: this.props.param, value });
+        this.props.onChange({ name: this.props.name, param: this.props.param, value: Asset.File(value) });
     }
 
     onChangeFile = (e: React.ChangeEvent<HTMLInputElement>) => {

src/mol-plugin/behavior/static/state.ts

@@ -20,6 +20,7 @@ import { zip } from '../../../mol-util/zip/zip';
 import { utf8Write, utf8ByteCount } from '../../../mol-io/common/utf8';
 import { objectForEach } from '../../../mol-util/object';
 import { UUID } from '../../../mol-util';
+import { Asset } from '../../../mol-util/assets';
 
 export function registerDefault(ctx: PluginContext) {
     SyncBehaviors(ctx);
@@ -184,7 +185,7 @@ export function Snapshots(ctx: PluginContext) {
     });
 
     PluginCommands.State.Snapshots.DownloadToFile.subscribe(ctx, async ({ name, type }) => {
-        const json = JSON.stringify(ctx.state.getSnapshot(), ctx.managers.asset.replacer, 2);
+        const json = JSON.stringify(ctx.state.getSnapshot(), null, 2);
         name = `mol-star_state_${(name || getFormattedTime())}`;
 
         if (type === 'json') {
@@ -197,10 +198,22 @@ export function Snapshots(ctx: PluginContext) {
             const zipDataObj: { [k: string]: Uint8Array } = {
                 'state.json': state
             };
-            for (const [file, id] of ctx.managers.asset.list) {
 
-                zipDataObj[`${id}/${file.name}`] = new Uint8Array(await file.arrayBuffer());
+            const assets: any[] = [];
+
+            for (const { asset, file } of ctx.managers.asset.assets) {
+                const id = Asset.isFile(asset) ? asset.id : UUID.create22();
+                assets.push([id, asset]);
+                zipDataObj[`assets/${id}`] = new Uint8Array(await file.arrayBuffer());
+            }
+
+            if (assets.length > 0) {
+                const index = JSON.stringify(assets, null, 2);
+                const data = new Uint8Array(utf8ByteCount(index));
+                utf8Write(data, 0, index);
+                zipDataObj['assets.json'] = data;
             }
+
             const zipFile = zip(zipDataObj);
 
             const blob = new Blob([zipFile], {type : 'application/zip'});
@@ -211,22 +224,30 @@ export function Snapshots(ctx: PluginContext) {
     PluginCommands.State.Snapshots.OpenFile.subscribe(ctx, async ({ file }) => {
         try {
             if (file.name.toLowerCase().endsWith('json')) {
-                const data = await readFromFile(file, 'string').run();
+                const data = await ctx.runTask(readFromFile(file, 'string'));
                 const snapshot = JSON.parse(data);
                 return ctx.state.setSnapshot(snapshot);
             } else {
-                const data = await readFromFile(file, 'zip').run();
-                objectForEach(data, (v, k) => {
-                    if (k === 'state.json') return;
+                const data = await ctx.runTask(readFromFile(file, 'zip'));
+                const assets = Object.create(null);
 
-                    const slash = k.indexOf('/');
-                    const id = k.substring(0, slash) as UUID;
-                    const name = k.substring(slash + 1);
-                    const file = new File([v], name);
-                    ctx.managers.asset.set(id, file);
+                objectForEach(data, (v, k) => {
+                    if (k === 'state.json' || k === 'assets.json') return;
+                    const name = k.substring(k.indexOf('/') + 1);
+                    assets[name] = new File([v], name);
                 });
                 const stateFile = new File([data['state.json']], 'state.json');
-                const stateData = await readFromFile(stateFile, 'string').run();
+                const stateData = await ctx.runTask(readFromFile(stateFile, 'string'));
+
+                if (data['assets.json']) {
+                    const file = new File([data['assets.json']], 'assets.json');
+                    const json = JSON.parse(await ctx.runTask(readFromFile(file, 'string')));
+
+                    for (const [id, asset] of json) {
+                        ctx.managers.asset.set(asset, assets[id]);
+                    }
+                }
+
                 const snapshot = JSON.parse(stateData);
                 return ctx.state.setSnapshot(snapshot);
             }

src/mol-plugin/context.ts

@@ -53,7 +53,7 @@ import { TaskManager } from './util/task-manager';
 import { PluginToastManager } from './util/toast';
 import { ViewportScreenshotHelper } from './util/viewport-screenshot';
 import { PLUGIN_VERSION, PLUGIN_VERSION_DATE } from './version';
-import { AssetManager } from './util/asset-manager';
+import { AssetManager } from '../mol-util/assets';
 
 export class PluginContext {
     runTask = <T>(task: Task<T>) => this.tasks.run(task);
@@ -152,7 +152,7 @@ export class PluginContext {
         camera: new CameraManager(this),
         lociLabels: void 0 as any as LociLabelManager,
         toast: new PluginToastManager(this),
-        asset: new AssetManager(this)
+        asset: new AssetManager()
     } as const
 
     readonly customModelProperties = new CustomProperty.Registry<Model>();

src/mol-plugin/util/asset-manager.ts

@@ -1,73 +0,0 @@
-/**
- * Copyright (c) 2020 mol* contributors, licensed under MIT, See LICENSE file for more info.
- *
- * @author Alexander Rose <alexander.rose@weirdbyte.de>
- */
-
-import { UUID } from '../../mol-util';
-import { PluginContext } from '../context';
-import { iterableToArray } from '../../mol-data/util';
-
-export { AssetManager };
-
-class AssetManager {
-    private files = new Map<UUID, File>()
-    private ids = new Map<File, UUID>()
-
-    get list() {
-        return iterableToArray(this.ids.entries());
-    }
-
-    set(id: UUID, file: File) {
-        this.files.set(id, file);
-        this.ids.set(file, id);
-    }
-
-    remove(id: UUID) {
-        if (this.files.has(id)) {
-            const file = this.files.get(id)!;
-            this.files.delete(id);
-            this.ids.delete(file);
-        }
-    }
-
-    has(id: UUID) {
-        return this.files.has(id);
-    }
-
-    get(id: UUID) {
-        return this.files.get(id);
-    }
-
-    /** For use with `JSON.stringify` */
-    replacer = (key: string, value: any) => {
-        if (value instanceof File) {
-            const id = this.ids.get(value);
-            if (!id) {
-                // TODO throw?
-                console.warn(`No asset found for '${value.name}'`);
-            }
-            return id ? AssetManager.Item(id, value.name) : {};
-        } else {
-            return value;
-        }
-    }
-
-    constructor(public ctx: PluginContext) {
-        ctx.state.data.events.object.removed.subscribe(e => {
-            const id = e.obj?.id;
-            if (id) ctx.managers.asset.remove(id);
-        });
-    }
-}
-
-namespace AssetManager {
-    export type Item = { kind: 'asset-item', id: UUID, name: string };
-    export function Item(id: UUID, name: string): Item {
-        return { kind: 'asset-item', id, name };
-    }
-
-    export function isItem(x?: any): x is Item {
-        return !!x && x?.kind === 'asset-item';
-    }
-}

src/mol-util/assets.ts

@@ -0,0 +1,105 @@
+/**
+ * Copyright (c) 2020 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
+ */
+
+import UUID from './uuid';
+import { iterableToArray } from '../mol-data/util';
+import { ajaxGet, DataType, DataResponse, readFromFile } from './data-source';
+import { Task } from '../mol-task';
+
+export { AssetManager, Asset };
+
+type _File = File;
+type Asset = Asset.Url | Asset.File
+
+namespace Asset {
+    export type Url = { url: string, title?: string, body?: string }
+    export type File = { id: UUID, name: string, file?: _File }
+
+    export function Url(url: string, options?: { body?: string, title?: string }): Url {
+        return { url, ...options };
+    }
+
+    export function File(file: _File): File {
+        return { id: UUID.create22(), name: file.name, file };
+    }
+
+    export function isUrl(x: Asset): x is Url {
+        return !!x && !!(x as any).url;
+    }
+
+    export function isFile(x: Asset): x is File {
+        return !!x && !!(x as any).id;
+    }
+}
+
+class AssetManager {
+    private _assets = new Map<string, { asset: Asset, file: File }>();
+
+    get assets() {
+        return iterableToArray(this._assets.values());
+    }
+
+    set(asset: Asset, file: File) {
+        if (Asset.isUrl(asset)) {
+            this._assets.set(getUrlKey(asset), { asset, file });
+        } else {
+            this._assets.set(asset.id, { asset, file });
+        }
+    }
+
+    resolve<T extends DataType>(asset: Asset, type: T, store = true): Task<DataResponse<T>> {
+        if (Asset.isUrl(asset)) {
+            const key = getUrlKey(asset);
+            if (this._assets.has(key)) {
+                return readFromFile(this._assets.get(key)!.file, type);
+            }
+
+            if (!store) {
+                return ajaxGet({ ...asset, type });
+            }
+
+            return Task.create(`Download ${asset.title || asset.url}`, async ctx => {
+                const data = await ajaxGet({ ...asset, type: 'binary' }).runInContext(ctx);
+                const file = new File([data], 'raw-data');
+                this._assets.set(key, { asset, file });
+                return await readFromFile(file, type).runInContext(ctx);
+            });
+        } else {
+            if (this._assets.has(asset.id)) return readFromFile(this._assets.get(asset.id)!.file, type);
+            if (!(asset.file instanceof File)) {
+                return Task.fail('Resolve asset', `Cannot resolve file asset '${asset.name}' (${asset.id})`);
+            }
+            if (store) {
+                this._assets.set(asset.id, { asset, file: asset.file });
+            }
+            return readFromFile(asset.file, type);
+        }
+    }
+
+    release(asset: Asset) {
+        if (Asset.isFile(asset)) {
+            this._assets.delete(asset.id);
+        } else {
+            this._assets.delete(getUrlKey(asset));
+        }
+    }
+}
+
+function getUrlKey(asset: Asset.Url) {
+    return asset.body ? `${asset.url}_${asset.body || ''}` : asset.url;
+}
+
+namespace AssetManager {
+    export type Item = { kind: 'asset-item', id: UUID, name: string };
+    export function Item(id: UUID, name: string): Item {
+        return { kind: 'asset-item', id, name };
+    }
+
+    export function isItem(x?: any): x is Item {
+        return !!x && x?.kind === 'asset-item';
+    }
+}

src/mol-util/data-source.ts

@@ -28,9 +28,9 @@ export enum DataCompressionMethod {
     Zip,
 }
 
-type DataType = 'json' | 'xml' | 'string' | 'binary' | 'zip'
-type DataValue = 'string' | any | XMLDocument | Uint8Array
-type DataResponse<T extends DataType> =
+export type DataType = 'json' | 'xml' | 'string' | 'binary' | 'zip'
+export type DataValue = 'string' | any | XMLDocument | Uint8Array
+export type DataResponse<T extends DataType> =
     T extends 'json' ? any :
         T extends 'xml' ? XMLDocument :
             T extends 'string' ? string :

src/mol-util/param-definition.ts

@@ -13,6 +13,7 @@ import { Script as ScriptData } from '../mol-script/script';
 import { Legend } from './legend';
 import { stringToWords } from './string';
 import { getColorListFromName, ColorListName } from './color/lists';
+import { Asset } from './assets';
 
 export namespace ParamDefinition {
     export interface Info {
@@ -147,7 +148,7 @@ export namespace ParamDefinition {
         return setInfo<Mat4>({ type: 'mat4', defaultValue }, info);
     }
 
-    export interface FileParam extends Base<File | null> {
+    export interface FileParam extends Base<Asset.File | null> {
         type: 'file'
         accept?: string
     }