data asset handling improvements

src/apps/state-docs/pd-to-md.ts

@@ -22,6 +22,7 @@ function paramInfo(param: PD.Any, offset: number): string {
         case 'color-list': return `A list of colors as 0xrrggbb`;
         case 'vec3': return `3D vector [x, y, z]`;
         case 'mat4': return `4x4 transformation matrix`;
+        case 'url': return `URL couple with unique identifier`;
         case 'file': return `JavaScript File Handle`;
         case 'file-list': return `JavaScript FileList Handle`;
         case 'select': return `One of ${oToS(param.options)}`;

src/apps/viewer/index.ts

@@ -17,6 +17,7 @@ import { PluginConfig } from '../../mol-plugin/config';
 import { CellPack } from '../../extensions/cellpack';
 import { RCSBAssemblySymmetry, RCSBValidationReport } from '../../extensions/rcsb';
 import { PDBeStructureQualityReport } from '../../extensions/pdbe';
+import { Asset } from '../../mol-util/assets';
 require('mol-plugin-ui/skin/light.scss');
 
 function getParam(name: string, regex: string): string {
@@ -88,7 +89,7 @@ async function tryLoadFromUrl(ctx: PluginContext) {
             source: {
                 name: 'url',
                 params: {
-                    url,
+                    url: Asset.Url(url),
                     format: format as any,
                     isBinary,
                     options: params.source.params.options,

src/examples/basic-wrapper/index.ts

@@ -19,6 +19,7 @@ import { CustomToastMessage } from './controls';
 import './index.html';
 import { buildStaticSuperposition, dynamicSuperpositionTest, StaticSuperpositionTestData } from './superposition';
 import { PDBeStructureQualityReport } from '../../extensions/pdbe';
+import { Asset } from '../../mol-util/assets';
 require('mol-plugin-ui/skin/light.scss');
 
 type LoadParams = { url: string, format?: BuiltInTrajectoryFormat, isBinary?: boolean, assemblyId?: string }
@@ -51,7 +52,7 @@ class BasicWrapper {
     async load({ url, format = 'mmcif', isBinary = false, assemblyId = '' }: LoadParams) {
         await this.plugin.clear();
 
-        const data = await this.plugin.builders.data.download({ url, isBinary }, { state: { isGhost: true } });
+        const data = await this.plugin.builders.data.download({ url: Asset.Url(url), isBinary }, { state: { isGhost: true } });
         const trajectory = await this.plugin.builders.structure.parseTrajectory(data, format);
 
         await this.plugin.builders.structure.hierarchy.applyPreset(trajectory, 'default', {

src/examples/basic-wrapper/superposition.ts

@@ -15,6 +15,7 @@ import { compile } from '../../mol-script/runtime/query/compiler';
 import { StateObjectRef } from '../../mol-state';
 import { BuiltInTrajectoryFormat } from '../../mol-plugin-state/formats/trajectory';
 import { StateTransforms } from '../../mol-plugin-state/transforms';
+import { Asset } from '../../mol-util/assets';
 
 export type SuperpositionTestInput = {
     pdbId: string,
@@ -97,7 +98,7 @@ async function siteVisual(plugin: PluginContext, s: StateObjectRef<PSO.Molecule.
 }
 
 async function loadStructure(plugin: PluginContext, url: string, format: BuiltInTrajectoryFormat, assemblyId?: string) {
-    const data = await plugin.builders.data.download({ url });
+    const data = await plugin.builders.data.download({ url: Asset.Url(url) });
     const trajectory = await plugin.builders.structure.parseTrajectory(data, format);
     const model = await plugin.builders.structure.createModel(trajectory);
     const structure = await plugin.builders.structure.createStructure(model, assemblyId ? { name: 'assembly', params: { id: assemblyId } } : void 0);

src/examples/lighting/index.ts

@@ -10,6 +10,7 @@ import { BuiltInTrajectoryFormat } from '../../mol-plugin-state/formats/trajecto
 import { PluginCommands } from '../../mol-plugin/commands';
 import { PluginContext } from '../../mol-plugin/context';
 import './index.html';
+import { Asset } from '../../mol-util/assets';
 require('mol-plugin-ui/skin/light.scss');
 
 type LoadParams = { url: string, format?: BuiltInTrajectoryFormat, isBinary?: boolean, assemblyId?: string }
@@ -101,7 +102,7 @@ class LightingDemo {
     async load({ url, format = 'mmcif', isBinary = false, assemblyId = '' }: LoadParams) {
         await this.plugin.clear();
 
-        const data = await this.plugin.builders.data.download({ url, isBinary }, { state: { isGhost: true } });
+        const data = await this.plugin.builders.data.download({ url: Asset.Url(url), isBinary }, { state: { isGhost: true } });
         const trajectory = await this.plugin.builders.structure.parseTrajectory(data, format);
         const model = await this.plugin.builders.structure.createModel(trajectory);
         const structure = await this.plugin.builders.structure.createStructure(model, assemblyId ? { name: 'assembly', params: { id: assemblyId } } : { name: 'deposited', params: { } });

src/examples/proteopedia-wrapper/index.ts

@@ -28,6 +28,7 @@ import { DefaultCanvas3DParams, Canvas3DProps } from '../../mol-canvas3d/canvas3
 import { createStructureRepresentationParams } from '../../mol-plugin-state/helpers/structure-representation-params';
 import { download } from '../../mol-util/download';
 import { getFormattedTime } from '../../mol-util/date';
+import { Asset } from '../../mol-util/assets';
 require('../../mol-plugin-ui/skin/light.scss');
 
 class MolStarProteopediaWrapper {
@@ -74,7 +75,7 @@ class MolStarProteopediaWrapper {
     }
 
     private download(b: StateBuilder.To<PSO.Root>, url: string) {
-        return b.apply(StateTransforms.Data.Download, { url, isBinary: false });
+        return b.apply(StateTransforms.Data.Download, { url: Asset.Url(url), isBinary: false });
     }
 
     private model(b: StateBuilder.To<PSO.Data.Binary | PSO.Data.String>, format: SupportedFormats) {

src/extensions/cellpack/model.ts

@@ -395,7 +395,7 @@ async function loadMembrane(name: string, plugin: PluginContext, runtime: Runtim
         const file = ingredientFiles[fname];
         b = b.apply(StateTransforms.Data.ReadFile, { file: Asset.File(file), isBinary: true, label: file.name }, { state: { isGhost: true } });
     } else {
-        const url = `${params.baseUrl}/membranes/${name}.bcif`;
+        const url = Asset.Url(`${params.baseUrl}/membranes/${name}.bcif`);
         b = b.apply(StateTransforms.Data.Download, { url, isBinary: true, label: name }, { state: { isGhost: true } });
     }
 
@@ -412,7 +412,7 @@ async function loadMembrane(name: string, plugin: PluginContext, runtime: Runtim
 }
 
 async function loadHivMembrane(plugin: PluginContext, runtime: RuntimeContext, state: State, params: LoadCellPackModelParams) {
-    const url = `${params.baseUrl}/membranes/hiv_lipids.bcif`;
+    const url = Asset.Url(`${params.baseUrl}/membranes/hiv_lipids.bcif`);
     const membrane = await state.build().toRoot()
         .apply(StateTransforms.Data.Download, { label: 'hiv_lipids', url, isBinary: true }, { state: { isGhost: true } })
         .apply(StateTransforms.Data.ParseCif, undefined, { state: { isGhost: true } })
@@ -431,7 +431,7 @@ async function loadHivMembrane(plugin: PluginContext, runtime: RuntimeContext, s
 async function loadPackings(plugin: PluginContext, runtime: RuntimeContext, state: State, params: LoadCellPackModelParams) {
     let cellPackJson: StateBuilder.To<PSO.Format.Json, StateTransformer<PSO.Data.String, PSO.Format.Json>>;
     if (params.source.name === 'id') {
-        const url = getCellPackModelUrl(params.source.params, params.baseUrl);
+        const url = Asset.Url(getCellPackModelUrl(params.source.params, params.baseUrl));
         cellPackJson = state.build().toRoot()
             .apply(StateTransforms.Data.Download, { url, isBinary: false, label: params.source.params }, { state: { isGhost: true } });
     } else {

src/mol-plugin-state/actions/structure.ts

@@ -16,6 +16,7 @@ import { PluginStateObject } from '../objects';
 import { StateTransforms } from '../transforms';
 import { Download } from '../transforms/data';
 import { CustomModelProperties, CustomStructureProperties, TrajectoryFromModelAndCoordinates } from '../transforms/model';
+import { Asset } from '../../mol-util/assets';
 
 const DownloadModelRepresentationOptions = (plugin: PluginContext) => PD.Group({
     type: RootStructureDefinition.getParams(void 0, 'auto').type,
@@ -65,7 +66,7 @@ const DownloadStructure = StateAction.build({
                     options
                 }, { isFlat: true, label: 'PubChem', description: 'Loads 3D conformer from PubChem.' }),
                 'url': PD.Group({
-                    url: PD.Text(''),
+                    url: PD.Url(''),
                     format: PD.Select<BuiltInTrajectoryFormat>('mmcif', PD.arrayToOptions(BuiltInTrajectoryFormats.map(f => f[0]), f => f)),
                     isBinary: PD.Boolean(false),
                     options
@@ -160,7 +161,7 @@ function getDownloadParams(src: string, url: (id: string) => string, label: (id:
     const ids = src.split(',').map(id => id.trim()).filter(id => !!id && (id.length >= 4 || /^[1-9][0-9]*$/.test(id)));
     const ret: StateTransformer.Params<Download>[] = [];
     for (const id of ids) {
-        ret.push({ url: url(id), isBinary, label: label(id) });
+        ret.push({ url: Asset.Url(url(id)), isBinary, label: label(id) });
     }
     return ret;
 }

src/mol-plugin-state/actions/volume.ts

@@ -13,6 +13,7 @@ import { ParamDefinition as PD } from '../../mol-util/param-definition';
 import { PluginStateObject } from '../objects';
 import { Download } from '../transforms/data';
 import { DataFormatProvider } from '../formats/provider';
+import { Asset } from '../../mol-util/assets';
 
 export { DownloadDensity };
 type DownloadDensity = typeof DownloadDensity
@@ -45,7 +46,7 @@ const DownloadDensity = StateAction.build({
                     detail: PD.Numeric(3, { min: 0, max: 10, step: 1 }, { label: 'Detail' }),
                 }, { isFlat: true }),
                 'url': PD.Group({
-                    url: PD.Text(''),
+                    url: PD.Url(''),
                     isBinary: PD.Boolean(false),
                     format: PD.Select('auto', options),
                 }, { isFlat: true })
@@ -70,37 +71,37 @@ const DownloadDensity = StateAction.build({
             break;
         case 'pdb-xray':
             downloadParams = src.params.provider.server === 'pdbe' ? {
-                url: src.params.type === '2fofc'
+                url: Asset.Url(src.params.type === '2fofc'
                     ? `http://www.ebi.ac.uk/pdbe/coordinates/files/${src.params.provider.id.toLowerCase()}.ccp4`
-                    : `http://www.ebi.ac.uk/pdbe/coordinates/files/${src.params.provider.id.toLowerCase()}_diff.ccp4`,
+                    : `http://www.ebi.ac.uk/pdbe/coordinates/files/${src.params.provider.id.toLowerCase()}_diff.ccp4`),
                 isBinary: true,
                 label: `PDBe X-ray map: ${src.params.provider.id}`
             } : {
-                url: src.params.type === '2fofc'
+                url: Asset.Url(src.params.type === '2fofc'
                     ? `https://edmaps.rcsb.org/maps/${src.params.provider.id.toLowerCase()}_2fofc.dsn6`
-                    : `https://edmaps.rcsb.org/maps/${src.params.provider.id.toLowerCase()}_fofc.dsn6`,
+                    : `https://edmaps.rcsb.org/maps/${src.params.provider.id.toLowerCase()}_fofc.dsn6`),
                 isBinary: true,
                 label: `RCSB X-ray map: ${src.params.provider.id}`
             };
             break;
         case 'pdb-emd-ds':
             downloadParams = src.params.provider.server === 'pdbe' ? {
-                url: `https://www.ebi.ac.uk/pdbe/densities/emd/${src.params.provider.id.toLowerCase()}/cell?detail=${src.params.detail}`,
+                url: Asset.Url(`https://www.ebi.ac.uk/pdbe/densities/emd/${src.params.provider.id.toLowerCase()}/cell?detail=${src.params.detail}`),
                 isBinary: true,
                 label: `PDBe EMD Density Server: ${src.params.provider.id}`
             } : {
-                url: `https://maps.rcsb.org/em/${src.params.provider.id.toLowerCase()}/cell?detail=${src.params.detail}`,
+                url: Asset.Url(`https://maps.rcsb.org/em/${src.params.provider.id.toLowerCase()}/cell?detail=${src.params.detail}`),
                 isBinary: true,
                 label: `RCSB PDB EMD Density Server: ${src.params.provider.id}`
             };
             break;
         case 'pdb-xray-ds':
             downloadParams = src.params.provider.server === 'pdbe' ? {
-                url: `https://www.ebi.ac.uk/pdbe/densities/x-ray/${src.params.provider.id.toLowerCase()}/cell?detail=${src.params.detail}`,
+                url: Asset.Url(`https://www.ebi.ac.uk/pdbe/densities/x-ray/${src.params.provider.id.toLowerCase()}/cell?detail=${src.params.detail}`),
                 isBinary: true,
                 label: `PDBe X-ray Density Server: ${src.params.provider.id}`
             } : {
-                url: `https://maps.rcsb.org/x-ray/${src.params.provider.id.toLowerCase()}/cell?detail=${src.params.detail}`,
+                url: Asset.Url(`https://maps.rcsb.org/x-ray/${src.params.provider.id.toLowerCase()}/cell?detail=${src.params.detail}`),
                 isBinary: true,
                 label: `RCSB PDB X-ray Density Server: ${src.params.provider.id}`
             };
@@ -113,7 +114,7 @@ const DownloadDensity = StateAction.build({
     switch (src.name) {
         case 'url':
             downloadParams = src.params;
-            provider = src.params.format === 'auto' ? plugin.dataFormats.auto(getFileInfo(downloadParams.url), data.cell?.obj!) : plugin.dataFormats.get(src.params.format);
+            provider = src.params.format === 'auto' ? plugin.dataFormats.auto(getFileInfo(downloadParams.url.url), data.cell?.obj!) : plugin.dataFormats.get(src.params.format);
             break;
         case 'pdb-xray':
             provider = src.params.provider.server === 'pdbe'

src/mol-plugin-state/transforms/data.ts

@@ -32,6 +32,7 @@ export { ParseDsn6 };
 export { ImportString };
 export { ImportJson };
 export { ParseJson };
+
 type Download = typeof Download
 const Download = PluginStateTransform.BuiltIn({
     name: 'download',
@@ -39,29 +40,27 @@ const Download = PluginStateTransform.BuiltIn({
     from: [SO.Root],
     to: [SO.Data.String, SO.Data.Binary],
     params: {
-        url: PD.Text('https://www.ebi.ac.uk/pdbe/static/entry/1cbs_updated.cif', { description: 'Resource URL. Must be the same domain or support CORS.' }),
+        url: PD.Url('https://www.ebi.ac.uk/pdbe/static/entry/1cbs_updated.cif', { description: 'Resource URL. Must be the same domain or support CORS.' }),
         label: PD.Optional(PD.Text('')),
-        isBinary: PD.Optional(PD.Boolean(false, { description: 'If true, download data as binary (string otherwise)' })),
-        body: PD.Optional(PD.Text(''))
+        isBinary: PD.Optional(PD.Boolean(false, { description: 'If true, download data as binary (string otherwise)' }))
     }
 })({
-    apply({ params: p }, plugin: PluginContext) {
+    apply({ params: p, cache }, plugin: PluginContext) {
         return Task.create('Download', async ctx => {
-            const data = await plugin.managers.asset.resolve(Asset.Url(p.url, { body: p.body }), p.isBinary ? 'binary' : 'string').runInContext(ctx);
+            const asset = await plugin.managers.asset.resolve(p.url, p.isBinary ? 'binary' : 'string').runInContext(ctx);
+            (cache as any).asset = asset;
             return p.isBinary
-                ? new SO.Data.Binary(data as Uint8Array, { label: p.label ? p.label : p.url })
-                : new SO.Data.String(data as string, { label: p.label ? p.label : p.url });
+                ? new SO.Data.Binary(asset.data as Uint8Array, { label: p.label ? p.label : p.url.url })
+                : new SO.Data.String(asset.data as string, { label: p.label ? p.label : p.url.url });
         });
     },
-    dispose({ params: p }, plugin: PluginContext) {
-        if (p) {
-            plugin.managers.asset.release(Asset.Url(p.url, { body: p.body }));
-        }
+    dispose({ cache }) {
+        ((cache as any)?.asset as Asset.Wrapper | undefined)?.dispose();
     },
     update({ oldParams, newParams, b }) {
         if (oldParams.url !== newParams.url || oldParams.isBinary !== newParams.isBinary) return StateTransformer.UpdateResult.Recreate;
         if (oldParams.label !== newParams.label) {
-            b.label = newParams.label || newParams.url;
+            b.label = newParams.label || newParams.url.url;
             return StateTransformer.UpdateResult.Updated;
         }
         return StateTransformer.UpdateResult.Unchanged;
@@ -77,35 +76,39 @@ const DownloadBlob = PluginStateTransform.BuiltIn({
     params: {
         sources: PD.ObjectList({
             id: PD.Text('', { label: 'Unique ID' }),
-            url: PD.Text('https://www.ebi.ac.uk/pdbe/static/entry/1cbs_updated.cif', { description: 'Resource URL. Must be the same domain or support CORS.' }),
+            url: PD.Url('https://www.ebi.ac.uk/pdbe/static/entry/1cbs_updated.cif', { description: 'Resource URL. Must be the same domain or support CORS.' }),
             isBinary: PD.Optional(PD.Boolean(false, { description: 'If true, download data as binary (string otherwise)' })),
-            body: PD.Optional(PD.Text('')),
             canFail: PD.Optional(PD.Boolean(false, { description: 'Indicate whether the download can fail and not be included in the blob as a result.' }))
         }, e => `${e.id}: ${e.url}`),
         maxConcurrency: PD.Optional(PD.Numeric(4, { min: 1, max: 12, step: 1 }, { description: 'The maximum number of concurrent downloads.' }))
     }
 })({
-    apply({ params }, plugin: PluginContext) {
+    apply({ params, cache }, plugin: PluginContext) {
         return Task.create('Download Blob', async ctx => {
             const entries: SO.Data.BlobEntry[] = [];
-            const data = await ajaxGetMany(ctx, params.sources, params.maxConcurrency || 4, plugin.managers.asset);
+            const data = await ajaxGetMany(ctx, plugin.managers.asset, params.sources, params.maxConcurrency || 4);
+
+            const assets: Asset.Wrapper[] = [];
 
             for (let i = 0; i < data.length; i++) {
                 const r = data[i], src = params.sources[i];
                 if (r.kind === 'error') plugin.log.warn(`Download ${r.id} (${src.url}) failed: ${r.error}`);
                 else {
+                    assets.push(r.result);
                     entries.push(src.isBinary
-                        ? { id: r.id, kind: 'binary', data: r.result as Uint8Array }
-                        : { id: r.id, kind: 'string', data: r.result as string });
+                        ? { id: r.id, kind: 'binary', data: r.result.data as Uint8Array }
+                        : { id: r.id, kind: 'string', data: r.result.data as string });
                 }
             }
+            (cache as any).assets = assets;
             return new SO.Data.Blob(entries, { label: 'Data Blob', description: `${entries.length} ${entries.length === 1 ? 'entry' : 'entries'}` });
         });
     },
-    dispose({ params }, plugin: PluginContext) {
-        if (!params) return;
-        for (const s of params.sources) {
-            plugin.managers.asset.release({ url: s.url, body: s.body });
+    dispose({ cache }, plugin: PluginContext) {
+        const assets: Asset.Wrapper[] | undefined = (cache as any)?.assets;
+        if (!assets) return;
+        for (const a of assets) {
+            a.dispose();
         }
     }
     // TODO: ??
@@ -163,25 +166,24 @@ const ReadFile = PluginStateTransform.BuiltIn({
         isBinary: PD.Optional(PD.Boolean(false, { description: 'If true, open file as as binary (string otherwise)' }))
     }
 })({
-    apply({ params: p }, plugin: PluginContext) {
+    apply({ params: p, cache }, plugin: PluginContext) {
         return Task.create('Open File', async ctx => {
             if (p.file === null) {
                 plugin.log.error('No file(s) selected');
                 return StateObject.Null;
             }
 
-            const data = await plugin.managers.asset.resolve(p.file, p.isBinary ? 'binary' : 'string').runInContext(ctx);
+            const asset = await plugin.managers.asset.resolve(p.file, p.isBinary ? 'binary' : 'string').runInContext(ctx);
+            (cache as any).asset = asset;
             const o = p.isBinary
-                ? new SO.Data.Binary(data as Uint8Array, { label: p.label ? p.label : p.file.name })
-                : new SO.Data.String(data as string, { label: p.label ? p.label : p.file.name });
+                ? new SO.Data.Binary(asset.data as Uint8Array, { label: p.label ? p.label : p.file.name })
+                : new SO.Data.String(asset.data as string, { label: p.label ? p.label : p.file.name });
 
             return o;
         });
     },
-    dispose({ params }, plugin: PluginContext) {
-        if (params?.file) {
-            plugin.managers.asset.release(params.file);
-        }
+    dispose({ cache }) {
+        ((cache as any)?.asset as Asset.Wrapper | undefined)?.dispose();
     },
     update({ oldParams, newParams, b }) {
         if (oldParams.label !== newParams.label) {

src/mol-plugin-ui/controls/parameters.tsx

@@ -181,6 +181,7 @@ function controlFor(param: PD.Any): ParamControl | undefined {
         case 'color-list': return ColorListControl;
         case 'vec3': return Vec3Control;
         case 'mat4': return Mat4Control;
+        case 'url': return UrlControl;
         case 'file': return FileControl;
         case 'file-list': return FileListControl;
         case 'select': return SelectControl;
@@ -786,6 +787,33 @@ export class Mat4Control extends React.PureComponent<ParamProps<PD.Mat4>, { isEx
     }
 }
 
+export class UrlControl extends SimpleParam<PD.UrlParam> {
+    onChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+        const value = e.target.value;
+        if (value !== this.props.value.url) {
+            this.update(Asset.Url(value));
+        }
+    }
+
+    onKeyPress = (e: React.KeyboardEvent<HTMLInputElement>) => {
+        if ((e.keyCode === 13 || e.charCode === 13)) {
+            if (this.props.onEnter) this.props.onEnter();
+        }
+        e.stopPropagation();
+    }
+
+    renderControl() {
+        const placeholder = this.props.param.label || camelCaseToWords(this.props.name);
+        return <input type='text'
+            value={this.props.value.url || ''}
+            placeholder={placeholder}
+            onChange={this.onChange}
+            onKeyPress={this.props.onEnter ? this.onKeyPress : void 0}
+            disabled={this.props.isDisabled}
+        />;
+    }
+}
+
 export class FileControl extends React.PureComponent<ParamProps<PD.FileParam>> {
     change(value: File) {
         this.props.onChange({ name: this.props.name, param: this.props.param, value: Asset.File(value) });

src/mol-plugin/behavior/static/state.ts

@@ -5,22 +5,20 @@
  * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
-import { PluginCommands } from '../../commands';
-import { PluginContext } from '../../context';
-import { StateTree, StateTransform, State } from '../../../mol-state';
-import { PluginStateSnapshotManager } from '../../../mol-plugin-state/snapshots';
+import { utf8ByteCount, utf8Write } from '../../../mol-io/common/utf8';
+import { Structure } from '../../../mol-model/structure';
 import { PluginStateObject as SO } from '../../../mol-plugin-state/objects';
-import { getFormattedTime } from '../../../mol-util/date';
+import { PluginStateSnapshotManager } from '../../../mol-plugin-state/snapshots';
+import { State, StateTransform, StateTree } from '../../../mol-state';
 import { readFromFile } from '../../../mol-util/data-source';
+import { getFormattedTime } from '../../../mol-util/date';
 import { download } from '../../../mol-util/download';
-import { Structure } from '../../../mol-model/structure';
+import { objectForEach } from '../../../mol-util/object';
 import { urlCombine } from '../../../mol-util/url';
-import { PluginConfig } from '../../config';
 import { zip } from '../../../mol-util/zip/zip';
-import { utf8Write, utf8ByteCount } from '../../../mol-io/common/utf8';
-import { objectForEach } from '../../../mol-util/object';
-import { UUID } from '../../../mol-util';
-import { Asset } from '../../../mol-util/assets';
+import { PluginCommands } from '../../commands';
+import { PluginConfig } from '../../config';
+import { PluginContext } from '../../context';
 
 export function registerDefault(ctx: PluginContext) {
     SyncBehaviors(ctx);
@@ -201,10 +199,10 @@ export function Snapshots(ctx: PluginContext) {
 
             const assets: any[] = [];
 
+            // TODO: there can be duplicate entries: check for this?
             for (const { asset, file } of ctx.managers.asset.assets) {
-                const id = Asset.isFile(asset) ? asset.id : UUID.create22();
-                assets.push([id, asset]);
-                zipDataObj[`assets/${id}`] = new Uint8Array(await file.arrayBuffer());
+                assets.push([asset.id, asset]);
+                zipDataObj[`assets/${asset.id}`] = new Uint8Array(await file.arrayBuffer());
             }
 
             if (assets.length > 0) {

src/mol-util/assets.ts

@@ -16,27 +16,40 @@ type _File = File;
 type Asset = Asset.Url | Asset.File
 
 namespace Asset {
-    export type Url = { url: string, title?: string, body?: string }
-    export type File = { id: UUID, name: string, file?: _File }
+    export type Url = { kind: 'url', id: UUID, url: string, title?: string, body?: string }
+    export type File = { kind: 'file', id: UUID, name: string, file?: _File }
 
     export function Url(url: string, options?: { body?: string, title?: string }): Url {
-        return { url, ...options };
+        return { kind: 'url', id: UUID.create22(), url, ...options };
     }
 
     export function File(file: _File): File {
-        return { id: UUID.create22(), name: file.name, file };
+        return { kind: 'file', id: UUID.create22(), name: file.name, file };
     }
 
-    export function isUrl(x: Asset): x is Url {
-        return !!x && !!(x as any).url;
+    export function isUrl(x?: Asset): x is Url {
+        return x?.kind === 'url';
     }
 
-    export function isFile(x: Asset): x is File {
-        return !!x && !!(x as any).id;
+    export function isFile(x?: Asset): x is File {
+        return x?.kind === 'file';
+    }
+
+    export class Wrapper<T extends DataType = DataType> {
+        dispose() {
+            this.manager.release(this.asset);
+        }
+
+        constructor(public readonly data: DataResponse<T>, private asset: Asset, private manager: AssetManager) {
+
+        }
     }
 }
 
 class AssetManager {
+    // TODO: add URL based ref-counted cache?
+    // TODO: when serializing, check for duplicates?
+
     private _assets = new Map<string, { asset: Asset, file: File }>();
 
     get assets() {
@@ -44,51 +57,42 @@ class AssetManager {
     }
 
     set(asset: Asset, file: File) {
-        if (Asset.isUrl(asset)) {
-            this._assets.set(getUrlKey(asset), { asset, file });
-        } else {
-            this._assets.set(asset.id, { asset, file });
-        }
+        this._assets.set(asset.id, { asset, file });
     }
 
-    resolve<T extends DataType>(asset: Asset, type: T, store = true): Task<DataResponse<T>> {
+    resolve<T extends DataType>(asset: Asset, type: T, store = true): Task<Asset.Wrapper<T>> {
         if (Asset.isUrl(asset)) {
-            const key = getUrlKey(asset);
-            if (this._assets.has(key)) {
-                return readFromFile(this._assets.get(key)!.file, type);
-            }
+            return Task.create(`Download ${asset.title || asset.url}`, async ctx => {
+                if (this._assets.has(asset.id)) {
+                    return new Asset.Wrapper(await readFromFile(this._assets.get(asset.id)!.file, type).runInContext(ctx), asset, this);
+                }
 
-            if (!store) {
-                return ajaxGet({ ...asset, type });
-            }
+                if (!store) {
+                    return new Asset.Wrapper(await ajaxGet({ ...asset, type }).runInContext(ctx), asset, this);
+                }
 
-            return Task.create(`Download ${asset.title || asset.url}`, async ctx => {
                 const data = await ajaxGet({ ...asset, type: 'binary' }).runInContext(ctx);
                 const file = new File([data], 'raw-data');
-                this._assets.set(key, { asset, file });
-                return await readFromFile(file, type).runInContext(ctx);
+                this._assets.set(asset.id, { asset, file });
+                return new Asset.Wrapper(await readFromFile(file, type).runInContext(ctx), asset, this);
             });
         } else {
-            if (this._assets.has(asset.id)) return readFromFile(this._assets.get(asset.id)!.file, type);
-            if (!(asset.file instanceof File)) {
-                return Task.fail('Resolve asset', `Cannot resolve file asset '${asset.name}' (${asset.id})`);
-            }
-            if (store) {
-                this._assets.set(asset.id, { asset, file: asset.file });
-            }
-            return readFromFile(asset.file, type);
+            return Task.create(`Read ${asset.name}`, async ctx => {
+                if (this._assets.has(asset.id)) {
+                    return new Asset.Wrapper(await readFromFile(this._assets.get(asset.id)!.file, type).runInContext(ctx), asset, this);
+                }
+                if (!(asset.file instanceof File)) {
+                    throw new Error(`Cannot resolve file asset '${asset.name}' (${asset.id})`);
+                }
+                if (store) {
+                    this._assets.set(asset.id, { asset, file: asset.file });
+                }
+                return new Asset.Wrapper(await readFromFile(asset.file, type).runInContext(ctx), asset, this);
+            });
         }
     }
 
     release(asset: Asset) {
-        if (Asset.isFile(asset)) {
-            this._assets.delete(asset.id);
-        } else {
-            this._assets.delete(getUrlKey(asset));
-        }
+        this._assets.delete(asset.id);
     }
-}
-
-function getUrlKey(asset: Asset.Url) {
-    return asset.body ? `${asset.url}_${asset.body || ''}` : asset.url;
 }

src/mol-util/data-source.ts

@@ -10,7 +10,7 @@
 import { Task, RuntimeContext } from '../mol-task';
 import { unzip, ungzip } from './zip/zip';
 import { utf8Read } from '../mol-io/common/utf8';
-import { AssetManager } from './assets';
+import { AssetManager, Asset } from './assets';
 
 // polyfill XMLHttpRequest in node.js
 const XHR = typeof document === 'undefined' ? require('xhr2') as {
@@ -281,23 +281,19 @@ function ajaxGetInternal<T extends DataType>(title: string | undefined, url: str
     });
 }
 
-export type AjaxGetManyEntry<T> = { kind: 'ok', id: string, result: T } | { kind: 'error', id: string, error: any }
-export async function ajaxGetMany(ctx: RuntimeContext, sources: { id: string, url: string, isBinary?: boolean, body?: string, canFail?: boolean }[], maxConcurrency: number, assetManager?: AssetManager) {
+export type AjaxGetManyEntry = { kind: 'ok', id: string, result: Asset.Wrapper<'string' | 'binary'> } | { kind: 'error', id: string, error: any }
+export async function ajaxGetMany(ctx: RuntimeContext, assetManager: AssetManager, sources: { id: string, url: Asset.Url, isBinary?: boolean, canFail?: boolean }[], maxConcurrency: number) {
     const len = sources.length;
-    const slots: AjaxGetManyEntry<string | Uint8Array>[] = new Array(sources.length);
+    const slots: AjaxGetManyEntry[] = new Array(sources.length);
 
     await ctx.update({ message: 'Downloading...', current: 0, max: len });
-    let promises: Promise<AjaxGetManyEntry<any> & { index: number }>[] = [], promiseKeys: number[] = [];
+    let promises: Promise<AjaxGetManyEntry & { index: number }>[] = [], promiseKeys: number[] = [];
     let currentSrc = 0;
     for (let _i = Math.min(len, maxConcurrency); currentSrc < _i; currentSrc++) {
         const current = sources[currentSrc];
 
-        if (assetManager) {
-            promises.push(wrapPromise(currentSrc, current.id,
-                assetManager.resolve({ url: current.url, body: current.body }, current.isBinary ? 'binary' : 'string').runAsChild(ctx)));
-        } else {
-            promises.push(wrapPromise(currentSrc, current.id, ajaxGet({ url: current.url, type: current.isBinary ? 'binary' : 'string' }).runAsChild(ctx)));
-        }
+        promises.push(wrapPromise(currentSrc, current.id,
+            assetManager.resolve(current.url, current.isBinary ? 'binary' : 'string').runAsChild(ctx)));
         promiseKeys.push(currentSrc);
     }
 
@@ -319,7 +315,8 @@ export async function ajaxGetMany(ctx: RuntimeContext, sources: { id: string, ur
         promiseKeys = promiseKeys.filter(_filterRemoveIndex, idx);
         if (currentSrc < len) {
             const current = sources[currentSrc];
-            promises.push(wrapPromise(currentSrc, current.id, ajaxGet({ url: current.url, type: current.isBinary ? 'binary' : 'string', body: current.body }).runAsChild(ctx)));
+            const asset = assetManager.resolve(current.url, current.isBinary ? 'binary' : 'string').runAsChild(ctx);
+            promises.push(wrapPromise(currentSrc, current.id, asset));
             promiseKeys.push(currentSrc);
             currentSrc++;
         }
@@ -332,7 +329,7 @@ function _filterRemoveIndex(this: number, _: any, i: number) {
     return this !== i;
 }
 
-async function wrapPromise<T>(index: number, id: string, p: Promise<T>): Promise<AjaxGetManyEntry<T> & { index: number }> {
+async function wrapPromise(index: number, id: string, p: Promise<Asset.Wrapper<'string' | 'binary'>>): Promise<AjaxGetManyEntry & { index: number }> {
     try {
         const result = await p;
         return { kind: 'ok', result, index, id };

src/mol-util/param-definition.ts

@@ -148,6 +148,15 @@ export namespace ParamDefinition {
         return setInfo<Mat4>({ type: 'mat4', defaultValue }, info);
     }
 
+    export interface UrlParam extends Base<Asset.Url> {
+        type: 'url'
+    }
+    export function Url(url: string | { url: string, body?: string }, info?: Info): UrlParam {
+        const defaultValue = typeof url === 'string' ? Asset.Url(url) : Asset.Url(url.url, { body: url.body });
+        const ret = setInfo<UrlParam>({ type: 'url', defaultValue }, info);
+        return ret;
+    }
+
     export interface FileParam extends Base<Asset.File | null> {
         type: 'file'
         accept?: string
@@ -298,7 +307,7 @@ export namespace ParamDefinition {
     }
 
     export type Any =
-        | Value<any> | Select<any> | MultiSelect<any> | BooleanParam | Text | Color | Vec3 | Mat4 | Numeric | FileParam | FileListParam | Interval | LineGraph
+        | Value<any> | Select<any> | MultiSelect<any> | BooleanParam | Text | Color | Vec3 | Mat4 | Numeric | FileParam | UrlParam | FileListParam | Interval | LineGraph
         | ColorList | Group<any> | Mapped<any> | Converted<any, any> | Conditioned<any, any, any> | Script | ObjectList
 
     export type Params = { [k: string]: Any }