wip: data/format blobs

src/apps/state-docs/pd-to-md.ts

@@ -28,7 +28,7 @@ function paramInfo(param: PD.Any, offset: number): string {
         case 'group': return `Object with:\n${getParams(param.params, offset + 2)}`;
         case 'mapped': return `Object { name: string, params: object } where name+params are:\n${getMapped(param, offset + 2)}`;
         case 'line-graph': return `A list of 2d vectors [xi, yi][]`;
-        case 'list': return `Array of\n${paramInfo(param.element, offset + 2)}`;
+        case 'object-list': return `Array of\n${paramInfo(PD.Group(param.element), offset + 2)}`;
         // TODO: support more languages
         case 'script-expression': return `An expression in the specified language { language: 'mol-script', expressiong: string }`;
         default:

src/mol-plugin/index.ts

@@ -25,6 +25,8 @@ export const DefaultPluginSpec: PluginSpec = {
         PluginSpec.Action(StateActions.Structure.CreateComplexRepresentation),
         PluginSpec.Action(StateActions.Structure.EnableModelCustomProps),
 
+        PluginSpec.Action(StateActions.Structure.TestBlob),
+
         // Volume streaming
         PluginSpec.Action(InitVolumeStreaming),
         PluginSpec.Action(BoxifyVolumeStreaming),

src/mol-plugin/state/actions/structure.ts

@@ -261,3 +261,24 @@ export const StructureFromSelection = StateAction.build({
     const root = state.build().to(ref).apply(StructureSelection, { query, label: params.label });
     return state.updateTree(root);
 });
+
+
+export const TestBlob = StateAction.build({
+    display: { name: 'Test Blob'},
+    from: PluginStateObject.Root
+})(({ ref, state }, ctx: PluginContext) => {
+
+    const ids = '5B6V,5B6W,5H2H,5H2I,5H2J,5B6X,5H2K,5H2L,5H2M,5B6Y,5H2N,5H2O,5H2P,5B6Z'.split(',').map(u => u.toLowerCase());
+
+    const root = state.build().to(ref)
+        .apply(StateTransforms.Data.DownloadBlob, {
+            sources: ids.map(id => ({ id, url: `https://webchem.ncbr.muni.cz/ModelServer/static/bcif/${id}`, isBinary: true })),
+            maxConcurrency: 4
+        }).apply(StateTransforms.Data.ParseBlob, {
+            formats: ids.map(id => ({ id, format: 'cif' as 'cif' }))
+        })
+        .apply(StateTransforms.Model.TrajectoryFromBlob)
+        .apply(StateTransforms.Model.ModelFromTrajectory, { modelIndex: 0 });
+    createStructureTree(ctx, root, false);
+    return state.updateTree(root);
+});

src/mol-plugin/state/objects.ts

@@ -51,10 +51,11 @@ export namespace PluginStateObject {
         export class String extends Create<string>({ name: 'String Data', typeClass: 'Data', }) { }
         export class Binary extends Create<Uint8Array>({ name: 'Binary Data', typeClass: 'Data' }) { }
 
-        // TODO
-        // export class MultipleRaw extends Create<{
-        //     [key: string]: { type: 'String' | 'Binary', data: string | Uint8Array }
-        // }>({ name: 'Data', typeClass: 'Data', shortName: 'MD', description: 'Multiple Keyed Data.' }) { }
+        export type BlobEntry = { id: string } &
+            ( { kind: 'string', data: string }
+            | { kind: 'binary', data: Uint8Array })
+        export type BlobData = BlobEntry[]
+        export class Blob extends Create<BlobData>({ name: 'Data Blob', typeClass: 'Data' }) { }
     }
 
     export namespace Format {
@@ -62,6 +63,18 @@ export namespace PluginStateObject {
         export class Cif extends Create<CifFile>({ name: 'CIF File', typeClass: 'Data' }) { }
         export class Ccp4 extends Create<Ccp4File>({ name: 'CCP4/MRC/MAP File', typeClass: 'Data' }) { }
         export class Dsn6 extends Create<Dsn6File>({ name: 'DSN6/BRIX File', typeClass: 'Data' }) { }
+
+        export type BlobEntry = { id: string } &
+            ( { kind: 'json', data: unknown }
+            | { kind: 'string', data: string }
+            | { kind: 'binary', data: Uint8Array }
+            | { kind: 'cif', data: CifFile }
+            | { kind: 'ccp4', data: Ccp4File }
+            | { kind: 'dsn6', data: Dsn6File }
+            // For non-build in extensions
+            | { kind: 'custom', data: unknown, tag: string })
+        export type BlobData = BlobEntry[]
+        export class Blob extends Create<BlobData>({ name: 'Format Blob', typeClass: 'Data' }) { }
     }
 
     export namespace Molecule {

src/mol-plugin/state/transforms/data.ts

@@ -12,7 +12,7 @@ import CIF from 'mol-io/reader/cif'
 import { PluginContext } from 'mol-plugin/context';
 import { ParamDefinition as PD } from 'mol-util/param-definition';
 import { StateTransformer } from 'mol-state';
-import { readFromFile } from 'mol-util/data-source';
+import { readFromFile, ajaxGetMany } from 'mol-util/data-source';
 import * as CCP4 from 'mol-io/reader/ccp4/parser'
 import * as DSN6 from 'mol-io/reader/dsn6/parser'
 
@@ -47,6 +47,52 @@ const Download = PluginStateTransform.BuiltIn({
     }
 });
 
+export { DownloadBlob }
+type DownloadBlob = typeof DownloadBlob
+const DownloadBlob = PluginStateTransform.BuiltIn({
+    name: 'download-blob',
+    display: { name: 'Download Blob', description: 'Download multiple string or binary data from the specified URLs.' },
+    from: SO.Root,
+    to: SO.Data.Blob,
+    params: {
+        sources: PD.ObjectList({
+            id: PD.Text('', { label: 'Unique ID' }),
+            url: PD.Text('https://www.ebi.ac.uk/pdbe/static/entry/1cbs_updated.cif', { description: 'Resource URL. Must be the same domain or support CORS.' }),
+            isBinary: PD.makeOptional(PD.Boolean(false, { description: 'If true, download data as binary (string otherwise)' })),
+            canFail: PD.makeOptional(PD.Boolean(false, { description: 'Indicate whether the download can fail and not be included in the blob as a result.' }))
+        }, e => `${e.id}: ${e.url}`),
+        maxConcurrency: PD.makeOptional(PD.Numeric(4, { min: 1, max: 12, step: 1 }, { description: 'The maximum number of concurrent downloads.' }))
+    }
+})({
+    apply({ params }, plugin: PluginContext) {
+        return Task.create('Download Blob', async ctx => {
+            const entries: SO.Data.BlobEntry[] = [];
+            const data = await ajaxGetMany(ctx, params.sources, params.maxConcurrency || 4);
+
+            for (let i = 0; i < data.length; i++) {
+                const r = data[i], src = params.sources[i];
+                if (r.kind === 'error') plugin.log.warn(`Download ${r.id} (${src.url}) failed: ${r.error}`);
+                else {
+                    entries.push(src.isBinary
+                        ? { id: r.id, kind: 'binary', data: r.result as Uint8Array }
+                        : { id: r.id, kind: 'string', data: r.result as string });
+                }
+            }
+            return new SO.Data.Blob(entries, { label: 'Data Blob', description: `${entries.length} ${entries.length === 1 ? 'entry' : 'entries'}` });
+        });
+    },
+    // TODO: ??
+    // update({ oldParams, newParams, b }) {
+    //     return 0 as any;
+    //     // if (oldParams.url !== newParams.url || oldParams.isBinary !== newParams.isBinary) return StateTransformer.UpdateResult.Recreate;
+    //     // if (oldParams.label !== newParams.label) {
+    //     //     (b.label as string) = newParams.label || newParams.url;
+    //     //     return StateTransformer.UpdateResult.Updated;
+    //     // }
+    //     // return StateTransformer.UpdateResult.Unchanged;
+    // }
+});
+
 export { ReadFile }
 type ReadFile = typeof ReadFile
 const ReadFile = PluginStateTransform.BuiltIn({
@@ -78,6 +124,50 @@ const ReadFile = PluginStateTransform.BuiltIn({
     isSerializable: () => ({ isSerializable: false, reason: 'Cannot serialize user loaded files.' })
 });
 
+export { ParseBlob }
+type ParseBlob = typeof ParseBlob
+const ParseBlob = PluginStateTransform.BuiltIn({
+    name: 'parse-blob',
+    display: { name: 'Parse Blob', description: 'Parse multiple data enties' },
+    from: SO.Data.Blob,
+    to: SO.Format.Blob,
+    params: {
+        formats: PD.ObjectList({
+            id: PD.Text('', { label: 'Unique ID' }),
+            format: PD.Select<'cif'>('cif', [['cif', 'cif']])
+        }, e => `${e.id}: ${e.format}`)
+    }
+})({
+    apply({ a, params }, plugin: PluginContext) {
+        return Task.create('Parse Blob', async ctx => {
+            const map = new Map<string, string>();
+            for (const f of params.formats) map.set(f.id, f.format);
+
+            const entries: SO.Format.BlobEntry[] = [];
+
+            for (const e of a.data) {
+                if (!map.has(e.id)) continue;
+
+                const parsed = await (e.kind === 'string' ? CIF.parse(e.data) : CIF.parseBinary(e.data)).runInContext(ctx);
+                if (parsed.isError) throw new Error(`${e.id}: ${parsed.message}`);
+                entries.push({ id: e.id, kind: 'cif', data: parsed.result });
+            }
+
+            return new SO.Format.Blob(entries, { label: 'Format Blob', description: `${entries.length} ${entries.length === 1 ? 'entry' : 'entries'}` });
+        });
+    },
+    // TODO: ??
+    // update({ oldParams, newParams, b }) {
+    //     return 0 as any;
+    //     // if (oldParams.url !== newParams.url || oldParams.isBinary !== newParams.isBinary) return StateTransformer.UpdateResult.Recreate;
+    //     // if (oldParams.label !== newParams.label) {
+    //     //     (b.label as string) = newParams.label || newParams.url;
+    //     //     return StateTransformer.UpdateResult.Updated;
+    //     // }
+    //     // return StateTransformer.UpdateResult.Unchanged;
+    // }
+});
+
 export { ParseCif }
 type ParseCif = typeof ParseCif
 const ParseCif = PluginStateTransform.BuiltIn({

src/mol-plugin/state/transforms/model.ts

@@ -25,6 +25,7 @@ import { parseGRO } from 'mol-io/reader/gro/parser';
 import { parseMolScript } from 'mol-script/language/parser';
 import { transpileMolScript } from 'mol-script/script/mol-script/symbols';
 
+export { TrajectoryFromBlob };
 export { TrajectoryFromMmCif };
 export { TrajectoryFromPDB };
 export { TrajectoryFromGRO };
@@ -37,6 +38,30 @@ export { UserStructureSelection };
 export { StructureComplexElement };
 export { CustomModelProperties };
 
+type TrajectoryFromBlob = typeof TrajectoryFromBlob
+const TrajectoryFromBlob = PluginStateTransform.BuiltIn({
+    name: 'trajectory-from-blob',
+    display: { name: 'Parse Blob', description: 'Parse format blob into a single trajectory.' },
+    from: SO.Format.Blob,
+    to: SO.Molecule.Trajectory
+})({
+    apply({ a }) {
+        return Task.create('Parse Format Blob', async ctx => {
+            const models: Model[] = [];
+            for (const e of a.data) {
+                if (e.kind !== 'cif') continue;
+                const block = e.data.blocks[0];
+                const xs = await trajectoryFromMmCIF(block).runInContext(ctx);
+                if (xs.length === 0) throw new Error('No models found.');
+                for (const x of xs) models.push(x);
+            }
+
+            const props = { label: `Trajectory`, description: `${models.length} model${models.length === 1 ? '' : 's'}` };
+            return new SO.Molecule.Trajectory(models, props);
+        });
+    }
+});
+
 type TrajectoryFromMmCif = typeof TrajectoryFromMmCif
 const TrajectoryFromMmCif = PluginStateTransform.BuiltIn({
     name: 'trajectory-from-mmcif',

src/mol-plugin/ui/controls/parameters.tsx

@@ -64,7 +64,7 @@ function controlFor(param: PD.Any): ParamControl | undefined {
         case 'mapped': return MappedControl;
         case 'line-graph': return LineGraphControl;
         case 'script-expression': return ScriptExpressionControl;
-        case 'list': return ListControl;
+        case 'object-list': return ObjectListControl;
         default:
             const _: never = param;
             console.warn(`${_} has no associated UI component`);
@@ -513,7 +513,7 @@ export class MappedControl extends React.PureComponent<ParamProps<PD.Mapped<any>
 }
 
 
-export class ListControl extends React.PureComponent<ParamProps<PD.List>, { isExpanded: boolean }> {
+export class ObjectListControl extends React.PureComponent<ParamProps<PD.ObjectList>, { isExpanded: boolean }> {
     // state = { isExpanded: !!this.props.param.isExpanded }
 
     // change(value: any) {

src/mol-util/data-source.ts

@@ -194,4 +194,58 @@ function ajaxGetInternal(title: string | undefined, url: string, type: 'json' |
     }, () => {
         if (xhttp) xhttp.abort();
     });
+}
+
+export type AjaxGetManyEntry<T> = { kind: 'ok', id: string, result: T } | { kind: 'error', id: string, error: any }
+export async function ajaxGetMany(ctx: RuntimeContext, sources: { id: string, url: string, isBinary?: boolean, canFail?: boolean }[], maxConcurrency: number) {
+    const len = sources.length;
+    const slots: AjaxGetManyEntry<string | Uint8Array>[] = new Array(sources.length);
+
+    await ctx.update({ message: 'Downloading...', current: 0, max: len });
+    let promises: Promise<AjaxGetManyEntry<any> & { index: number }>[] = [], promiseKeys: number[] = [];
+    let currentSrc = 0;
+    for (let _i = Math.min(len, maxConcurrency); currentSrc < _i; currentSrc++) {
+        const current = sources[currentSrc];
+        promises.push(wrapPromise(currentSrc, current.id, ajaxGet({ url: current.url, type: current.isBinary ? 'binary' : 'string' }).runAsChild(ctx)));
+        promiseKeys.push(currentSrc);
+    }
+
+    let done = 0;
+    while (promises.length > 0) {
+        const r = await Promise.race(promises);
+        const src = sources[r.index];
+        const idx = promiseKeys.indexOf(r.index);
+        done++;
+        if (r.kind === 'error' && !src.canFail) {
+            // TODO: cancel other downloads
+            throw new Error(`${src.url}: ${r.error}`);
+        }
+        if (ctx.shouldUpdate) {
+            await ctx.update({ message: 'Downloading...', current: done, max: len });
+        }
+        slots[r.index] = r;
+        promises = promises.filter(_filterRemoveIndex, idx);
+        promiseKeys = promiseKeys.filter(_filterRemoveIndex, idx);
+        if (currentSrc < len) {
+            const current = sources[currentSrc];
+            promises.push(wrapPromise(currentSrc, current.id, ajaxGet({ url: current.url, type: current.isBinary ? 'binary' : 'string' }).runAsChild(ctx)));
+            promiseKeys.push(currentSrc);
+            currentSrc++;
+        }
+    }
+
+    return slots;
+}
+
+function _filterRemoveIndex(this: number, _: any, i: number) {
+    return this !== i;
+}
+
+async function wrapPromise<T>(index: number, id: string, p: Promise<T>): Promise<AjaxGetManyEntry<T> & { index: number }> {
+    try {
+        const result = await p;
+        return { kind: 'ok', result, index, id };
+    } catch (error) {
+        return { kind: 'error', error, index, id }
+    }
 }

src/mol-util/param-definition.ts

@@ -32,6 +32,10 @@ export namespace ParamDefinition {
         defaultValue: T
     }
 
+    export interface Optional<T extends Any = Any> extends Base<T['defaultValue'] | undefined> {
+        type: T['type']
+    }
+
     export function makeOptional<T>(p: Base<T>): Base<T | undefined> {
         p.isOptional = true;
         return p;
@@ -189,13 +193,13 @@ export namespace ParamDefinition {
         }, info);
     }
 
-    export interface List<T = any> extends Base<T[]> {
-        type: 'list',
-        element: Any,
+    export interface ObjectList<T = any> extends Base<T[]> {
+        type: 'object-list',
+        element: Params,
         getLabel(t: T): string
     }
-    export function List<T extends Any>(element: T, getLabel: (e: T['defaultValue']) => string, info?: Info & { defaultValue?: T['defaultValue'][] }): List<T['defaultValue']> {
-        return setInfo<List<T['defaultValue']>>({ type: 'list', element, getLabel, defaultValue: (info && info.defaultValue) || []  });
+    export function ObjectList<T>(element: For<T>, getLabel: (e: T) => string, info?: Info & { defaultValue?: T[] }): ObjectList<Normalize<T>> {
+        return setInfo<ObjectList<Normalize<T>>>({ type: 'object-list', element: element as any as Params, getLabel, defaultValue: (info && info.defaultValue) || []  });
     }
 
     export interface Converted<T, C> extends Base<T> {
@@ -231,7 +235,7 @@ export namespace ParamDefinition {
 
     export type Any =
         | Value<any> | Select<any> | MultiSelect<any> | Boolean | Text | Color | Vec3 | Numeric | FileParam | Interval | LineGraph
-        | ColorScale<any> | Group<any> | Mapped<any> | Converted<any, any> | Conditioned<any, any, any> | ScriptExpression | List
+        | ColorScale<any> | Group<any> | Mapped<any> | Converted<any, any> | Conditioned<any, any, any> | ScriptExpression | ObjectList
 
     export type Params = { [k: string]: Any }
     export type Values<T extends Params> = { [k in keyof T]: T[k]['defaultValue'] }
@@ -322,12 +326,12 @@ export namespace ParamDefinition {
         } else if (p.type === 'script-expression') {
             const u = a as ScriptExpression['defaultValue'], v = b as ScriptExpression['defaultValue'];
             return u.language === v.language && u.expression === v.expression;
-        } else if (p.type === 'list') {
-            const u = a as List['defaultValue'], v = b as List['defaultValue'];
+        } else if (p.type === 'object-list') {
+            const u = a as ObjectList['defaultValue'], v = b as ObjectList['defaultValue'];
             const l = u.length;
             if (l !== v.length) return false;
             for (let i = 0; i < l; i++) {
-                if (!isParamEqual(p.element, u[i], v[i])) return false;
+                if (!areEqual(p.element, u[i], v[i])) return false;
             }
             return true;
         } else if (typeof a === 'object' && typeof b === 'object') {