mol-plugin: DataManager [wip]

package.json

@@ -14,11 +14,11 @@
     "lint": "tslint src/**/*.ts",
     "test": "npm run lint && jest",
     "build": "npm run build-tsc && npm run build-extra && npm run build-webpack",
-    "build-tsc": "tsc",
+    "build-tsc": "tsc --incremental",
     "build-extra": "cpx \"src/**/*.{scss,woff,woff2,ttf,otf,eot,svg,html,ico}\" lib/",
     "build-webpack": "webpack --mode production",
     "watch": "concurrently --kill-others \"npm:watch-tsc\" \"npm:watch-extra\" \"npm:watch-webpack\"",
-    "watch-tsc": "tsc -watch",
+    "watch-tsc": "tsc --watch --incremental",
     "watch-extra": "cpx \"src/**/*.{scss,woff,woff2,ttf,otf,eot,svg,html,ico}\" lib/ --watch",
     "watch-webpack": "webpack -w --mode development --display minimal",
     "serve": "http-server -p 1338",

src/mol-plugin/context.ts

@@ -44,6 +44,7 @@ import { PluginToastManager } from './state/toast';
 import { StructureMeasurementManager } from './util/structure-measurement';
 import { ViewportScreenshotHelper } from './util/viewport-screenshot';
 import { StructureRepresentationManager } from './state/representation/structure';
+import { DataManager } from './state/manager/data';
 
 interface Log {
     entries: List<LogEntry>
@@ -132,6 +133,10 @@ export class PluginContext {
     readonly customStructureProperties = new CustomStructureProperty.Registry();
     readonly customParamEditors = new Map<string, StateTransformParameters.Class>();
 
+    readonly managers = {
+        data: new DataManager(this),
+    } as const;
+
     readonly helpers = {
         measurement: new StructureMeasurementManager(this),
         structureSelectionManager: new StructureElementSelectionManager(this),

src/mol-plugin/state/manager/base.ts

@@ -6,12 +6,16 @@
 
 // TODO: primites
 
-// import { StateObject, State, StateObjectCell, StateBuilder, StateTransformer } from '../../../mol-state';
-// import { RuntimeContext } from '../../../mol-task';
-// import { PluginContext } from '../../context';
+import { StateObject, State, StateObjectCell, StateBuilder, StateTransformer, StateTransform } from '../../../mol-state';
+import { RuntimeContext } from '../../../mol-task';
+import { PluginContext } from '../../context';
 
-// export type StateAction<P = any, O extends StateObject = StateObject, R = {}> =
-//     (ctx: RuntimeContext, state: State, cell: StateObjectCell<O>, params: P, plugin: PluginContext) => Promise<R> | R;
+export { StateAction, BuilderAction }
 
-// export type BuilderAction<P = any, O extends StateObject = StateObject, T extends StateTransformer = StateTransformer, R = {}> =
-//     (builder: StateBuilder.To<O, T>, params: P, plugin: PluginContext) => R;
+type StateAction<P = any, O extends StateObject = StateObject, R = {}> =
+    (cell: StateObjectCell<O>, params: P, ctx: { ctx: RuntimeContext, state: State, plugin: PluginContext }) => Promise<R> | R;
+function StateAction<P = any, O extends StateObject = StateObject, R = {}>(action: StateAction<P, O, R>) { return action; }
+
+type BuilderAction<P = any, O extends StateObject = StateObject, T extends StateTransformer = StateTransformer, R = {}> =
+    (builder: StateBuilder.To<O, T>, params: P, ctx: { options?: Partial<StateTransform.Options>, plugin: PluginContext }) => R;
+function BuilderAction<P = any, O extends StateObject = StateObject, T extends StateTransformer = StateTransformer, R = {}>(action: BuilderAction<P, O, T, R>) { return action; }

src/mol-plugin/state/manager/data.ts

@@ -4,4 +4,55 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
-// TODO: data manager (download / open file / parse)
+import { StateTransformer, StateTransform, StateObjectSelector, StateObjectCell } from '../../../mol-state';
+import { PluginContext } from '../../context';
+import { Download, ReadFile } from '../transforms/data';
+import { getFileInfo } from '../../../mol-util/file-info';
+import { DataFormatProvider } from './data/provider';
+import { BuiltInDataFormats } from './data/formats';
+import { objectForEach } from '../../../mol-util/object';
+
+export class DataManager {
+    readonly formats: DataFormatProvider[] = [];
+
+    addFormat(p: DataFormatProvider) {
+        this.formats.push(p);
+    }
+
+    get dataState() {
+        return this.plugin.state.dataState;
+    }
+
+    async download(params: StateTransformer.Params<Download>, options?: Partial<StateTransform.Options>) {
+        const data = this.dataState.build().toRoot().apply(Download, params, options);
+        await this.plugin.runTask(this.dataState.updateTree(data));
+        return { data: data.selector };
+    }
+
+    async readFile(params: StateTransformer.Params<ReadFile>, options?: Partial<StateTransform.Options>) {
+        const data = this.dataState.build().toRoot().apply(ReadFile, params, options);
+        const fileInfo = getFileInfo(params.file);
+        await this.plugin.runTask(this.dataState.updateTree(data));
+        return { data: data.selector, fileInfo };
+    }
+
+    async parse<K extends keyof BuiltInDataFormats, P extends BuiltInDataFormats[K]>(provider: K, data: StateObjectSelector<DataFormatProvider.Data<P>> | StateObjectCell | string, params?: DataFormatProvider.Params<P>): Promise<DataFormatProvider.Ret<P>>
+    async parse<P extends DataFormatProvider>(provider: P, data: StateObjectSelector<DataFormatProvider.Data<P>> | StateObjectCell | string, params?: DataFormatProvider.Params<P>): Promise<DataFormatProvider.Ret<P>>
+    async parse<P extends DataFormatProvider>(providerOrBuildIn: P | string, data: StateObjectSelector<DataFormatProvider.Data<P>> | StateObjectCell | StateTransform.Ref, params?: DataFormatProvider.Params<P>) {
+        const provider: P = typeof providerOrBuildIn === 'string' ? BuiltInDataFormats[providerOrBuildIn as keyof BuiltInDataFormats] as unknown as P : providerOrBuildIn as P;
+        const cell = StateObjectCell.resolve(this.dataState, data);
+        if (!cell) {
+            throw new Error('Could not resolve data cell.');
+        }
+        return provider.apply({ state: this.dataState, plugin: this.plugin }, cell, params);
+    }
+
+    // async test() {
+    //     const { data } = await this.download({ url: '' });
+    //     const cif = await this.parse('mmcif', data);
+    // }
+
+    constructor(public plugin: PluginContext) {
+        objectForEach(BuiltInDataFormats, f => this.formats.push(f));
+    }
+}

src/mol-plugin/state/manager/data/actions.ts

@@ -0,0 +1,17 @@
+/**
+ * Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import { BuilderAction } from '../base';
+import { StateTransformer } from '../../../../mol-state';
+import { Download as DownloadData, ReadFile } from '../../transforms/data';
+
+export const Download = BuilderAction((builder, params: StateTransformer.Params<DownloadData>, { options }) => {
+    return builder.apply(DownloadData, params, options);
+});
+
+export const OpenFile = BuilderAction((builder, params: StateTransformer.Params<ReadFile>, { options }) => {
+    return builder.apply(ReadFile, params, options);
+});

src/mol-plugin/state/manager/data/formats.ts

@@ -0,0 +1,50 @@
+/**
+ * Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import { DataFormatProvider } from './provider';
+import { PluginContext } from '../../../context';
+import { FileInfo } from '../../../../mol-util/file-info';
+import { StateTransforms } from '../../transforms';
+
+export const MmcifFormatProvider = DataFormatProvider({
+    id: 'mmcif',
+    display: { name: 'mmCIF', group: 'Molecule' },
+    extensions: { text: ['cif', 'mmcif', 'mcif'], binary: ['bcif'] }
+})({
+    isApplicable(plugin: PluginContext, data: string | Uint8Array, info?: FileInfo): boolean {
+        // TODO: check CIF variants
+        return true;
+    },
+    async apply({ plugin, state }, data) {
+        const dictionary = state.build().to(data).apply(StateTransforms.Data.ParseCif, void 0, { state: { isGhost: true } });
+        const trajectory = dictionary.apply(StateTransforms.Model.TrajectoryFromMmCif);
+        await plugin.runTask(state.updateTree(trajectory));
+        return { dictionary: dictionary.selector, trajectory: trajectory.selector };
+    }
+});
+
+export const PdbFormatProvider = DataFormatProvider({
+    id: 'pdb',
+    display: { name: 'PDB', group: 'Molecule' },
+    extensions: { text: ['pdb', 'ent'] }
+})({
+    async apply({ plugin, state }, data) {
+        const trajectory = state.build().to(data).apply(StateTransforms.Model.TrajectoryFromPDB);
+        await plugin.runTask(state.updateTree(trajectory));
+        return { trajectory: trajectory.selector };
+    }
+});
+
+export const BuiltInDataFormats = {
+    'mmcif': MmcifFormatProvider,
+    'pdb': PdbFormatProvider
+}
+export type BuiltInDataFormats = typeof BuiltInDataFormats
+
+// export const TrajectoryFormatProviders = {
+//     'mmcif': MmcifFormatProvider,
+//     'pdb': PdbFormatProvider
+// }

src/mol-plugin/state/manager/data/parsers.ts

@@ -1,8 +0,0 @@
-/**
- * Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
- *
- * @author David Sehnal <david.sehnal@gmail.com>
- */
-
-// TODO: build in format parsers, allow to define custom formats
-// basically state actions that take data and return parsed format in appropriate built in representation (e.g. string|binary => )

src/mol-plugin/state/manager/data/provider.ts

@@ -0,0 +1,50 @@
+/**
+ * Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import { PluginContext } from '../../../context';
+import { State, StateObjectCell } from '../../../../mol-state';
+import { ParamDefinition as PD } from '../../../../mol-util/param-definition';
+import { PluginStateObject } from '../../objects';
+import { FileInfo } from '../../../../mol-util/file-info';
+
+export { DataFormatProvider }
+
+type DataFormatProvider<Id extends string = string, D extends PluginStateObject.Data.Binary | PluginStateObject.Data.String = PluginStateObject.Data.Binary | PluginStateObject.Data.String, P = any, S = {}> =
+    DataFormatProvider.Base<Id, P> & DataFormatProvider.Definition<D, P, S>
+
+function DataFormatProvider<Id extends string, P = {}>(provider: DataFormatProvider.Base<Id, P>) {
+    return function<D extends PluginStateObject.Data.Binary | PluginStateObject.Data.String, S>(p: DataFormatProvider.Definition<D, P, S>): DataFormatProvider<Id, D, P, S> {
+        return { ...provider, ...p };
+    };
+}
+
+namespace DataFormatProvider {
+    export type Ret<P extends DataFormatProvider> = P extends DataFormatProvider<any, any, any, infer T> ? T : never
+    export type Data<P extends DataFormatProvider> = P extends DataFormatProvider<any, infer T, any, any> ? T : never
+    export type Params<P extends DataFormatProvider> = P extends DataFormatProvider<any, any, infer T, any> ? T : never
+
+    export interface Base<Id extends string = string, P = any> {
+        id: Id,
+        display: { name: string, group?: string, description?: string },
+        extensions: { text?: string[], binary?: string[] },
+        params?(plugin: PluginContext, data: string | Uint8Array, info?: FileInfo): PD.Def<P>
+        // TODO: default representation
+        // defaultRepresentation?: RepresenatationProvider
+    }
+    export interface Definition<D extends PluginStateObject.Data.Binary | PluginStateObject.Data.String = PluginStateObject.Data.Binary | PluginStateObject.Data.String, P = any, S = any> {
+        isApplicable?(plugin: PluginContext, data: string | Uint8Array, info?: FileInfo): boolean,
+        apply(ctx: { state: State, plugin: PluginContext }, data: StateObjectCell<D>, params: P): Promise<S>
+    }
+}
+
+// interface DataSourceProvider<Id extends string = string, Format extends DataFormatProvider = DataFormatProvider, P = any, S = {}> {
+//     id: Id,
+//     display: { name: string, group?: string, description?: string },
+//     format: Format,
+//     apply(ctx: { ctx: RuntimeContext, state: State, plugin: PluginContext }, params: P): Promise<S>,
+//     params(plugin: PluginContext): PD.Def<P>,
+// }
+// function DataSourceProvider<Id extends string, Format extends DataFormatProvider, P, S>(provider: DataSourceProvider<Id, Format, P, S>) { return provider; }

src/mol-plugin/state/manager/data/sources.ts

@@ -4,5 +4,24 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
+// import { DataSourceProvider } from './provider';
+// import { MmcifFormatProvider } from './formats';
+// import { ParamDefinition as PD } from '../../../../mol-util/param-definition';
+// import { Download } from './actions';
+
 // TODO: basic types string, binary (both download and from file)
-// TODO: decompress functionality (string|binary --> string|binary)
+// TODO: decompress functionality (string|binary --> string|binary)
+
+// export const PDBeUpdatedMmcifDataSource = DataSourceProvider({
+//     id: 'pdbe-updated-mmcif',
+//     display: { name: 'PDBe Updated mmCIF', group: 'Molecule' },
+//     format: MmcifFormatProvider,
+//     params() {
+//         return { id: PD.Text('1cbs', { label: 'PDB Id(s)', description: 'One or more comma separated PDB ids.' }) }
+//     },
+//     async apply(ctx, params) {
+//         const download = Download(ctx.state.build().toRoot(), { url: `https://www.ebi.ac.uk/pdbe/static/entry/${params.id.toLowerCase()}_updated.cif` }, ctx);
+//         await ctx.state.updateTree(download).runInContext(ctx.ctx);
+//         return 0;
+//     }
+// })

src/mol-plugin/util/task-manager.ts

@@ -4,9 +4,11 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
-import { Task, Progress } from '../../mol-task';
+import { Task, Progress, RuntimeContext } from '../../mol-task';
 import { RxEventHelper } from '../../mol-util/rx-event-helper';
 import { now } from '../../mol-util/now';
+import { CreateObservableCtx, ExecuteInContext } from '../../mol-task/execution/observable';
+import { arrayRemoveInPlace } from '../../mol-util/array';
 
 export { TaskManager }
 
@@ -14,6 +16,7 @@ class TaskManager {
     private ev = RxEventHelper.create();
     private id = 0;
     private abortRequests = new Map<number, string | undefined>();
+    private currentContext: { ctx: RuntimeContext, refCount: number }[] = [];
 
     readonly events = {
         progress: this.ev<TaskManager.ProgressEvent>(),
@@ -34,14 +37,26 @@ class TaskManager {
         };
     }
 
-    async run<T>(task: Task<T>): Promise<T> {
+    async run<T>(task: Task<T>, createNewContext = false): Promise<T> {
         const id = this.id++;
+
+        let ctx: TaskManager['currentContext'][0];
+
+        if (createNewContext || this.currentContext.length === 0) {
+            ctx = { ctx: CreateObservableCtx(task, this.track(id, task.id), 100), refCount: 1 };
+        } else {
+            ctx = this.currentContext[this.currentContext.length - 1];
+            ctx.refCount++;
+        }
+
         try {
-            const ret = await task.run(this.track(id, task.id), 100);
+            const ret = await ExecuteInContext(ctx.ctx, task);
             return ret;
         } finally {
             this.events.finished.next({ id });
             this.abortRequests.delete(task.id);
+            ctx.refCount--;
+            if (ctx.refCount === 0) arrayRemoveInPlace(this.currentContext, ctx);
         }
     }
 

src/mol-state/object.ts

@@ -93,6 +93,17 @@ namespace StateObjectCell {
         const c: StateObjectCell = o;
         return !!c && !!c.transform && !!c.parent && !!c.status;
     }
+
+    export type Ref = StateTransform.Ref | StateObjectCell | StateObjectSelector
+
+    export function resolve(state: State, refOrCellOrSelector: StateTransform.Ref | StateObjectCell | StateObjectSelector) {
+        const ref = typeof refOrCellOrSelector === 'string'
+            ? refOrCellOrSelector
+            : StateObjectCell.is(refOrCellOrSelector)
+            ? refOrCellOrSelector.transform.ref
+            : refOrCellOrSelector.ref;
+        return state.cells.get(ref);
+    }
 }
 
 // TODO: improve the API?

src/mol-task/execution/observable.ts

@@ -23,6 +23,11 @@ export function ExecuteObservable<T>(task: Task<T>, observer: Progress.Observer,
     return execute(task as ExposedTask<T>, ctx);
 }
 
+export function CreateObservableCtx<T>(task: Task<T>, observer: Progress.Observer, updateRateMs = 250) {
+    const info = ProgressInfo(task, observer, updateRateMs);
+    return new ObservableRuntimeContext(info, info.root);
+}
+
 export function ExecuteInContext<T>(ctx: RuntimeContext, task: Task<T>) {
     return execute(task as ExposedTask<T>, ctx as ObservableRuntimeContext);
 }