Merge branch 'master' into cartoon-repr

src/mol-app/ui/visualization/sequence-view.tsx

@@ -34,14 +34,14 @@ function createQuery(entityId: string, label_seq_id: number) {
 // TODO: this is really ineffective and should be done using a canvas.
 class EntitySequence extends React.Component<{ ctx: Context, seq: StructureSequence.Entity, structure: Structure }> {
 
-    async raiseInteractityEvent(seqId?: number) {
+    raiseInteractityEvent(seqId?: number) {
         if (typeof seqId === 'undefined') {
             InteractivityEvents.HighlightLoci.dispatch(this.props.ctx, EmptyLoci);
             return;
         }
 
         const query = createQuery(this.props.seq.entityId, seqId);
-        const loci = StructureSelection.toLoci(await StructureQuery.run(query, this.props.structure));
+        const loci = StructureSelection.toLoci(StructureQuery.run(query, this.props.structure));
         if (loci.elements.length === 0) InteractivityEvents.HighlightLoci.dispatch(this.props.ctx, EmptyLoci);
         else InteractivityEvents.HighlightLoci.dispatch(this.props.ctx, loci);
     }

src/mol-model/structure/export/categories/atom_site.ts

@@ -39,7 +39,7 @@ const atom_site_fields: CifField<StructureElement>[] = [
 
     CifField.int('pdbx_PDB_model_num', P.unit.model_num, { encoder: E.deltaRLE }),
     CifField.str<StructureElement, Structure>('operator_name', P.unit.operator_name, {
-        shouldInclude: structure => { console.log(!!structure); return structure.units.some(u => !u.conformation.operator.isIdentity) }
+        shouldInclude: structure => structure.units.some(u => !u.conformation.operator.isIdentity)
     })
 ];
 

src/mol-model/structure/model/properties/utils/atomic-ranges.ts

@@ -65,9 +65,6 @@ export function getAtomicRanges(data: AtomicData, segments: AtomicSegments, chem
         }
     }
 
-    console.log('polymerRanges', polymerRanges)
-    console.log('gapRanges', gapRanges)
-
     return {
         polymerRanges: SortedRanges.ofSortedRanges(polymerRanges as ElementIndex[]),
         gapRanges: SortedRanges.ofSortedRanges(gapRanges as ElementIndex[])

src/mol-model/structure/model/properties/utils/coarse-ranges.ts

@@ -22,7 +22,6 @@ export function getCoarseRanges(data: CoarseElementData, chemicalComponentMap: M
 
     while (chainIt.hasNext) {
         const { start, end } = chainIt.move();
-        console.log('chain', start, end)
 
         let startIndex = -1
         let prevSeqEnd = -1
@@ -45,8 +44,6 @@ export function getCoarseRanges(data: CoarseElementData, chemicalComponentMap: M
         }
     }
 
-    console.log(polymerRanges, gapRanges)
-
     return {
         polymerRanges: SortedRanges.ofSortedRanges(polymerRanges as ElementIndex[]),
         gapRanges: SortedRanges.ofSortedRanges(gapRanges as ElementIndex[])

src/mol-model/structure/query.ts

@@ -7,8 +7,8 @@
 import { StructureSelection } from './query/selection'
 import { StructureQuery } from './query/query'
 export * from './query/context'
-import * as generators from './query/generators'
-import * as modifiers from './query/modifiers'
+import * as generators from './query/queries/generators'
+import * as modifiers from './query/queries/modifiers'
 import pred from './query/predicates'
 
 export const Queries = {

src/mol-model/structure/query/context.ts

@@ -4,35 +4,57 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
-import { RuntimeContext } from 'mol-task';
 import { Structure, StructureElement } from '../structure';
+import { now } from 'mol-task';
 
 export interface QueryContextView {
     readonly element: StructureElement;
+    readonly currentStructure: Structure;
 }
 
 export class QueryContext implements QueryContextView {
-    private currentStack: StructureElement[] = [];
+    private currentElementStack: StructureElement[] = [];
+    private currentStructureStack: Structure[] = [];
+    private timeCreated = now();
+    private timeoutMs: number;
 
-    readonly structure: Structure;
-    readonly taskCtx: RuntimeContext;
+    readonly inputStructure: Structure;
 
     /** Current element */
     readonly element: StructureElement = StructureElement.create();
+    currentStructure: Structure = void 0 as any;
 
     pushCurrentElement(): StructureElement {
-        this.currentStack[this.currentStack.length] = this.element;
+        this.currentElementStack[this.currentElementStack.length] = this.element;
         (this.element as StructureElement) = StructureElement.create();
         return this.element;
     }
 
     popCurrentElement() {
-        (this.element as StructureElement) = this.currentStack.pop()!;
+        (this.element as StructureElement) = this.currentElementStack.pop()!;
     }
 
-    constructor(structure: Structure, taskCtx: RuntimeContext) {
-        this.structure = structure;
-        this.taskCtx = taskCtx;
+    pushCurrentStructure() {
+        if (this.currentStructure) this.currentStructureStack.push(this.currentStructure);
+    }
+
+    popCurrentStructure() {
+        if (this.currentStructureStack.length) (this.currentStructure as Structure) = this.currentStructureStack.pop()!;
+        else (this.currentStructure as Structure) = void 0 as any;
+    }
+
+    throwIfTimedOut() {
+        if (this.timeoutMs === 0) return;
+        if (now() - this.timeCreated > this.timeoutMs) {
+            throw new Error(`The query took too long to execute (> ${this.timeoutMs / 1000}s).`);
+        }
+    }
+
+    // todo timeout
+
+    constructor(structure: Structure, timeoutMs = 0) {
+        this.inputStructure = structure;
+        this.timeoutMs = timeoutMs;
     }
 }
 

src/mol-model/structure/query/queries/combinators.ts

@@ -0,0 +1,23 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import { StructureQuery } from '../query';
+import { StructureSelection } from '../selection';
+
+export function merge(queries: ArrayLike<StructureQuery>): StructureQuery {
+    return ctx => {
+        const ret = StructureSelection.UniqueBuilder(ctx.inputStructure);
+        for (let i = 0; i < queries.length; i++) {
+            StructureSelection.forEach(queries[i](ctx), (s, j) => {
+                ret.add(s);
+                if (i % 100) ctx.throwIfTimedOut();
+            });
+        }
+        return ret.getSelection();
+    }
+}
+
+// TODO: intersect, distanceCluster

src/mol-model/structure/query/queries/filters.ts

@@ -0,0 +1,104 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import { isSuperset } from 'mol-util/set';
+import { Unit } from '../../structure';
+import { QueryContext, QueryFn, QueryPredicate } from '../context';
+import { StructureQuery } from '../query';
+import { StructureSelection } from '../selection';
+import { structureAreIntersecting } from '../utils/structure';
+
+export function pick(query: StructureQuery, pred: QueryPredicate): StructureQuery {
+    return ctx => {
+        const sel = query(ctx);
+        const ret = StructureSelection.LinearBuilder(ctx.inputStructure);
+        ctx.pushCurrentElement();
+        StructureSelection.forEach(sel, (s, i) => {
+            ctx.currentStructure = s;
+            if (pred(ctx)) ret.add(s);
+            if (i % 100) ctx.throwIfTimedOut();
+        });
+        ctx.popCurrentStructure();
+        return ret.getSelection();
+    };
+}
+
+export interface UnitTypeProperties { atomic?: QueryFn, coarse?: QueryFn }
+
+export function getCurrentStructureProperties(ctx: QueryContext, props: UnitTypeProperties, set: Set<any>) {
+    const { units } = ctx.currentStructure;
+    const l = ctx.pushCurrentElement();
+
+    for (const unit of units) {
+        l.unit = unit;
+        const elements = unit.elements;
+
+        let fn;
+        if (Unit.isAtomic(unit)) fn = props.atomic;
+        else fn = props.coarse;
+        if (!fn) continue;
+
+        for (let j = 0, _j = elements.length; j < _j; j++) {
+            l.element = elements[j];
+            set.add(fn(ctx));
+        }
+
+        ctx.throwIfTimedOut();
+    }
+    ctx.popCurrentElement();
+    return set;
+}
+
+function getSelectionProperties(ctx: QueryContext, query: StructureQuery, props: UnitTypeProperties) {
+    const set = new Set();
+
+    const sel = query(ctx);
+    ctx.pushCurrentElement();
+    StructureSelection.forEach(sel, (s, i) => {
+        ctx.currentStructure = s;
+        getCurrentStructureProperties(ctx, props, set);
+
+        if (i % 10) ctx.throwIfTimedOut();
+    });
+    ctx.popCurrentElement();
+    return set;
+}
+
+export function withSameAtomProperties(query: StructureQuery, propertySource: StructureQuery, props: UnitTypeProperties): StructureQuery {
+    return ctx => {
+        const sel = query(ctx);
+        const propSet = getSelectionProperties(ctx, propertySource, props);
+
+        const ret = StructureSelection.LinearBuilder(ctx.inputStructure);
+        ctx.pushCurrentStructure();
+        StructureSelection.forEach(sel, (s, i) => {
+            ctx.currentStructure = s;
+            const currentProps = getCurrentStructureProperties(ctx, props, new Set());
+            if (isSuperset(currentProps, propSet)) {
+                ret.add(s);
+            }
+
+            if (i % 10) ctx.throwIfTimedOut();
+        });
+        ctx.popCurrentStructure();
+        return ret.getSelection();
+    };
+}
+
+export function areIntersectedBy(query: StructureQuery, by: StructureQuery): StructureQuery {
+    return ctx => {
+        const mask = StructureSelection.unionStructure(by(ctx));
+        const ret = StructureSelection.LinearBuilder(ctx.inputStructure);
+
+        StructureSelection.forEach(query(ctx), (s, i) => {
+            if (structureAreIntersecting(mask, s)) ret.add(s);
+            if (i % 10) ctx.throwIfTimedOut();
+        });
+        return ret.getSelection();
+    };
+}
+
+// TODO: within, isConnectedTo

src/mol-model/structure/query/generators.ts → src/mol-model/structure/query/queries/generators.ts

@@ -4,14 +4,14 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
-import { StructureQuery } from './query'
-import { StructureSelection } from './selection'
-import { Unit, StructureProperties as P } from '../structure'
+import { StructureQuery } from '../query'
+import { StructureSelection } from '../selection'
+import { Unit, StructureProperties as P } from '../../structure'
 import { Segmentation } from 'mol-data/int'
-import { LinearGroupingBuilder } from './utils/builders';
-import { QueryPredicate, QueryFn, QueryContextView } from './context';
+import { LinearGroupingBuilder } from '../utils/builders';
+import { QueryPredicate, QueryFn, QueryContextView } from '../context';
 
-export const all: StructureQuery = async (ctx) => StructureSelection.Singletons(ctx.structure, ctx.structure);
+export const all: StructureQuery = ctx => StructureSelection.Singletons(ctx.inputStructure, ctx.inputStructure);
 
 export interface AtomsQueryParams {
     entityTest: QueryPredicate,
@@ -44,13 +44,12 @@ export function atoms(params?: Partial<AtomsQueryParams>): StructureQuery {
 }
 
 function atomGroupsLinear(atomTest: QueryPredicate): StructureQuery {
-    return async (ctx) => {
-        const { structure } = ctx;
-        const { units } = structure;
+    return ctx => {
+        const { inputStructure } = ctx;
+        const { units } = inputStructure;
         const l = ctx.pushCurrentElement();
-        const builder = structure.subsetBuilder(true);
+        const builder = inputStructure.subsetBuilder(true);
 
-        let progress = 0;
         for (const unit of units) {
             l.unit = unit;
             const elements = unit.elements;
@@ -62,22 +61,20 @@ function atomGroupsLinear(atomTest: QueryPredicate): StructureQuery {
             }
             builder.commitUnit();
 
-            progress++;
-            if (ctx.taskCtx.shouldUpdate) await ctx.taskCtx.update({ message: 'Atom Groups', current: progress, max: units.length });
+            ctx.throwIfTimedOut();
         }
         ctx.popCurrentElement();
-        return StructureSelection.Singletons(structure, builder.getStructure());
+        return StructureSelection.Singletons(inputStructure, builder.getStructure());
     };
 }
 
 function atomGroupsSegmented({ entityTest, chainTest, residueTest, atomTest }: AtomsQueryParams): StructureQuery {
-    return async (ctx) => {
-        const { structure } = ctx;
-        const { units } = structure;
+    return ctx => {
+        const { inputStructure } = ctx;
+        const { units } = inputStructure;
         const l = ctx.pushCurrentElement();
-        const builder = structure.subsetBuilder(true);
+        const builder = inputStructure.subsetBuilder(true);
 
-        let progress = 0;
         for (const unit of units) {
             if (unit.kind !== Unit.Kind.Atomic) continue;
 
@@ -111,22 +108,20 @@ function atomGroupsSegmented({ entityTest, chainTest, residueTest, atomTest }: A
             }
             builder.commitUnit();
 
-            progress++;
-            if (ctx.taskCtx.shouldUpdate) await ctx.taskCtx.update({ message: 'Atom Groups', current: progress, max: units.length });
+            ctx.throwIfTimedOut();
         }
         ctx.popCurrentElement();
-        return StructureSelection.Singletons(structure, builder.getStructure());
+        return StructureSelection.Singletons(inputStructure, builder.getStructure());
     };
 }
 
 function atomGroupsGrouped({ entityTest, chainTest, residueTest, atomTest, groupBy }: AtomsQueryParams): StructureQuery {
-    return async (ctx) => {
-        const { structure } = ctx;
-        const { units } = structure;
+    return ctx => {
+        const { inputStructure } = ctx;
+        const { units } = inputStructure;
         const l = ctx.pushCurrentElement();
-        const builder = new LinearGroupingBuilder(structure);
+        const builder = new LinearGroupingBuilder(inputStructure);
 
-        let progress = 0;
         for (const unit of units) {
             if (unit.kind !== Unit.Kind.Atomic) continue;
 
@@ -156,8 +151,7 @@ function atomGroupsGrouped({ entityTest, chainTest, residueTest, atomTest, group
                 }
             }
 
-            progress++;
-            if (ctx.taskCtx.shouldUpdate) await ctx.taskCtx.update({ message: 'Atom Groups', current: progress, max: units.length });
+            ctx.throwIfTimedOut();
         }
         ctx.popCurrentElement();
         return builder.getSelection();

src/mol-model/structure/query/modifiers.ts → src/mol-model/structure/query/queries/modifiers.ts

@@ -5,14 +5,14 @@
  */
 
 import { Segmentation } from 'mol-data/int';
-import { RuntimeContext } from 'mol-task';
-import { Structure, Unit } from '../structure';
-import { StructureQuery } from './query';
-import { StructureSelection } from './selection';
-import { UniqueStructuresBuilder } from './utils/builders';
-import { StructureUniqueSubsetBuilder } from '../structure/util/unique-subset-builder';
+import { Structure, Unit } from '../../structure';
+import { StructureQuery } from '../query';
+import { StructureSelection } from '../selection';
+import { UniqueStructuresBuilder } from '../utils/builders';
+import { StructureUniqueSubsetBuilder } from '../../structure/util/unique-subset-builder';
+import { QueryContext } from '../context';
 
-function getWholeResidues(ctx: RuntimeContext, source: Structure, structure: Structure) {
+function getWholeResidues(ctx: QueryContext, source: Structure, structure: Structure) {
     const builder = source.subsetBuilder(true);
     for (const unit of structure.units) {
         if (unit.kind !== Unit.Kind.Atomic) {
@@ -33,22 +33,21 @@ function getWholeResidues(ctx: RuntimeContext, source: Structure, structure: Str
             }
         }
         builder.commitUnit();
+
+        ctx.throwIfTimedOut();
     }
     return builder.getStructure();
 }
 
 export function wholeResidues(query: StructureQuery, isFlat: boolean): StructureQuery {
-    return async (ctx) => {
-        const inner = await query(ctx);
+    return ctx => {
+        const inner = query(ctx);
         if (StructureSelection.isSingleton(inner)) {
-            return StructureSelection.Singletons(ctx.structure, getWholeResidues(ctx.taskCtx, ctx.structure, inner.structure));
+            return StructureSelection.Singletons(ctx.inputStructure, getWholeResidues(ctx, ctx.inputStructure, inner.structure));
         } else {
-            const builder = new UniqueStructuresBuilder(ctx.structure);
-            let progress = 0;
+            const builder = new UniqueStructuresBuilder(ctx.inputStructure);
             for (const s of inner.structures) {
-                builder.add(getWholeResidues(ctx.taskCtx, ctx.structure, s));
-                progress++;
-                if (ctx.taskCtx.shouldUpdate) await ctx.taskCtx.update({ message: 'Whole Residues', current: progress, max: inner.structures.length });
+                builder.add(getWholeResidues(ctx, ctx.inputStructure, s));
             }
             return builder.getSelection();
         }
@@ -64,12 +63,11 @@ export interface IncludeSurroundingsParams {
     wholeResidues?: boolean
 }
 
-async function getIncludeSurroundings(ctx: RuntimeContext, source: Structure, structure: Structure, params: IncludeSurroundingsParams) {
+function getIncludeSurroundings(ctx: QueryContext, source: Structure, structure: Structure, params: IncludeSurroundingsParams) {
     const builder = new StructureUniqueSubsetBuilder(source);
     const lookup = source.lookup3d;
     const r = params.radius;
 
-    let progress = 0;
     for (const unit of structure.units) {
         const { x, y, z } = unit.conformation;
         const elements = unit.elements;
@@ -77,23 +75,23 @@ async function getIncludeSurroundings(ctx: RuntimeContext, source: Structure, st
             const e = elements[i];
             lookup.findIntoBuilder(x(e), y(e), z(e), r, builder);
         }
-        progress++;
-        if (progress % 2500 === 0 && ctx.shouldUpdate) await ctx.update({ message: 'Include Surroudnings', isIndeterminate: true });
+
+        ctx.throwIfTimedOut();
     }
     return !!params.wholeResidues ? getWholeResidues(ctx, source, builder.getStructure()) : builder.getStructure();
 }
 
 export function includeSurroundings(query: StructureQuery, params: IncludeSurroundingsParams): StructureQuery {
-    return async (ctx) => {
-        const inner = await query(ctx);
+    return ctx => {
+        const inner = query(ctx);
         if (StructureSelection.isSingleton(inner)) {
-            const surr = await getIncludeSurroundings(ctx.taskCtx, ctx.structure, inner.structure, params);
-            const ret = StructureSelection.Singletons(ctx.structure, surr);
+            const surr = getIncludeSurroundings(ctx, ctx.inputStructure, inner.structure, params);
+            const ret = StructureSelection.Singletons(ctx.inputStructure, surr);
             return ret;
         } else {
-            const builder = new UniqueStructuresBuilder(ctx.structure);
+            const builder = new UniqueStructuresBuilder(ctx.inputStructure);
             for (const s of inner.structures) {
-                builder.add(await getIncludeSurroundings(ctx.taskCtx, ctx.structure, s, params));
+                builder.add(getIncludeSurroundings(ctx, ctx.inputStructure, s, params));
             }
             return builder.getSelection();
         }

src/mol-model/structure/query/query.ts

@@ -4,19 +4,14 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
-import { RuntimeContext, Task } from 'mol-task'
 import { Structure } from '../structure'
 import { StructureSelection } from './selection'
 import { QueryContext } from './context';
 
-interface StructureQuery { (ctx: QueryContext): Promise<StructureSelection> }
+interface StructureQuery { (ctx: QueryContext): StructureSelection }
 namespace StructureQuery {
-    export function run(query: StructureQuery, structure: Structure, ctx?: RuntimeContext) {
-        return query(new QueryContext(structure, ctx || RuntimeContext.Synchronous))
-    }
-
-    export function asTask(query: StructureQuery, structure: Structure) {
-        return Task.create('Structure Query', ctx => query(new QueryContext(structure, ctx)));
+    export function run(query: StructureQuery, structure: Structure, timeoutMs = 0) {
+        return query(new QueryContext(structure, timeoutMs));
     }
 }
 

src/mol-model/structure/query/selection.ts

@@ -101,6 +101,24 @@ namespace StructureSelection {
     export function LinearBuilder(structure: Structure): Builder { return new LinearBuilderImpl(structure); }
     export function UniqueBuilder(structure: Structure): Builder { return new HashBuilderImpl(structure); }
 
+    export function forEach(sel: StructureSelection, fn: (s: Structure, i: number) => void) {
+        let idx = 0;
+        if (StructureSelection.isSingleton(sel)) {
+            for (const unit of sel.structure.units) {
+                const { elements } = unit;
+                for (let i = 0, _i = elements.length; i < _i; i++) {
+                    // TODO: optimize this somehow???
+                    const s = Structure.create([unit.getChild(SortedArray.ofSingleton(elements[i]))]);
+                    fn(s, idx++);
+                }
+            }
+        } else {
+            for (const s of sel.structures) {
+                fn(s, idx++);
+            }
+        }
+    }
+
     // TODO: spatial lookup
 }
 

src/mol-model/structure/structure/symmetry.ts

@@ -25,10 +25,10 @@ namespace StructureSymmetry {
 
             const assembler = Structure.Builder();
 
-            const queryCtx = new QueryContext(structure, ctx);
+            const queryCtx = new QueryContext(structure);
 
             for (const g of assembly.operatorGroups) {
-                const selection = await g.selector(queryCtx);
+                const selection = g.selector(queryCtx);
                 if (StructureSelection.structureCount(selection) === 0) {
                     continue;
                 }

src/perf-tests/structure.ts

@@ -331,7 +331,7 @@ export namespace PropertyAccess {
             radius: 5,
             wholeResidues: true
         });
-        const surr = StructureSelection.unionStructure(await StructureQuery.run(q1, a));
+        const surr = StructureSelection.unionStructure(StructureQuery.run(q1, a));
         console.timeEnd('symmetry')
 
         // for (const u of surr.units) {
@@ -444,13 +444,13 @@ export namespace PropertyAccess {
         //console.log(to_mmCIF('test', Selection.union(q0r)));
 
         console.time('q1')
-        await query(q1, structures[0]);
+        query(q1, structures[0]);
         console.timeEnd('q1')
         console.time('q1')
-        await query(q1, structures[0]);
+        query(q1, structures[0]);
         console.timeEnd('q1')
         console.time('q2')
-        const q2r = await query(q2, structures[0]);
+        const q2r = query(q2, structures[0]);
         console.timeEnd('q2')
         console.log(StructureSelection.structureCount(q2r));
         //console.log(q1(structures[0]));
@@ -461,8 +461,8 @@ export namespace PropertyAccess {
             //.add('test q', () => q1(structures[0]))
             //.add('test q', () => q(structures[0]))
             .add('test int', () => sumProperty(structures[0], l => col(l.element)))
-            .add('test q1', async () => await query(q1, structures[0]))
-            .add('test q3', async () => await query(q3, structures[0]))
+            .add('test q1', async () => query(q1, structures[0]))
+            .add('test q3', async () => query(q3, structures[0]))
             // .add('sum residue', () => sumPropertyResidue(structures[0], l => l.unit.hierarchy.residues.auth_seq_id.value(l.unit.residueIndex[l.atom])))
 
             // .add('baseline', () =>  baseline(models[0]))

src/perf-tests/tasks.ts

@@ -98,20 +98,80 @@ export namespace Tasks {
             .on('cycle', (e: any) => console.log(String(e.target)))
             .run();
     }
+
+    function add(x: number, y: number) {
+        return x + y;
+    }
+
+    // async function addAs(x: number, y: number) {
+    //     return x + y;
+    // }
+
+    async function opAsync(n: number) {
+        let ret = 0;
+        for (let i = 0; i < n; i++) {
+            const v = add(i, i + 1);
+            ret += (v as any).then ? await v : v;
+        }
+        return ret;
+    }
+
+    function opNormal(n: number) {
+        let ret = 0;
+        for (let i = 0; i < n; i++) {
+            ret += add(i, i + 1);
+        }
+        return ret;
+    }
+
+    export async function awaitF() {
+        const N = 10000000;
+
+        console.time('async');
+        console.log(await opAsync(N));
+        console.timeEnd('async');
+
+        console.time('async');
+        console.log(await opAsync(N));
+        console.timeEnd('async');
+
+        console.time('async');
+        console.log(await opAsync(N));
+        console.timeEnd('async');
+
+        console.time('normal');
+        console.log(opNormal(N));
+        console.timeEnd('normal');
+        console.time('normal');
+        console.log(opNormal(N));
+        console.timeEnd('normal');
+        console.time('normal');
+        console.log(opNormal(N));
+        console.timeEnd('normal');
+
+        // const suite = new B.Suite();
+        // suite
+        //     .add(`async`, async () => { return await opAsync(100000); })
+        //     .add(`normal`, () => { return opNormal(100000); })
+        //     .on('cycle', (e: any) => console.log(String(e.target)))
+        //     .run();
+    }
 }
 
 (async function() {
     // await Tasks.testImmediate();
     // await Tasks.testImmediate();
 
-    await Tasks.baseline();
-    await Tasks.yielding();
-    await Tasks.yielding1();
-    await Tasks.testYielding();
-    await Tasks.baseline();
-    await Tasks.yielding();
-    await Tasks.yielding1();
-    await Tasks.testYielding();
+    // await Tasks.baseline();
+    // await Tasks.yielding();
+    // await Tasks.yielding1();
+    // await Tasks.testYielding();
+    // await Tasks.baseline();
+    // await Tasks.yielding();
+    // await Tasks.yielding1();
+    // await Tasks.testYielding();
+
+    await Tasks.awaitF();
 }())
 
 // console.time('test')

src/servers/model/config.ts

@@ -44,9 +44,12 @@ const config = {
      */
     maxQueryTimeInMs: 5 * 1000,
 
+    /** Maximum number of requests before "server busy" */
+    maxQueueLength: 30,
+
     /**
      * Maps a request identifier to a filename.
-     * 
+     *
      * @param source
      *   Source of the data.
      * @param id

src/servers/model/local.ts

@@ -0,0 +1,62 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import * as fs from 'fs'
+import Version from './version';
+import { LocalInput, runLocal } from './server/api-local';
+
+console.log(`Mol* ModelServer (${Version}), (c) 2018 Mol* authors`);
+console.log(``);
+
+let exampleWorkload: LocalInput = [{
+        input: 'c:/test/quick/1tqn.cif',
+        output: 'c:/test/quick/localapi/1tqn_full.cif',
+        query: 'full', // same as defined in Api/Queries
+    }, {
+        input: 'c:/test/quick/1tqn.cif',
+        output: 'c:/test/quick/localapi/1tqn_full.bcif',
+        query: 'full',
+        params: { binary: true }
+    }, {
+        input: 'c:/test/quick/1cbs_updated.cif',
+        output: 'c:/test/quick/localapi/1cbs_ligint.cif',
+        query: 'residueInteraction', // action is case sensitive
+        params: { label_comp_id: 'REA' }
+    }, {
+        input: 'c:/test/quick/1cbs_updated.cif', // multiple files that are repeated will only be parsed once
+        output: 'c:/test/quick/localapi/1cbs_ligint.bcif',
+        query: 'residueInteraction',
+        params: { label_comp_id: 'REA', binary: true } // parameters are just a JSON version of the query string
+    }
+];
+
+
+if (process.argv.length !== 3) {
+    let help = [
+        `Usage: `,
+        ``,
+        `   node local jobs.json`,
+        ``,
+        `jobs.json is a JSON version of the WebAPI. Query names are case sensitive.`,
+        `The jobs are automatically sorted by inputFilenama and the given file is only loaded once.`,
+        `All processing errors are sent to stderr.`,
+        ``,
+        `Jobs example:`,
+        ``,
+        JSON.stringify(exampleWorkload, null, 2)
+    ];
+
+    console.log(help.join('\n'));
+} else {
+    try {
+        const input = JSON.parse(fs.readFileSync(process.argv[2], 'utf8'));
+        runLocal(input);
+    } catch (e) {
+        console.error(e);
+    }
+}
+
+// TODO: write utility that splits jobs into multiple chunks?

src/servers/model/server.ts

@@ -9,7 +9,7 @@ import * as compression from 'compression'
 import ServerConfig from './config'
 import { ConsoleLogger } from 'mol-util/console-logger';
 import { PerformanceMonitor } from 'mol-util/performance-monitor';
-import { initWebApi } from './server/web-api';
+import { initWebApi } from './server/api-web';
 import Version from './version'
 
 function setupShutdown() {

src/servers/model/server/api-local.ts

@@ -0,0 +1,107 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+import { JobManager, Job } from './jobs';
+import { ConsoleLogger } from 'mol-util/console-logger';
+import { resolveJob } from './query';
+import { StructureCache } from './structure-wrapper';
+import { now } from 'mol-task';
+import { PerformanceMonitor } from 'mol-util/performance-monitor';
+
+export type LocalInput = {
+    input: string,
+    output: string,
+    query: string,
+    params?: any
+}[];
+
+export async function runLocal(input: LocalInput) {
+    if (!input.length) {
+        ConsoleLogger.error('Local', 'No input');
+        return;
+    }
+
+    for (const job of input) {
+        JobManager.add('_local_', job.input, job.query, job.params || { }, job.output);
+    }
+    JobManager.sort();
+
+    const started = now();
+
+    let job: Job | undefined = JobManager.getNext();
+    let key = job.key;
+    let progress = 0;
+    while (job) {
+        try {
+            const encoder = await resolveJob(job);
+            const writer = wrapFile(job.outputFilename!);
+            encoder.writeTo(writer);
+            writer.end();
+            ConsoleLogger.logId(job.id, 'Query', 'Written.');
+
+            if (JobManager.hasNext()) {
+                job = JobManager.getNext();
+                if (key !== job.key) StructureCache.expire(key);
+                key = job.key;
+            } else {
+                break;
+            }
+        } catch (e) {
+            ConsoleLogger.errorId(job.id, e);
+        }
+        ConsoleLogger.log('Progress', `[${++progress}/${input.length}] after ${PerformanceMonitor.format(now() - started)}.`);
+    }
+
+    ConsoleLogger.log('Progress', `Done in ${PerformanceMonitor.format(now() - started)}.`);
+    StructureCache.expireAll();
+}
+
+function wrapFile(fn: string) {
+    const w = {
+        open(this: any) {
+            if (this.opened) return;
+            makeDir(path.dirname(fn));
+            this.file = fs.openSync(fn, 'w');
+            this.opened = true;
+        },
+        writeBinary(this: any, data: Uint8Array) {
+            this.open();
+            fs.writeSync(this.file, new Buffer(data));
+            return true;
+        },
+        writeString(this: any, data: string) {
+            this.open();
+            fs.writeSync(this.file, data);
+            return true;
+        },
+        end(this: any) {
+            if (!this.opened || this.ended) return;
+            fs.close(this.file, function () { });
+            this.ended = true;
+        },
+        file: 0,
+        ended: false,
+        opened: false
+    };
+
+    return w;
+}
+
+function makeDir(path: string, root?: string): boolean {
+    let dirs = path.split(/\/|\\/g),
+        dir = dirs.shift();
+
+    root = (root || '') + dir + '/';
+
+    try { fs.mkdirSync(root); }
+    catch (e) {
+        if (!fs.statSync(root).isDirectory()) throw new Error(e);
+    }
+
+    return !dirs.length || makeDir(dirs.join('/'), root);
+}

src/servers/model/server/web-api.ts → src/servers/model/server/api-web.ts

@@ -8,7 +8,9 @@ import * as express from 'express';
 import Config from '../config';
 import { QueryDefinition, QueryList } from './api';
 import { ConsoleLogger } from 'mol-util/console-logger';
-import { createRequest, resolveRequest } from './query';
+import { resolveJob } from './query';
+import { JobManager } from './jobs';
+import { UUID } from 'mol-util';
 
 function makePath(p: string) {
     return Config.appPrefix + '/' + p;
@@ -16,9 +18,9 @@ function makePath(p: string) {
 
 function wrapResponse(fn: string, res: express.Response) {
     const w = {
-        do404(this: any) {
+        doError(this: any, code = 404, message = 'Not Found.') {
             if (!this.headerWritten) {
-                res.writeHead(404);
+                res.status(code).send(message);
                 this.headerWritten = true;
             }
             this.end();
@@ -53,15 +55,45 @@ function wrapResponse(fn: string, res: express.Response) {
     return w;
 }
 
+const responseMap = new Map<UUID, express.Response>();
+
+async function processNextJob() {
+    if (!JobManager.hasNext()) return;
+
+    const job = JobManager.getNext();
+    const response = responseMap.get(job.id)!;
+    responseMap.delete(job.id);
+
+    const filenameBase = `${job.entryId}_${job.queryDefinition.name.replace(/\s/g, '_')}`
+    const writer = wrapResponse(job.responseFormat.isBinary ? `${filenameBase}.bcif` : `${filenameBase}.cif`, response);
+
+    try {
+        const encoder = await resolveJob(job);
+        writer.writeHeader(job.responseFormat.isBinary);
+        encoder.writeTo(writer);
+    } catch (e) {
+        ConsoleLogger.errorId(job.id, '' + e);
+        writer.doError(404, '' + e);
+    } finally {
+        writer.end();
+        ConsoleLogger.logId(job.id, 'Query', 'Finished.');
+        setImmediate(processNextJob);
+    }
+}
+
 function mapQuery(app: express.Express, queryName: string, queryDefinition: QueryDefinition) {
-    app.get(makePath(':entryId/' + queryName), async (req, res) => {
+    app.get(makePath(':entryId/' + queryName), (req, res) => {
         ConsoleLogger.log('Server', `Query '${req.params.entryId}/${queryName}'...`);
 
-        const request = createRequest('pdb', req.params.entryId, queryName, req.query);
-        const writer = wrapResponse(request.responseFormat.isBinary ? 'result.bcif' : 'result.cif', res);
-        writer.writeHeader(request.responseFormat.isBinary);
-        await resolveRequest(request, writer);
-        writer.end();
+        if (JobManager.size >= Config.maxQueueLength) {
+            res.status(503).send('Too many queries, please try again later.');
+            res.end();
+            return;
+        }
+
+        const jobId = JobManager.add('pdb', req.params.entryId, queryName, req.query);
+        responseMap.set(jobId, res);
+        if (JobManager.size === 1) processNextJob();
     });
 }
 

src/servers/model/server/cache.ts

@@ -48,18 +48,24 @@ export class Cache<T> {
     private refresh(e: CacheNode<T>) {
         this.clearTimeout(e);
 
-        e.value.timeoutId = setTimeout(() => this.expire(e), ServerConfig.cacheParams.entryTimeoutInMs);
+        e.value.timeoutId = setTimeout(() => this.expireNode(e), ServerConfig.cacheParams.entryTimeoutInMs);
         this.entries.remove(e);
         this.entries.addFirst(e.value);
     }
 
-    private expire(e: CacheNode<T>, notify = true) {
+    private expireNode(e: CacheNode<T>, notify = true) {
         if (notify) ConsoleLogger.log('Cache', `${e.value.key} expired.`);
         this.dispose(e);
     }
 
     expireAll() {
-        for (let e = this.entries.first; e; e = e.next) this.expire(e, false);
+        for (let e = this.entries.first; e; e = e.next) this.expireNode(e, false);
+    }
+
+    expire(key: string) {
+        const entry = this.entryMap.get(key);
+        if (!entry) return;
+        this.expireNode(entry);
     }
 
     add(item: T) {
@@ -86,7 +92,6 @@ export class Cache<T> {
         return this.entryMap.has(key);
     }
 
-
     get(key: string) {
         if (!this.entryMap.has(key)) return void 0;
         let e = this.entryMap.get(key)!;

src/servers/model/server/jobs.ts

@@ -0,0 +1,88 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import { UUID } from 'mol-util';
+import { getQueryByName, normalizeQueryParams, QueryDefinition } from './api';
+import { LinkedList } from 'mol-data/generic';
+
+export interface ResponseFormat {
+    isBinary: boolean
+}
+
+export interface Job {
+    id: UUID,
+    datetime_utc: string,
+
+    sourceId: '_local_' | string,
+    entryId: string,
+    key: string,
+
+    queryDefinition: QueryDefinition,
+    normalizedParams: any,
+    responseFormat: ResponseFormat,
+
+    outputFilename?: string
+}
+
+export function createJob(sourceId: '_local_' | string, entryId: string, queryName: string, params: any, outputFilename?: string): Job {
+    const queryDefinition = getQueryByName(queryName);
+    if (!queryDefinition) throw new Error(`Query '${queryName}' is not supported.`);
+
+    const normalizedParams = normalizeQueryParams(queryDefinition, params);
+
+    return {
+        id: UUID.create(),
+        datetime_utc: `${new Date().toISOString().replace(/T/, ' ').replace(/\..+/, '')}`,
+        key: `${sourceId}/${entryId}`,
+        sourceId,
+        entryId,
+        queryDefinition,
+        normalizedParams,
+        responseFormat: { isBinary: !!params.binary },
+        outputFilename
+    };
+}
+
+class _JobQueue {
+    private list: LinkedList<Job> = LinkedList();
+
+    get size() {
+        return this.list.count;
+    }
+
+    add(sourceId: '_local_' | string, entryId: string, queryName: string, params: any, outputFilename?: string) {
+        const job = createJob(sourceId, entryId, queryName, params, outputFilename);
+        this.list.addLast(job);
+        return job.id;
+    }
+
+    hasNext(): boolean {
+        return this.list.count > 0;
+    }
+
+    getNext(): Job {
+        return this.list.removeFirst()!;
+    }
+
+    /** Sort the job list by key = sourceId/entryId */
+    sort() {
+        if (this.list.count === 0) return;
+
+        const jobs: Job[] = [];
+        for (let j = this.list.first; !!j; j = j.next) {
+            jobs[jobs.length] = j.value;
+        }
+
+        jobs.sort((a, b) => a.key < b.key ? -1 : 1);
+
+        this.list = LinkedList();
+        for (const j of jobs) {
+            this.list.addLast(j);
+        }
+    }
+}
+
+export const JobManager = new _JobQueue();

src/servers/model/server/query.ts

@@ -4,35 +4,18 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
-import { UUID } from 'mol-util';
-import { getQueryByName, normalizeQueryParams, QueryDefinition } from './api';
-import { getStructure, StructureWrapper } from './structure-wrapper';
-import Config from '../config';
-import { Progress, now } from 'mol-task';
-import { ConsoleLogger } from 'mol-util/console-logger';
-import Writer from 'mol-io/writer/writer';
-import { CifWriter } from 'mol-io/writer/cif'
-import { encode_mmCIF_categories } from 'mol-model/structure/export/mmcif';
-import { StructureSelection, StructureQuery } from 'mol-model/structure';
-import Version from '../version'
 import { Column } from 'mol-data/db';
+import { CifWriter } from 'mol-io/writer/cif';
+import { StructureQuery, StructureSelection } from 'mol-model/structure';
+import { encode_mmCIF_categories } from 'mol-model/structure/export/mmcif';
+import { now, Progress } from 'mol-task';
+import { ConsoleLogger } from 'mol-util/console-logger';
 import { PerformanceMonitor } from 'mol-util/performance-monitor';
-
-export interface ResponseFormat {
-    isBinary: boolean
-}
-
-export interface Request {
-    id: UUID,
-    datetime_utc: string,
-
-    sourceId: '_local_' | string,
-    entryId: string,
-
-    queryDefinition: QueryDefinition,
-    normalizedParams: any,
-    responseFormat: ResponseFormat
-}
+import Config from '../config';
+import Version from '../version';
+import { Job } from './jobs';
+import { getStructure, StructureWrapper } from './structure-wrapper';
+import CifField = CifWriter.Field
 
 export interface Stats {
     structure: StructureWrapper,
@@ -40,66 +23,58 @@ export interface Stats {
     encodeTimeMs: number
 }
 
-export function createRequest(sourceId: '_local_' | string, entryId: string, queryName: string, params: any): Request {
-    const queryDefinition = getQueryByName(queryName);
-    if (!queryDefinition) throw new Error(`Query '${queryName}' is not supported.`);
-
-    const normalizedParams = normalizeQueryParams(queryDefinition, params);
-
-    return {
-        id: UUID.create(),
-        datetime_utc: `${new Date().toISOString().replace(/T/, ' ').replace(/\..+/, '')}`,
-        sourceId,
-        entryId,
-        queryDefinition,
-        normalizedParams,
-        responseFormat: { isBinary: !!params.binary }
-    };
-}
-
 const perf = new PerformanceMonitor();
 
-export async function resolveRequest(req: Request, writer: Writer) {
-    ConsoleLogger.logId(req.id, 'Query', 'Starting.');
-
-    const wrappedStructure = await getStructure(req.sourceId, req.entryId);
-
-    perf.start('query');
-    const structure = req.queryDefinition.structureTransform
-        ? await req.queryDefinition.structureTransform(req.normalizedParams, wrappedStructure.structure)
-        : wrappedStructure.structure;
-    const query = req.queryDefinition.query(req.normalizedParams, structure);
-    const result = StructureSelection.unionStructure(await StructureQuery.asTask(query, structure).run(abortingObserver, 250));
-    perf.end('query');
-
-    ConsoleLogger.logId(req.id, 'Query', 'Query finished.');
-
-    const encoder = CifWriter.createEncoder({ binary: req.responseFormat.isBinary, encoderName: `ModelServer ${Version}` });
-
-    perf.start('encode');
-    encoder.startDataBlock(structure.units[0].model.label.toUpperCase());
-    encoder.writeCategory(_model_server_result, [req]);
-    encoder.writeCategory(_model_server_params, [req]);
-
-    // encoder.setFilter(mmCIF_Export_Filters.onlyPositions);
-    encode_mmCIF_categories(encoder, result);
-    // encoder.setFilter();
-    perf.end('encode');
-
-    ConsoleLogger.logId(req.id, 'Query', 'Encoded.');
-
-    const stats: Stats = {
-        structure: wrappedStructure,
-        queryTimeMs: perf.time('query'),
-        encodeTimeMs: perf.time('encode')
-    };
+export async function resolveJob(job: Job): Promise<CifWriter.Encoder<any>> {
+    ConsoleLogger.logId(job.id, 'Query', 'Starting.');
+
+    const wrappedStructure = await getStructure(job);
+
+    try {
+        const encoder = CifWriter.createEncoder({ binary: job.responseFormat.isBinary, encoderName: `ModelServer ${Version}` });
+        perf.start('query');
+        const structure = job.queryDefinition.structureTransform
+            ? await job.queryDefinition.structureTransform(job.normalizedParams, wrappedStructure.structure)
+            : wrappedStructure.structure;
+        const query = job.queryDefinition.query(job.normalizedParams, structure);
+        const result = await StructureSelection.unionStructure(StructureQuery.run(query, structure, Config.maxQueryTimeInMs));
+        perf.end('query');
+
+        ConsoleLogger.logId(job.id, 'Query', 'Query finished.');
+
+        perf.start('encode');
+        encoder.startDataBlock(structure.units[0].model.label.toUpperCase());
+        encoder.writeCategory(_model_server_result, [job]);
+        encoder.writeCategory(_model_server_params, [job]);
+
+        // encoder.setFilter(mmCIF_Export_Filters.onlyPositions);
+        encode_mmCIF_categories(encoder, result);
+        // encoder.setFilter();
+        perf.end('encode');
+
+        const stats: Stats = {
+            structure: wrappedStructure,
+            queryTimeMs: perf.time('query'),
+            encodeTimeMs: perf.time('encode')
+        };
+
+        encoder.writeCategory(_model_server_stats, [stats]);
+        encoder.encode();
+        ConsoleLogger.logId(job.id, 'Query', 'Encoded.');
+        return encoder;
+    } catch (e) {
+        ConsoleLogger.errorId(job.id, e);
+        return doError(job, e);
+    }
+}
 
-    encoder.writeCategory(_model_server_stats, [stats]);
+function doError(job: Job, e: any) {
+    const encoder = CifWriter.createEncoder({ binary: job.responseFormat.isBinary, encoderName: `ModelServer ${Version}` });
+    encoder.writeCategory(_model_server_result, [job]);
+    encoder.writeCategory(_model_server_params, [job]);
+    encoder.writeCategory(_model_server_error, ['' + e]);
     encoder.encode();
-
-    encoder.writeTo(writer);
-
-    ConsoleLogger.logId(req.id, 'Query', 'Written.');
+    return encoder;
 }
 
 const maxTime = Config.maxQueryTimeInMs;
@@ -109,8 +84,6 @@ export function abortingObserver(p: Progress) {
     }
 }
 
-import CifField = CifWriter.Field
-
 function string<T>(name: string, str: (data: T, i: number) => string, isSpecified?: (data: T) => boolean): CifField<number, T> {
     if (isSpecified) {
         return CifField.str(name, (i, d) => str(d, i), { valueKind: (i, d) => isSpecified(d) ? Column.ValueKind.Present : Column.ValueKind.NotPresent });
@@ -122,13 +95,13 @@ function int32<T>(name: string, value: (data: T) => number): CifField<number, T>
     return CifField.int(name, (i, d) => value(d));
 }
 
-const _model_server_result_fields: CifField<number, Request>[] = [
-    string<Request>('request_id', ctx => '' + ctx.id),
-    string<Request>('datetime_utc', ctx => ctx.datetime_utc),
-    string<Request>('server_version', ctx => Version),
-    string<Request>('query_name', ctx => ctx.queryDefinition.name),
-    string<Request>('source_id', ctx => ctx.sourceId),
-    string<Request>('entry_id', ctx => ctx.entryId),
+const _model_server_result_fields: CifField<any, Job>[] = [
+    string<Job>('job_id', ctx => '' + ctx.id),
+    string<Job>('datetime_utc', ctx => ctx.datetime_utc),
+    string<Job>('server_version', ctx => Version),
+    string<Job>('query_name', ctx => ctx.queryDefinition.name),
+    string<Job>('source_id', ctx => ctx.sourceId),
+    string<Job>('entry_id', ctx => ctx.entryId),
 ];
 
 const _model_server_params_fields: CifField<number, string[]>[] = [
@@ -136,6 +109,10 @@ const _model_server_params_fields: CifField<number, string[]>[] = [
     string<string[]>('value', (ctx, i) => ctx[i][1])
 ];
 
+const _model_server_error_fields: CifField<number, string>[] = [
+    string<string>('message', (ctx, i) => ctx)
+];
+
 const _model_server_stats_fields: CifField<number, Stats>[] = [
     int32<Stats>('io_time_ms', ctx => ctx.structure.info.readTime | 0),
     int32<Stats>('parse_time_ms', ctx => ctx.structure.info.parseTime | 0),
@@ -144,18 +121,22 @@ const _model_server_stats_fields: CifField<number, Stats>[] = [
     int32<Stats>('encode_time_ms', ctx => ctx.encodeTimeMs | 0)
 ];
 
-
-const _model_server_result: CifWriter.Category<Request> = {
+const _model_server_result: CifWriter.Category<Job> = {
     name: 'model_server_result',
-    instance: (request) => ({ data: request, fields: _model_server_result_fields, rowCount: 1 })
+    instance: (job) => ({ data: job, fields: _model_server_result_fields, rowCount: 1 })
+};
+
+const _model_server_error: CifWriter.Category<string> = {
+    name: 'model_server_error',
+    instance: (message) => ({ data: message, fields: _model_server_error_fields, rowCount: 1 })
 };
 
-const _model_server_params: CifWriter.Category<Request> = {
+const _model_server_params: CifWriter.Category<Job> = {
     name: 'model_server_params',
-    instance(request) {
+    instance(job) {
         const params: string[][] = [];
-        for (const k of Object.keys(request.normalizedParams)) {
-            params.push([k, '' + request.normalizedParams[k]]);
+        for (const k of Object.keys(job.normalizedParams)) {
+            params.push([k, '' + job.normalizedParams[k]]);
         }
         return {
             data: params,

src/servers/model/server/structure-wrapper.ts

@@ -12,6 +12,8 @@ import CIF from 'mol-io/reader/cif'
 import * as util from 'util'
 import * as fs from 'fs'
 import * as zlib from 'zlib'
+import { Job } from './jobs';
+import { ConsoleLogger } from 'mol-util/console-logger';
 
 require('util.promisify').shim();
 
@@ -38,13 +40,12 @@ export class StructureWrapper {
     structure: Structure;
 }
 
-export async function getStructure(sourceId: '_local_' | string, entryId: string): Promise<StructureWrapper> {
-    const key = `${sourceId}/${entryId}`;
+export async function getStructure(job: Job): Promise<StructureWrapper> {
     if (Config.cacheParams.useCache) {
-        const ret = StructureCache.get(key);
+        const ret = StructureCache.get(job.key);
         if (ret) return ret;
     }
-    const ret = await readStructure(key, sourceId, entryId);
+    const ret = await readStructure(job.key, job.sourceId, job.entryId);
     if (Config.cacheParams.useCache) {
         StructureCache.add(ret);
     }
@@ -83,10 +84,18 @@ async function parseCif(data: string|Uint8Array) {
 
 async function readStructure(key: string, sourceId: string, entryId: string) {
     const filename = sourceId === '_local_' ? entryId : Config.mapFile(sourceId, entryId);
-    if (!filename) throw new Error(`Entry '${key}' not found.`);
+    if (!filename) throw new Error(`Cound not map '${key}' to a valid filename.`);
+    if (!fs.existsSync(filename)) throw new Error(`Could not find source file for '${key}'.`);
 
     perf.start('read');
-    const data = await readFile(filename);
+    let data;
+    try {
+        data = await readFile(filename);
+    } catch (e) {
+        ConsoleLogger.error(key, '' + e);
+        throw new Error(`Could not read the file for '${key}' from disk.`);
+    }
+
     perf.end('read');
     perf.start('parse');
     const frame = (await parseCif(data)).blocks[0];

src/servers/model/test.ts

@@ -1,6 +1,7 @@
-import { createRequest, resolveRequest } from './server/query';
+import { resolveJob } from './server/query';
 import * as fs from 'fs'
 import { StructureCache } from './server/structure-wrapper';
+import { createJob } from './server/jobs';
 
 function wrapFile(fn: string) {
     const w = {
@@ -34,9 +35,10 @@ function wrapFile(fn: string) {
 
 async function run() {
     try {
-        const request = createRequest('_local_', 'e:/test/quick/1cbs_updated.cif', 'residueInteraction', { label_comp_id: 'REA' });
+        const request = createJob('_local_', 'e:/test/quick/1cbs_updated.cif', 'residueInteraction', { label_comp_id: 'REA' });
+        const encoder = await resolveJob(request);
         const writer = wrapFile('e:/test/mol-star/1cbs_full.cif');
-        await resolveRequest(request, writer);
+        encoder.writeTo(writer);
         writer.end();
     } finally {
         StructureCache.expireAll();