ModelServer test working

src/mol-model/structure/export/mmcif.ts

@@ -118,17 +118,21 @@ function atomSiteProvider({ structure }: Context): Encoder.CategoryInstance {
     }
 }
 
-function to_mmCIF(name: string, structure: Structure, asBinary = false) {
+/** Doesn't start a data block */
+export function encode_mmCIF_categories(encoder: Encoder.EncoderInstance, structure: Structure) {
     const models = Structure.getModels(structure);
-    if (models.length !== 1) throw 'cant export stucture composed from multiple models.';
+    if (models.length !== 1) throw 'Can\'t export stucture composed from multiple models.';
     const model = models[0];
 
     const ctx: Context = { structure, model };
-    const w = Encoder.create({ binary: asBinary });
+    encoder.writeCategory(entityProvider, [ctx]);
+    encoder.writeCategory(atomSiteProvider, [ctx]);
+}
 
+function to_mmCIF(name: string, structure: Structure, asBinary = false) {
+    const w = Encoder.create({ binary: asBinary });
     w.startDataBlock(name);
-    w.writeCategory(entityProvider, [ctx]);
-    w.writeCategory(atomSiteProvider, [ctx]);
+    encode_mmCIF_categories(w, structure);
     return w.getData();
 }
 

src/mol-model/structure/structure/structure.ts

@@ -195,6 +195,8 @@ namespace Structure {
         private advance() {
             if (this.idx < this.maxIdx) {
                 this.idx++;
+
+                if (this.idx === this.maxIdx) this.hasNext = this.unitIndex + 1 < this.structure.units.length;
                 return;
             }
 

src/mol-model/structure/structure/symmetry.ts

@@ -76,6 +76,7 @@ namespace StructureSymmetry {
                 }
             }
 
+
             return assembler.getStructure();
         });
     }

src/mol-util/console-logger.ts

@@ -25,7 +25,7 @@ export namespace ConsoleLogger {
         console.log(`[${tag}] ${msg}`);
     }
 
-    export function logId(guid: string, tag: string, msg: string) {
+    export function logId(guid: string | String, tag: string, msg: string) {
         console.log(`[${guid}][${tag}] ${msg}`);
     }
 
@@ -35,7 +35,7 @@ export namespace ConsoleLogger {
     }
 
 
-    export function errorId(guid: string, e: any) {
+    export function errorId(guid: string | String, e: any) {
         console.error(`[${guid}][Error] ${e}`);
         if (e.stack) console.error(e.stack);
     }

src/servers/model/server/api.ts

@@ -25,7 +25,7 @@ export interface QueryParamInfo {
 export interface QueryDefinition {
     niceName: string,
     exampleId: string, // default is 1cbs
-    query: (params: any, originalStructure: Structure, transformedStructure: Structure) => Query.Provider,
+    query: (params: any, structure: Structure) => Query,
     description: string,
     params: QueryParamInfo[],
     structureTransform?: (params: any, s: Structure) => Promise<Structure>
@@ -37,13 +37,11 @@ const AtomSiteParameters = {
     label_asym_id: <QueryParamInfo>{ name: 'label_asym_id', type: QueryParamType.String, description: 'Corresponds to the \'_atom_site.label_asym_id\' field.' },
     auth_asym_id: <QueryParamInfo>{ name: 'auth_asym_id', type: QueryParamType.String, exampleValue: 'A', description: 'Corresponds to the \'_atom_site.auth_asym_id\' field.' },
 
+    label_seq_id: <QueryParamInfo>{ name: 'label_seq_id', type: QueryParamType.Integer, description: 'Residue seq. number. Corresponds to the \'_atom_site.label_seq_id\' field.' },
+    auth_seq_id: <QueryParamInfo>{ name: 'auth_seq_id', type: QueryParamType.Integer, exampleValue: '200', description: 'Author residue seq. number. Corresponds to the \'_atom_site.auth_seq_id\' field.' },
     label_comp_id: <QueryParamInfo>{ name: 'label_comp_id', type: QueryParamType.String, description: 'Residue name. Corresponds to the \'_atom_site.label_comp_id\' field.' },
     auth_comp_id: <QueryParamInfo>{ name: 'auth_comp_id', type: QueryParamType.String, exampleValue: 'REA', description: 'Author residue name. Corresponds to the \'_atom_site.auth_comp_id\' field.' },
-
     pdbx_PDB_ins_code: <QueryParamInfo>{ name: 'pdbx_PDB_ins_code', type: QueryParamType.String, description: 'Corresponds to the \'_atom_site.pdbx_PDB_ins_code\' field.' },
-
-    label_seq_id: <QueryParamInfo>{ name: 'label_seq_id', type: QueryParamType.Integer, description: 'Residue seq. number. Corresponds to the \'_atom_site.label_seq_id\' field.' },
-    auth_seq_id: <QueryParamInfo>{ name: 'auth_seq_id', type: QueryParamType.Integer, exampleValue: '200', description: 'Author residue seq. number. Corresponds to the \'_atom_site.auth_seq_id\' field.' },
 };
 
 // function entityTest(params: any): Element.Predicate | undefined {
@@ -72,6 +70,29 @@ function chainTest(params: any): Element.Predicate | undefined {
 }
 
 function residueTest(params: any): Element.Predicate | undefined {
+    const props: Element.Property<any>[] = [], values: any[] = [];
+
+    if (typeof params.label_seq_id !== 'undefined') {
+        props.push(Queries.props.residue.label_seq_id);
+        values.push(+params.label_seq_id);
+    }
+
+    if (typeof params.auth_seq_id !== 'undefined') {
+        props.push(Queries.props.residue.auth_seq_id);
+        values.push(+params.auth_seq_id);
+    }
+
+    if (typeof params.label_comp_id !== 'undefined') {
+        props.push(Queries.props.residue.label_comp_id);
+        values.push(params.label_comp_id);
+    }
+
+    if (typeof params.auth_comp_id !== 'undefined') {
+        props.push(Queries.props.residue.auth_comp_id);
+        values.push(params.auth_comp_id);
+    }
+
+
     if (typeof params.label_seq_id !== 'undefined') {
         const p = Queries.props.residue.label_seq_id, id = +params.label_seq_id;
         if (typeof params.pdbx_PDB_ins_code !== 'undefined') {
@@ -96,16 +117,16 @@ function residueTest(params: any): Element.Predicate | undefined {
 // }
 
 const QueryMap: { [id: string]: Partial<QueryDefinition> } = {
-    'full': { niceName: 'Full Structure', query: () => Queries.generators.all, description: 'The full structure.' },
+    'full': { niceName: 'Full Structure', query: () => Query(Queries.generators.all), description: 'The full structure.' },
     'residueInteraction': {
         niceName: 'Residues Inside a Sphere',
         description: 'Identifies all residues within the given radius from the source residue.',
         query(p) {
             const center = Queries.generators.atoms({ entityTest: entityTest1_555(p), chainTest: chainTest(p), residueTest: residueTest(p) });
-            return Queries.modifiers.includeSurroundings(center, { radius: p.radius, wholeResidues: true });
+            return Query(Queries.modifiers.includeSurroundings(center, { radius: p.radius, wholeResidues: true }));
         },
         structureTransform(p, s) {
-            return StructureSymmetry.builderSymmetryMates(p, p. radius).run();
+            return StructureSymmetry.builderSymmetryMates(s, p.radius).run();
         },
         params: [
             AtomSiteParameters.entity_id,
@@ -130,10 +151,10 @@ const QueryMap: { [id: string]: Partial<QueryDefinition> } = {
             },
         ]
     },
-}
+};
 
-export function getQueryByName(name: string) {
-    return QueryMap[name];
+export function getQueryByName(name: string): QueryDefinition {
+    return QueryMap[name] as QueryDefinition;
 }
 
 export const QueryList = (function () {
@@ -143,21 +164,30 @@ export const QueryList = (function () {
     return list;
 })();
 
+// normalize the queries
+(function () {
+    for (let q of QueryList) {
+        const m = q.definition;
+        m.params = m.params || [];
+    }
+})();
+
 function _normalizeQueryParams(params: { [p: string]: string }, paramList: QueryParamInfo[]): { [p: string]: string | number | boolean } {
     const ret: any = {};
     for (const p of paramList) {
         const key = p.name;
+        const value = params[key];
 
-        if (typeof params[key] === 'undefined' || (params[key] !== null && params[key]['length'] === 0)) {
+        if (typeof value === 'undefined' || (typeof value !== 'undefined' && value !== null && value['length'] === 0)) {
             if (p.required) {
                 throw `The parameter '${key}' is required.`;
             }
             ret[key] = p.defaultValue;
         } else {
             switch (p.type) {
-                case QueryParamType.String: ret[key] = params[key]; break;
-                case QueryParamType.Integer: ret[key] = parseInt(params[key]); break;
-                case QueryParamType.Float: ret[key] = parseFloat(params[key]); break;
+                case QueryParamType.String: ret[key] = value; break;
+                case QueryParamType.Integer: ret[key] = parseInt(value); break;
+                case QueryParamType.Float: ret[key] = parseFloat(value); break;
             }
 
             if (p.validation) p.validation(ret[key]);

src/servers/model/server/query.ts

@@ -5,27 +5,74 @@
  */
 
 import { UUID } from 'mol-util';
+import { getQueryByName, normalizeQueryParams, QueryDefinition } from './api';
+import { getStructure } from './structure-wrapper';
+import Config from '../config';
+import { Progress, now } from 'mol-task';
+import { ConsoleLogger } from 'mol-util/console-logger';
+import Writer from 'mol-io/writer/writer';
+import * as Encoder from 'mol-io/writer/cif'
+import { encode_mmCIF_categories } from 'mol-model/structure/export/mmcif';
+import { Selection } from 'mol-model/structure';
+import Version from '../version'
 
 export interface ResponseFormat {
-
+    isBinary: boolean
 }
 
-export interface Query {
+export interface Request {
     id: UUID,
 
-    sourceId: 'file' | string,
+    sourceId: '_local_' | string,
     entryId: string,
 
-    kind: string,
-    params: any,
-
+    queryDefinition: QueryDefinition,
+    normalizedParams: any,
     responseFormat: ResponseFormat
 }
 
-// export class QueryQueue {
+export function createRequest(sourceId: '_local_' | string, entryId: string, queryName: string, params: any): Request {
+    const queryDefinition = getQueryByName(queryName);
+    if (!queryDefinition) throw new Error(`Query '${queryName}' is not supported.`);
+
+    const normalizedParams = normalizeQueryParams(queryDefinition, params);
+
+    return {
+        id: UUID.create(),
+        sourceId,
+        entryId,
+        queryDefinition,
+        normalizedParams,
+        responseFormat: { isBinary: !!params.binary }
+    };
+}
+
+export async function resolveRequest(req: Request, writer: Writer) {
+    ConsoleLogger.logId(req.id, 'Query', 'Starting.');
 
-// }
+    const wrappedStructure = await getStructure(req.sourceId, req.entryId);
+    const structure = req.queryDefinition.structureTransform
+        ? await req.queryDefinition.structureTransform(req.normalizedParams, wrappedStructure.structure)
+        : wrappedStructure.structure;
+    const query = req.queryDefinition.query(req.normalizedParams, structure);
+    const result = Selection.unionStructure(await query(structure).run(abortingObserver, 250));
 
-export function resolveQuery() {
+    ConsoleLogger.logId(req.id, 'Query', 'Query finished.');
+
+    const encoder = Encoder.create({ binary: req.responseFormat.isBinary, encoderName: `ModelServer ${Version}` });
+    encoder.startDataBlock('result');
+    encode_mmCIF_categories(encoder, result);
+
+    ConsoleLogger.logId(req.id, 'Query', 'Encoded.');
+
+    encoder.writeTo(writer);
+
+    ConsoleLogger.logId(req.id, 'Query', 'Written.');
+}
 
+const maxTime = Config.maxQueryTimeInMs;
+export function abortingObserver(p: Progress) {
+    if (now() - p.root.progress.startedTime > maxTime) {
+        p.requestAbort(`Exceeded maximum allowed time for a query (${maxTime}ms)`);
+    }
 }

src/servers/model/server/structure-wrapper.ts

@@ -4,7 +4,16 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
-import { Structure } from 'mol-model/structure';
+import { Structure, Model } from 'mol-model/structure';
+import { PerformanceMonitor } from 'mol-util/performance-monitor';
+import { Cache } from './cache';
+import Config from '../config';
+import CIF from 'mol-io/reader/cif'
+import * as util from 'util'
+import * as fs from 'fs'
+import * as zlib from 'zlib'
+
+require('util.promisify').shim();
 
 export enum StructureSourceType {
     File,
@@ -15,10 +24,10 @@ export interface StructureInfo {
     sourceType: StructureSourceType;
     readTime: number;
     parseTime: number;
+    createModelTime: number;
 
     sourceId: string,
-    entryId: string,
-    filename: string
+    entryId: string
 }
 
 export class StructureWrapper {
@@ -29,8 +38,78 @@ export class StructureWrapper {
     structure: Structure;
 }
 
-export function getStructure(filename: string): Promise<StructureWrapper>
-export function getStructure(sourceId: string, entryId: string): Promise<StructureWrapper>
-export function getStructure(sourceIdOrFilename: string, entryId?: string): Promise<StructureWrapper> {
-    return 0 as any;
+export async function getStructure(sourceId: '_local_' | string, entryId: string): Promise<StructureWrapper> {
+    const key = `${sourceId}/${entryId}`;
+    if (Config.cacheParams.useCache) {
+        const ret = StructureCache.get(key);
+        if (ret) return ret;
+    }
+    const ret = await readStructure(key, sourceId, entryId);
+    if (Config.cacheParams.useCache) {
+        StructureCache.add(ret);
+    }
+    return ret;
+}
+
+export const StructureCache = new Cache<StructureWrapper>(s => s.key, s => s.approximateSize);
+const perf = new PerformanceMonitor();
+
+const readFileAsync = util.promisify(fs.readFile);
+const unzipAsync = util.promisify<zlib.InputType, Buffer>(zlib.unzip);
+
+async function readFile(filename: string) {
+    const isGz = /\.gz$/i.test(filename);
+    if (filename.match(/\.bcif/)) {
+        let input = await readFileAsync(filename)
+        if (isGz) input = await unzipAsync(input);
+        const data = new Uint8Array(input.byteLength);
+        for (let i = 0; i < input.byteLength; i++) data[i] = input[i];
+        return data;
+    } else {
+        if (isGz) {
+            const data = await unzipAsync(await readFileAsync(filename));
+            return data.toString('utf8');
+        }
+        return readFileAsync(filename, 'utf8');
+    }
+}
+
+async function parseCif(data: string|Uint8Array) {
+    const comp = CIF.parse(data);
+    const parsed = await comp.run();
+    if (parsed.isError) throw parsed;
+    return parsed.result;
+}
+
+async function readStructure(key: string, sourceId: string, entryId: string) {
+    const filename = sourceId === '_local_' ? entryId : Config.mapFile(sourceId, entryId);
+    if (!filename) throw new Error(`Entry '${key}' not found.`);
+
+    perf.start('read');
+    const data = await readFile(filename);
+    perf.end('read');
+    perf.start('parse');
+    const mmcif = CIF.schema.mmCIF((await parseCif(data)).blocks[0]);
+    perf.end('parse');
+    perf.start('createModel');
+    const models = await Model.create({ kind: 'mmCIF', data: mmcif }).run();
+    perf.end('createModel');
+
+    const structure = Structure.ofModel(models[0]);
+
+    const ret: StructureWrapper = {
+        info: {
+            sourceType: StructureSourceType.File,
+            readTime: perf.time('read'),
+            parseTime: perf.time('parse'),
+            createModelTime: perf.time('createModel'),
+            sourceId,
+            entryId
+        },
+        key,
+        approximateSize: typeof data === 'string' ? 2 * data.length : data.length,
+        structure
+    };
+
+    return ret;
 }

src/servers/model/test.ts

@@ -0,0 +1,46 @@
+import { createRequest, resolveRequest } from './server/query';
+import * as fs from 'fs'
+import { StructureCache } from './server/structure-wrapper';
+
+function wrapFile(fn: string) {
+    const w = {
+        open(this: any) {
+            if (this.opened) return;
+            this.file = fs.openSync(fn, 'w');
+            this.opened = true;
+        },
+        writeBinary(this: any, data: Uint8Array) {
+            this.open();
+            fs.writeSync(this.file, new Buffer(data));
+            return true;
+        },
+        writeString(this: any, data: string) {
+            this.open();
+            fs.writeSync(this.file, data);
+            return true;
+        },
+        end(this: any) {
+            if (!this.opened || this.ended) return;
+            fs.close(this.file, function () { });
+            this.ended = true;
+        },
+        file: 0,
+        ended: false,
+        opened: false
+    };
+
+    return w;
+}
+
+async function run() {
+    try {
+        const request = createRequest('_local_', 'e:/test/quick/1cbs_updated.cif', 'residueInteraction', { label_comp_id: 'REA' });
+        const writer = wrapFile('e:/test/mol-star/1cbs_full.cif');
+        await resolveRequest(request, writer);
+        writer.end();
+    } finally {
+        StructureCache.expireAll();
+    }
+}
+
+run();