Basic ModelServer web api

package.json

@@ -17,7 +17,14 @@
     "watch-extra": "cpx \"src/**/*.{vert,frag,glsl,scss,woff,woff2,ttf,otf,eot,svg,html}\" build/node_modules/ --watch",
     "test": "jest",
     "build-viewer": "webpack build/node_modules/apps/viewer/index.js --mode development -o build/viewer/index.js",
-    "watch-viewer": "webpack build/node_modules/apps/viewer/index.js -w --mode development -o build/viewer/index.js"
+    "watch-viewer": "webpack build/node_modules/apps/viewer/index.js -w --mode development -o build/viewer/index.js",
+    "run-model-server": "node build/node_modules/servers/model/server.js",
+    "run-model-server-watch": "nodemon --watch build/node_modules build/node_modules/servers/model/server.js"
+  },
+  "nodemonConfig": {
+    "ignoreRoot": ["./node_modules", ".git"],
+    "ignore": [],
+    "delay": "2500"
   },
   "jest": {
     "moduleFileExtensions": [

src/mol-model/structure/export/mmcif.ts

@@ -27,7 +27,7 @@ function int<K, D = any>(name: string, value: (k: K, d: D) => number, valueKind?
 }
 
 function float<K, D = any>(name: string, value: (k: K, d: D) => number, valueKind?: (k: K) => Column.ValueKind): Encoder.FieldDefinition<K, D> {
-    return { name, type: Encoder.FieldType.Float, value, valueKind }
+    return { name, type: Encoder.FieldType.Float, value, valueKind, digitCount: 3 }
 }
 
 // function col<K, D>(name: string, c: (data: D) => Column<any>): Encoder.FieldDefinition<K, D> {

src/servers/model/server.ts

@@ -0,0 +1,73 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import * as express from 'express'
+import * as compression from 'compression'
+import ServerConfig from './config'
+import { ConsoleLogger } from 'mol-util/console-logger';
+import { PerformanceMonitor } from 'mol-util/performance-monitor';
+import { initWebApi } from './server/web-api';
+import Version from './version'
+
+function setupShutdown() {
+    if (ServerConfig.shutdownParams.timeoutVarianceMinutes > ServerConfig.shutdownParams.timeoutMinutes) {
+        ConsoleLogger.log('Server', 'Shutdown timeout variance is greater than the timer itself, ignoring.');
+    } else {
+        let tVar = 0;
+        if (ServerConfig.shutdownParams.timeoutVarianceMinutes > 0) {
+            tVar = 2 * (Math.random() - 0.5) * ServerConfig.shutdownParams.timeoutVarianceMinutes;
+        }
+        let tMs = (ServerConfig.shutdownParams.timeoutMinutes + tVar) * 60 * 1000;
+
+        console.log(`----------------------------------------------------------------------------`);
+        console.log(`  The server will shut down in ${PerformanceMonitor.format(tMs)} to prevent slow performance.`);
+        console.log(`  Please make sure a daemon is running that will automatically restart it.`);
+        console.log(`----------------------------------------------------------------------------`);
+        console.log();
+
+        setTimeout(() => {
+            /*if (WebApi.ApiState.pendingQueries > 0) {
+                WebApi.ApiState.shutdownOnZeroPending = true;
+            } else*/ {
+                ConsoleLogger.log('Server', `Shut down due to timeout.`);
+                process.exit(0);
+            }
+        }, tMs);
+    }
+}
+
+const port = process.env.port || ServerConfig.defaultPort;
+
+function startServer() {
+    let app = express();
+    app.use(compression(<any>{ level: 6, memLevel: 9, chunkSize: 16 * 16384, filter: () => true }));
+
+    // app.get(ServerConfig.appPrefix + '/documentation', (req, res) => {
+    //     res.writeHead(200, { 'Content-Type': 'text/html' });
+    //     res.write(Documentation.getHTMLDocs(ServerConfig.appPrefix));
+    //     res.end();
+    // });
+
+    initWebApi(app);
+
+    // app.get('*', (req, res) => {
+    //     res.writeHead(200, { 'Content-Type': 'text/html' });
+    //     res.write(Documentation.getHTMLDocs(ServerConfig.appPrefix));
+    //     res.end();
+    // });
+
+    app.listen(port);
+}
+
+startServer();
+console.log(`Mol* ModelServer ${Version}`);
+console.log(``);
+console.log(`The server is running on port ${port}.`);
+console.log(``);
+
+if (ServerConfig.shutdownParams && ServerConfig.shutdownParams.timeoutMinutes > 0) {
+    setupShutdown();
+}

src/servers/model/server/api.ts

@@ -23,6 +23,7 @@ export interface QueryParamInfo {
 }
 
 export interface QueryDefinition {
+    name: string,
     niceName: string,
     exampleId: string, // default is 1cbs
     query: (params: any, structure: Structure) => Query,
@@ -92,24 +93,24 @@ function residueTest(params: any): Element.Predicate | undefined {
         values.push(params.auth_comp_id);
     }
 
-
-    if (typeof params.label_seq_id !== 'undefined') {
-        const p = Queries.props.residue.label_seq_id, id = +params.label_seq_id;
-        if (typeof params.pdbx_PDB_ins_code !== 'undefined') {
-            const p1 = Queries.props.residue.label_seq_id, id1 = params.pdbx_PDB_ins_code;
-            return Element.property(l => p(l) === id && p1(l) === id1);
-        }
-        return Element.property(l => p(l) === id);
+    if (typeof params.pdbx_PDB_ins_code !== 'undefined') {
+        props.push(Queries.props.residue.pdbx_PDB_ins_code);
+        values.push(params.pdbx_PDB_ins_code);
     }
-    if (typeof params.auth_seq_id !== 'undefined') {
-        const p = Queries.props.residue.auth_seq_id, id = +params.auth_seq_id;
-        if (typeof params.pdbx_PDB_ins_code !== 'undefined') {
-            const p1 = Queries.props.residue.label_seq_id, id1 = params.pdbx_PDB_ins_code;
-            return Element.property(l => p(l) === id && p1(l) === id1);
+
+    switch (props.length) {
+        case 0: return void 0;
+        case 1: return Element.property(l => props[0](l) === values[0]);
+        case 2: return Element.property(l => props[0](l) === values[0] && props[1](l) === values[1]);
+        case 3: return Element.property(l => props[0](l) === values[0] && props[1](l) === values[1] && props[2](l) === values[2]);
+        default: {
+            const len = props.length;
+            return Element.property(l => {
+                for (let i = 0; i < len; i++) if (!props[i](l) !== values[i]) return false;
+                return true;
+            });
         }
-        return Element.property(l => p(l) === id);
     }
-    return void 0;
 }
 
 // function buildResiduesQuery(params: any): Query.Provider {
@@ -168,6 +169,7 @@ export const QueryList = (function () {
 (function () {
     for (let q of QueryList) {
         const m = q.definition;
+        m.name = q.name;
         m.params = m.params || [];
     }
 })();
@@ -177,12 +179,11 @@ function _normalizeQueryParams(params: { [p: string]: string }, paramList: Query
     for (const p of paramList) {
         const key = p.name;
         const value = params[key];
-
         if (typeof value === 'undefined' || (typeof value !== 'undefined' && value !== null && value['length'] === 0)) {
             if (p.required) {
                 throw `The parameter '${key}' is required.`;
             }
-            ret[key] = p.defaultValue;
+            if (typeof p.defaultValue !== 'undefined') ret[key] = p.defaultValue;
         } else {
             switch (p.type) {
                 case QueryParamType.String: ret[key] = value; break;

src/servers/model/server/query.ts

@@ -6,7 +6,7 @@
 
 import { UUID } from 'mol-util';
 import { getQueryByName, normalizeQueryParams, QueryDefinition } from './api';
-import { getStructure } from './structure-wrapper';
+import { getStructure, StructureWrapper } from './structure-wrapper';
 import Config from '../config';
 import { Progress, now } from 'mol-task';
 import { ConsoleLogger } from 'mol-util/console-logger';
@@ -15,6 +15,9 @@ import * as Encoder from 'mol-io/writer/cif'
 import { encode_mmCIF_categories } from 'mol-model/structure/export/mmcif';
 import { Selection } from 'mol-model/structure';
 import Version from '../version'
+import { Column } from 'mol-data/db';
+import { Iterator } from 'mol-data';
+import { PerformanceMonitor } from 'mol-util/performance-monitor';
 
 export interface ResponseFormat {
     isBinary: boolean
@@ -22,6 +25,7 @@ export interface ResponseFormat {
 
 export interface Request {
     id: UUID,
+    datetime_utc: string,
 
     sourceId: '_local_' | string,
     entryId: string,
@@ -31,6 +35,12 @@ export interface Request {
     responseFormat: ResponseFormat
 }
 
+export interface Stats {
+    structure: StructureWrapper,
+    queryTimeMs: number,
+    encodeTimeMs: number
+}
+
 export function createRequest(sourceId: '_local_' | string, entryId: string, queryName: string, params: any): Request {
     const queryDefinition = getQueryByName(queryName);
     if (!queryDefinition) throw new Error(`Query '${queryName}' is not supported.`);
@@ -39,6 +49,7 @@ export function createRequest(sourceId: '_local_' | string, entryId: string, que
 
     return {
         id: UUID.create(),
+        datetime_utc: `${new Date().toISOString().replace(/T/, ' ').replace(/\..+/, '')}`,
         sourceId,
         entryId,
         queryDefinition,
@@ -47,24 +58,42 @@ export function createRequest(sourceId: '_local_' | string, entryId: string, que
     };
 }
 
+const perf = new PerformanceMonitor();
+
 export async function resolveRequest(req: Request, writer: Writer) {
     ConsoleLogger.logId(req.id, 'Query', 'Starting.');
 
     const wrappedStructure = await getStructure(req.sourceId, req.entryId);
+
+    perf.start('query');
     const structure = req.queryDefinition.structureTransform
         ? await req.queryDefinition.structureTransform(req.normalizedParams, wrappedStructure.structure)
         : wrappedStructure.structure;
     const query = req.queryDefinition.query(req.normalizedParams, structure);
     const result = Selection.unionStructure(await query(structure).run(abortingObserver, 250));
+    perf.end('query');
 
     ConsoleLogger.logId(req.id, 'Query', 'Query finished.');
 
     const encoder = Encoder.create({ binary: req.responseFormat.isBinary, encoderName: `ModelServer ${Version}` });
+
+    perf.start('encode');
     encoder.startDataBlock('result');
+    encoder.writeCategory(_model_server_result, [req]);
+    encoder.writeCategory(_model_server_params, [req]);
     encode_mmCIF_categories(encoder, result);
+    perf.end('encode');
 
     ConsoleLogger.logId(req.id, 'Query', 'Encoded.');
 
+    const stats: Stats = {
+        structure: wrappedStructure,
+        queryTimeMs: perf.time('query'),
+        encodeTimeMs: perf.time('encode')
+    };
+
+    encoder.writeCategory(_model_server_stats, [stats]);
+
     encoder.writeTo(writer);
 
     ConsoleLogger.logId(req.id, 'Query', 'Written.');
@@ -75,4 +104,72 @@ export function abortingObserver(p: Progress) {
     if (now() - p.root.progress.startedTime > maxTime) {
         p.requestAbort(`Exceeded maximum allowed time for a query (${maxTime}ms)`);
     }
+}
+
+type FieldDesc<T> = Encoder.FieldDefinition<number, T>
+type CategoryInstance = Encoder.CategoryInstance
+
+function string<T>(name: string, str: (data: T, i: number) => string, isSpecified?: (data: T) => boolean): FieldDesc<T> {
+    if (isSpecified) {
+        return { name, type: Encoder.FieldType.Str, value: (i, d) => str(d, i), valueKind: (i, d) => isSpecified(d) ? Column.ValueKind.Present : Column.ValueKind.NotPresent };
+    }
+    return { name, type: Encoder.FieldType.Str, value: (i, d) => str(d, i) };
+}
+
+function int32<T>(name: string, value: (data: T) => number): FieldDesc<T> {
+    return { name, type: Encoder.FieldType.Int, value: (i, d) => value(d) };
+}
+
+const _model_server_result_fields: FieldDesc<Request>[] = [
+    string<Request>('request_id', ctx => '' + ctx.id),
+    string<Request>('datetime_utc', ctx => ctx.datetime_utc),
+    string<Request>('server_version', ctx => Version),
+    string<Request>('query_name', ctx => ctx.queryDefinition.name),
+    string<Request>('source_id', ctx => ctx.sourceId),
+    string<Request>('entry_id', ctx => ctx.entryId),
+];
+
+const _model_server_params_fields: FieldDesc<string[]>[] = [
+    string<string[]>('name', (ctx, i) => ctx[i][0]),
+    string<string[]>('value', (ctx, i) => ctx[i][1])
+];
+
+const _model_server_stats_fields: FieldDesc<Stats>[] = [
+    int32<Stats>('io_time_ms', ctx => ctx.structure.info.readTime | 0),
+    int32<Stats>('parse_time_ms', ctx => ctx.structure.info.parseTime | 0),
+    int32<Stats>('create_model_time_ms', ctx => ctx.structure.info.createModelTime | 0),
+    int32<Stats>('query_time_ms', ctx => ctx.queryTimeMs | 0),
+    int32<Stats>('encode_time_ms', ctx => ctx.encodeTimeMs | 0)
+];
+
+
+function _model_server_result(request: Request): CategoryInstance {
+    return {
+        data: request,
+        definition: { name: 'model_server_result', fields: _model_server_result_fields },
+        keys: () => Iterator.Value(0),
+        rowCount: 1
+    };
+}
+
+function _model_server_params(request: Request): CategoryInstance {
+    const params: string[][] = [];
+    for (const k of Object.keys(request.normalizedParams)) {
+        params.push([k, '' + request.normalizedParams[k]]);
+    }
+    return {
+        data: params,
+        definition: { name: 'model_server_params', fields: _model_server_params_fields },
+        keys: () => Iterator.Range(0, params.length - 1),
+        rowCount: params.length
+    };
+}
+
+function _model_server_stats(stats: Stats): CategoryInstance {
+    return {
+        data: stats,
+        definition: { name: 'model_server_stats', fields: _model_server_stats_fields },
+        keys: () => Iterator.Value(0),
+        rowCount: 1
+    };
 }

src/servers/model/server/web-api.ts

@@ -0,0 +1,72 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import * as express from 'express';
+import Config from '../config';
+import { QueryDefinition, QueryList } from './api';
+import { ConsoleLogger } from 'mol-util/console-logger';
+import { createRequest, resolveRequest } from './query';
+
+function makePath(p: string) {
+    return Config.appPrefix + '/' + p;
+}
+
+function wrapResponse(fn: string, res: express.Response) {
+    const w = {
+        do404(this: any) {
+            if (!this.headerWritten) {
+                res.writeHead(404);
+                this.headerWritten = true;
+            }
+            this.end();
+        },
+        writeHeader(this: any, binary: boolean) {
+            if (this.headerWritten) return;
+            res.writeHead(200, {
+                'Content-Type': binary ? 'application/octet-stream' : 'text/plain; charset=utf-8',
+                'Access-Control-Allow-Origin': '*',
+                'Access-Control-Allow-Headers': 'X-Requested-With',
+                'Content-Disposition': `inline; filename="${fn}"`
+            });
+            this.headerWritten = true;
+        },
+        writeBinary(this: any, data: Uint8Array) {
+            if (!this.headerWritten) this.writeHeader(true);
+            return res.write(new Buffer(data.buffer));
+        },
+        writeString(this: any, data: string) {
+            if (!this.headerWritten) this.writeHeader(false);
+            return res.write(data);
+        },
+        end(this: any) {
+            if (this.ended) return;
+            res.end();
+            this.ended = true;
+        },
+        ended: false,
+        headerWritten: false
+    };
+
+    return w;
+}
+
+function mapQuery(app: express.Express, queryName: string, queryDefinition: QueryDefinition) {
+    app.get(makePath(':entryId/' + queryName), async (req, res) => {
+        ConsoleLogger.log('Server', `Query '${req.params.entryId}/${queryName}'...`);
+
+        const request = createRequest('pdb', req.params.entryId, queryName, req.query);
+        const writer = wrapResponse(request.responseFormat.isBinary ? 'result.bcif' : 'result.cif', res);
+        writer.writeHeader(request.responseFormat.isBinary);
+        await resolveRequest(request, writer);
+        writer.end();
+    });
+}
+
+export function initWebApi(app: express.Express) {
+    for (const q of QueryList) {
+        mapQuery(app, q.name, q.definition);
+    }
+}