model-server: query-many (wip)

+ some UI changes

src/mol-io/writer/cif.ts

@@ -6,7 +6,7 @@
  */
 
 import TextEncoder from './cif/encoder/text'
-import BinaryEncoder, { EncodingProvider } from './cif/encoder/binary'
+import BinaryEncoder, { BinaryEncodingProvider } from './cif/encoder/binary'
 import * as _Encoder from './cif/encoder'
 import { ArrayEncoding, ArrayEncoder } from '../common/binary-cif';
 import { CifFrame } from '../reader/cif';
@@ -20,7 +20,7 @@ export namespace CifWriter {
     export interface EncoderParams {
         binary?: boolean,
         encoderName?: string,
-        binaryEncodingPovider?: EncodingProvider,
+        binaryEncodingPovider?: BinaryEncodingProvider,
         binaryAutoClassifyEncoding?: boolean
     }
 
@@ -44,7 +44,7 @@ export namespace CifWriter {
         return { fields, source: [source] };
     }
 
-    export function createEncodingProviderFromCifFrame(frame: CifFrame): EncodingProvider {
+    export function createEncodingProviderFromCifFrame(frame: CifFrame): BinaryEncodingProvider {
         return {
             get(c, f) {
                 const cat = frame.categories[c];
@@ -55,7 +55,7 @@ export namespace CifWriter {
         }
     };
 
-    export function createEncodingProviderFromJsonConfig(hints: EncodingStrategyHint[]): EncodingProvider {
+    export function createEncodingProviderFromJsonConfig(hints: EncodingStrategyHint[]): BinaryEncodingProvider {
         return {
             get(c, f) {
                 for (let i = 0; i < hints.length; i++) {

src/mol-io/writer/cif/encoder.ts

@@ -10,6 +10,7 @@ import { Column, Table, Database, DatabaseCollection } from '../../../mol-data/d
 import { Tensor } from '../../../mol-math/linear-algebra'
 import EncoderBase from '../encoder'
 import { ArrayEncoder, ArrayEncoding } from '../../common/binary-cif';
+import { BinaryEncodingProvider } from './encoder/binary';
 
 // TODO: support for "coordinate fields", make "coordinate precision" a parameter of the encoder
 // TODO: automatically detect "precision" of floating point arrays.
@@ -227,7 +228,9 @@ export interface Encoder<T = string | Uint8Array> extends EncoderBase {
 
     startDataBlock(header: string): void,
     writeCategory<Ctx>(category: Category<Ctx>, context?: Ctx, options?: Encoder.WriteCategoryOptions): void,
-    getData(): T
+    getData(): T,
+
+    binaryEncodingProvider: BinaryEncodingProvider | undefined;
 }
 
 export namespace Encoder {

src/mol-io/writer/cif/encoder/binary.ts

@@ -17,7 +17,7 @@ import { getIncludedFields, getCategoryInstanceData, CategoryInstanceData } from
 import { classifyIntArray, classifyFloatArray } from '../../../common/binary-cif/classifier';
 import { ArrayCtor } from '../../../../mol-util/type-helpers';
 
-export interface EncodingProvider {
+export interface BinaryEncodingProvider {
     get(category: string, field: string): ArrayEncoder | undefined;
 }
 
@@ -28,6 +28,8 @@ export default class BinaryEncoder implements Encoder<Uint8Array> {
     private filter: Category.Filter = Category.DefaultFilter;
     private formatter: Category.Formatter = Category.DefaultFormatter;
 
+    binaryEncodingProvider: BinaryEncodingProvider | undefined = void 0;
+
     setFilter(filter?: Category.Filter) {
         this.filter = filter || Category.DefaultFilter;
     }
@@ -68,7 +70,7 @@ export default class BinaryEncoder implements Encoder<Uint8Array> {
             if (!this.filter.includeField(category.name, f.name)) continue;
 
             const format = this.formatter.getFormat(category.name, f.name);
-            cat.columns.push(encodeField(category.name, f, source, rowCount, format, this.encodingProvider, this.autoClassify));
+            cat.columns.push(encodeField(category.name, f, source, rowCount, format, this.binaryEncodingProvider, this.autoClassify));
         }
         // no columns included.
         if (!cat.columns.length) return;
@@ -92,7 +94,8 @@ export default class BinaryEncoder implements Encoder<Uint8Array> {
         return this.encodedData;
     }
 
-    constructor(encoder: string, private encodingProvider: EncodingProvider | undefined, private autoClassify: boolean) {
+    constructor(encoder: string, encodingProvider: BinaryEncodingProvider | undefined, private autoClassify: boolean) {
+        this.binaryEncodingProvider = encodingProvider;
         this.data = {
             encoder,
             version: VERSION,
@@ -114,7 +117,7 @@ function getDefaultEncoder(type: Field.Type): ArrayEncoder {
     return ArrayEncoder.by(E.byteArray);
 }
 
-function tryGetEncoder(categoryName: string, field: Field, format: Field.Format | undefined, provider: EncodingProvider | undefined) {
+function tryGetEncoder(categoryName: string, field: Field, format: Field.Format | undefined, provider: BinaryEncodingProvider | undefined) {
     if (format && format.encoder) {
         return format.encoder;
     } else if (field.defaultFormat && field.defaultFormat.encoder) {
@@ -133,7 +136,7 @@ function classify(type: Field.Type, data: ArrayLike<any>) {
 }
 
 function encodeField(categoryName: string, field: Field, data: CategoryInstanceData['source'], totalCount: number,
-    format: Field.Format | undefined, encoderProvider: EncodingProvider | undefined, autoClassify: boolean): EncodedColumn {
+    format: Field.Format | undefined, encoderProvider: BinaryEncodingProvider | undefined, autoClassify: boolean): EncodedColumn {
 
     const { array, allPresent, mask } = getFieldData(field, getArrayCtor(field, format), totalCount, data);
 

src/mol-io/writer/cif/encoder/text.ts

@@ -19,6 +19,8 @@ export default class TextEncoder implements Encoder<string> {
     private filter: Category.Filter = Category.DefaultFilter;
     private formatter: Category.Formatter = Category.DefaultFormatter;
 
+    binaryEncodingProvider = void 0;
+
     setFilter(filter?: Category.Filter) {
         this.filter = filter || Category.DefaultFilter;
     }

src/mol-plugin-state/formats/trajectory.ts

@@ -11,7 +11,7 @@ import { guessCifVariant, DataFormatProvider, DataFormatRegistry } from './regis
 import { StateTransformer, StateObjectRef } from '../../mol-state';
 import { PluginStateObject } from '../objects';
 
-export interface TrajectoryFormatProvider<P extends { trajectoryTags?: string | string[] } = { trajectoryTags?: string | string[] }, R extends { trajectory: StateObjectRef<PluginStateObject.Molecule.Trajectory> } = { trajectory: StateObjectRef<PluginStateObject.Molecule.Trajectory> }> 
+export interface TrajectoryFormatProvider<P extends { trajectoryTags?: string | string[] } = { trajectoryTags?: string | string[] }, R extends { trajectory: StateObjectRef<PluginStateObject.Molecule.Trajectory> } = { trajectory: StateObjectRef<PluginStateObject.Molecule.Trajectory> }>
     extends DataFormatProvider<P, R> {
 }
 
@@ -32,10 +32,13 @@ export const MmcifProvider: TrajectoryFormatProvider = {
     },
     parse: async (plugin, data, params) => {
         const state = plugin.state.data;
-        const trajectory = state.build().to(data)
+        const cif = state.build().to(data)
             .apply(StateTransforms.Data.ParseCif, void 0, { state: { isGhost: true } })
-            .apply(StateTransforms.Model.TrajectoryFromMmCif, void 0, { tags: params?.trajectoryTags })
+        const trajectory = cif.apply(StateTransforms.Model.TrajectoryFromMmCif, void 0, { tags: params?.trajectoryTags })
         await plugin.updateDataState(trajectory, { revertOnError: true });
+        if (cif.selector.cell?.obj?.data.blocks.length || 0 > 1) {
+            plugin.state.data.updateCellState(cif.ref, { isGhost: false });
+        }
         return { trajectory: trajectory.selector };
     }
 }

src/mol-plugin-state/manager/structure/hierarchy.ts

@@ -159,9 +159,6 @@ export class StructureHierarchyManager extends PluginComponent {
                 if (t.models.length > 0) {
                     await this.clearTrajectory(t);
                 }
-
-                if (t.models.length === 0) return;
-
                 await this.plugin.builders.structure.hierarchy.applyPreset(t.cell, provider, params);
             }
         });

src/mol-plugin-ui/sequence/sequence.tsx

@@ -13,7 +13,6 @@ import { SequenceWrapper } from './wrapper';
 import { StructureElement, StructureProperties, Unit } from '../../mol-model/structure';
 import { Subject } from 'rxjs';
 import { debounceTime } from 'rxjs/operators';
-import { Color } from '../../mol-util/color';
 import { OrderedSet } from '../../mol-data/int';
 import { Representation } from '../../mol-repr/representation';
 

src/mol-plugin-ui/skin/base/components/controls-base.scss

@@ -23,6 +23,14 @@
     padding: 0;
     text-align: center;
 
+    &:hover {
+        color: $hover-font-color;
+        background-color: color-increase-contrast($msp-form-control-background, 5%);
+        border: none;
+        outline-offset: -1px !important;
+        outline: 1px solid color-increase-contrast($msp-form-control-background, 20%) !important;
+    }
+
     &[disabled], &[disabled]:hover, &[disabled]:active {
         color: $msp-btn-link-toggle-off-font-color;
     }
@@ -37,6 +45,14 @@
     padding: 0;
     text-align: center;
 
+    &:hover {
+        color: $hover-font-color;
+        background-color: color-increase-contrast($msp-form-control-background, 5%);
+        border: none;
+        outline-offset: -1px !important;
+        outline: 1px solid color-increase-contrast($msp-form-control-background, 20%) !important;
+    }
+
     &[disabled], &[disabled]:hover, &[disabled]:active {
         color: $msp-btn-link-toggle-off-font-color;
     }

src/mol-plugin-ui/skin/base/components/temp.scss

@@ -205,6 +205,12 @@
     margin-left: $control-spacing;
 }
 
+.msp-tree-updates-wrapper {
+    .msp-control-group-header:last-child {
+        margin-bottom: 1px;
+    }
+}
+
 .msp-log-list {
     list-style: none;
 

src/mol-plugin-ui/state/tree.tsx

@@ -256,7 +256,7 @@ class StateTreeNodeLabel extends PluginUIComponent<{ cell: StateObjectCell, dept
             decorators!.push(<UpdateTransformControl key={`${d.transform.transformer.id}-${i}`} state={cell.parent} transform={d.transform} noMargin wrapInExpander expanderHeaderLeftMargin={margin} />);
         }
 
-        return decorators;
+        return <div className='msp-tree-updates-wrapper'>{decorators}</div>;
     }
 
     render() {
@@ -285,7 +285,7 @@ class StateTreeNodeLabel extends PluginUIComponent<{ cell: StateObjectCell, dept
         const children = cell.parent.tree.children.get(this.ref);
         const cellState = cell.state;
 
-        const visibility = <button onClick={this.toggleVisible} className={`msp-btn msp-btn-link msp-tree-visibility${cellState.isHidden ? ' msp-tree-visibility-hidden' : ''}`}>
+        const visibility = <button onClick={this.toggleVisible} className={`msp-btn msp-btn-link msp-btn-icon msp-tree-visibility${cellState.isHidden ? ' msp-tree-visibility-hidden' : ''}`}>
             <Icon name='visual-visibility' />
         </button>;
 
@@ -300,7 +300,7 @@ class StateTreeNodeLabel extends PluginUIComponent<{ cell: StateObjectCell, dept
             {children.size > 0 &&  <button onClick={this.toggleExpanded} className='msp-btn msp-btn-link msp-tree-toggle-exp-button'>
                 <Icon name={cellState.isCollapsed ? 'expand' : 'collapse'} />
             </button>}
-            {!cell.state.isLocked && <button onClick={this.remove} className='msp-btn msp-btn-link msp-tree-remove-button'>
+            {!cell.state.isLocked && <button onClick={this.remove} className='msp-btn msp-btn-link msp-btn-icon msp-tree-remove-button'>
                 <Icon name='remove' />
             </button>}{visibility}
         </div>;
@@ -319,11 +319,10 @@ class StateTreeNodeLabel extends PluginUIComponent<{ cell: StateObjectCell, dept
         if (this.state.action === 'options') {
             const actions = this.actions;
             const updates = this.updates(marginStyle.marginLeft as string);
-            // TODO: fix 1px extra margin when updates are empty
             return <div style={{ marginBottom: '1px' }}>
                 {row}
                 {updates}
-                {actions && <div style={{ marginLeft: marginStyle.marginLeft }}>
+                {actions && <div style={{ marginLeft: marginStyle.marginLeft, marginTop: '-1px' }}>
                     <ActionMenu items={actions} onSelect={this.selectAction} />
                 </div>}
             </div>

src/mol-plugin-ui/state/update-transform.tsx

@@ -29,7 +29,7 @@ namespace UpdateTransformControl {
 }
 
 class UpdateTransformControl extends TransformControlBase<UpdateTransformControl.Props, UpdateTransformControl.ComponentState> {
-    applyAction() { 
+    applyAction() {
         if (this.props.customUpdate) return this.props.customUpdate(this.state.params);
         return this.plugin.state.updateTransform(this.props.state, this.props.transform.ref, this.state.params);
     }

src/mol-plugin-ui/structure/source.tsx

@@ -88,7 +88,7 @@ export class StructureSourceControls extends CollapsableControls<{}, StructureSo
             ]);
         }
 
-        if (current.models.length > 1) {
+        if (current.models.length > 1 || current.trajectories.length > 1) {
             ret.push([
                 ActionMenu.Header('Models'),
                 ...current.models.map(this.item)

src/servers/model/query/schemas.ts

@@ -1,5 +1,5 @@
 /**
- * Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2019-2020 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * @author David Sehnal <david.sehnal@gmail.com>
  */
@@ -21,7 +21,10 @@ const InteractionCategories = new Set([
     'pdbx_struct_mod_residue',
     'chem_comp_bond',
     'atom_sites',
-    'atom_site'
+    'atom_site',
+    'pdbx_entity_branch',
+    'pdbx_entity_branch_link',
+    'pdbx_branch_scheme'
 ]);
 
 const AssemblyCategories = new Set([
@@ -41,7 +44,10 @@ const AssemblyCategories = new Set([
     'pdbx_struct_mod_residue',
     'chem_comp_bond',
     'atom_sites',
-    'atom_site'
+    'atom_site',
+    'pdbx_entity_branch',
+    'pdbx_entity_branch_link',
+    'pdbx_branch_scheme'
 ]);
 
 export const QuerySchemas = {

src/servers/model/server/api-local.ts

@@ -6,7 +6,7 @@
 
 import * as fs from 'fs';
 import * as path from 'path';
-import { JobManager, Job } from './jobs';
+import { JobManager, Job, JobEntry } from './jobs';
 import { ConsoleLogger } from '../../../mol-util/console-logger';
 import { resolveJob } from './query';
 import { StructureCache } from './structure-wrapper';
@@ -33,11 +33,13 @@ export async function runLocal(input: LocalInput) {
     for (const job of input) {
         const binary = /\.bcif/.test(job.output);
         JobManager.add({
-            entryId: job.input,
-            queryName: job.query,
-            queryParams: job.params || { },
-            options: {
+            entries: [JobEntry({
+                entryId: job.input,
+                queryName: job.query,
+                queryParams: job.params || { },
                 modelNums: job.modelNums,
+            })],
+            options: {
                 outputFilename: job.output,
                 binary
             }
@@ -48,7 +50,7 @@ export async function runLocal(input: LocalInput) {
     const started = now();
 
     let job: Job | undefined = JobManager.getNext();
-    let key = job.key;
+    let key = job.entries[0].key;
     let progress = 0;
     while (job) {
         try {
@@ -60,8 +62,8 @@ export async function runLocal(input: LocalInput) {
 
             if (JobManager.hasNext()) {
                 job = JobManager.getNext();
-                if (key !== job.key) StructureCache.expire(key);
-                key = job.key;
+                if (key !== job.entries[0].key) StructureCache.expire(key);
+                key = job.entries[0].key;
             } else {
                 break;
             }

src/servers/model/server/api-schema.ts

@@ -7,6 +7,7 @@
 import VERSION from '../version'
 import { QueryParamInfo, QueryParamType, QueryDefinition, CommonQueryParamsInfo, QueryList } from './api';
 import { ModelServerConfig as ServerConfig } from '../config';
+import { MultipleQuerySpec } from './api-web-multiple';
 
 export const shortcutIconLink = `<link rel='shortcut icon' href='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAMAAABEpIrGAAAABGdBTUEAALGPC/xhBQAAAAFzUkdCAK7OHOkAAAAnUExURQAAAMIrHrspHr0oH7soILonHrwqH7onILsoHrsoH7soH7woILwpIKgVokoAAAAMdFJOUwAQHzNxWmBHS5XO6jdtAmoAAACZSURBVDjLxZNRCsQgDAVNXmwb9f7nXZEaLRgXloXOhwQdjMYYwpOLw55fBT46KhbOKhmRR2zLcFJQj8UR+HxFgArIF5BKJbEncC6NDEdI5SatBRSDJwGAoiFDONrEJXWYhGMIcRJGCrb1TOtDahfUuQXd10jkFYq0ViIrbUpNcVT6redeC1+b9tH2WLR93Sx2VCzkv/7NjfABxjQHksGB7lAAAAAASUVORK5CYII=' />`
 
@@ -46,11 +47,70 @@ function getPaths() {
     for (const { name, definition } of QueryList) {
         ret[`${ServerConfig.apiPrefix}/v1/{id}/${name}`] = getQueryInfo(definition);
     }
+
+    const queryManySummary = 'Executes multiple queries at the same time and writes them as separate data blocks.';
+    const queryManyExample: MultipleQuerySpec = {
+        queries: [
+            { entryId: '1cbs', query: 'residueInteraction', params: { atom_site: [{ label_comp_id: 'REA' }], radius: 5 } },
+            { entryId: '1tqn', query: 'full' }
+        ],
+        encoding: 'cif'
+    };
+    ret[`${ServerConfig.apiPrefix}/v1/query-many`] = {
+        get: {
+            tags: ['General'],
+            summary: queryManySummary,
+            operationId: 'query-many',
+            parameters: [{
+                name: 'query',
+                in: 'query',
+                description: 'URI encoded JSON object with the query definiton.',
+                required: true,
+                schema: {
+                    type: 'string',
+                },
+                example: JSON.stringify(queryManyExample),
+                style: 'simple'
+            }],
+            responses: {
+                200: {
+                    description: 'Separate CIF data blocks with the result',
+                    content: {
+                        'text/plain': {},
+                        'application/octet-stream': {},
+                    }
+                }
+            }
+        },
+        post: {
+            tags: ['General'],
+            summary: queryManySummary,
+            operationId: 'query-many-post',
+            parameters: [],
+            requestBody: {
+                content: {
+                    'application/json': {
+                        schema: { type: 'object' },
+                        example: queryManyExample
+                    }
+                }
+            },
+            responses: {
+                200: {
+                    description: 'Separate CIF data blocks with the result',
+                    content: {
+                        'text/plain': {},
+                        'application/octet-stream': {},
+                    }
+                }
+            }
+        }
+    };
+
     return ret;
 }
 
 function getQueryInfo(def: QueryDefinition) {
-
     const jsonExample: any = {};
     def.jsonParams.forEach(p => {
         if (!p.exampleValues || !p.exampleValues.length) return;

src/servers/model/server/api-web-multiple.ts

@@ -0,0 +1,20 @@
+/**
+ * Copyright (c) 2018-2020 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import { QueryName, QueryParams } from './api';
+
+export interface MultipleQueryEntry<Name extends QueryName = QueryName> {
+    data_source?: string,
+    entryId: string,
+    query: Name,
+    params?: QueryParams<Name>,
+    model_nums?: number[]
+}
+
+export interface MultipleQuerySpec {
+    queries: MultipleQueryEntry[],
+    encoding?: 'cif' | 'bcif'
+}

src/servers/model/server/api-web.ts

@@ -1,5 +1,5 @@
 /**
- * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2018-2020 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * @author David Sehnal <david.sehnal@gmail.com>
  */
@@ -11,11 +11,12 @@ import * as bodyParser from 'body-parser'
 import { ModelServerConfig as Config, ModelServerConfig, mapSourceAndIdToFilename } from '../config';
 import { ConsoleLogger } from '../../../mol-util/console-logger';
 import { resolveJob } from './query';
-import { JobManager } from './jobs';
+import { JobManager, JobEntry } from './jobs';
 import { UUID } from '../../../mol-util';
 import { QueryDefinition, normalizeRestQueryParams, normalizeRestCommonParams, QueryList } from './api';
 import { getApiSchema, shortcutIconLink } from './api-schema';
 import { swaggerUiAssetsHandler, swaggerUiIndexHandler } from '../../common/swagger-ui';
+import { MultipleQuerySpec } from './api-web-multiple';
 
 function makePath(p: string) {
     return Config.apiPrefix + '/' + p;
@@ -69,7 +70,9 @@ async function processNextJob() {
     const response = responseMap.get(job.id)!;
     responseMap.delete(job.id);
 
-    const filenameBase = `${job.entryId}_${job.queryDefinition.name.replace(/\s/g, '_')}`
+    const filenameBase = job.entries.length === 1
+        ? `${job.entries[0].entryId}_${job.entries[0].queryDefinition.name.replace(/\s/g, '_')}`
+        : `result`;
     const writer = wrapResponse(job.responseFormat.isBinary ? `${filenameBase}.bcif` : `${filenameBase}.cif`, response);
 
     try {
@@ -87,35 +90,31 @@ async function processNextJob() {
 }
 
 function mapQuery(app: express.Express, queryName: string, queryDefinition: QueryDefinition) {
-    app.get(makePath('v1/:id/' + queryName), (req, res) => {
-        // console.log({ queryName, params: req.params, query: req.query });
+    function createJob(queryParams: any, req: express.Request, res: express.Response) {
         const entryId = req.params.id;
-        const queryParams = normalizeRestQueryParams(queryDefinition, req.query);
         const commonParams = normalizeRestCommonParams(req.query);
         const jobId = JobManager.add({
-            sourceId: commonParams.data_source || ModelServerConfig.defaultSource,
-            entryId,
-            queryName: queryName as any,
-            queryParams,
-            options: { modelNums: commonParams.model_nums, binary: commonParams.encoding === 'bcif' }
+            entries: [JobEntry({
+                sourceId: commonParams.data_source || ModelServerConfig.defaultSource,
+                entryId,
+                queryName: queryName as any,
+                queryParams,
+                modelNums: commonParams.model_nums
+            })],
+            options: { binary: commonParams.encoding === 'bcif' }
         });
         responseMap.set(jobId, res);
         if (JobManager.size === 1) processNextJob();
+    }
+
+    app.get(makePath('v1/:id/' + queryName), (req, res) => {
+        const queryParams = normalizeRestQueryParams(queryDefinition, req.query);
+        createJob(queryParams, req, res);
     });
 
     app.post(makePath('v1/:id/' + queryName), (req, res) => {
-        const entryId = req.params.id;
         const queryParams = req.body;
-        const commonParams = normalizeRestCommonParams(req.query);
-        const jobId = JobManager.add({
-            sourceId: commonParams.data_source || ModelServerConfig.defaultSource,
-            entryId,
-            queryName: queryName as any,
-            queryParams,
-            options: { modelNums: commonParams.model_nums, binary: commonParams.encoding === 'bcif' }
-        });
-        responseMap.set(jobId, res);
-        if (JobManager.size === 1) processNextJob();
+        createJob(queryParams, req, res);
     });
 }
 
@@ -154,28 +153,36 @@ function serveStatic(req: express.Request, res: express.Response) {
     });
 }
 
+function createMultiJob(spec: MultipleQuerySpec, res: express.Response) {
+    const jobId = JobManager.add({
+        entries: spec.queries.map(q => JobEntry({
+            sourceId: q.data_source || ModelServerConfig.defaultSource,
+            entryId: q.entryId,
+            queryName: q.query,
+            queryParams: q.params || { },
+            modelNums: q.model_nums
+        })),
+        options: { binary: spec.encoding?.toLowerCase() === 'bcif' }
+    });
+    responseMap.set(jobId, res);
+    if (JobManager.size === 1) processNextJob();
+}
+
 export function initWebApi(app: express.Express) {
     app.use(bodyParser.json({ limit: '1mb' }));
 
     app.get(makePath('static/:source/:id'), (req, res) => serveStatic(req, res));
     app.get(makePath('v1/static/:source/:id'), (req, res) => serveStatic(req, res));
 
-    // app.get(makePath('v1/json'), (req, res) => {
-    //     const query = /\?(.*)$/.exec(req.url)![1];
-    //     const args = JSON.parse(decodeURIComponent(query));
-    //     const name = args.name;
-    //     const entryId = args.id;
-    //     const queryParams = args.params || { };
-    //     const jobId = JobManager.add({
-    //         sourceId: 'pdb',
-    //         entryId,
-    //         queryName: name,
-    //         queryParams,
-    //         options: { modelNums: args.modelNums, binary: args.binary }
-    //     });
-    //     responseMap.set(jobId, res);
-    //     if (JobManager.size === 1) processNextJob();
-    // });
+    app.get(makePath('v1/query-many'), (req, res) => {
+        const query = /\?query=(.*)$/.exec(req.url)![1];
+        const params = JSON.parse(decodeURIComponent(query));
+        createMultiJob(params, res);
+    });
+    app.post(makePath('v1/query-many'), (req, res) => {
+        const params = req.body;
+        createMultiJob(params, res);
+    });
 
     app.use(bodyParser.json({ limit: '20mb' }));
 
@@ -201,8 +208,4 @@ export function initWebApi(app: express.Express) {
         title: 'ModelServer API',
         shortcutIconLink
     }));
-
-    // app.get('*', (req, res) => {
-    //     res.send(LandingPage);
-    // });
 }

src/servers/model/server/jobs.ts

@@ -1,5 +1,5 @@
 /**
- * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2018-2020 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * @author David Sehnal <david.sehnal@gmail.com>
  */
@@ -16,44 +16,64 @@ export interface Job {
     id: UUID,
     datetime_utc: string,
 
+    entries: JobEntry[],
+
+    responseFormat: ResponseFormat,
+    outputFilename?: string
+}
+
+export interface JobEntry {
+    job: Job,
     sourceId: '_local_' | string,
     entryId: string,
     key: string,
 
     queryDefinition: QueryDefinition,
     normalizedParams: any,
-    responseFormat: ResponseFormat,
-    modelNums?: number[],
-
-    outputFilename?: string
+    modelNums?: number[]
 }
 
-export interface JobDefinition<Name extends QueryName> {
+interface JobEntryDefinition<Name extends QueryName> {
     sourceId?: string, // = '_local_',
     entryId: string,
     queryName: Name,
     queryParams: QueryParams<Name>,
-    options?: { modelNums?: number[], outputFilename?: string, binary?: boolean }
+    modelNums?: number[]
 }
 
-export function createJob<Name extends QueryName>(definition: JobDefinition<Name>): Job {
+export function JobEntry<Name extends QueryName>(definition: JobEntryDefinition<Name>): JobEntry {
     const queryDefinition = getQueryByName(definition.queryName);
     if (!queryDefinition) throw new Error(`Query '${definition.queryName}' is not supported.`);
 
     const normalizedParams = definition.queryParams;
     const sourceId = definition.sourceId || '_local_';
+
     return {
-        id: UUID.create22(),
-        datetime_utc: `${new Date().toISOString().replace(/T/, ' ').replace(/\..+/, '')}`,
+        job: void 0 as any,
         key: `${sourceId}/${definition.entryId}`,
         sourceId,
         entryId: definition.entryId,
         queryDefinition,
         normalizedParams,
+        modelNums: definition.modelNums
+    }
+}
+
+export interface JobDefinition {
+    entries: JobEntry[],
+    options?: { outputFilename?: string, binary?: boolean }
+}
+
+export function createJob(definition: JobDefinition): Job {
+    const job: Job = {
+        id: UUID.create22(),
+        datetime_utc: `${new Date().toISOString().replace(/T/, ' ').replace(/\..+/, '')}`,
+        entries: definition.entries,
         responseFormat: { isBinary: !!(definition.options && definition.options.binary) },
-        modelNums: definition.options && definition.options.modelNums,
         outputFilename: definition.options && definition.options.outputFilename
     };
+    definition.entries.forEach(e => e.job = job);
+    return job;
 }
 
 class _JobQueue {
@@ -63,7 +83,7 @@ class _JobQueue {
         return this.list.count;
     }
 
-    add<Name extends QueryName>(definition: JobDefinition<Name>) {
+    add(definition: JobDefinition) {
         const job = createJob(definition);
         this.list.addLast(job);
         return job.id;
@@ -86,7 +106,7 @@ class _JobQueue {
             jobs[jobs.length] = j.value;
         }
 
-        jobs.sort((a, b) => a.key < b.key ? -1 : 1);
+        jobs.sort((a, b) => a.entries[0]?.key < b.entries[0]?.key ? -1 : 1);
 
         this.list = LinkedList();
         for (const j of jobs) {

src/servers/model/server/query.ts

@@ -1,5 +1,5 @@
 /**
- * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2018-2020 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * @author David Sehnal <david.sehnal@gmail.com>
  */
@@ -14,8 +14,8 @@ import { ConsoleLogger } from '../../../mol-util/console-logger';
 import { PerformanceMonitor } from '../../../mol-util/performance-monitor';
 import { ModelServerConfig as Config } from '../config';
 import Version from '../version';
-import { Job } from './jobs';
-import { createStructureWrapperFromJob, StructureWrapper, resolveStructures } from './structure-wrapper';
+import { Job, JobEntry } from './jobs';
+import { createStructureWrapperFromJobEntry, StructureWrapper, resolveStructures } from './structure-wrapper';
 import CifField = CifWriter.Field
 import { createModelPropertiesProviderFromConfig, ModelPropertiesProvider } from '../property-provider';
 
@@ -38,23 +38,46 @@ function propertyProvider() {
 export async function resolveJob(job: Job): Promise<CifWriter.Encoder<any>> {
     ConsoleLogger.logId(job.id, 'Query', 'Starting.');
 
-    const wrappedStructure = await createStructureWrapperFromJob(job, propertyProvider());
+    const encoder = CifWriter.createEncoder({
+        binary: job.responseFormat.isBinary,
+        encoderName: `ModelServer ${Version}`,
+        binaryAutoClassifyEncoding: true
+    });
+
+    // TODO: how to handle missing entries?
+    for (const entry of job.entries) {
+        const structure = await createStructureWrapperFromJobEntry(entry, propertyProvider());
+
+        // TODO: this should be unique in case the same structure is queried twice
+        // const data = (entry.sourceId === '_local_' ? path.basename(entry.entryId) : entry.entryId).replace(/[^a-z0-9\_]/ig, '').toUpperCase();
+        encoder.startDataBlock(structure.cifFrame.header);
+        await resolveJobEntry(entry, structure, encoder);
+    }
+
+    ConsoleLogger.logId(job.id, 'Query', 'Encoding.');
+    encoder.encode();
+
+    return encoder;
+}
+
+async function resolveJobEntry(entry: JobEntry, structure: StructureWrapper, encoder: CifWriter.Encoder<any>) {
+    ConsoleLogger.logId(entry.job.id, 'Query', `Start ${entry.key}/${entry.queryDefinition.name}.`);
 
     try {
         perf.start('query');
-        const sourceStructures = await resolveStructures(wrappedStructure, job.modelNums);
+        const sourceStructures = await resolveStructures(structure, entry.modelNums);
         if (!sourceStructures.length) throw new Error('Model not available');
 
         let structures: Structure[] = sourceStructures;
 
-        if (job.queryDefinition.structureTransform) {
+        if (entry.queryDefinition.structureTransform) {
             structures = [];
             for (const s of sourceStructures) {
-                structures.push(await job.queryDefinition.structureTransform(job.normalizedParams, s));
+                structures.push(await entry.queryDefinition.structureTransform(entry.normalizedParams, s));
             }
         }
 
-        const queries = structures.map(s => job.queryDefinition.query(job.normalizedParams, s));
+        const queries = structures.map(s => entry.queryDefinition.query(entry.normalizedParams, s));
         const result: Structure[] = [];
         for (let i = 0; i < structures.length; i++) {
             const s = await StructureSelection.unionStructure(StructureQuery.run(queries[i], structures[i], { timeoutMs: Config.queryTimeoutMs }))
@@ -62,39 +85,36 @@ export async function resolveJob(job: Job): Promise<CifWriter.Encoder<any>> {
         }
         perf.end('query');
 
-        const encoder = CifWriter.createEncoder({
-            binary: job.responseFormat.isBinary,
-            encoderName: `ModelServer ${Version}`,
-            binaryEncodingPovider: getEncodingProvider(wrappedStructure),
-            binaryAutoClassifyEncoding: true
-        });
-
-        ConsoleLogger.logId(job.id, 'Query', 'Query finished.');
+        ConsoleLogger.logId(entry.job.id, 'Query', `Queried ${entry.key}/${entry.queryDefinition.name}.`);
 
         perf.start('encode');
-        encoder.startDataBlock(sourceStructures[0].models[0].entryId.toUpperCase());
-        encoder.writeCategory(_model_server_result, job);
-        encoder.writeCategory(_model_server_params, job);
 
-        if (job.queryDefinition.filter) encoder.setFilter(job.queryDefinition.filter);
+        encoder.binaryEncodingProvider = getEncodingProvider(structure);
+
+        // TODO: this actually needs to "reversible" in case of error.
+        encoder.writeCategory(_model_server_result, entry);
+        encoder.writeCategory(_model_server_params, entry);
+
+        if (entry.queryDefinition.filter) encoder.setFilter(entry.queryDefinition.filter);
         if (result.length > 0) encode_mmCIF_categories(encoder, result);
-        if (job.queryDefinition.filter) encoder.setFilter();
+        if (entry.queryDefinition.filter) encoder.setFilter();
         perf.end('encode');
 
         const stats: Stats = {
-            structure: wrappedStructure,
+            structure: structure,
             queryTimeMs: perf.time('query'),
             encodeTimeMs: perf.time('encode'),
             resultSize: result.reduce((n, s) => n + s.elementCount, 0)
         };
 
         encoder.writeCategory(_model_server_stats, stats);
-        encoder.encode();
-        ConsoleLogger.logId(job.id, 'Query', 'Encoded.');
+        ConsoleLogger.logId(entry.job.id, 'Query', `Written ${entry.key}/${entry.queryDefinition.name}.`);
         return encoder;
     } catch (e) {
-        ConsoleLogger.errorId(job.id, e);
-        return doError(job, e);
+        ConsoleLogger.errorId(entry.job.id, e);
+        doError(entry, encoder, e);
+    } finally {
+        encoder.binaryEncodingProvider = void 0;
     }
 }
 
@@ -103,13 +123,10 @@ function getEncodingProvider(structure: StructureWrapper) {
     return CifWriter.createEncodingProviderFromCifFrame(structure.cifFrame);
 }
 
-function doError(job: Job, e: any) {
-    const encoder = CifWriter.createEncoder({ binary: job.responseFormat.isBinary, encoderName: `ModelServer ${Version}` });
-    encoder.writeCategory(_model_server_result, job);
-    encoder.writeCategory(_model_server_params, job);
+function doError(entry: JobEntry, encoder: CifWriter.Encoder<any>, e: any) {
+    encoder.writeCategory(_model_server_result, entry);
+    encoder.writeCategory(_model_server_params, entry);
     encoder.writeCategory(_model_server_error, '' + e);
-    encoder.encode();
-    return encoder;
 }
 
 const maxTime = Config.queryTimeoutMs;
@@ -130,13 +147,13 @@ function int32<T>(name: string, value: (data: T) => number): CifField<number, T>
     return CifField.int(name, (i, d) => value(d));
 }
 
-const _model_server_result_fields: CifField<any, Job>[] = [
-    string<Job>('job_id', ctx => '' + ctx.id),
-    string<Job>('datetime_utc', ctx => ctx.datetime_utc),
-    string<Job>('server_version', ctx => Version),
-    string<Job>('query_name', ctx => ctx.queryDefinition.name),
-    string<Job>('source_id', ctx => ctx.sourceId),
-    string<Job>('entry_id', ctx => ctx.entryId),
+const _model_server_result_fields: CifField<any, JobEntry>[] = [
+    string<JobEntry>('job_id', ctx => '' + ctx.job.id),
+    string<JobEntry>('datetime_utc', ctx => ctx.job.datetime_utc),
+    string<JobEntry>('server_version', ctx => Version),
+    string<JobEntry>('query_name', ctx => ctx.queryDefinition.name),
+    string<JobEntry>('source_id', ctx => ctx.sourceId),
+    string<JobEntry>('entry_id', ctx => ctx.entryId),
 ];
 
 const _model_server_params_fields: CifField<number, string[]>[] = [
@@ -158,7 +175,7 @@ const _model_server_stats_fields: CifField<number, Stats>[] = [
     int32<Stats>('element_count', ctx => ctx.resultSize | 0),
 ];
 
-const _model_server_result: CifWriter.Category<Job> = {
+const _model_server_result: CifWriter.Category<JobEntry> = {
     name: 'model_server_result',
     instance: (job) => CifWriter.categoryInstance(_model_server_result_fields, { data: job, rowCount: 1 })
 };
@@ -168,7 +185,7 @@ const _model_server_error: CifWriter.Category<string> = {
     instance: (message) => CifWriter.categoryInstance(_model_server_error_fields, { data: message, rowCount: 1 })
 };
 
-const _model_server_params: CifWriter.Category<Job> = {
+const _model_server_params: CifWriter.Category<JobEntry> = {
     name: 'model_server_params',
     instance(job) {
         const params: string[][] = [];
@@ -179,7 +196,6 @@ const _model_server_params: CifWriter.Category<Job> = {
     }
 };
 
-
 const _model_server_stats: CifWriter.Category<Stats> = {
     name: 'model_server_stats',
     instance: (stats) => CifWriter.categoryInstance(_model_server_stats_fields, { data: stats, rowCount: 1 })

src/servers/model/server/structure-wrapper.ts

@@ -1,5 +1,5 @@
 /**
- * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2018-2020 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * @author David Sehnal <david.sehnal@gmail.com>
  */
@@ -12,7 +12,7 @@ import { CIF, CifFrame, CifBlock } from '../../../mol-io/reader/cif'
 import * as util from 'util'
 import * as fs from 'fs'
 import * as zlib from 'zlib'
-import { Job } from './jobs';
+import { JobEntry } from './jobs';
 import { ConsoleLogger } from '../../../mol-util/console-logger';
 import { ModelPropertiesProvider } from '../property-provider';
 import { trajectoryFromMmCIF } from '../../../mol-model-formats/structure/mmcif';
@@ -49,12 +49,12 @@ export interface StructureWrapper {
     cache: object
 }
 
-export async function createStructureWrapperFromJob(job: Job, propertyProvider: ModelPropertiesProvider | undefined, allowCache = true): Promise<StructureWrapper> {
+export async function createStructureWrapperFromJobEntry(entry: JobEntry, propertyProvider: ModelPropertiesProvider | undefined, allowCache = true): Promise<StructureWrapper> {
     if (allowCache && Config.cacheMaxSizeInBytes > 0) {
-        const ret = StructureCache.get(job.key);
+        const ret = StructureCache.get(entry.key);
         if (ret) return ret;
     }
-    const ret = await readStructureWrapper(job.key, job.sourceId, job.entryId, job.id, propertyProvider);
+    const ret = await readStructureWrapper(entry.key, entry.sourceId, entry.entryId, entry.job.id, propertyProvider);
     if (allowCache && Config.cacheMaxSizeInBytes > 0) {
         StructureCache.add(ret);
     }

src/servers/model/test.ts

@@ -1,86 +1,86 @@
-import { resolveJob } from './server/query';
-import * as fs from 'fs'
-import * as path from 'path'
-import { StructureCache } from './server/structure-wrapper';
-import { createJob } from './server/jobs';
+// import { resolveJob } from './server/query';
+// import * as fs from 'fs'
+// import * as path from 'path'
+// import { StructureCache } from './server/structure-wrapper';
+// import { createJob } from './server/jobs';
 
-function wrapFile(fn: string) {
-    const w = {
-        open(this: any) {
-            if (this.opened) return;
-            this.file = fs.openSync(fn, 'w');
-            this.opened = true;
-        },
-        writeBinary(this: any, data: Uint8Array) {
-            this.open();
-            fs.writeSync(this.file, Buffer.from(data));
-            return true;
-        },
-        writeString(this: any, data: string) {
-            this.open();
-            fs.writeSync(this.file, data);
-            return true;
-        },
-        end(this: any) {
-            if (!this.opened || this.ended) return;
-            fs.close(this.file, function () { });
-            this.ended = true;
-        },
-        file: 0,
-        ended: false,
-        opened: false
-    };
+// function wrapFile(fn: string) {
+//     const w = {
+//         open(this: any) {
+//             if (this.opened) return;
+//             this.file = fs.openSync(fn, 'w');
+//             this.opened = true;
+//         },
+//         writeBinary(this: any, data: Uint8Array) {
+//             this.open();
+//             fs.writeSync(this.file, Buffer.from(data));
+//             return true;
+//         },
+//         writeString(this: any, data: string) {
+//             this.open();
+//             fs.writeSync(this.file, data);
+//             return true;
+//         },
+//         end(this: any) {
+//             if (!this.opened || this.ended) return;
+//             fs.close(this.file, function () { });
+//             this.ended = true;
+//         },
+//         file: 0,
+//         ended: false,
+//         opened: false
+//     };
 
-    return w;
-}
+//     return w;
+// }
 
-export const basePath = path.join(__dirname, '..', '..', '..', '..')
-export const examplesPath = path.join(basePath, 'examples')
-export const outPath = path.join(basePath, 'build', 'test')
-if (!fs.existsSync(outPath)) fs.mkdirSync(outPath);
+// export const basePath = path.join(__dirname, '..', '..', '..', '..')
+// export const examplesPath = path.join(basePath, 'examples')
+// export const outPath = path.join(basePath, 'build', 'test')
+// if (!fs.existsSync(outPath)) fs.mkdirSync(outPath);
 
-async function run() {
-    try {
-        // const testFile = '1crn.cif'
-        // const testFile = '1grm_updated.cif'
-        // const testFile = 'C:/Projects/mol-star/molstar/build/test/in/1grm_updated.cif'
-        // const request = createJob({
-        //     entryId: testFile,
-        //     queryName: 'full',
-        //     queryParams: { },
-        // });
-        const testFile = '1cbs_updated.cif'
-        const request = createJob({
-            entryId: path.join(examplesPath, testFile),
-            queryName: 'full',
-            queryParams: { }
-        });
+// async function run() {
+//     try {
+//         // const testFile = '1crn.cif'
+//         // const testFile = '1grm_updated.cif'
+//         // const testFile = 'C:/Projects/mol-star/molstar/build/test/in/1grm_updated.cif'
+//         // const request = createJob({
+//         //     entryId: testFile,
+//         //     queryName: 'full',
+//         //     queryParams: { },
+//         // });
+//         const testFile = '1cbs_updated.cif'
+//         const request = createJob({
+//             entryId: path.join(examplesPath, testFile),
+//             queryName: 'full',
+//             queryParams: { }
+//         });
 
-        // const request = createJob({
-        //     entryId: path.join(examplesPath, testFile),
-        //     queryName: 'atoms',
-        //     queryParams: {
-        //         atom_site: { label_comp_id: 'ALA' }
-        //     }
-        // });
-        // const request = createJob({
-        //     entryId: path.join(examplesPath, testFile),
-        //     queryName: 'residueInteraction',
-        //     queryParams: {
-        //         atom_site: { label_comp_id: 'REA' },
-        //         radius: 5
-        //     }
-        // });
-        const encoder = await resolveJob(request);
-        const writer = wrapFile(path.join(outPath, testFile));
-        // const writer = wrapFile(path.join(outPath, '1grm_test.cif'));
-        encoder.writeTo(writer);
-        writer.end();
-    } catch (e) {
-        console.error(e)
-    } finally {
-        StructureCache.expireAll();
-    }
-}
+//         // const request = createJob({
+//         //     entryId: path.join(examplesPath, testFile),
+//         //     queryName: 'atoms',
+//         //     queryParams: {
+//         //         atom_site: { label_comp_id: 'ALA' }
+//         //     }
+//         // });
+//         // const request = createJob({
+//         //     entryId: path.join(examplesPath, testFile),
+//         //     queryName: 'residueInteraction',
+//         //     queryParams: {
+//         //         atom_site: { label_comp_id: 'REA' },
+//         //         radius: 5
+//         //     }
+//         // });
+//         const encoder = await resolveJob(request);
+//         const writer = wrapFile(path.join(outPath, testFile));
+//         // const writer = wrapFile(path.join(outPath, '1grm_test.cif'));
+//         encoder.writeTo(writer);
+//         writer.end();
+//     } catch (e) {
+//         console.error(e)
+//     } finally {
+//         StructureCache.expireAll();
+//     }
+// }
 
-run();
+// run();