Merge branch 'master' into gl-lines

# Conflicts:
#	package-lock.json

docs/model-server/readme.md

@@ -0,0 +1,130 @@
+Model Server
+============
+
+Model Server is a tool for preprocessing and querying macromolecular structure data.
+
+Installing and Running
+=====================
+
+Getting the code (use node 8+):
+```
+git clone https://github.com/molstar/molstar-proto
+npm install
+```
+
+Customize configuration at ``src/server/model/config.ts`` to point to your data and which custom properties to include (see the [Custom Properties](#custom-properties) section). Alternatively, the config can be edited in the compiled version in ``build/node_modules/servers/model/config.js``.
+
+Running the server locally for testing:
+```
+npm run model-server
+```
+or
+```
+node build/node_modules/servers/model/server
+```
+
+In production it is a good idea to use a service that will keep the server running, such as [forever.js](https://github.com/foreverjs/forever).
+
+
+## Memory issues
+
+Sometimes nodejs might run into problems with memory. This is usually resolved by adding the ``--max-old-space-size=8192`` parameter.
+
+Preprocessor
+============
+
+The preprocessor application allows to add custom data to CIF files and/or convert CIF to BinaryCIF. See the [Custom Properties](#custom-properties) section for providing custom properties.
+
+## Usage
+
+The app works in two modes: single files and folders.
+
+Single files:
+
+```
+node build\node_modules\servers\model\preprocess -i input.cif [-oc output.cif] [-ob output.bcif] [--cfg config.json]
+```
+
+Folder: 
+```
+node build\node_modules\servers\model\preprocess -fin input_folder [-foc output_cif_folder] [-fob output_bcif_folder] [--cfg config.json]
+```
+
+## Config
+
+The config speficies the maximum number of processes to use (in case of folder processing) and defines sources and parameters for custom properties.
+
+Example:
+```json
+{
+    "numProcesses": 4,
+    "customProperties": {
+        "sources": [
+            "./properties/pdbe"
+        ],
+        "params": {
+            "PDBe": {
+                "UseFileSource": false,
+                "API": {
+                    "residuewise_outlier_summary": "https://www.ebi.ac.uk/pdbe/api/validation/residuewise_outlier_summary/entry",
+                    "preferred_assembly": "https://www.ebi.ac.uk/pdbe/api/pdb/entry/summary",
+                    "struct_ref_domain": "https://www.ebi.ac.uk/pdbe/api/mappings/sequence_domains"
+                }
+            }
+        }
+    }
+}
+```
+
+Custom Properties
+=================
+
+It is possible to provide property descriptors that transform data to internal representation and define how it should be exported into one or mode CIF categories. Examples of this are located in the ``mol-model-props`` module and are linked to the server in the config and ``servers/model/properties``.
+
+Local Mode
+==========
+
+The server can be run in local/file based mode:
+
+```
+node build/node_modules/servers/model/server jobs.json
+```
+
+where ``jobs.json`` is an array of 
+
+```ts
+type LocalInput = {
+    input: string,
+    output: string,
+    query: QueryName,
+    modelNums?: number[],
+    params?: any,
+    binary?: boolean
+}[]
+```
+
+For example
+
+```json
+[
+  {
+    "input": "c:/test/quick/1tqn.cif",
+    "output": "c:/test/quick/localapi/1tqn_full.cif",
+    "query": "full"
+  },
+  {
+    "input": "c:/test/quick/1tqn.cif",
+    "output": "c:/test/quick/localapi/1tqn_full.bcif",
+    "query": "full",
+    "params": {}
+  },
+  {
+    "input": "c:/test/quick/1cbs_updated.cif",
+    "output": "c:/test/quick/localapi/1cbs_ligint.cif",
+    "query": "residueInteraction",
+    "params": {
+      "atom_site": { "label_comp_id": "REA" }
+    }
+  }
+]
+```

package.json

@@ -73,15 +73,15 @@
   "author": "",
   "license": "MIT",
   "devDependencies": {
-    "@types/argparse": "^1.0.34",
+    "@types/argparse": "^1.0.35",
     "@types/benchmark": "^1.0.31",
     "@types/compression": "0.0.36",
     "@types/express": "^4.16.0",
-    "@types/jest": "^23.3.2",
-    "@types/node": "^10.10.1",
+    "@types/jest": "^23.3.3",
+    "@types/node": "^10.11.4",
     "@types/node-fetch": "^2.1.2",
     "@types/react": "^16.4.14",
-    "@types/react-dom": "^16.0.7",
+    "@types/react-dom": "^16.0.8",
     "benchmark": "^2.1.4",
     "cpx": "^1.5.0",
     "css-loader": "^1.0.0",
@@ -89,8 +89,8 @@
     "file-loader": "^2.0.0",
     "glslify-import": "^3.1.0",
     "glslify-loader": "^1.0.2",
-    "graphql-code-generator": "^0.12.5",
-    "graphql-codegen-typescript-template": "^0.12.5",
+    "graphql-code-generator": "^0.12.6",
+    "graphql-codegen-typescript-template": "^0.12.6",
     "jest": "^23.6.0",
     "jest-raw-loader": "^1.0.1",
     "mini-css-extract-plugin": "^0.4.3",
@@ -99,13 +99,13 @@
     "resolve-url-loader": "^3.0.0",
     "sass-loader": "^7.1.0",
     "style-loader": "^0.23.0",
-    "ts-jest": "^23.10.0",
+    "ts-jest": "^23.10.3",
     "tslint": "^5.11.0",
-    "typescript": "^3.0.3",
+    "typescript": "^3.1.1",
     "uglify-js": "^3.4.9",
     "util.promisify": "^1.0.0",
-    "webpack": "^4.19.1",
-    "webpack-cli": "^3.1.0"
+    "webpack": "^4.20.2",
+    "webpack-cli": "^3.1.2"
   },
   "dependencies": {
     "argparse": "^1.0.10",

src/apps/cif2bcif/converter.ts

@@ -27,7 +27,7 @@ async function getCIF(ctx: RuntimeContext, path: string) {
 function getCategoryInstanceProvider(cat: CifCategory, fields: CifWriter.Field[]): CifWriter.Category {
     return {
         name: cat.name,
-        instance: () => ({ data: cat, fields, rowCount: cat.rowCount })
+        instance: () => CifWriter.categoryInstance(fields, { data: cat, rowCount: cat.rowCount })
     };
 }
 

src/mol-io/writer/cif.ts

@@ -40,6 +40,10 @@ export namespace CifWriter {
         fixedPoint3: E.by(E.fixedPoint(1000)).and(E.delta).and(E.integerPacking),
     };
 
+    export function categoryInstance<Key, Data>(fields: Field<Key, Data>[], source: Category.DataSource): Category.Instance {
+        return { fields, source: [source] };
+    }
+
     export function createEncodingProviderFromCifFrame(frame: CifFrame): EncodingProvider {
         return {
             get(c, f) {

src/mol-io/writer/cif/encoder.ts

@@ -36,7 +36,11 @@ export namespace Field {
         typedArray?: ArrayEncoding.TypedArrayCtor
     }
 
-    export type ParamsBase<K, D> = { valueKind?: (k: K, d: D) => Column.ValueKind, encoder?: ArrayEncoder, shouldInclude?: (data: D) => boolean }
+    export type ParamsBase<K, D> = {
+        valueKind?: (k: K, d: D) => Column.ValueKind,
+        encoder?: ArrayEncoder,
+        shouldInclude?: (data: D) => boolean
+    }
 
     export function str<K, D = any>(name: string, value: (k: K, d: D, index: number) => string, params?: ParamsBase<K, D>): Field<K, D> {
         return { name, type: Type.Str, value, valueKind: params && params.valueKind, defaultFormat: params && params.encoder ? { encoder: params.encoder } : void 0, shouldInclude: params && params.shouldInclude };
@@ -91,6 +95,11 @@ export namespace Field {
             return this;
         }
 
+        many(fields: ArrayLike<Field<K, D>>) {
+            for (let i = 0; i < fields.length; i++) this.fields.push(fields[i]);
+            return this;
+        }
+
         getFields() { return this.fields; }
     }
 
@@ -105,15 +114,19 @@ export interface Category<Ctx = any> {
 }
 
 export namespace Category {
-    export const Empty: Instance = { fields: [], rowCount: 0 };
+    export const Empty: Instance = { fields: [], source: [] };
 
-    export interface Instance<Key = any, Data = any> {
-        fields: Field[],
+    export interface DataSource<Key = any, Data = any> {
         data?: Data,
         rowCount: number,
         keys?: () => Iterator<Key>
     }
 
+    export interface Instance<Key = any, Data = any> {
+        fields: Field[],
+        source: DataSource<Key, Data>[]
+    }
+
     export interface Filter {
         includeCategory(categoryName: string): boolean,
         includeField(categoryName: string, fieldName: string): boolean,
@@ -134,9 +147,15 @@ export namespace Category {
 
     export function ofTable(table: Table<Table.Schema>, indices?: ArrayLike<number>): Category.Instance {
         if (indices) {
-            return { fields: cifFieldsFromTableSchema(table._schema), data: table, rowCount: indices.length, keys: () => Iterator.Array(indices) };
+            return {
+                fields: cifFieldsFromTableSchema(table._schema),
+                source: [{ data: table, rowCount: indices.length, keys: () => Iterator.Array(indices) }]
+            };
         }
-        return { fields: cifFieldsFromTableSchema(table._schema), data: table, rowCount: table._rowCount };
+        return {
+            fields: cifFieldsFromTableSchema(table._schema),
+            source: [{ data: table, rowCount: table._rowCount }]
+        };
     }
 }
 
@@ -145,7 +164,7 @@ export interface Encoder<T = string | Uint8Array> extends EncoderBase {
     setFormatter(formatter?: Category.Formatter): void,
 
     startDataBlock(header: string): void,
-    writeCategory<Ctx>(category: Category<Ctx>, contexts?: Ctx[]): void,
+    writeCategory<Ctx>(category: Category<Ctx>, context?: Ctx): void,
     getData(): T
 }
 

src/mol-io/writer/cif/encoder/binary.ts

@@ -6,7 +6,6 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
-import { Iterator } from 'mol-data'
 import { Column } from 'mol-data/db'
 import encodeMsgPack from '../../../common/msgpack/encode'
 import {
@@ -14,7 +13,7 @@ import {
 } from '../../../common/binary-cif'
 import { Field, Category, Encoder } from '../encoder'
 import Writer from '../../writer'
-import { getIncludedFields } from './util';
+import { getIncludedFields, getCategoryInstanceData, CategoryInstanceData } from './util';
 import { classifyIntArray, classifyFloatArray } from '../../../common/binary-cif/classifier';
 
 export interface EncodingProvider {
@@ -43,7 +42,7 @@ export default class BinaryEncoder implements Encoder<Uint8Array> {
         });
     }
 
-    writeCategory<Ctx>(category: Category<Ctx>, contexts?: Ctx[]) {
+    writeCategory<Ctx>(category: Category<Ctx>, context?: Ctx) {
         if (!this.data) {
             throw new Error('The writer contents have already been encoded, no more writing.');
         }
@@ -54,22 +53,17 @@ export default class BinaryEncoder implements Encoder<Uint8Array> {
 
         if (!this.filter.includeCategory(category.name)) return;
 
-        const src = !contexts || !contexts.length ? [category.instance(<any>void 0)] : contexts.map(c => category.instance(c));
-        const instances = src.filter(c => c && c.rowCount > 0);
-        if (!instances.length) return;
+        const { instance, rowCount, source } = getCategoryInstanceData(category, context);
+        if (!rowCount) return;
 
-        const count = instances.reduce((a, c) => a + c.rowCount, 0);
-        if (!count) return;
-
-        const cat: EncodedCategory = { name: '_' + category.name, columns: [], rowCount: count };
-        const data = instances.map(c => ({ data: c.data, keys: () => c.keys ? c.keys() : Iterator.Range(0, c.rowCount - 1) }));
-        const fields = getIncludedFields(instances[0]);
+        const cat: EncodedCategory = { name: '_' + category.name, columns: [], rowCount };
+        const fields = getIncludedFields(instance);
 
         for (const f of fields) {
             if (!this.filter.includeField(category.name, f.name)) continue;
 
             const format = this.formatter.getFormat(category.name, f.name);
-            cat.columns.push(encodeField(category.name, f, data, count, format, this.encodingProvider, this.autoClassify));
+            cat.columns.push(encodeField(category.name, f, source, rowCount, format, this.encodingProvider, this.autoClassify));
         }
         // no columns included.
         if (!cat.columns.length) return;
@@ -133,7 +127,7 @@ function classify(type: Field.Type, data: ArrayLike<any>) {
     return classifyFloatArray(data);
 }
 
-function encodeField(categoryName: string, field: Field, data: { data: any, keys: () => Iterator<any> }[], totalCount: number, 
+function encodeField(categoryName: string, field: Field, data: CategoryInstanceData['source'], totalCount: number,
     format: Field.Format | undefined, encoderProvider: EncodingProvider | undefined, autoClassify: boolean): EncodedColumn {
 
     const { array, allPresent, mask } = getFieldData(field, getArrayCtor(field, format), totalCount, data);
@@ -163,7 +157,7 @@ function encodeField(categoryName: string, field: Field, data: { data: any, keys
     };
 }
 
-function getFieldData(field: Field<any, any>, arrayCtor: Helpers.ArrayCtor<string | number>, totalCount: number, data: { data: any; keys: () => Iterator<any>; }[]) {
+function getFieldData(field: Field<any, any>, arrayCtor: Helpers.ArrayCtor<string | number>, totalCount: number, data: CategoryInstanceData['source']) {
     const isStr = field.type === Field.Type.Str;
     const array = new arrayCtor(totalCount);
     const mask = new Uint8Array(totalCount);

src/mol-io/writer/cif/encoder/text.ts

@@ -6,12 +6,11 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
-import { Iterator } from 'mol-data'
 import { Column } from 'mol-data/db'
 import StringBuilder from 'mol-util/string-builder'
 import { Category, Field, Encoder } from '../encoder'
 import Writer from '../../writer'
-import { getFieldDigitCount, getIncludedFields } from './util';
+import { getFieldDigitCount, getIncludedFields, getCategoryInstanceData, CategoryInstanceData } from './util';
 
 export default class TextEncoder implements Encoder<string> {
     private builder = StringBuilder.create();
@@ -33,7 +32,7 @@ export default class TextEncoder implements Encoder<string> {
         StringBuilder.write(this.builder, `data_${(header || '').replace(/[ \n\t]/g, '').toUpperCase()}\n#\n`);
     }
 
-    writeCategory<Ctx>(category: Category<Ctx>, contexts?: Ctx[]) {
+    writeCategory<Ctx>(category: Category<Ctx>, context?: Ctx) {
         if (this.encoded) {
             throw new Error('The writer contents have already been encoded, no more writing.');
         }
@@ -43,19 +42,13 @@ export default class TextEncoder implements Encoder<string> {
         }
 
         if (!this.filter.includeCategory(category.name)) return;
-
-        const src = !contexts || !contexts.length ? [category.instance(<any>void 0)] : contexts.map(c => category.instance(c));
-        const instances = src.filter(c => c && c.rowCount > 0);
-        if (!instances.length) return;
-
-        const rowCount = instances.reduce((v, c) => v + c.rowCount, 0);
-
-        if (rowCount === 0) return;
+        const { instance, rowCount, source } = getCategoryInstanceData(category, context);
+        if (!rowCount) return;
 
         if (rowCount === 1) {
-            writeCifSingleRecord(category, instances[0]!, this.builder, this.filter, this.formatter);
+            writeCifSingleRecord(category, instance, source, this.builder, this.filter, this.formatter);
         } else {
-            writeCifLoop(category, instances, this.builder, this.filter, this.formatter);
+            writeCifLoop(category, instance, source, this.builder, this.filter, this.formatter);
         }
     }
 
@@ -110,18 +103,18 @@ function getFloatPrecisions(categoryName: string, fields: Field[], formatter: Ca
     return ret;
 }
 
-function writeCifSingleRecord(category: Category, instance: Category.Instance, builder: StringBuilder, filter: Category.Filter, formatter: Category.Formatter) {
+function writeCifSingleRecord(category: Category, instance: Category.Instance, source: CategoryInstanceData['source'], builder: StringBuilder, filter: Category.Filter, formatter: Category.Formatter) {
     const fields = getIncludedFields(instance);
-    const data = instance.data;
+    const src = source[0];
+    const data = src.data;
     let width = fields.reduce((w, f) => filter.includeField(category.name, f.name) ? Math.max(w, f.name.length) : 0, 0);
 
     // this means no field from this category is included.
     if (width === 0) return;
     width += category.name.length + 6;
 
-    const it = instance.keys ? instance.keys() : Iterator.Range(0, instance.rowCount - 1);
+    const it = src.keys();
     const key = it.move();
-
     const precisions = getFloatPrecisions(category.name, instance.fields, formatter);
 
     for (let _f = 0; _f < fields.length; _f++) {
@@ -135,8 +128,8 @@ function writeCifSingleRecord(category: Category, instance: Category.Instance, b
     StringBuilder.write(builder, '#\n');
 }
 
-function writeCifLoop(category: Category, instances: Category.Instance[], builder: StringBuilder, filter: Category.Filter, formatter: Category.Formatter) {
-    const fieldSource = getIncludedFields(instances[0]);
+function writeCifLoop(category: Category, instance: Category.Instance, source: CategoryInstanceData['source'], builder: StringBuilder, filter: Category.Filter, formatter: Category.Formatter) {
+    const fieldSource = getIncludedFields(instance);
     const fields = filter === Category.DefaultFilter ? fieldSource : fieldSource.filter(f => filter.includeField(category.name, f.name));
     const fieldCount = fields.length;
     if (fieldCount === 0) return;
@@ -149,13 +142,13 @@ function writeCifLoop(category: Category, instances: Category.Instance[], builde
     }
 
     let index = 0;
-    for (let _c = 0; _c < instances.length; _c++) {
-        const instance = instances[_c];
-        const data = instance.data;
+    for (let _c = 0; _c < source.length; _c++) {
+        const src = source[_c];
+        const data = src.data;
 
-        if (instance.rowCount === 0) continue;
+        if (src.rowCount === 0) continue;
 
-        const it = instance.keys ? instance.keys() : Iterator.Range(0, instance.rowCount - 1);
+        const it = src.keys();
         while (it.hasNext)  {
             const key = it.move();
 
@@ -171,7 +164,7 @@ function writeCifLoop(category: Category, instances: Category.Instance[], builde
 }
 
 function isMultiline(value: string) {
-    return !!value && value.indexOf('\n') >= 0;
+    return typeof value === 'string' && value.indexOf('\n') >= 0;
 }
 
 function writeLine(builder: StringBuilder, val: string) {
@@ -203,7 +196,7 @@ function writeChecked(builder: StringBuilder, val: string) {
         return false;
     }
 
-    let escape = false, escapeCharStart = '\'', escapeCharEnd = '\' ';
+    let escape = val.charCodeAt(0) === 95 /* _ */, escapeCharStart = '\'', escapeCharEnd = '\' ';
     let hasWhitespace = false;
     let hasSingle = false;
     let hasDouble = false;

src/mol-io/writer/cif/encoder/util.ts

@@ -4,6 +4,7 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
+import { Iterator } from 'mol-data'
 import { Field, Category } from '../encoder';
 
 export function getFieldDigitCount(field: Field) {
@@ -13,6 +14,27 @@ export function getFieldDigitCount(field: Field) {
 
 export function getIncludedFields(category: Category.Instance) {
     return category.fields.some(f => !!f.shouldInclude)
-        ? category.fields.filter(f => !f.shouldInclude || f.shouldInclude(category.data))
+        ? category.fields.filter(f => !f.shouldInclude || category.source.some(src => f.shouldInclude!(src.data)))
         : category.fields;
+}
+
+export interface CategoryInstanceData<Ctx = any> {
+    instance: Category.Instance<Ctx>,
+    rowCount: number,
+    source: { data: any, keys: () => Iterator<any>, rowCount: number }[]
+}
+
+export function getCategoryInstanceData<Ctx>(category: Category<Ctx>, ctx?: Ctx): CategoryInstanceData<Ctx> {
+    const instance = category.instance(ctx as any);
+    let sources = instance.source.filter(s => s.rowCount > 0);
+    if (!sources.length) return { instance, rowCount: 0, source: [] };
+
+    const rowCount = sources.reduce((a, c) => a + c.rowCount, 0);
+    const source = sources.map(c => ({
+        data: c.data,
+        keys: () => c.keys ? c.keys() : Iterator.Range(0, c.rowCount - 1),
+        rowCount: c.rowCount
+    }));
+
+    return { instance, rowCount, source };
 }

src/mol-model-props/common/wrapper.ts

@@ -0,0 +1,54 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import { CifWriter } from 'mol-io/writer/cif';
+import { Model } from 'mol-model/structure';
+import { dateToUtcString } from 'mol-util/date';
+
+interface PropertyWrapper<Data> {
+    info: PropertyWrapper.Info,
+    data: Data
+}
+
+namespace PropertyWrapper {
+    export interface Info {
+        timestamp_utc: string
+    }
+
+    export function createInfo(): Info {
+        return { timestamp_utc: dateToUtcString(new Date()) };
+    }
+
+    export function defaultInfoCategory<Ctx>(name: string, getter: (ctx: Ctx) => Info | undefined): CifWriter.Category<Ctx> {
+        return {
+            name,
+            instance(ctx) {
+                const info = getter(ctx);
+                return {
+                    fields: _info_fields,
+                    source: [{ data: info, rowCount: 1 }]
+                };
+            }
+        }
+    }
+
+    const _info_fields: CifWriter.Field<number, Info>[] = [
+        CifWriter.Field.str('updated_datetime_utc', (_, date) => date.timestamp_utc)
+    ];
+
+    export function tryGetInfoFromCif(categoryName: string, model: Model): Info | undefined {
+        if (model.sourceData.kind !== 'mmCIF' || !model.sourceData.frame.categoryNames.includes(categoryName)) {
+            return;
+        }
+
+        const timestampField = model.sourceData.frame.categories[categoryName].getField('updated_datetime_utc');
+        if (!timestampField || timestampField.rowCount === 0) return;
+
+        return { timestamp_utc: timestampField.str(0) || dateToUtcString(new Date()) };
+    }
+}
+
+export { PropertyWrapper }

src/mol-model-props/pdbe/preferred-assembly.ts

@@ -0,0 +1,89 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import { Column, Table } from 'mol-data/db';
+import { toTable } from 'mol-io/reader/cif/schema';
+import { CifWriter } from 'mol-io/writer/cif';
+import { Model, ModelPropertyDescriptor } from 'mol-model/structure';
+
+export namespace PDBePreferredAssembly {
+    export type Property = string
+
+    export function getFirstFromModel(model: Model): Property {
+        const asm = model.symmetry.assemblies;
+        return asm.length ? asm[0].id : '';
+    }
+
+    export function get(model: Model): Property {
+        return model._staticPropertyData.__PDBePreferredAssebly__ || getFirstFromModel(model);
+    }
+    function set(model: Model, prop: Property) {
+        (model._staticPropertyData.__PDBePreferredAssebly__ as Property) = prop;
+    }
+
+    export const Schema = {
+        pdbe_preferred_assembly: {
+            assembly_id: Column.Schema.str
+        }
+    };
+    export type Schema = typeof Schema
+
+    export const Descriptor = ModelPropertyDescriptor({
+        isStatic: true,
+        name: 'pdbe_preferred_assembly',
+        cifExport: {
+            prefix: 'pdbe',
+            context(ctx): Property { return get(ctx.firstModel); },
+            categories: [{
+                name: 'pdbe_preferred_assembly',
+                instance(ctx: Property) {
+                    return CifWriter.Category.ofTable(Table.ofArrays(Schema.pdbe_preferred_assembly, { assembly_id: [ctx] }));
+                }
+            }]
+        }
+    });
+
+    function fromCifData(model: Model): string | undefined {
+        if (model.sourceData.kind !== 'mmCIF') return void 0;
+        const cat = model.sourceData.frame.categories.pdbe_preferred_assembly;
+        if (!cat) return void 0;
+        return toTable(Schema.pdbe_preferred_assembly, cat).assembly_id.value(0) || getFirstFromModel(model);
+    }
+
+    export async function attachFromCifOrApi(model: Model, params: {
+        // optional JSON source
+        PDBe_apiSourceJson?: (model: Model) => Promise<any>
+    }) {
+        if (model.customProperties.has(Descriptor)) return true;
+
+        let asmName: string | undefined = fromCifData(model);
+        if (asmName === void 0 &&  params.PDBe_apiSourceJson) {
+            const data = await params.PDBe_apiSourceJson(model);
+            if (!data) return false;
+            asmName = asmNameFromJson(model, data);
+        } else {
+            return false;
+        }
+
+        if (!asmName) return false;
+
+        model.customProperties.add(Descriptor);
+        set(model, asmName);
+        return true;
+    }
+}
+
+function asmNameFromJson(modelData: Model, data: any): string {
+    const assemblies = data[0] && data[0].assemblies;
+    if (!assemblies || !assemblies.length) return PDBePreferredAssembly.getFirstFromModel(modelData);
+
+    for (const asm of assemblies) {
+        if (asm.preferred) {
+            return asm.assembly_id;
+        }
+    }
+    return assemblies[0].assembly_id;
+}

src/mol-model-props/pdbe/struct-ref-domain.ts

@@ -0,0 +1,120 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import { Column, Table } from 'mol-data/db';
+import { toTable } from 'mol-io/reader/cif/schema';
+import { CifWriter } from 'mol-io/writer/cif';
+import { Model, ModelPropertyDescriptor } from 'mol-model/structure';
+import { PropertyWrapper } from '../common/wrapper';
+
+export namespace PDBeStructRefDomain {
+    export type Property = PropertyWrapper<Table<Schema['pdbe_struct_ref_domain']> | undefined>
+
+    export function get(model: Model): Property | undefined {
+        return model._staticPropertyData.__PDBeStructRefSeq__;
+    }
+    function set(model: Model, prop: Property) {
+        (model._staticPropertyData.__PDBeStructRefSeq__ as Property) = prop;
+    }
+
+    export const Schema = {
+        pdbe_struct_ref_domain: {
+            id: Column.Schema.int,
+            db_name: Column.Schema.str,
+            db_code: Column.Schema.str,
+
+            identifier: Column.Schema.str,
+            name: Column.Schema.str,
+
+            label_entity_id: Column.Schema.str,
+            label_asym_id: Column.Schema.str,
+            beg_label_seq_id: Column.Schema.int,
+            beg_pdbx_PDB_ins_code: Column.Schema.str,
+            end_label_seq_id: Column.Schema.int,
+            end_pdbx_PDB_ins_code: Column.Schema.str
+        }
+    };
+    export type Schema = typeof Schema
+
+    export const Descriptor = ModelPropertyDescriptor({
+        isStatic: true,
+        name: 'pdbe_struct_ref_domain',
+        cifExport: {
+            prefix: 'pdbe',
+            context(ctx): Property { return get(ctx.firstModel)!; },
+            categories: [
+                PropertyWrapper.defaultInfoCategory<Property>('pdbe_struct_ref_domain_info', ctx => ctx.info),
+                {
+                    name: 'pdbe_struct_ref_domain',
+                    instance(ctx: Property) {
+                        if (!ctx || !ctx.data) return CifWriter.Category.Empty;
+                        return CifWriter.Category.ofTable(ctx.data);
+                    }
+                }]
+        }
+    });
+
+    function fromCifData(model: Model): Property['data'] {
+        if (model.sourceData.kind !== 'mmCIF') return void 0;
+        const cat = model.sourceData.frame.categories.pdbe_struct_ref_domain;
+        if (!cat) return void 0;
+        return toTable(Schema.pdbe_struct_ref_domain, cat);
+    }
+
+    export async function attachFromCifOrApi(model: Model, params: {
+        // optional JSON source
+        PDBe_apiSourceJson?: (model: Model) => Promise<any>
+    }) {
+        if (model.customProperties.has(Descriptor)) return true;
+
+
+        let table: Property['data'];
+        let info = PropertyWrapper.tryGetInfoFromCif('pdbe_struct_ref_domain_info', model);
+        if (info) {
+            table = fromCifData(model);
+        } else if (params.PDBe_apiSourceJson) {
+            const data = await params.PDBe_apiSourceJson(model);
+            if (!data) return false;
+            info = PropertyWrapper.createInfo();
+            table = fromPDBeJson(model, data);
+        } else {
+            return false;
+        }
+
+        model.customProperties.add(Descriptor);
+        set(model, { info, data: table });
+        return true;
+    }
+}
+
+function fromPDBeJson(modelData: Model, data: any): PDBeStructRefDomain.Property['data'] {
+    const rows: Table.Row<PDBeStructRefDomain.Schema['pdbe_struct_ref_domain']>[] = [];
+
+    let id = 1;
+    for (const db_name of Object.keys(data)) {
+        const db = data[db_name];
+        for (const db_code of Object.keys(db)) {
+            const domain = db[db_code];
+            for (const map of domain.mappings) {
+                rows.push({
+                    id: id++,
+                    db_name,
+                    db_code,
+                    identifier: domain.identifier,
+                    name: domain.name,
+                    label_entity_id: '' + map.entity_id,
+                    label_asym_id: map.struct_asym_id,
+                    beg_label_seq_id: map.start.residue_number,
+                    beg_pdbx_PDB_ins_code: map.start.author_insertion_code,
+                    end_label_seq_id: map.end.residue_number,
+                    end_pdbx_PDB_ins_code: map.end.author_insertion_code,
+                })
+            }
+        }
+    }
+
+    return Table.ofRows(PDBeStructRefDomain.Schema.pdbe_struct_ref_domain, rows) as PDBeStructRefDomain.Property['data'];
+}

src/mol-model-props/pdbe/structure-quality-report.ts

@@ -4,100 +4,27 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
-import { CifWriter } from 'mol-io/writer/cif';
-import { Model, ModelPropertyDescriptor, ResidueIndex, Unit, ResidueCustomProperty, StructureProperties as P } from 'mol-model/structure';
-import { residueIdFields } from 'mol-model/structure/export/categories/atom_site';
-import CifField = CifWriter.Field;
-import { mmCIF_residueId_schema } from 'mol-io/reader/cif/schema/mmcif-extras';
 import { Column, Table } from 'mol-data/db';
 import { toTable } from 'mol-io/reader/cif/schema';
-import { StructureElement } from 'mol-model/structure/structure';
-
-
-import { QuerySymbolRuntime } from 'mol-script/runtime/query/compiler';
+import { mmCIF_residueId_schema } from 'mol-io/reader/cif/schema/mmcif-extras';
+import { CifWriter } from 'mol-io/writer/cif';
+import { Model, ModelPropertyDescriptor, ResidueIndex, Unit, IndexedCustomProperty } from 'mol-model/structure';
+import { residueIdFields } from 'mol-model/structure/export/categories/atom_site';
+import { StructureElement, CifExportContext } from 'mol-model/structure/structure';
 import { CustomPropSymbol } from 'mol-script/language/symbol';
 import Type from 'mol-script/language/type';
+import { QuerySymbolRuntime } from 'mol-script/runtime/query/compiler';
+import { PropertyWrapper } from '../common/wrapper';
 
-type IssueMap = ResidueCustomProperty<string[]>
-
-const _Descriptor = ModelPropertyDescriptor({
-    isStatic: false,
-    name: 'structure_quality_report',
-    cifExport: {
-        prefix: 'pdbe',
-        categories: [{
-            name: 'pdbe_structure_quality_report',
-            instance() {
-                return { fields: _structure_quality_report_fields, rowCount: 1 }
-            }
-        }, {
-            name: 'pdbe_structure_quality_report_issues',
-            instance(ctx) {
-                const issues = StructureQualityReport.get(ctx.model);
-                if (!issues) return CifWriter.Category.Empty;
-                return ResidueCustomProperty.createCifCategory(ctx, issues, _structure_quality_report_issues_fields);
-            }
-        }]
-    },
-    symbols: {
-        issueCount: QuerySymbolRuntime.Dynamic(CustomPropSymbol('pdbe', 'structure-quality.issue-count', Type.Num),
-            ctx => StructureQualityReport.getIssues(ctx.element).length),
-        // TODO: add (hasIssue :: IssueType(extends string) -> boolean) symbol
-    }
-})
-
-type ExportCtx = ResidueCustomProperty.ExportCtx<string[]>
-const _structure_quality_report_issues_fields: CifField<number, ExportCtx>[] = [
-    CifField.index('id'),
-    ...residueIdFields<number, ExportCtx>((i, d) => d.elements[i]),
-    CifField.int<number, ExportCtx>('pdbx_PDB_model_num', (i, d) => P.unit.model_num(d.elements[i])),
-    CifField.str<number, ExportCtx>('issues', (i, d) => d.property(i).join(','))
-];
-
-const _structure_quality_report_fields: CifField<ResidueIndex, ExportCtx>[] = [
-    CifField.str('updated_datetime_utc', () => `${new Date().toISOString().replace(/T/, ' ').replace(/\..+/, '')}`)
-];
-
-function createIssueMapFromJson(modelData: Model, data: any): IssueMap | undefined {
-    const ret = new Map<ResidueIndex, string[]>();
-    if (!data.molecules) return;
-
-    for (const entity of data.molecules) {
-        const entity_id = entity.entity_id.toString();
-        for (const chain of entity.chains) {
-            const asym_id = chain.struct_asym_id.toString();
-            for (const model of chain.models) {
-                const model_id = model.model_id.toString();
-                if (+model_id !== modelData.modelNum) continue;
-
-                for (const residue of model.residues) {
-                    const auth_seq_id = residue.author_residue_number, ins_code = residue.author_insertion_code || '';
-                    const idx = modelData.atomicHierarchy.index.findResidue(entity_id, asym_id, auth_seq_id, ins_code);
-                    ret.set(idx, residue.outlier_types);
-                }
-            }
-        }
-    }
-
-    return ResidueCustomProperty.fromMap(ret, Unit.Kind.Atomic);
-}
-
-function createIssueMapFromCif(modelData: Model, data: Table<typeof StructureQualityReport.Schema.pdbe_structure_quality_report_issues>): IssueMap | undefined {
-    const ret = new Map<ResidueIndex, string[]>();
-    const { label_entity_id, label_asym_id, auth_seq_id, pdbx_PDB_ins_code, issues, pdbx_PDB_model_num, _rowCount } = data;
+export namespace StructureQualityReport {
+    export type IssueMap = IndexedCustomProperty.Residue<string[]>
+    export type Property = PropertyWrapper<IssueMap | undefined>
 
-    for (let i = 0; i < _rowCount; i++) {
-        if (pdbx_PDB_model_num.value(i) !== modelData.modelNum) continue;
-        const idx = modelData.atomicHierarchy.index.findResidue(label_entity_id.value(i), label_asym_id.value(i), auth_seq_id.value(i), pdbx_PDB_ins_code.value(i));
-        ret.set(idx, issues.value(i));
+    export function get(model: Model): Property | undefined {
+        // must be defined before the descriptor so it's not undefined.
+        return model._dynamicPropertyData.__PDBeStructureQualityReport__;
     }
 
-    return ResidueCustomProperty.fromMap(ret, Unit.Kind.Atomic);
-}
-
-export namespace StructureQualityReport {
-    export const Descriptor = _Descriptor;
-
     export const Schema = {
         pdbe_structure_quality_report: {
             updated_datetime_utc: Column.Schema.str
@@ -106,46 +33,206 @@ export namespace StructureQualityReport {
             id: Column.Schema.int,
             ...mmCIF_residueId_schema,
             pdbx_PDB_model_num: Column.Schema.int,
-            issues: Column.Schema.List(',', x => x)
+            issue_type_group_id: Column.Schema.int
+        },
+        pdbe_structure_quality_report_issue_types: {
+            group_id: Column.Schema.int,
+            issue_type: Column.Schema.str
+        }
+    };
+    export type Schema = typeof Schema
+
+    export const Descriptor = ModelPropertyDescriptor({
+        isStatic: false,
+        name: 'pdbe_structure_quality_report',
+        cifExport: {
+            prefix: 'pdbe',
+            context(ctx) {
+                return createExportContext(ctx);
+            },
+            categories: [
+                PropertyWrapper.defaultInfoCategory<ReportExportContext>('pdbe_structure_quality_report', ctx => ctx.info),
+                {
+                    name: 'pdbe_structure_quality_report_issues',
+                    instance(ctx: ReportExportContext) {
+                        return {
+                            fields: _structure_quality_report_issues_fields,
+                            source: ctx.models.map(data => ({ data, rowCount: data.elements.length }))
+                        }
+                    }
+                }, {
+                    name: 'pdbe_structure_quality_report_issue_types',
+                    instance(ctx: ReportExportContext) {
+                        return CifWriter.Category.ofTable(ctx.issueTypes);
+                    }
+                }]
+        },
+        symbols: {
+            issueCount: QuerySymbolRuntime.Dynamic(CustomPropSymbol('pdbe', 'structure-quality.issue-count', Type.Num),
+                ctx => StructureQualityReport.getIssues(ctx.element).length),
+            // TODO: add (hasIssue :: IssueType(extends string) -> boolean) symbol
+        }
+    });
+
+    function getCifData(model: Model) {
+        if (model.sourceData.kind !== 'mmCIF') throw new Error('Data format must be mmCIF.');
+        return {
+            residues: toTable(Schema.pdbe_structure_quality_report_issues, model.sourceData.frame.categories.pdbe_structure_quality_report_issues),
+            groups: toTable(Schema.pdbe_structure_quality_report_issue_types, model.sourceData.frame.categories.pdbe_structure_quality_report_issue_types),
         }
     }
 
     export async function attachFromCifOrApi(model: Model, params: {
-        // provide JSON from api
+        // optional JSON source
         PDBe_apiSourceJson?: (model: Model) => Promise<any>
     }) {
         if (get(model)) return true;
 
-        let issueMap;
-
-        if (model.sourceData.kind === 'mmCIF' && model.sourceData.frame.categoryNames.includes('pdbe_structure_quality_report')) {
-            const data = toTable(Schema.pdbe_structure_quality_report_issues, model.sourceData.frame.categories.pdbe_structure_quality_report);
-            issueMap = createIssueMapFromCif(model, data);
+        let issueMap: IssueMap | undefined;
+        let info = PropertyWrapper.tryGetInfoFromCif('pdbe_structure_quality_report', model);
+        if (info) {
+            const data = getCifData(model);
+            issueMap = createIssueMapFromCif(model, data.residues, data.groups);
         } else if (params.PDBe_apiSourceJson) {
-            const id = model.label.toLowerCase();
-            const json = await params.PDBe_apiSourceJson(model);
-            const data = json[id];
+            const data = await params.PDBe_apiSourceJson(model);
             if (!data) return false;
+            info = PropertyWrapper.createInfo();
             issueMap = createIssueMapFromJson(model, data);
         } else {
             return false;
         }
 
         model.customProperties.add(Descriptor);
-        model._dynamicPropertyData.__StructureQualityReport__ = issueMap;
+        set(model, { info, data: issueMap });
         return true;
     }
 
-    export function get(model: Model): IssueMap | undefined {
-        return model._dynamicPropertyData.__StructureQualityReport__;
+    function set(model: Model, prop: Property) {
+        (model._dynamicPropertyData.__PDBeStructureQualityReport__ as Property) = prop;
+    }
+
+    export function getIssueMap(model: Model): IssueMap | undefined {
+        const prop = get(model);
+        return prop && prop.data;
     }
 
     const _emptyArray: string[] = [];
     export function getIssues(e: StructureElement) {
         if (!Unit.isAtomic(e.unit)) return _emptyArray;
-        const issues = StructureQualityReport.get(e.unit.model);
-        if (!issues) return _emptyArray;
+        const prop = StructureQualityReport.get(e.unit.model);
+        if (!prop || !prop.data) return _emptyArray;
         const rI = e.unit.residueIndex[e.element];
-        return issues.has(rI) ? issues.get(rI)! : _emptyArray;
+        return prop.data.has(rI) ? prop.data.get(rI)! : _emptyArray;
+    }
+}
+
+const _structure_quality_report_issues_fields = CifWriter.fields<number, ReportExportContext['models'][0]>()
+    .index('id')
+    .many(residueIdFields((i, d) => d.elements[i], { includeModelNum: true }))
+    .int('issue_type_group_id', (i, d) => d.groupId[i])
+    .getFields();
+
+interface ReportExportContext {
+    models: {
+        elements: StructureElement[],
+        groupId: number[]
+    }[],
+    info: PropertyWrapper.Info,
+    issueTypes: Table<StructureQualityReport.Schema['pdbe_structure_quality_report_issue_types']>,
+}
+
+function createExportContext(ctx: CifExportContext): ReportExportContext {
+    const groupMap = new Map<string, number>();
+    const models: ReportExportContext['models'] = [];
+    const group_id: number[] = [], issue_type: string[] = [];
+    let info: PropertyWrapper.Info = PropertyWrapper.createInfo();
+
+    for (const s of ctx.structures) {
+        const prop = StructureQualityReport.get(s.model);
+        if (prop) info = prop.info;
+        if (!prop || !prop.data) continue;
+
+        const { elements, property } = prop.data.getElements(s);
+        if (elements.length === 0) continue;
+
+        const elementGroupId: number[] = [];
+        for (let i = 0; i < elements.length; i++) {
+            const issues = property(i);
+            const key = issues.join(',');
+            if (!groupMap.has(key)) {
+                const idx = groupMap.size + 1;
+                groupMap.set(key, idx);
+                for (const issue of issues) {
+                    group_id.push(idx);
+                    issue_type.push(issue);
+                }
+            }
+            elementGroupId[i] = groupMap.get(key)!;
+        }
+        models.push({ elements, groupId: elementGroupId });
+    }
+
+    return {
+        info,
+        models,
+        issueTypes: Table.ofArrays(StructureQualityReport.Schema.pdbe_structure_quality_report_issue_types, { group_id, issue_type })
+    }
+}
+
+function createIssueMapFromJson(modelData: Model, data: any): StructureQualityReport.IssueMap | undefined {
+    const ret = new Map<ResidueIndex, string[]>();
+    if (!data.molecules) return;
+
+    for (const entity of data.molecules) {
+        const entity_id = entity.entity_id.toString();
+        for (const chain of entity.chains) {
+            const asym_id = chain.struct_asym_id.toString();
+            for (const model of chain.models) {
+                const model_id = model.model_id.toString();
+                if (+model_id !== modelData.modelNum) continue;
+
+                for (const residue of model.residues) {
+                    const auth_seq_id = residue.author_residue_number, ins_code = residue.author_insertion_code || '';
+                    const idx = modelData.atomicHierarchy.index.findResidue(entity_id, asym_id, auth_seq_id, ins_code);
+                    ret.set(idx, residue.outlier_types);
+                }
+            }
+        }
+    }
+
+    return IndexedCustomProperty.fromResidueMap(ret);
+}
+
+function createIssueMapFromCif(modelData: Model,
+    residueData: Table<typeof StructureQualityReport.Schema.pdbe_structure_quality_report_issues>,
+    groupData: Table<typeof StructureQualityReport.Schema.pdbe_structure_quality_report_issue_types>): StructureQualityReport.IssueMap | undefined {
+
+    const ret = new Map<ResidueIndex, string[]>();
+    const { label_entity_id, label_asym_id, auth_seq_id, pdbx_PDB_ins_code, issue_type_group_id, pdbx_PDB_model_num, _rowCount } = residueData;
+
+    const groups = parseIssueTypes(groupData);
+
+    for (let i = 0; i < _rowCount; i++) {
+        if (pdbx_PDB_model_num.value(i) !== modelData.modelNum) continue;
+        const idx = modelData.atomicHierarchy.index.findResidue(label_entity_id.value(i), label_asym_id.value(i), auth_seq_id.value(i), pdbx_PDB_ins_code.value(i));
+        ret.set(idx, groups.get(issue_type_group_id.value(i))!);
+    }
+
+    return IndexedCustomProperty.fromResidueMap(ret);
+}
+
+function parseIssueTypes(groupData: Table<typeof StructureQualityReport.Schema.pdbe_structure_quality_report_issue_types>): Map<number, string[]> {
+    const ret = new Map<number, string[]>();
+    const { group_id, issue_type } = groupData;
+    for (let i = 0; i < groupData._rowCount; i++) {
+        let group: string[];
+        const id = group_id.value(i);
+        if (ret.has(id)) group = ret.get(id)!;
+        else {
+            group = [];
+            ret.set(id, group);
+        }
+        group.push(issue_type.value(i));
     }
+    return ret;
 }

src/mol-model-props/rcsb/symmetry.ts

@@ -22,7 +22,7 @@ const { str, int, float, Aliased, Vector, List } = Column.Schema;
 
 function getInstance(name: keyof AssemblySymmetry.Schema): (ctx: CifExportContext) => CifWriter.Category.Instance<any, any> {
     return function(ctx: CifExportContext) {
-        const assemblySymmetry = AssemblySymmetry.get(ctx.model);
+        const assemblySymmetry = AssemblySymmetry.get(ctx.structures[0].model);
         return assemblySymmetry ? Category.ofTable(assemblySymmetry.db[name]) : CifWriter.Category.Empty;
     }
 }
@@ -178,6 +178,9 @@ export namespace AssemblySymmetry {
 
         let db: Database
 
+        // TODO: there should be a "meta field" that indicates the property was added (see for example PDBe structure report)
+        // the reason for this is that the feature might not be present and therefore the "default" categories would
+        // not be created. This would result in an unnecessarily failed web request.
         if (model.sourceData.kind === 'mmCIF' && model.sourceData.frame.categoryNames.includes('rcsb_assembly_symmetry_feature')) {
             const rcsb_assembly_symmetry_feature = toTable(Schema.rcsb_assembly_symmetry_feature, model.sourceData.frame.categories.rcsb_assembly_symmetry_feature)
 

src/mol-model/structure/export/categories/atom_site.ts

@@ -51,28 +51,46 @@ const atom_site_fields = CifWriter.fields<StructureElement, Structure>()
 
 export const _atom_site: CifCategory<CifExportContext> = {
     name: 'atom_site',
-    instance({ structure }: CifExportContext) {
+    instance({ structures }: CifExportContext) {
         return {
             fields: atom_site_fields,
-            data: structure,
-            rowCount: structure.elementCount,
-            keys: () => structure.elementLocations()
+            source: structures.map(s => ({
+                data: s,
+                rowCount: s.elementCount,
+                keys: () => s.elementLocations()
+            }))
         };
     }
 }
 
-function prefixed(prefix: string, name: string) {
-    return prefix ? `${prefix}_${name}` : name;
+function prepostfixed(prefix: string | undefined, postfix: string | undefined, name: string) {
+    if (prefix && postfix) return `${prefix}_${name}_${postfix}`;
+    if (prefix) return `${prefix}_${name}`;
+    if (postfix) return `${name}_${postfix}`;
+    return name;
 }
 
 function mappedProp<K, D>(loc: (key: K, data: D) => StructureElement, prop: (e: StructureElement) => any) {
     return (k: K, d: D) => prop(loc(k, d));
 }
 
-export function residueIdFields<K, D>(getLocation: (key: K, data: D) => StructureElement, prefix = ''): CifField<K, D>[] {
-    return CifWriter.fields<K, D>()
-        .str(prefixed(prefix, `label_comp_id`), mappedProp(getLocation, P.residue.label_comp_id))
-        .int(prefixed(prefix, `label_seq_id`), mappedProp(getLocation, P.residue.label_seq_id), {
+function addModelNum<K, D>(fields: CifWriter.Field.Builder<K, D>, getLocation: (key: K, data: D) => StructureElement, options?: IdFieldsOptions) {
+    if (options && options.includeModelNum) {
+        fields.int('pdbx_PDB_model_num', mappedProp(getLocation, P.unit.model_num));
+    }
+}
+
+export interface IdFieldsOptions {
+    prefix?: string,
+    postfix?: string,
+    includeModelNum?: boolean
+}
+
+export function residueIdFields<K, D>(getLocation: (key: K, data: D) => StructureElement, options?: IdFieldsOptions): CifField<K, D>[] {
+    const prefix = options && options.prefix, postfix = options && options.postfix;
+    const ret = CifWriter.fields<K, D>()
+        .str(prepostfixed(prefix, postfix, `label_comp_id`), mappedProp(getLocation, P.residue.label_comp_id))
+        .int(prepostfixed(prefix, postfix, `label_seq_id`), mappedProp(getLocation, P.residue.label_seq_id), {
             encoder: E.deltaRLE,
             valueKind: (k, d) => {
                 const e = getLocation(k, d);
@@ -80,21 +98,63 @@ export function residueIdFields<K, D>(getLocation: (key: K, data: D) => Structur
                 return m.atomicHierarchy.residues.label_seq_id.valueKind(m.atomicHierarchy.residueAtomSegments.index[e.element]);
             }
         })
-        .str(prefixed(prefix, `pdbx_PDB_ins_code`), mappedProp(getLocation, P.residue.pdbx_PDB_ins_code))
+        .str(prepostfixed(prefix, postfix, `pdbx_PDB_ins_code`), mappedProp(getLocation, P.residue.pdbx_PDB_ins_code))
+
+        .str(prepostfixed(prefix, postfix, `label_asym_id`), mappedProp(getLocation, P.chain.label_asym_id))
+        .str(prepostfixed(prefix, postfix, `label_entity_id`), mappedProp(getLocation, P.chain.label_entity_id))
+
+        .str(prepostfixed(prefix, postfix, `auth_comp_id`), mappedProp(getLocation, P.residue.auth_comp_id))
+        .int(prepostfixed(prefix, postfix, `auth_seq_id`), mappedProp(getLocation, P.residue.auth_seq_id), { encoder: E.deltaRLE })
+        .str(prepostfixed(prefix, postfix, `auth_asym_id`), mappedProp(getLocation, P.chain.auth_asym_id));
+
+    addModelNum(ret, getLocation, options);
+    return ret.getFields();
+}
+
+export function chainIdFields<K, D>(getLocation: (key: K, data: D) => StructureElement, options?: IdFieldsOptions): CifField<K, D>[] {
+    const prefix = options && options.prefix, postfix = options && options.postfix;
+    const ret = CifField.build<K, D>()
+        .str(prepostfixed(prefix, postfix, `label_asym_id`), mappedProp(getLocation, P.chain.label_asym_id))
+        .str(prepostfixed(prefix, postfix, `label_entity_id`), mappedProp(getLocation, P.chain.label_entity_id))
+        .str(prepostfixed(prefix, postfix, `auth_asym_id`), mappedProp(getLocation, P.chain.auth_asym_id))
+
+    addModelNum(ret, getLocation, options);
+    return ret.getFields();
+}
 
-        .str(prefixed(prefix, `label_asym_id`), mappedProp(getLocation, P.chain.label_asym_id))
-        .str(prefixed(prefix, `label_entity_id`), mappedProp(getLocation, P.chain.label_entity_id))
+export function entityIdFields<K, D>(getLocation: (key: K, data: D) => StructureElement, options?: IdFieldsOptions): CifField<K, D>[] {
+    const prefix = options && options.prefix, postfix = options && options.postfix;
+    const ret = CifField.build<K, D>()
+        .str(prepostfixed(prefix, postfix, `label_entity_id`), mappedProp(getLocation, P.chain.label_entity_id))
 
-        .str(prefixed(prefix, `auth_comp_id`), mappedProp(getLocation, P.residue.auth_comp_id))
-        .int(prefixed(prefix, `auth_seq_id`), mappedProp(getLocation, P.residue.auth_seq_id), { encoder: E.deltaRLE })
-        .str(prefixed(prefix, `auth_asym_id`), mappedProp(getLocation, P.chain.auth_asym_id))
-        .getFields();
+    addModelNum(ret, getLocation, options);
+    return ret.getFields();
 }
 
-export function chainIdFields<K, D>(getLocation: (key: K, data: D) => StructureElement, prefix = ''): CifField<K, D>[] {
-    return CifField.build<K, D>()
-        .str(prefixed(prefix, `label_asym_id`), mappedProp(getLocation, P.chain.label_asym_id))
-        .str(prefixed(prefix, `label_entity_id`), mappedProp(getLocation, P.chain.label_entity_id))
-        .str(prefixed(prefix, `auth_asym_id`), mappedProp(getLocation, P.chain.auth_asym_id))
-        .getFields();
+export function atomIdFields<K, D>(getLocation: (key: K, data: D) => StructureElement, options?: IdFieldsOptions): CifField<K, D>[] {
+    const prefix = options && options.prefix, postfix = options && options.postfix;
+    const ret = CifWriter.fields<K, D>()
+        .str(prepostfixed(prefix, postfix, `label_atom_id`), mappedProp(getLocation, P.atom.label_atom_id))
+        .str(prepostfixed(prefix, postfix, `label_comp_id`), mappedProp(getLocation, P.residue.label_comp_id))
+        .int(prepostfixed(prefix, postfix, `label_seq_id`), mappedProp(getLocation, P.residue.label_seq_id), {
+            encoder: E.deltaRLE,
+            valueKind: (k, d) => {
+                const e = getLocation(k, d);
+                const m = e.unit.model;
+                return m.atomicHierarchy.residues.label_seq_id.valueKind(m.atomicHierarchy.residueAtomSegments.index[e.element]);
+            }
+        })
+        .str(prepostfixed(prefix, postfix, `label_alt_id`), mappedProp(getLocation, P.atom.label_alt_id))
+        .str(prepostfixed(prefix, postfix, `pdbx_PDB_ins_code`), mappedProp(getLocation, P.residue.pdbx_PDB_ins_code))
+
+        .str(prepostfixed(prefix, postfix, `label_asym_id`), mappedProp(getLocation, P.chain.label_asym_id))
+        .str(prepostfixed(prefix, postfix, `label_entity_id`), mappedProp(getLocation, P.chain.label_entity_id))
+
+        .str(prepostfixed(prefix, postfix, `auth_atom_id`), mappedProp(getLocation, P.atom.auth_atom_id))
+        .str(prepostfixed(prefix, postfix, `auth_comp_id`), mappedProp(getLocation, P.residue.auth_comp_id))
+        .int(prepostfixed(prefix, postfix, `auth_seq_id`), mappedProp(getLocation, P.residue.auth_seq_id), { encoder: E.deltaRLE })
+        .str(prepostfixed(prefix, postfix, `auth_asym_id`), mappedProp(getLocation, P.chain.auth_asym_id));
+
+    addModelNum(ret, getLocation, options);
+    return ret.getFields();
 }

src/mol-model/structure/export/categories/misc.ts

@@ -0,0 +1,35 @@
+/**
+ * Copyright (c) 2017-2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import { Column } from 'mol-data/db';
+import { CifWriter } from 'mol-io/writer/cif';
+import { CifExportContext } from '../mmcif';
+import { getModelMmCifCategory, getUniqueResidueNamesFromStructures } from './utils';
+import CifCategory = CifWriter.Category
+
+export const _chem_comp: CifCategory<CifExportContext> = {
+    name: 'chem_comp',
+    instance({ firstModel, structures, cache }) {
+        const chem_comp = getModelMmCifCategory(structures[0].model, 'chem_comp');
+        if (!chem_comp) return CifCategory.Empty;
+        const { id } = chem_comp;
+        const names = cache.uniqueResidueNames || (cache.uniqueResidueNames = getUniqueResidueNamesFromStructures(structures));
+        const indices = Column.indicesOf(id, id => names.has(id));
+        return CifCategory.ofTable(chem_comp, indices);
+    }
+}
+
+export const _pdbx_chem_comp_identifier: CifCategory<CifExportContext> = {
+    name: 'pdbx_chem_comp_identifier',
+    instance({ firstModel, structures, cache }) {
+        const pdbx_chem_comp_identifier = getModelMmCifCategory(firstModel, 'pdbx_chem_comp_identifier');
+        if (!pdbx_chem_comp_identifier) return CifCategory.Empty;
+        const { comp_id } = pdbx_chem_comp_identifier;
+        const names = cache.uniqueResidueNames || (cache.uniqueResidueNames = getUniqueResidueNamesFromStructures(structures));
+        const indices = Column.indicesOf(comp_id, id => names.has(id));
+        return CifCategory.ofTable(pdbx_chem_comp_identifier, indices);
+    }
+}

src/mol-model/structure/export/categories/modified-residues.ts

@@ -27,7 +27,9 @@ const pdbx_struct_mod_residue_fields: CifField<number, StructureElement[]>[] = [
     CifField.str('details', (i, xs) => xs[i].unit.model.properties.modifiedResidues.details.get(P.residue.label_comp_id(xs[i]))!)
 ];
 
-function getModifiedResidues({ model, structure }: CifExportContext): StructureElement[] {
+function getModifiedResidues({ structures }: CifExportContext): StructureElement[] {
+    // TODO: can different models have differnt modified residues?
+    const structure = structures[0], model = structure.model;
     const map = model.properties.modifiedResidues.parentId;
     if (!map.size) return [];
 
@@ -54,6 +56,9 @@ export const _pdbx_struct_mod_residue: CifCategory<CifExportContext> = {
     name: 'pdbx_struct_mod_residue',
     instance(ctx) {
         const residues = getModifiedResidues(ctx);
-        return { fields: pdbx_struct_mod_residue_fields, data: residues, rowCount: residues.length };
+        return {
+            fields: pdbx_struct_mod_residue_fields,
+            source: [{ data: residues, rowCount: residues.length }]
+        };
     }
 }

src/mol-model/structure/export/categories/secondary-structure.ts

@@ -18,7 +18,10 @@ export const _struct_conf: CifCategory<CifExportContext> = {
     name: 'struct_conf',
     instance(ctx) {
         const elements = findElements(ctx, 'helix');
-        return { fields: struct_conf_fields, data: elements, rowCount: elements.length };
+        return {
+            fields: struct_conf_fields,
+            source: [{ data: elements, rowCount: elements.length }]
+        };
     }
 };
 
@@ -26,7 +29,10 @@ export const _struct_sheet_range: CifCategory<CifExportContext> = {
     name: 'struct_sheet_range',
     instance(ctx) {
         const elements = (findElements(ctx, 'sheet') as SSElement<SecondaryStructure.Sheet>[]).sort(compare_ssr);
-        return { fields: struct_sheet_range_fields, data: elements, rowCount: elements.length };
+        return {
+            fields: struct_sheet_range_fields,
+            source: [{ data: elements, rowCount: elements.length }]
+        };
     }
 };
 
@@ -38,8 +44,8 @@ function compare_ssr(x: SSElement<SecondaryStructure.Sheet>, y: SSElement<Second
 const struct_conf_fields: CifField[] = [
     CifField.str<number, SSElement<SecondaryStructure.Helix>[]>('conf_type_id', (i, data) => data[i].element.type_id),
     CifField.str<number, SSElement<SecondaryStructure.Helix>[]>('id', (i, data, idx) => `${data[i].element.type_id}${idx + 1}`),
-    ...residueIdFields<number, SSElement<SecondaryStructure.Helix>[]>((i, e) => e[i].start, 'beg'),
-    ...residueIdFields<number, SSElement<SecondaryStructure.Helix>[]>((i, e) => e[i].end, 'end'),
+    ...residueIdFields<number, SSElement<SecondaryStructure.Helix>[]>((i, e) => e[i].start, { prefix: 'beg' }),
+    ...residueIdFields<number, SSElement<SecondaryStructure.Helix>[]>((i, e) => e[i].end, { prefix: 'end' }),
     CifField.str<number, SSElement<SecondaryStructure.Helix>[]>('pdbx_PDB_helix_class', (i, data) => data[i].element.helix_class),
     CifField.str<number, SSElement<SecondaryStructure.Helix>[]>('details', (i, data) => data[i].element.details || '', {
         valueKind: (i, d) => !!d[i].element.details ? Column.ValueKind.Present : Column.ValueKind.Unknown
@@ -50,8 +56,8 @@ const struct_conf_fields: CifField[] = [
 const struct_sheet_range_fields: CifField[] = [
     CifField.str<number, SSElement<SecondaryStructure.Sheet>[]>('sheet_id', (i, data) => data[i].element.sheet_id),
     CifField.index('id'),
-    ...residueIdFields<number, SSElement<SecondaryStructure.Sheet>[]>((i, e) => e[i].start, 'beg'),
-    ...residueIdFields<number, SSElement<SecondaryStructure.Sheet>[]>((i, e) => e[i].end, 'end'),
+    ...residueIdFields<number, SSElement<SecondaryStructure.Sheet>[]>((i, e) => e[i].start, { prefix: 'beg' }),
+    ...residueIdFields<number, SSElement<SecondaryStructure.Sheet>[]>((i, e) => e[i].end, { prefix: 'end' }),
     CifField.str('symmetry', (i, data) => '', { valueKind: (i, d) => Column.ValueKind.Unknown })
 ];
 
@@ -63,11 +69,12 @@ interface SSElement<T extends SecondaryStructure.Element> {
 }
 
 function findElements<T extends SecondaryStructure.Element>(ctx: CifExportContext, kind: SecondaryStructure.Element['kind']) {
-    const { key, elements } = ctx.model.properties.secondaryStructure;
+    // TODO: encode secondary structure for different models?
+    const { key, elements } = ctx.structures[0].model.properties.secondaryStructure;
 
     const ssElements: SSElement<any>[] = [];
 
-    for (const unit of ctx.structure.units) {
+    for (const unit of ctx.structures[0].units) {
         // currently can only support this for "identity" operators.
         if (!Unit.isAtomic(unit) || !unit.conformation.operator.isIdentity) continue;
 

src/mol-model/structure/export/categories/sequence.ts

@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) 2017-2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import { Column } from 'mol-data/db';
+import { CifWriter } from 'mol-io/writer/cif';
+import { Structure } from '../../structure';
+import { CifExportContext } from '../mmcif';
+import { getModelMmCifCategory, getUniqueEntityIdsFromStructures } from './utils';
+import CifCategory = CifWriter.Category
+
+export const _struct_asym: CifCategory<CifExportContext> = createCategory('struct_asym');
+export const _entity_poly: CifCategory<CifExportContext> = createCategory('entity_poly');
+export const _entity_poly_seq: CifCategory<CifExportContext> = createCategory('entity_poly_seq');
+
+function createCategory(categoryName: 'struct_asym' | 'entity_poly' | 'entity_poly_seq'): CifCategory<CifExportContext> {
+    return {
+        name: categoryName,
+        instance({ structures, cache }) {
+            return getCategoryInstance(structures, categoryName, cache);
+        }
+    };
+}
+
+function getCategoryInstance(structures: Structure[], categoryName: 'struct_asym' | 'entity_poly' | 'entity_poly_seq', cache: any) {
+    const category = getModelMmCifCategory(structures[0].model, categoryName);
+    if (!category) return CifCategory.Empty;
+    const { entity_id } = category;
+    const names = cache.uniqueEntityIds || (cache.uniqueEntityIds = getUniqueEntityIdsFromStructures(structures));
+    const indices = Column.indicesOf(entity_id, id => names.has(id));
+    return CifCategory.ofTable(category, indices);
+
+}
+

src/mol-model/structure/export/categories/utils.ts

@@ -0,0 +1,42 @@
+/**
+ * Copyright (c) 2017-2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import { mmCIF_Database, mmCIF_Schema } from 'mol-io/reader/cif/schema/mmcif';
+import { unionMany } from 'mol-util/set';
+import { Model } from '../../model';
+import { Structure } from '../../structure';
+import { EntityIndex } from '../../model/indexing';
+import { UniqueArray } from 'mol-data/generic';
+import { sortArray } from 'mol-data/util';
+
+export function getModelMmCifCategory<K extends keyof mmCIF_Schema>(model: Model, name: K): mmCIF_Database[K] | undefined {
+    if (model.sourceData.kind !== 'mmCIF') return;
+    return model.sourceData.data[name];
+}
+
+export function getUniqueResidueNamesFromStructures(structures: Structure[]) {
+    return unionMany(structures.map(s => s.uniqueResidueNames));
+}
+
+export function getUniqueEntityIdsFromStructures(structures: Structure[]): Set<string> {
+    if (structures.length === 0) return new Set();
+
+    const names = structures[0].model.entities.data.id;
+    return new Set(getUniqueEntityIndicesFromStructures(structures).map(i => names.value(i)));
+}
+
+export function getUniqueEntityIndicesFromStructures(structures: Structure[]): ReadonlyArray<EntityIndex> {
+    if (structures.length === 0) return [];
+    if (structures.length === 1) return structures[0].entityIndices;
+    const ret = UniqueArray.create<EntityIndex, EntityIndex>();
+    for (const s of structures) {
+        for (const e of s.entityIndices) {
+            UniqueArray.add(ret, e, e);
+        }
+    }
+    sortArray(ret.array);
+    return ret.array;
+}

src/mol-model/structure/export/mmcif.ts

@@ -8,29 +8,38 @@
 import { CifWriter } from 'mol-io/writer/cif'
 import { mmCIF_Schema } from 'mol-io/reader/cif/schema/mmcif'
 import { Structure } from '../structure'
-import { Model } from '../model'
 import { _atom_site } from './categories/atom_site';
 import CifCategory = CifWriter.Category
 import { _struct_conf, _struct_sheet_range } from './categories/secondary-structure';
 import { _pdbx_struct_mod_residue } from './categories/modified-residues';
+import { _chem_comp, _pdbx_chem_comp_identifier } from './categories/misc';
+import { Model } from '../model';
+import { getUniqueEntityIndicesFromStructures } from './categories/utils';
+import { _struct_asym, _entity_poly, _entity_poly_seq } from './categories/sequence';
+import { ModelPropertyDescriptor } from '../model/properties/custom';
 
 export interface CifExportContext {
-    structure: Structure,
-    model: Model,
+    structures: Structure[],
+    firstModel: Model,
     cache: any
 }
 
 export namespace CifExportContext {
-    export function create(structures: Structure | Structure[]): CifExportContext[] {
-        if (Array.isArray(structures)) return structures.map(structure => ({ structure, model: structure.models[0], cache: Object.create(null) }));
-        return [{ structure: structures, model: structures.models[0], cache: Object.create(null) }];
+    export function create(structures: Structure | Structure[]): CifExportContext {
+        const structureArray = Array.isArray(structures) ? structures : [structures];
+        return {
+            structures: structureArray,
+            firstModel: structureArray[0].model,
+            cache: Object.create(null)
+        };
     }
 }
 
 function copy_mmCif_category(name: keyof mmCIF_Schema): CifCategory<CifExportContext> {
     return {
         name,
-        instance({ model }) {
+        instance({ structures }) {
+            const model = structures[0].model;
             if (model.sourceData.kind !== 'mmCIF') return CifCategory.Empty;
             const table = model.sourceData.data[name];
             if (!table || !table._rowCount) return CifCategory.Empty;
@@ -41,9 +50,9 @@ function copy_mmCif_category(name: keyof mmCIF_Schema): CifCategory<CifExportCon
 
 const _entity: CifCategory<CifExportContext> = {
     name: 'entity',
-    instance({ structure, model}) {
-        const keys = Structure.getEntityKeys(structure);
-        return CifCategory.ofTable(model.entities.data, keys);
+    instance({ structures }) {
+        const indices = getUniqueEntityIndicesFromStructures(structures);
+        return CifCategory.ofTable(structures[0].model.entities.data, indices);
     }
 }
 
@@ -67,9 +76,9 @@ const Categories = [
     _struct_sheet_range,
 
     // Sequence
-    copy_mmCif_category('struct_asym'), // TODO: filter only present chains?
-    copy_mmCif_category('entity_poly'),
-    copy_mmCif_category('entity_poly_seq'),
+    _struct_asym,
+    _entity_poly,
+    _entity_poly_seq,
 
     // Branch
     copy_mmCif_category('pdbx_entity_branch'),
@@ -78,8 +87,8 @@ const Categories = [
 
     // Misc
     // TODO: filter for actual present residues?
-    copy_mmCif_category('chem_comp'),
-    copy_mmCif_category('pdbx_chem_comp_identifier'),
+    _chem_comp,
+    _pdbx_chem_comp_identifier,
     copy_mmCif_category('atom_sites'),
 
     _pdbx_struct_mod_residue,
@@ -100,13 +109,13 @@ export const mmCIF_Export_Filters = {
 }
 
 /** Doesn't start a data block */
-export function encode_mmCIF_categories(encoder: CifWriter.Encoder, structures: Structure | Structure[], params?: { skipCategoryNames?: Set<string>, exportCtx?: CifExportContext[] }) {
+export function encode_mmCIF_categories(encoder: CifWriter.Encoder, structures: Structure | Structure[], params?: { skipCategoryNames?: Set<string>, exportCtx?: CifExportContext }) {
     const first = Array.isArray(structures) ? structures[0] : (structures as Structure);
     const models = first.models;
     if (models.length !== 1) throw 'Can\'t export stucture composed from multiple models.';
 
     const _params = params || { };
-    const ctx: CifExportContext[] = params && params.exportCtx ? params.exportCtx : CifExportContext.create(structures);
+    const ctx: CifExportContext = params && params.exportCtx ? params.exportCtx : CifExportContext.create(structures);
 
     for (const cat of Categories) {
         if (_params.skipCategoryNames && _params.skipCategoryNames.has(cat.name)) continue;
@@ -118,10 +127,20 @@ export function encode_mmCIF_categories(encoder: CifWriter.Encoder, structures:
 
         const prefix = customProp.cifExport.prefix;
         const cats = customProp.cifExport.categories;
+
+        let propCtx = ctx;
+        if (customProp.cifExport.context) {
+            const propId = ModelPropertyDescriptor.getUUID(customProp);
+            if (ctx.cache[propId + '__ctx']) propCtx = ctx.cache[propId + '__ctx'];
+            else {
+                propCtx = customProp.cifExport.context(ctx) || ctx;
+                ctx.cache[propId + '__ctx'] = propCtx;
+            }
+        }
         for (const cat of cats) {
             if (_params.skipCategoryNames && _params.skipCategoryNames.has(cat.name)) continue;
             if (cat.name.indexOf(prefix) !== 0) throw new Error(`Custom category '${cat.name}' name must start with prefix '${prefix}.'`);
-            encoder.writeCategory(cat, ctx);
+            encoder.writeCategory(cat, propCtx);
         }
     }
 }

src/mol-model/structure/model/formats/mmcif/bonds/comp.ts

@@ -26,10 +26,10 @@ export namespace ComponentBond {
             categories: [{
                 name: 'chem_comp_bond',
                 instance(ctx) {
-                    const chem_comp_bond = getChemCompBond(ctx.model);
+                    const chem_comp_bond = getChemCompBond(ctx.structures[0].model);
                     if (!chem_comp_bond) return CifWriter.Category.Empty;
 
-                    const comp_names = getUniqueResidueNames(ctx.structure);
+                    const comp_names = getUniqueResidueNames(ctx.structures[0]);
                     const { comp_id, _rowCount } = chem_comp_bond;
                     const indices: number[] = [];
                     for (let i = 0; i < _rowCount; i++) {

src/mol-model/structure/model/formats/mmcif/bonds/struct_conn.ts

@@ -31,10 +31,11 @@ export namespace StructConn {
             categories: [{
                 name: 'struct_conn',
                 instance(ctx) {
-                    const struct_conn = getStructConn(ctx.model);
+                    const structure = ctx.structures[0], model = structure.model;
+                    const struct_conn = getStructConn(model);
                     if (!struct_conn) return CifWriter.Category.Empty;
 
-                    const strConn = get(ctx.model);
+                    const strConn = get(model);
                     if (!strConn || strConn.entries.length === 0) return CifWriter.Category.Empty;
 
                     const foundAtoms = new Set<ElementIndex>();
@@ -45,7 +46,7 @@ export namespace StructConn {
                         for (let i = 0, _i = partners.length; i < _i; i++) {
                             const atom = partners[i].atomIndex;
                             if (foundAtoms.has(atom)) continue;
-                            if (hasAtom(ctx.structure, atom)) {
+                            if (hasAtom(structure, atom)) {
                                 foundAtoms.add(atom);
                             } else {
                                 hasAll = false;

src/mol-model/structure/model/properties/atomic/hierarchy.ts

@@ -154,10 +154,23 @@ export interface AtomicIndex {
      * Find the residue index where the spefied residue should be inserted to maintain the ordering (entity_id, asym_id, seq_id, ins_code).
      * Useful for determining ranges for sequence-level annotations.
      * @param pdbx_PDB_ins_code Empty string for undefined
-     * @returns index or -1 if entity or chain is not present.
      */
     findResidueInsertion(key: AtomicIndex.ResidueLabelKey): ResidueIndex,
 
+    /**
+     * Find element index of an atom.
+     * @param key
+     * @returns index or -1 if the atom is not present.
+     */
+    findAtom(key: AtomicIndex.AtomKey): ElementIndex,
+
+    /**
+     * Find element index of an atom.
+     * @param key
+     * @returns index or -1 if the atom is not present.
+     */
+    findAtomAuth(key: AtomicIndex.AtomAuthKey): ElementIndex
+
     // TODO: add indices that support comp_id?
 }
 
@@ -170,6 +183,9 @@ export namespace AtomicIndex {
 
     export interface ResidueAuthKey { auth_asym_id: string, auth_comp_id: string, auth_seq_id: number, pdbx_PDB_ins_code?: string }
     export interface ResidueLabelKey { label_entity_id: string, label_asym_id: string, label_seq_id: number, pdbx_PDB_ins_code?: string }
+
+    export interface AtomKey extends ResidueKey { label_atom_id: string, label_alt_id?: string }
+    export interface AtomAuthKey extends ResidueAuthKey { auth_atom_id: string, label_alt_id?: string }
 }
 
 export interface AtomicRanges {

src/mol-model/structure/model/properties/custom.ts

@@ -6,4 +6,4 @@
 
 export * from './custom/descriptor'
 export * from './custom/collection'
-export * from './custom/residue'
+export * from './custom/indexed'

src/mol-model/structure/model/properties/custom/chain.ts

@@ -0,0 +1,91 @@
+// /**
+//  * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+//  *
+//  * @author David Sehnal <david.sehnal@gmail.com>
+//  */
+
+// import { ChainIndex } from '../../indexing';
+// import { Unit, Structure, StructureElement } from '../../../structure';
+// import { Segmentation } from 'mol-data/int';
+// import { UUID } from 'mol-util';
+// import { CifWriter } from 'mol-io/writer/cif';
+
+// export interface ChainCustomProperty<T = any> {
+//     readonly id: UUID,
+//     readonly kind: Unit.Kind,
+//     has(idx: ChainIndex): boolean
+//     get(idx: ChainIndex): T | undefined
+// }
+
+// export namespace ChainCustomProperty {
+//     export interface ExportCtx<T> {
+//         elements: StructureElement[],
+//         property(index: number): T
+//     };
+
+//     function getExportCtx<T>(prop: ChainCustomProperty<T>, structure: Structure): ExportCtx<T> {
+//         const chainIndex = structure.model.atomicHierarchy.chainAtomSegments.index;
+//         const elements = getStructureElements(structure, prop);
+//         return { elements, property: i => prop.get(chainIndex[elements[i].element])! };
+//     }
+
+//     export function getCifDataSource<T>(structure: Structure, prop: ChainCustomProperty<T> | undefined, cache: any): CifWriter.Category.Instance['source'][0] {
+//         if (!prop) return { rowCount: 0 };
+//         if (cache && cache[prop.id]) return cache[prop.id];
+//         const data = getExportCtx(prop, structure);
+//         const ret = { data, rowCount: data.elements.length };
+//         if (cache) cache[prop.id] = ret;
+//         return ret;
+//     }
+
+//     class FromMap<T> implements ChainCustomProperty<T> {
+//         readonly id = UUID.create();
+
+//         has(idx: ChainIndex): boolean {
+//             return this.map.has(idx);
+//         }
+
+//         get(idx: ChainIndex) {
+//             return this.map.get(idx);
+//         }
+
+//         constructor(private map: Map<ChainIndex, T>, public kind: Unit.Kind) {
+//         }
+//     }
+
+//     export function fromMap<T>(map: Map<ChainIndex, T>, kind: Unit.Kind) {
+//         return new FromMap(map, kind);
+//     }
+
+//     /**
+//      * Gets all StructureElements that correspond to 1st atoms of residues that have an property assigned.
+//      * Only works correctly for structures with a single model.
+//      */
+//     export function getStructureElements(structure: Structure, property: ChainCustomProperty) {
+//         const models = structure.models;
+//         if (models.length !== 1) throw new Error(`Only works on structures with a single model.`);
+
+//         const seenChains = new Set<ChainIndex>();
+//         const unitGroups = structure.unitSymmetryGroups;
+//         const loci: StructureElement[] = [];
+
+//         for (const unitGroup of unitGroups) {
+//             const unit = unitGroup.units[0];
+//             if (unit.kind !== property.kind) {
+//                 continue;
+//             }
+
+//             const chains = Segmentation.transientSegments(unit.model.atomicHierarchy.chainAtomSegments, unit.elements);
+//             while (chains.hasNext) {
+//                 const seg = chains.move();
+//                 if (!property.has(seg.index) || seenChains.has(seg.index)) continue;
+
+//                 seenChains.add(seg.index);
+//                 loci[loci.length] = StructureElement.create(unit, unit.elements[seg.start]);
+//             }
+//         }
+
+//         loci.sort((x, y) => x.element - y.element);
+//         return loci;
+//     }
+// }

src/mol-model/structure/model/properties/custom/collection.ts

@@ -14,14 +14,14 @@ export class CustomProperties {
         return this._list;
     }
 
-    add(desc: ModelPropertyDescriptor) {
+    add(desc: ModelPropertyDescriptor<any>) {
         if (this._set.has(desc)) return;
 
         this._list.push(desc);
         this._set.add(desc);
     }
 
-    has(desc: ModelPropertyDescriptor): boolean {
+    has(desc: ModelPropertyDescriptor<any>): boolean {
         return this._set.has(desc);
     }
 }

src/mol-model/structure/model/properties/custom/descriptor.ts

@@ -7,23 +7,34 @@
 import { CifWriter } from 'mol-io/writer/cif'
 import { CifExportContext } from '../../../export/mmcif';
 import { QuerySymbolRuntime } from 'mol-script/runtime/query/compiler';
+import { UUID } from 'mol-util';
 
-interface ModelPropertyDescriptor<Symbols extends { [name: string]: QuerySymbolRuntime } = { }> {
+interface ModelPropertyDescriptor<ExportCtx = CifExportContext, Symbols extends { [name: string]: QuerySymbolRuntime } = { }> {
     readonly isStatic: boolean,
     readonly name: string,
 
     cifExport?: {
         // Prefix enforced during export.
         prefix: string,
-        categories: CifWriter.Category<CifExportContext>[]
+        context?: (ctx: CifExportContext) => ExportCtx | undefined,
+        categories: CifWriter.Category<ExportCtx>[]
     },
 
     // TODO: add aliases when lisp-like mol-script is done
     symbols?: Symbols
 }
 
-function ModelPropertyDescriptor<Desc extends ModelPropertyDescriptor>(desc: Desc) {
+function ModelPropertyDescriptor<Ctx, Desc extends ModelPropertyDescriptor<Ctx>>(desc: Desc) {
     return desc;
 }
 
+namespace ModelPropertyDescriptor {
+    export function getUUID(prop: ModelPropertyDescriptor): UUID {
+        if (!(prop as any).__key) {
+            (prop as any).__key = UUID.create();
+        }
+        return (prop as any).__key;
+    }
+}
+
 export { ModelPropertyDescriptor }

src/mol-model/structure/model/properties/custom/indexed.ts

@@ -0,0 +1,223 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import { ResidueIndex, ChainIndex, ElementIndex, EntityIndex } from '../../indexing';
+import { Unit, Structure, StructureElement } from '../../../structure';
+import { Segmentation } from 'mol-data/int';
+import { UUID } from 'mol-util';
+import { CifWriter } from 'mol-io/writer/cif';
+import { Model } from '../../model';
+
+export interface IndexedCustomProperty<Idx extends IndexedCustomProperty.Index, T = any> {
+    readonly id: UUID,
+    readonly kind: Unit.Kind,
+    readonly level: IndexedCustomProperty.Level,
+    has(idx: Idx): boolean,
+    get(idx: Idx): T | undefined,
+    getElements(structure: Structure): IndexedCustomProperty.Elements<T>
+}
+
+export namespace IndexedCustomProperty {
+    export type Index = ElementIndex | ResidueIndex | ChainIndex | EntityIndex
+    export type Level = 'atom' | 'residue' | 'chain' | 'entity'
+
+    export interface Elements<T> {
+        elements: StructureElement[],
+        property(index: number): T
+    }
+
+    export function getCifDataSource<Idx extends Index, T>(structure: Structure, prop: IndexedCustomProperty<Idx, T> | undefined, cache: any): CifWriter.Category.Instance['source'][0] {
+        if (!prop) return { rowCount: 0 };
+        if (cache && cache[prop.id]) return cache[prop.id];
+        const data = prop.getElements(structure);
+        const ret = { data, rowCount: data.elements.length };
+        if (cache) cache[prop.id] = ret;
+        return ret;
+    }
+
+    export type Atom<T> = IndexedCustomProperty<ElementIndex, T>
+    export function fromAtomMap<T>(map: Map<ElementIndex, T>): Atom<T> {
+        return new ElementMappedCustomProperty(map);
+    }
+
+    export function fromAtomArray<T>(array: ArrayLike<T>): Atom<T> {
+        // TODO: create "array based custom property" as optimization
+        return new ElementMappedCustomProperty(arrayToMap(array));
+    }
+
+    export type Residue<T> = IndexedCustomProperty<ResidueIndex, T>
+    const getResidueSegments = (model: Model) => model.atomicHierarchy.residueAtomSegments;
+    export function fromResidueMap<T>(map: Map<ResidueIndex, T>): Residue<T> {
+        return new SegmentedMappedIndexedCustomProperty('residue', map, getResidueSegments, Unit.Kind.Atomic);
+    }
+
+    export function fromResidueArray<T>(array: ArrayLike<T>): Residue<T> {
+        // TODO: create "array based custom property" as optimization
+        return new SegmentedMappedIndexedCustomProperty('residue', arrayToMap(array), getResidueSegments, Unit.Kind.Atomic);
+    }
+
+    export type Chain<T> = IndexedCustomProperty<ChainIndex, T>
+    const getChainSegments = (model: Model) => model.atomicHierarchy.chainAtomSegments;
+    export function fromChainMap<T>(map: Map<ChainIndex, T>): Chain<T> {
+        return new SegmentedMappedIndexedCustomProperty('chain', map, getChainSegments, Unit.Kind.Atomic);
+    }
+
+    export function fromChainArray<T>(array: ArrayLike<T>): Chain<T> {
+        // TODO: create "array based custom property" as optimization
+        return new SegmentedMappedIndexedCustomProperty('chain', arrayToMap(array), getChainSegments, Unit.Kind.Atomic);
+    }
+
+    export type Entity<T> = IndexedCustomProperty<EntityIndex, T>
+    export function fromEntityMap<T>(map: Map<EntityIndex, T>): Entity<T> {
+        return new EntityMappedCustomProperty(map);
+    }
+}
+
+function arrayToMap<Idx extends IndexedCustomProperty.Index, T>(array: ArrayLike<T>): Map<Idx, T> {
+    const ret = new Map<Idx, T>();
+    for (let i = 0 as Idx, _i = array.length; i < _i; i++) ret.set(i, array[i as number]);
+    return ret;
+}
+
+class SegmentedMappedIndexedCustomProperty<Idx extends IndexedCustomProperty.Index, T = any> implements IndexedCustomProperty<Idx, T> {
+    readonly id: UUID = UUID.create();
+    readonly kind: Unit.Kind;
+    has(idx: Idx): boolean { return this.map.has(idx); }
+    get(idx: Idx) { return this.map.get(idx); }
+
+    private getStructureElements(structure: Structure) {
+        const models = structure.models;
+        if (models.length !== 1) throw new Error(`Only works on structures with a single model.`);
+
+        const seenIndices = new Set<Idx>();
+        const unitGroups = structure.unitSymmetryGroups;
+        const loci: StructureElement[] = [];
+
+        const segments = this.segmentGetter(models[0]);
+
+        for (const unitGroup of unitGroups) {
+            const unit = unitGroup.units[0];
+            if (unit.kind !== this.kind) {
+                continue;
+            }
+
+            const chains = Segmentation.transientSegments(segments, unit.elements);
+            while (chains.hasNext) {
+                const seg = chains.move();
+                if (!this.has(seg.index) || seenIndices.has(seg.index)) continue;
+                seenIndices.add(seg.index);
+                loci[loci.length] = StructureElement.create(unit, unit.elements[seg.start]);
+            }
+        }
+
+        loci.sort((x, y) => x.element - y.element);
+        return loci;
+    }
+
+    getElements(structure: Structure): IndexedCustomProperty.Elements<T> {
+        const index = this.segmentGetter(structure.model).index;
+        const elements = this.getStructureElements(structure);
+        return { elements, property: i => this.get(index[elements[i].element])! };
+    }
+
+    constructor(public level: 'residue' | 'chain', private map: Map<Idx, T>, private segmentGetter: (model: Model) => Segmentation<ElementIndex, Idx>, kind: Unit.Kind) {
+        this.kind = kind;
+    }
+}
+
+class ElementMappedCustomProperty<T = any> implements IndexedCustomProperty<ElementIndex, T> {
+    readonly id: UUID = UUID.create();
+    readonly kind: Unit.Kind;
+    readonly level = 'atom';
+    has(idx: ElementIndex): boolean { return this.map.has(idx); }
+    get(idx: ElementIndex) { return this.map.get(idx); }
+
+    private getStructureElements(structure: Structure) {
+        const models = structure.models;
+        if (models.length !== 1) throw new Error(`Only works on structures with a single model.`);
+
+        const seenIndices = new Set<ElementIndex>();
+        const unitGroups = structure.unitSymmetryGroups;
+        const loci: StructureElement[] = [];
+
+        for (const unitGroup of unitGroups) {
+            const unit = unitGroup.units[0];
+            if (unit.kind !== this.kind) {
+                continue;
+            }
+
+            const elements = unit.elements;
+            for (let i = 0, _i = elements.length; i < _i; i++) {
+                const e = elements[i];
+                if (!this.has(e) || seenIndices.has(e)) continue;
+                seenIndices.add(elements[i]);
+                loci[loci.length] = StructureElement.create(unit, e);
+            }
+        }
+
+        loci.sort((x, y) => x.element - y.element);
+        return loci;
+    }
+
+    getElements(structure: Structure): IndexedCustomProperty.Elements<T> {
+        const elements = this.getStructureElements(structure);
+        return { elements, property: i => this.get(elements[i].element)! };
+    }
+
+    constructor(private map: Map<ElementIndex, T>) {
+        this.kind = Unit.Kind.Atomic;
+    }
+}
+
+class EntityMappedCustomProperty<T = any> implements IndexedCustomProperty<EntityIndex, T> {
+    readonly id: UUID = UUID.create();
+    readonly kind: Unit.Kind;
+    readonly level = 'entity';
+    has(idx: EntityIndex): boolean { return this.map.has(idx); }
+    get(idx: EntityIndex) { return this.map.get(idx); }
+
+    private getStructureElements(structure: Structure) {
+        const models = structure.models;
+        if (models.length !== 1) throw new Error(`Only works on structures with a single model.`);
+
+        const index = models[0].atomicHierarchy.index;
+        const seenIndices = new Set<EntityIndex>();
+        const unitGroups = structure.unitSymmetryGroups;
+        const loci: StructureElement[] = [];
+
+        const segments = models[0].atomicHierarchy.chainAtomSegments;
+
+        for (const unitGroup of unitGroups) {
+            const unit = unitGroup.units[0];
+            if (unit.kind !== this.kind) {
+                continue;
+            }
+
+            const chains = Segmentation.transientSegments(segments, unit.elements);
+            while (chains.hasNext) {
+                const seg = chains.move();
+                const eI = index.getEntityFromChain(seg.index);
+                if (!this.has(eI) || seenIndices.has(eI)) continue;
+                seenIndices.add(eI);
+                loci[loci.length] = StructureElement.create(unit, unit.elements[seg.start]);
+            }
+        }
+
+        loci.sort((x, y) => x.element - y.element);
+        return loci;
+    }
+
+    getElements(structure: Structure): IndexedCustomProperty.Elements<T> {
+        const elements = this.getStructureElements(structure);
+        const chainIndex = structure.model.atomicHierarchy.chainAtomSegments.index;
+        const index = structure.model.atomicHierarchy.index;
+        return { elements, property: i => this.get(index.getEntityFromChain(chainIndex[elements[i].element]))! };
+    }
+
+    constructor(private map: Map<EntityIndex, T>) {
+        this.kind = Unit.Kind.Atomic;
+    }
+}

src/mol-model/structure/model/properties/custom/residue.ts

@@ -1,94 +1,91 @@
-/**
- * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
- *
- * @author David Sehnal <david.sehnal@gmail.com>
- */
-
-import { ResidueIndex } from '../../indexing';
-import { Unit, Structure, StructureElement } from '../../../structure';
-import { Segmentation } from 'mol-data/int';
-import { CifExportContext } from '../../../export/mmcif';
-import { UUID } from 'mol-util';
-import { CifWriter } from 'mol-io/writer/cif';
-
-export interface ResidueCustomProperty<T = any> {
-    readonly id: UUID,
-    readonly kind: Unit.Kind,
-    has(idx: ResidueIndex): boolean
-    get(idx: ResidueIndex): T | undefined
-}
-
-export namespace ResidueCustomProperty {
-    export interface ExportCtx<T> {
-        exportCtx: CifExportContext,
-        elements: StructureElement[],
-        property(index: number): T
-    };
-
-    function getExportCtx<T>(exportCtx: CifExportContext, prop: ResidueCustomProperty<T>): ExportCtx<T> {
-        if (exportCtx.cache[prop.id]) return exportCtx.cache[prop.id];
-        const residueIndex = exportCtx.model.atomicHierarchy.residueAtomSegments.index;
-        const elements = getStructureElements(exportCtx.structure, prop);
-        return {
-            exportCtx,
-            elements,
-            property: i => prop.get(residueIndex[elements[i].element])!
-        }
-    }
-
-    export function createCifCategory<T>(ctx: CifExportContext, prop: ResidueCustomProperty<T>, fields: CifWriter.Field<number, ExportCtx<T>>[]): CifWriter.Category.Instance {
-        const data = getExportCtx(ctx, prop);
-        return { fields, data, rowCount: data.elements.length };
-    }
-
-    class FromMap<T> implements ResidueCustomProperty<T> {
-        readonly id = UUID.create();
-
-        has(idx: ResidueIndex): boolean {
-            return this.map.has(idx);
-        }
-
-        get(idx: ResidueIndex) {
-            return this.map.get(idx);
-        }
-
-        constructor(private map: Map<ResidueIndex, T>, public kind: Unit.Kind) {
-        }
-    }
-
-    export function fromMap<T>(map: Map<ResidueIndex, T>, kind: Unit.Kind) {
-        return new FromMap(map, kind);
-    }
-
-    /**
-     * Gets all StructureElements that correspond to 1st atoms of residues that have an property assigned.
-     * Only works correctly for structures with a single model.
-     */
-    export function getStructureElements(structure: Structure, property: ResidueCustomProperty) {
-        const models = structure.models;
-        if (models.length !== 1) throw new Error(`Only works on structures with a single model.`);
-
-        const seenResidues = new Set<ResidueIndex>();
-        const unitGroups = structure.unitSymmetryGroups;
-        const loci: StructureElement[] = [];
-
-        for (const unitGroup of unitGroups) {
-            const unit = unitGroup.units[0];
-            if (unit.kind !== property.kind) {
-                continue;
-            }
-
-            const residues = Segmentation.transientSegments(unit.model.atomicHierarchy.residueAtomSegments, unit.elements);
-            while (residues.hasNext) {
-                const seg = residues.move();
-                if (!property.has(seg.index) || seenResidues.has(seg.index)) continue;
-
-                seenResidues.add(seg.index);
-                loci[loci.length] = StructureElement.create(unit, unit.elements[seg.start]);
-            }
-        }
-
-        loci.sort((x, y) => x.element - y.element);
-        return loci;
-    }
-}
+// /**
+//  * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+//  *
+//  * @author David Sehnal <david.sehnal@gmail.com>
+//  */
+
+// import { ResidueIndex } from '../../indexing';
+// import { Unit, Structure, StructureElement } from '../../../structure';
+// import { Segmentation } from 'mol-data/int';
+// import { UUID } from 'mol-util';
+// import { CifWriter } from 'mol-io/writer/cif';
+
+// export interface ResidueCustomProperty<T = any> {
+//     readonly id: UUID,
+//     readonly kind: Unit.Kind,
+//     has(idx: ResidueIndex): boolean
+//     get(idx: ResidueIndex): T | undefined
+// }
+
+// export namespace ResidueCustomProperty {
+//     export interface ExportCtx<T> {
+//         elements: StructureElement[],
+//         property(index: number): T
+//     };
+
+//     function getExportCtx<T>(prop: ResidueCustomProperty<T>, structure: Structure): ExportCtx<T> {
+//         const residueIndex = structure.model.atomicHierarchy.residueAtomSegments.index;
+//         const elements = getStructureElements(structure, prop);
+//         return { elements, property: i => prop.get(residueIndex[elements[i].element])! };
+//     }
+
+//     export function getCifDataSource<T>(structure: Structure, prop: ResidueCustomProperty<T> | undefined, cache: any): CifWriter.Category.Instance['source'][0] {
+//         if (!prop) return { rowCount: 0 };
+//         if (cache && cache[prop.id]) return cache[prop.id];
+//         const data = getExportCtx(prop, structure);
+//         const ret = { data, rowCount: data.elements.length };
+//         if (cache) cache[prop.id] = ret;
+//         return ret;
+//     }
+
+//     class FromMap<T> implements ResidueCustomProperty<T> {
+//         readonly id = UUID.create();
+
+//         has(idx: ResidueIndex): boolean {
+//             return this.map.has(idx);
+//         }
+
+//         get(idx: ResidueIndex) {
+//             return this.map.get(idx);
+//         }
+
+//         constructor(private map: Map<ResidueIndex, T>, public kind: Unit.Kind) {
+//         }
+//     }
+
+//     export function fromMap<T>(map: Map<ResidueIndex, T>, kind: Unit.Kind) {
+//         return new FromMap(map, kind);
+//     }
+
+//     /**
+//      * Gets all StructureElements that correspond to 1st atoms of residues that have an property assigned.
+//      * Only works correctly for structures with a single model.
+//      */
+//     export function getStructureElements(structure: Structure, property: ResidueCustomProperty) {
+//         const models = structure.models;
+//         if (models.length !== 1) throw new Error(`Only works on structures with a single model.`);
+
+//         const seenResidues = new Set<ResidueIndex>();
+//         const unitGroups = structure.unitSymmetryGroups;
+//         const loci: StructureElement[] = [];
+
+//         for (const unitGroup of unitGroups) {
+//             const unit = unitGroup.units[0];
+//             if (unit.kind !== property.kind) {
+//                 continue;
+//             }
+
+//             const residues = Segmentation.transientSegments(unit.model.atomicHierarchy.residueAtomSegments, unit.elements);
+//             while (residues.hasNext) {
+//                 const seg = residues.move();
+//                 if (!property.has(seg.index) || seenResidues.has(seg.index)) continue;
+
+//                 seenResidues.add(seg.index);
+//                 loci[loci.length] = StructureElement.create(unit, unit.elements[seg.start]);
+//             }
+//         }
+
+//         loci.sort((x, y) => x.element - y.element);
+//         return loci;
+//     }
+// }

src/mol-model/structure/model/properties/utils/atomic-index.ts

@@ -7,9 +7,10 @@
 import { AtomicData, AtomicSegments } from '../atomic'
 import { Interval, Segmentation, SortedArray } from 'mol-data/int'
 import { Entities } from '../common'
-import { ChainIndex, ResidueIndex, EntityIndex } from '../../indexing';
+import { ChainIndex, ResidueIndex, EntityIndex, ElementIndex } from '../../indexing';
 import { AtomicIndex, AtomicHierarchy } from '../atomic/hierarchy';
 import { cantorPairing } from 'mol-data/util';
+import { Column } from 'mol-data/db';
 
 function getResidueId(seq_id: number, ins_code: string) {
     if (!ins_code) return seq_id;
@@ -38,6 +39,9 @@ function missingEntity(k: string) {
 interface Mapping {
     entities: Entities,
     label_seq_id: SortedArray,
+    label_atom_id: Column<string>,
+    auth_atom_id: Column<string>,
+    label_alt_id: Column<string>,
     segments: AtomicSegments,
 
     chain_index_entity_index: EntityIndex[],
@@ -46,7 +50,7 @@ interface Mapping {
     chain_index_label_seq_id: Map<ChainIndex, Map<string | number, ResidueIndex>>,
 
     auth_asym_id: Map<string, ChainIndex>,
-    chain_index_auth_seq_id: Map<ChainIndex, Map<string | number, ResidueIndex>>,
+    chain_index_auth_seq_id: Map<ChainIndex, Map<string | number, ResidueIndex>>
 }
 
 function createMapping(entities: Entities, data: AtomicData, segments: AtomicSegments): Mapping {
@@ -54,6 +58,9 @@ function createMapping(entities: Entities, data: AtomicData, segments: AtomicSeg
         entities,
         segments,
         label_seq_id: SortedArray.ofSortedArray(data.residues.label_seq_id.toArray({ array: Int32Array })),
+        label_atom_id: data.atoms.label_atom_id,
+        auth_atom_id: data.atoms.auth_atom_id,
+        label_alt_id: data.atoms.label_alt_id,
         chain_index_entity_index: new Int32Array(data.chains._rowCount) as any,
         entity_index_label_asym_id: new Map(),
         chain_index_label_seq_id: new Map(),
@@ -125,11 +132,46 @@ class Index implements AtomicIndex {
         return idx;
     }
 
+    findAtom(key: AtomicIndex.AtomKey): ElementIndex {
+        const rI = this.findResidue(key);
+        if (rI < 0) return -1 as ElementIndex;
+        const offsets = this.map.segments.residueAtomSegments.offsets;
+        if (typeof key.label_alt_id === 'undefined') {
+            return findAtomByName(offsets[rI], offsets[rI + 1], this.map.label_atom_id, key.label_atom_id);
+        }
+        return findAtomByNameAndAltLoc(offsets[rI], offsets[rI + 1], this.map.label_atom_id, this.map.label_alt_id, key.label_atom_id, key.label_alt_id);
+    }
+
+    findAtomAuth(key: AtomicIndex.AtomAuthKey): ElementIndex {
+        const rI = this.findResidueAuth(key);
+        if (rI < 0) return -1 as ElementIndex;
+        const offsets = this.map.segments.residueAtomSegments.offsets;
+        if (typeof key.label_alt_id === 'undefined') {
+            return findAtomByName(offsets[rI], offsets[rI + 1], this.map.auth_atom_id, key.auth_atom_id);
+        }
+        return findAtomByNameAndAltLoc(offsets[rI], offsets[rI + 1], this.map.auth_atom_id, this.map.label_alt_id, key.auth_atom_id, key.label_alt_id);
+    }
+
     constructor(private map: Mapping) {
         this.entityIndex = map.entities.getEntityIndex;
     }
 }
 
+function findAtomByName(start: ElementIndex, end: ElementIndex, data: Column<string>, atomName: string): ElementIndex {
+    for (let i = start; i < end; i++) {
+        if (data.value(i) === atomName) return i;
+    }
+    return -1 as ElementIndex;
+}
+
+function findAtomByNameAndAltLoc(start: ElementIndex, end: ElementIndex, nameData: Column<string>, altLocData: Column<string>,
+    atomName: string, altLoc: string): ElementIndex {
+    for (let i = start; i < end; i++) {
+        if (nameData.value(i) === atomName && altLocData.value(i) === altLoc) return i;
+    }
+    return -1 as ElementIndex;
+}
+
 export function getAtomicIndex(data: AtomicData, entities: Entities, segments: AtomicSegments): AtomicIndex {
     const map = createMapping(entities, data, segments);
 

src/mol-model/structure/structure.ts

@@ -12,4 +12,5 @@ import { Link } from './structure/unit/links'
 import StructureProperties from './structure/properties'
 
 export { StructureElement, Link, Structure, Unit, StructureSymmetry, StructureProperties }
-export * from './structure/unit/rings'
+export * from './structure/unit/rings'
+export * from './export/mmcif'

src/mol-model/structure/structure/structure.ts

@@ -18,12 +18,13 @@ import { InterUnitBonds, computeInterUnitBonds } from './unit/links';
 import { PairRestraints, CrossLinkRestraint, extractCrossLinkRestraints } from './unit/pair-restraints';
 import StructureSymmetry from './symmetry';
 import StructureProperties from './properties';
-import { ResidueIndex, ChainIndex } from '../model/indexing';
+import { ResidueIndex, ChainIndex, EntityIndex } from '../model/indexing';
 import { Carbohydrates } from './carbohydrates/data';
 import { computeCarbohydrates } from './carbohydrates/compute';
 import { Vec3 } from 'mol-math/linear-algebra';
 import { idFactory } from 'mol-util/id-factory';
 import { GridLookup3D } from 'mol-math/geometry';
+import { UUID } from 'mol-util';
 
 class Structure {
     /** Maps unit.id to unit */
@@ -38,6 +39,10 @@ class Structure {
         unitSymmetryGroups?: ReadonlyArray<Unit.SymmetryGroup>,
         carbohydrates?: Carbohydrates,
         models?: ReadonlyArray<Model>,
+        model?: Model,
+        uniqueResidueNames?: Set<string>,
+        entityIndices?: ReadonlyArray<EntityIndex>,
+        uniqueAtomicResidueIndices?: ReadonlyMap<UUID, ReadonlyArray<ResidueIndex>>,
         hashCode: number,
         elementCount: number,
         polymerResidueCount: number,
@@ -128,6 +133,29 @@ class Structure {
         return this._props.models;
     }
 
+    get uniqueResidueNames() {
+        return this._props.uniqueResidueNames
+            || (this._props.uniqueResidueNames = getUniqueResidueNames(this));
+    }
+
+    get entityIndices() {
+        return this._props.entityIndices || (this._props.entityIndices = getEntityIndices(this));
+    }
+
+    get uniqueAtomicResidueIndices() {
+        return this._props.uniqueAtomicResidueIndices
+            || (this._props.uniqueAtomicResidueIndices = getUniqueAtomicResidueIndices(this));
+    }
+
+    /** If the structure is based on a single model, return it. Otherwise throw an exception. */
+    get model(): Model {
+        if (this._props.model) return this._props.model;
+        const models = this.models;
+        if (models.length > 1) throw new Error('The structre is based on multiple models.');
+        this._props.model = models[0];
+        return this._props.model;
+    }
+
     hasElement(e: StructureElement) {
         if (!this.unitMap.has(e.unit.id)) return false;
         return SortedArray.has(this.unitMap.get(e.unit.id).elements, e.element);
@@ -166,6 +194,81 @@ function getModels(s: Structure) {
     return arr.array;
 }
 
+function getUniqueResidueNames(s: Structure) {
+    const prop = StructureProperties.residue.label_comp_id;
+    const names = new Set<string>();
+    const loc = StructureElement.create();
+    for (const unit of s.units) {
+        // TODO: support coarse unit?
+        if (!Unit.isAtomic(unit)) continue;
+        const residues = Segmentation.transientSegments(unit.model.atomicHierarchy.residueAtomSegments, unit.elements);
+        loc.unit = unit;
+        while (residues.hasNext) {
+            const seg = residues.move();
+            loc.element = unit.elements[seg.start];
+            names.add(prop(loc));
+        }
+    }
+    return names;
+}
+
+function getEntityIndices(structure: Structure): ReadonlyArray<EntityIndex> {
+    const { units } = structure;
+    const l = StructureElement.create();
+    const keys = UniqueArray.create<number, EntityIndex>();
+
+    for (const unit of units) {
+        const prop = unit.kind === Unit.Kind.Atomic ? StructureProperties.entity.key : StructureProperties.coarse.entityKey;
+
+        l.unit = unit;
+        const elements = unit.elements;
+
+        const chainsIt = Segmentation.transientSegments(unit.model.atomicHierarchy.chainAtomSegments, elements);
+        while (chainsIt.hasNext) {
+            const chainSegment = chainsIt.move();
+            l.element = elements[chainSegment.start];
+            const key = prop(l);
+            UniqueArray.add(keys, key, key);
+        }
+    }
+
+    sortArray(keys.array);
+    return keys.array;
+}
+
+function getUniqueAtomicResidueIndices(structure: Structure): ReadonlyMap<UUID, ReadonlyArray<ResidueIndex>> {
+    const map = new Map<UUID, UniqueArray<ResidueIndex, ResidueIndex>>();
+    const modelIds: UUID[] = [];
+
+    const unitGroups = structure.unitSymmetryGroups;
+    for (const unitGroup of unitGroups) {
+        const unit = unitGroup.units[0];
+        if (!Unit.isAtomic(unit)) continue;
+
+        let uniqueResidues: UniqueArray<ResidueIndex, ResidueIndex>;
+        if (map.has(unit.model.id)) uniqueResidues = map.get(unit.model.id)!;
+        else {
+            uniqueResidues = UniqueArray.create<ResidueIndex, ResidueIndex>();
+            modelIds.push(unit.model.id);
+            map.set(unit.model.id, uniqueResidues);
+        }
+
+        const residues = Segmentation.transientSegments(unit.model.atomicHierarchy.residueAtomSegments, unit.elements);
+        while (residues.hasNext) {
+            const seg = residues.move();
+            UniqueArray.add(uniqueResidues, seg.index, seg.index);
+        }
+    }
+
+    const ret = new Map<UUID, ReadonlyArray<ResidueIndex>>();
+    for (const id of modelIds) {
+        const array = map.get(id)!.array;
+        sortArray(array);
+        ret.set(id, array)
+    }
+    return ret;
+}
+
 namespace Structure {
     export const Empty = new Structure([]);
 
@@ -336,49 +439,6 @@ namespace Structure {
         }
     }
 
-    export function getEntityKeys(structure: Structure) {
-        const { units } = structure;
-        const l = StructureElement.create();
-        const keys = UniqueArray.create<number, number>();
-
-        for (const unit of units) {
-            const prop = unit.kind === Unit.Kind.Atomic ? StructureProperties.entity.key : StructureProperties.coarse.entityKey;
-
-            l.unit = unit;
-            const elements = unit.elements;
-
-            const chainsIt = Segmentation.transientSegments(unit.model.atomicHierarchy.chainAtomSegments, elements);
-            while (chainsIt.hasNext) {
-                const chainSegment = chainsIt.move();
-                l.element = elements[chainSegment.start];
-                const key = prop(l);
-                UniqueArray.add(keys, key, key);
-            }
-        }
-
-        sortArray(keys.array);
-        return keys.array;
-    }
-
-    export function getUniqueAtomicResidueIndices(structure: Structure, model: Model): ReadonlyArray<ResidueIndex> {
-        const uniqueResidues = UniqueArray.create<ResidueIndex, ResidueIndex>();
-        const unitGroups = structure.unitSymmetryGroups;
-        for (const unitGroup of unitGroups) {
-            const unit = unitGroup.units[0];
-            if (unit.model !== model || !Unit.isAtomic(unit)) {
-                continue;
-            }
-
-            const residues = Segmentation.transientSegments(unit.model.atomicHierarchy.residueAtomSegments, unit.elements);
-            while (residues.hasNext) {
-                const seg = residues.move();
-                UniqueArray.add(uniqueResidues, seg.index, seg.index);
-            }
-        }
-        sortArray(uniqueResidues.array);
-        return uniqueResidues.array;
-    }
-
     const distVec = Vec3.zero();
     function unitElementMinDistance(unit: Unit, p: Vec3, eRadius: number) {
         const { elements, conformation: { position, r } } = unit, dV = distVec;

src/mol-script/runtime/query/compiler.ts

@@ -18,7 +18,7 @@ export class QueryRuntimeTable {
         this.map.set(runtime.symbol.id, runtime);
     }
 
-    addCustomProp(desc: ModelPropertyDescriptor) {
+    addCustomProp(desc: ModelPropertyDescriptor<any>) {
         if (!desc.symbols) return;
 
         for (const k of Object.keys(desc.symbols)) {

src/mol-task/util/scheduler.ts

@@ -129,7 +129,7 @@ function createImmediateActions() {
     }
 
     function installReadyStateChangeImplementation() {
-        const html = doc!.documentElement;
+        const html = doc!.documentElement!;
         registerImmediate = function(handle) {
             // Create a <script> element; its readystatechange event will be fired asynchronously once it is inserted
             // into the document. Do so, thus queuing up the task. Remember to clean up once it's been called.
@@ -178,8 +178,8 @@ const immediateActions = (function () {
     if (typeof setImmediate !== 'undefined') {
         if (typeof window !== 'undefined') {
             return {
-                setImmediate: (handler: any, ...args: any[]) => window.setImmediate(handler, ...args as any),
-                clearImmediate: (handle: any) => window.clearImmediate(handle)
+                setImmediate: (handler: any, ...args: any[]) => (window as any).setImmediate(handler, ...args as any) as number,
+                clearImmediate: (handle: any) => (window as any).clearImmediate(handle)
             };
         } else {
             return { setImmediate, clearImmediate }

src/mol-util/console-logger.ts

@@ -34,6 +34,9 @@ export namespace ConsoleLogger {
         if (e.stack) console.error(e.stack);
     }
 
+    export function warn(ctx: string, e: any) {
+        console.error(`[Warn] (${ctx}) ${e}`);
+    }
 
     export function errorId(guid: string | String, e: any) {
         console.error(`[${guid}][Error] ${e}`);

src/mol-util/date.ts

@@ -0,0 +1,9 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+export function dateToUtcString(date: Date) {
+    return date.toISOString().replace(/T/, ' ').replace(/\..+/, '');
+}

src/mol-util/input/input-observer.ts

@@ -189,7 +189,7 @@ namespace InputObserver {
         function attach () {
             element.addEventListener( 'contextmenu', onContextMenu, false )
 
-            element.addEventListener('wheel', onMouseWheel, false)
+            element.addEventListener('wheel', onMouseWheel as any, false)
             element.addEventListener('mousedown', onPointerDown as any, false)
             // for dragging to work outside canvas bounds,
             // mouse move/up events have to be added to a parent, i.e. window
@@ -214,7 +214,7 @@ namespace InputObserver {
 
             element.removeEventListener( 'contextmenu', onContextMenu, false )
 
-            element.removeEventListener('wheel', onMouseWheel, false)
+            element.removeEventListener('wheel', onMouseWheel as any, false)
             element.removeEventListener('mousedown', onMouseDown as any, false)
             window.removeEventListener('mousemove', onMouseMove as any, false)
             window.removeEventListener('mouseup', onMouseUp as any, false)

src/mol-util/make-dir.ts

@@ -0,0 +1,21 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import * as fs from 'fs';
+
+export function makeDir(path: string, root?: string): boolean {
+    let dirs = path.split(/\/|\\/g),
+        dir = dirs.shift();
+
+    root = (root || '') + dir + '/';
+
+    try { fs.mkdirSync(root); }
+    catch (e) {
+        if (!fs.statSync(root).isDirectory()) throw new Error(e);
+    }
+
+    return !dirs.length || makeDir(dirs.join('/'), root);
+}

src/mol-util/set.ts

@@ -21,6 +21,25 @@ export function union<T>(setA: Set<T>, setB: Set<T>): Set<T> {
     return union;
 }
 
+export function unionMany<T>(sets: Set<T>[]) {
+    if (sets.length === 0) return new Set<T>();
+    if (sets.length === 1) return sets[0];
+    const union = new Set(sets[0]);
+    for (let i = 1; i < sets.length; i++) {
+        for (const elem of Array.from(sets[i])) union.add(elem);
+    }
+    return union;
+}
+
+export function unionManyArrays<T>(arrays: T[][]) {
+    if (arrays.length === 0) return new Set<T>();
+    const union = new Set(arrays[0]);
+    for (let i = 1; i < arrays.length; i++) {
+        for (const elem of arrays[i]) union.add(elem);
+    }
+    return union;
+}
+
 /** Create set containing elements of set a that are also in set b. */
 export function intersection<T>(setA: Set<T>, setB: Set<T>): Set<T> {
     const intersection = new Set();

src/perf-tests/cif-encoder.ts

@@ -23,7 +23,7 @@ function getCat(name: string): CifWriter.Category {
     return {
         name,
         instance(ctx: { fields: CifWriter.Field[], rowCount: number }) {
-            return { data: void 0, fields: ctx.fields, rowCount: ctx.rowCount };
+            return { fields: ctx.fields, source: [{ rowCount: ctx.rowCount }] };
         }
     };
 }

src/servers/model/config.ts

@@ -48,12 +48,27 @@ const config = {
     maxQueueLength: 30,
 
     /**
-     * Paths (relative to the root directory of the model server) to JavaScript files that specify custom properties
+     * Provide a property config or a path a JSON file with the config.
      */
-    customPropertyProviders: [
-        './properties/pdbe',
-        './properties/rcsb'
-    ],
+    customProperties: <import('./property-provider').ModelPropertyProviderConfig | string>{
+        sources: [
+            './properties/pdbe',
+            './properties/rcsb'
+        ],
+        params: {
+            PDBe: {
+                UseFileSource: false,
+                API: {
+                    residuewise_outlier_summary: 'https://www.ebi.ac.uk/pdbe/api/validation/residuewise_outlier_summary/entry',
+                    preferred_assembly: 'https://www.ebi.ac.uk/pdbe/api/pdb/entry/summary',
+                    struct_ref_domain: 'https://www.ebi.ac.uk/pdbe/api/mappings/sequence_domains'
+                },
+                File: {
+                    residuewise_outlier_summary: 'e:/test/mol-star/model/props/'
+                }
+            }
+        }
+    },
 
     /**
      * Maps a request identifier to a filename.
@@ -66,7 +81,9 @@ const config = {
     mapFile(source: string, id: string) {
         switch (source.toLowerCase()) {
             // case 'pdb': return `e:/test/quick/${id}_updated.cif`;
-            case 'pdb': return `e:/test/mol-star/model/out/${id}_updated.bcif`;
+            case 'pdb': return `c:/test/mol-star/model/out/${id}_updated.bcif`;
+            case 'pdb-bcif': return `c:/test/mol-star/model/out/${id}_updated.bcif`;
+            case 'pdb-cif': return `c:/test/mol-star/model/out/${id}_updated.cif`;
             default: return void 0;
         }
     }

src/servers/model/local.ts

@@ -19,17 +19,17 @@ let exampleWorkload: LocalInput = [{
         input: 'c:/test/quick/1tqn.cif',
         output: 'c:/test/quick/localapi/1tqn_full.bcif',
         query: 'full',
-        params: { binary: true }
+        params: {}
     }, {
         input: 'c:/test/quick/1cbs_updated.cif',
         output: 'c:/test/quick/localapi/1cbs_ligint.cif',
         query: 'residueInteraction', // action is case sensitive
-        params: { label_comp_id: 'REA' }
+        params: { atom_site: { label_comp_id: 'REA' }, radius: 5 }
     }, {
         input: 'c:/test/quick/1cbs_updated.cif', // multiple files that are repeated will only be parsed once
         output: 'c:/test/quick/localapi/1cbs_ligint.bcif',
         query: 'residueInteraction',
-        params: { label_comp_id: 'REA', binary: true } // parameters are just a JSON version of the query string
+        params: { atom_site: { label_comp_id: 'REA' } } // parameters are just a JSON version of the query string
     }
 ];
 

src/servers/model/preprocess/converter.ts

@@ -12,7 +12,7 @@ import { Task } from 'mol-task';
 function getCategoryInstanceProvider(cat: CifCategory, fields: CifWriter.Field[]): CifWriter.Category {
     return {
         name: cat.name,
-        instance: () => ({ data: cat, fields, rowCount: cat.rowCount })
+        instance: () => CifWriter.categoryInstance(fields, { data: cat, rowCount: cat.rowCount })
     };
 }
 

src/servers/model/preprocess/master.ts

@@ -8,6 +8,7 @@ import * as fs from 'fs'
 import * as path from 'path'
 import * as argparse from 'argparse'
 import { runMaster, PreprocessEntry } from './parallel';
+import { ModelPropertyProviderConfig } from '../property-provider';
 
 const cmdParser = new argparse.ArgumentParser({
     addHelp: true,
@@ -37,13 +38,13 @@ interface CmdArgs {
 
 export interface PreprocessConfig {
     numProcesses?: number,
-    customPropertyProviders?: string[]
+    customProperties?: ModelPropertyProviderConfig | string
 }
 
 const cmdArgs = cmdParser.parseArgs() as CmdArgs;
 
 let entries: PreprocessEntry[] = []
-let config: PreprocessConfig = { numProcesses: 1, customPropertyProviders: [] }
+let config: PreprocessConfig = { numProcesses: 1, customProperties: void 0 }
 
 if (cmdArgs.input) entries.push({ source: cmdArgs.input, cif: cmdArgs.outCIF, bcif: cmdArgs.outBCIF });
 // else if (cmdArgs.bulk) runBulk(cmdArgs.bulk);

src/servers/model/preprocess/parallel.ts

@@ -9,7 +9,7 @@ import * as cluster from 'cluster'
 import { now } from 'mol-task';
 import { PerformanceMonitor } from 'mol-util/performance-monitor';
 import { preprocessFile } from './preprocess';
-import { createModelPropertiesProviderFromSources } from '../property-provider';
+import { createModelPropertiesProvider } from '../property-provider';
 
 type PreprocessConfig = import('./master').PreprocessConfig
 
@@ -50,7 +50,7 @@ export function runMaster(config: PreprocessConfig, entries: PreprocessEntry[])
 
 export function runChild() {
     process.on('message', async ({ entries, config }: { entries: PreprocessEntry[], config: PreprocessConfig }) => {
-        const props = createModelPropertiesProviderFromSources(config.customPropertyProviders || []);
+        const props = createModelPropertiesProvider(config.customProperties);
         for (const entry of entries) {
             try {
                 await preprocessFile(entry.source, props, entry.cif, entry.bcif);
@@ -64,7 +64,7 @@ export function runChild() {
 }
 
 async function runSingle(entry: PreprocessEntry, config: PreprocessConfig, onMessage: (msg: any) => void) {
-    const props = createModelPropertiesProviderFromSources(config.customPropertyProviders || []);
+    const props = createModelPropertiesProvider(config.customProperties);
     try {
         await preprocessFile(entry.source, props, entry.cif, entry.bcif);
     } catch (e) {

src/servers/model/preprocess/preprocess.ts

@@ -36,7 +36,7 @@ export async function preprocessFile(filename: string, propertyProvider?: ModelP
     }
 }
 
-function encode(structure: Structure, header: string, categories: CifWriter.Category[], encoder: CifWriter.Encoder, exportCtx: CifExportContext[], writer: Writer) {
+function encode(structure: Structure, header: string, categories: CifWriter.Category[], encoder: CifWriter.Encoder, exportCtx: CifExportContext, writer: Writer) {
     const skipCategoryNames = new Set<string>(categories.map(c => c.name));
     encoder.startDataBlock(header);
     for (const cat of categories) {

src/servers/model/properties/pdbe.ts

@@ -5,13 +5,15 @@
  * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
-import { Model } from 'mol-model/structure';
-import { PDBe_structureQualityReport } from './providers/pdbe';
+import { PDBe_structureQualityReport, PDBe_preferredAssembly, PDBe_structRefDomain } from './providers/pdbe';
+import { AttachModelProperties } from '../property-provider';
 
-export function attachModelProperties(model: Model, cache: object): Promise<any>[] {
+export const attachModelProperties: AttachModelProperties = (args) => {
     // return a list of promises that start attaching the props in parallel
     // (if there are downloads etc.)
     return [
-        PDBe_structureQualityReport(model, cache)
+        PDBe_structureQualityReport(args),
+        PDBe_preferredAssembly(args),
+        PDBe_structRefDomain(args)
     ];
 }

src/servers/model/properties/providers/pdbe.ts

@@ -4,20 +4,99 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
- import { Model } from 'mol-model/structure';
+import * as fs from 'fs'
+import * as path from 'path'
+import { Model } from 'mol-model/structure';
 import { StructureQualityReport } from 'mol-model-props/pdbe/structure-quality-report';
 import { fetchRetry } from '../../utils/fetch-retry';
 import { UUID } from 'mol-util';
+import { PDBePreferredAssembly } from 'mol-model-props/pdbe/preferred-assembly';
+import { PDBeStructRefDomain } from 'mol-model-props/pdbe/struct-ref-domain';
+import { AttachModelProperty } from '../../property-provider';
+import { ConsoleLogger } from 'mol-util/console-logger';
 
-const cacheKey = UUID.create();
-export function PDBe_structureQualityReport(model: Model, cache: any) {
-    return StructureQualityReport.attachFromCifOrApi(model, {
-        PDBe_apiSourceJson: async model => {
+export const PDBe_structureQualityReport: AttachModelProperty = ({ model, params, cache }) => {
+    const PDBe_apiSourceJson = useFileSource(params)
+        ? residuewise_outlier_summary.getDataFromAggregateFile(getFilePrefix(params, 'residuewise_outlier_summary'))
+        : apiQueryProvider(getApiUrl(params, 'residuewise_outlier_summary', 'https://www.ebi.ac.uk/pdbe/api/validation/residuewise_outlier_summary/entry'), cache);
+
+    return StructureQualityReport.attachFromCifOrApi(model, { PDBe_apiSourceJson });
+}
+
+export const PDBe_preferredAssembly: AttachModelProperty = ({ model, params, cache }) => {
+    const PDBe_apiSourceJson = apiQueryProvider(getApiUrl(params, 'preferred_assembly', 'https://www.ebi.ac.uk/pdbe/api/pdb/entry/summary'), cache);
+    return PDBePreferredAssembly.attachFromCifOrApi(model, { PDBe_apiSourceJson });
+}
+
+export const PDBe_structRefDomain: AttachModelProperty = ({ model, params, cache }) => {
+    const PDBe_apiSourceJson = apiQueryProvider(getApiUrl(params, 'struct_ref_domain', 'https://www.ebi.ac.uk/pdbe/api/mappings/sequence_domains'), cache);
+    return PDBeStructRefDomain.attachFromCifOrApi(model, { PDBe_apiSourceJson });
+}
+
+namespace residuewise_outlier_summary {
+    const json = new Map<string, any>();
+    export function getDataFromAggregateFile(pathPrefix: string) {
+        // This is for "testing" purposes and should probably only read
+        // a single file with the appropriate prop in the "production" version.
+        return async (model: Model) => {
+            const key = `${model.label[1]}${model.label[2]}`;
+            if (!json.has(key)) {
+                const fn = path.join(pathPrefix, `${key}.json`);
+                if (!fs.existsSync(fn)) json.set(key, { });
+                // TODO: use async readFile?
+                else json.set(key, JSON.parse(fs.readFileSync(fn, 'utf8')));
+            }
+            return json.get(key)![model.label.toLowerCase()] || { };
+        }
+    }
+}
+
+function getApiUrl(params: any, name: string, fallback: string) {
+    const url = getParam<string>(params, 'PDBe', 'API', name);
+    if (!url) return fallback;
+    if (url[url.length - 1] === '/') return url.substring(0, url.length - 1);
+    return url;
+}
+
+function getFilePrefix(params: any, name: string) {
+    const ret = getParam<string>(params, 'PDBe', 'File', name);
+    if (!ret) throw new Error(`PDBe file prefix '${name}' not set!`);
+    return ret;
+}
+
+function useFileSource(params: any) {
+    return !!getParam<boolean>(params, 'PDBe', 'UseFileSource')
+}
+
+function getParam<T>(params: any, ...path: string[]): T | undefined {
+    try {
+        let current = params;
+        for (const p of path) {
+            if (typeof current === 'undefined') return;
+            current = current[p];
+        }
+        return current;
+    } catch (e) {
+        ConsoleLogger.error('Config', `Unable to retrieve property ${path.join('.')} from ${JSON.stringify(params)}`);
+    }
+}
+
+
+function apiQueryProvider(urlPrefix: string, cache: any) {
+    const cacheKey = UUID.create();
+    return async (model: Model) => {
+        try {
             if (cache[cacheKey]) return cache[cacheKey];
-            const rawData = await fetchRetry(`https://www.ebi.ac.uk/pdbe/api/validation/residuewise_outlier_summary/entry/${model.label.toLowerCase()}`, 1500, 5);
-            const json = await rawData.json();
+            const rawData = await fetchRetry(`${urlPrefix}/${model.label.toLowerCase()}`, 1500, 5);
+            // TODO: is this ok?
+            if (rawData.status !== 200) return { };
+            const json = (await rawData.json())[model.label.toLowerCase()] || { };
             cache[cacheKey] = json;
             return json;
+        } catch (e) {
+            // TODO: handle better
+            ConsoleLogger.warn('Props', `Count not retrieve prop @${`${urlPrefix}/${model.label.toLowerCase()}`}`);
+            return { };
         }
-    });
+    }
 }

src/servers/model/properties/providers/rcsb.ts

@@ -4,9 +4,9 @@
  * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
-import { Model } from 'mol-model/structure';
 import { AssemblySymmetry } from 'mol-model-props/rcsb/symmetry';
+import { AttachModelProperty } from '../../property-provider';
 
-export function RCSB_assemblySymmetry(model: Model) {
+export const RCSB_assemblySymmetry: AttachModelProperty = ({ model }) => {
     return AssemblySymmetry.attachFromCifOrAPI(model)
 }

src/servers/model/properties/rcsb.ts

@@ -5,13 +5,13 @@
  * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
-import { Model } from 'mol-model/structure';
+import { AttachModelProperties } from '../property-provider';
 import { RCSB_assemblySymmetry } from './providers/rcsb';
 
-export function attachModelProperties(model: Model): Promise<any>[] {
+export const attachModelProperties: AttachModelProperties = (args) => {
     // return a list of promises that start attaching the props in parallel
     // (if there are downloads etc.)
     return [
-        RCSB_assemblySymmetry(model)
+        RCSB_assemblySymmetry(args)
     ];
 }

src/servers/model/property-provider.ts

@@ -4,27 +4,48 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
+import * as fs from 'fs'
 import { Model } from 'mol-model/structure';
 import Config from './config';
+import { ConsoleLogger } from 'mol-util/console-logger';
 
-export type ModelPropertiesProvider = (model: Model, cache: object) => Promise<any>[]
+export interface ModelPropertyProviderConfig {
+    sources: string[],
+    params?: { [name: string]: any }
+}
+
+export type AttachModelProperty = (args: { model: Model, params: any, cache: any }) => Promise<any>
+export type AttachModelProperties = (args: { model: Model, params: any, cache: any }) => Promise<any>[]
+export type ModelPropertiesProvider = (model: Model, cache: any) => Promise<any>[]
 
 export function createModelPropertiesProviderFromConfig(): ModelPropertiesProvider {
-    return createModelPropertiesProviderFromSources(Config.customPropertyProviders);
+    return createModelPropertiesProvider(Config.customProperties);
 }
 
-export function createModelPropertiesProviderFromSources(sources: string[]): ModelPropertiesProvider {
-    if (!sources || sources.length === 0) return () => [];
+export function createModelPropertiesProvider(configOrPath: ModelPropertyProviderConfig | string | undefined): ModelPropertiesProvider {
+    let config: ModelPropertyProviderConfig;
+    if (typeof configOrPath === 'string') {
+        try {
+            config = JSON.parse(fs.readFileSync(configOrPath, 'utf8'));
+        } catch {
+            ConsoleLogger.error('Config', `Could not read property provider config file '${configOrPath}', ignoring.`);
+            return () => [];
+        }
+    } else {
+        config = configOrPath!;
+    }
+
+    if (!config || !config.sources || config.sources.length === 0) return () => [];
 
-    const ps: ModelPropertiesProvider[] = [];
-    for (const p of sources) {
+    const ps: AttachModelProperties[] = [];
+    for (const p of config.sources) {
         ps.push(require(p).attachModelProperties);
     }
 
     return (model, cache) => {
         const ret: Promise<any>[] = [];
         for (const p of ps) {
-            for (const e of p(model, cache)) ret.push(e);
+            for (const e of p({ model, cache, params: config.params })) ret.push(e);
         }
         return ret;
     }

src/servers/model/query/atoms.ts

@@ -6,8 +6,9 @@
 
 import { QueryPredicate, StructureElement, StructureProperties as Props } from 'mol-model/structure';
 import { AtomsQueryParams } from 'mol-model/structure/query/queries/generators';
+import { AtomSiteSchema } from '../server/api';
 
-export function getAtomsTests(params: any): Partial<AtomsQueryParams>[] {
+export function getAtomsTests(params: AtomSiteSchema): Partial<AtomsQueryParams>[] {
     if (!params) return [{ }];
     if (Array.isArray(params)) {
         return params.map(p => atomsTest(p));
@@ -16,7 +17,7 @@ export function getAtomsTests(params: any): Partial<AtomsQueryParams>[] {
     }
 }
 
-function atomsTest(params: any): Partial<AtomsQueryParams> {
+function atomsTest(params: AtomSiteSchema): Partial<AtomsQueryParams> {
     return {
         entityTest: entityTest(params),
         chainTest: chainTest(params),
@@ -25,13 +26,13 @@ function atomsTest(params: any): Partial<AtomsQueryParams> {
     };
 }
 
-function entityTest(params: any): QueryPredicate | undefined {
-    if (!params || typeof params.entity_id === 'undefined') return void 0;
+function entityTest(params: AtomSiteSchema): QueryPredicate | undefined {
+    if (!params || typeof params.label_entity_id === 'undefined') return void 0;
     const p = Props.entity.id, id = '' + params.label_entity_id;
     return ctx => p(ctx.element) === id;
 }
 
-function chainTest(params: any): QueryPredicate | undefined {
+function chainTest(params: AtomSiteSchema): QueryPredicate | undefined {
     if (!params) return void 0;
 
     if (typeof params.label_asym_id !== 'undefined') {
@@ -45,7 +46,7 @@ function chainTest(params: any): QueryPredicate | undefined {
     return void 0;
 }
 
-function residueTest(params: any): QueryPredicate | undefined {
+function residueTest(params: AtomSiteSchema): QueryPredicate | undefined {
     if (!params) return void 0;
 
     const props: StructureElement.Property<any>[] = [], values: any[] = [];
@@ -78,7 +79,7 @@ function residueTest(params: any): QueryPredicate | undefined {
     return andEqual(props, values);
 }
 
-function atomTest(params: any): QueryPredicate | undefined {
+function atomTest(params: AtomSiteSchema): QueryPredicate | undefined {
     if (!params) return void 0;
 
     const props: StructureElement.Property<any>[] = [], values: any[] = [];

src/servers/model/server/api-local.ts

@@ -12,13 +12,15 @@ import { resolveJob } from './query';
 import { StructureCache } from './structure-wrapper';
 import { now } from 'mol-task';
 import { PerformanceMonitor } from 'mol-util/performance-monitor';
+import { QueryName } from './api';
 
 export type LocalInput = {
     input: string,
     output: string,
-    query: string,
+    query: QueryName,
     modelNums?: number[],
-    params?: any
+    params?: any,
+    binary?: boolean
 }[];
 
 export async function runLocal(input: LocalInput) {
@@ -28,7 +30,17 @@ export async function runLocal(input: LocalInput) {
     }
 
     for (const job of input) {
-        JobManager.add('_local_', job.input, job.query, job.params || { }, job.modelNums, job.output);
+        const binary = /\.bcif/.test(job.output);
+        JobManager.add({
+            entryId: job.input,
+            queryName: job.query,
+            queryParams: job.params || { },
+            options: {
+                modelNums: job.modelNums,
+                outputFilename: job.output,
+                binary
+            }
+        });
     }
     JobManager.sort();
 

src/servers/model/server/api-web.ts

@@ -4,12 +4,15 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
+import * as fs from 'fs';
+import * as path from 'path';
 import * as express from 'express';
 import Config from '../config';
 import { ConsoleLogger } from 'mol-util/console-logger';
 import { resolveJob } from './query';
 import { JobManager } from './jobs';
 import { UUID } from 'mol-util';
+import { LandingPage } from './landing';
 
 function makePath(p: string) {
     return Config.appPrefix + '/' + p;
@@ -97,17 +100,55 @@ async function processNextJob() {
 // }
 
 export function initWebApi(app: express.Express) {
-    app.get(makePath('query'), (req, res) => {
+    app.get(makePath('static/:format/:id'), async (req, res) => {
+        const binary = req.params.format === 'bcif';
+        const id = req.params.id;
+        const fn = Config.mapFile(binary ? 'pdb-bcif' : 'pdb-cif', id);
+        if (!fn || !fs.existsSync(fn)) {
+            res.status(404);
+            res.end();
+            return;
+        }
+        fs.readFile(fn, (err, data) => {
+            if (err) {
+                res.status(404);
+                res.end();
+                return;
+            }
+
+            const f = path.parse(fn);
+            res.writeHead(200, {
+                'Content-Type': binary ? 'application/octet-stream' : 'text/plain; charset=utf-8',
+                'Access-Control-Allow-Origin': '*',
+                'Access-Control-Allow-Headers': 'X-Requested-With',
+                'Content-Disposition': `inline; filename="${f.name}${f.ext}"`
+            });
+            res.write(data);
+            res.end();
+        });
+    })
+
+    app.get(makePath('api/v1'), (req, res) => {
         const query = /\?(.*)$/.exec(req.url)![1];
         const args = JSON.parse(decodeURIComponent(query));
         const name = args.name;
         const entryId = args.id;
-        const params = args.params || { };
-        const jobId = JobManager.add('pdb', entryId, name, params, args.modelNums);
+        const queryParams = args.params || { };
+        const jobId = JobManager.add({
+            sourceId: 'pdb',
+            entryId,
+            queryName: name,
+            queryParams,
+            options: { modelNums: args.modelNums, binary: args.binary }
+        });
         responseMap.set(jobId, res);
         if (JobManager.size === 1) processNextJob();
     });
 
+    app.get('*', (req, res) => {
+        res.send(LandingPage);
+    });
+
     // for (const q of QueryList) {
     //     mapQuery(app, q.name, q.definition);
     // }

src/servers/model/server/api.ts

@@ -24,28 +24,33 @@ export interface QueryParamInfo {
     validation?: (v: any) => void
 }
 
-export interface QueryDefinition {
+export interface QueryDefinition<Params = any> {
     name: string,
     niceName: string,
     exampleId: string, // default is 1cbs
     query: (params: any, structure: Structure) => StructureQuery,
     description: string,
     params: QueryParamInfo[],
-    structureTransform?: (params: any, s: Structure) => Promise<Structure>
+    structureTransform?: (params: any, s: Structure) => Promise<Structure>,
+    '@params': Params
 }
 
-// const AtomSiteParams = {
-//     entity_id: <QueryParamInfo>{ name: 'entity_id', type: QueryParamType.String, description: 'Corresponds to the \'_entity.id\' or \'*.label_entity_id\' field, depending on the context.' },
+export interface AtomSiteSchema {
+    label_entity_id?: string,
 
-//     label_asym_id: <QueryParamInfo>{ name: 'label_asym_id', type: QueryParamType.String, description: 'Corresponds to the \'_atom_site.label_asym_id\' field.' },
-//     auth_asym_id: <QueryParamInfo>{ name: 'auth_asym_id', type: QueryParamType.String, exampleValue: 'A', description: 'Corresponds to the \'_atom_site.auth_asym_id\' field.' },
+    label_asym_id?: string,
+    auth_asym_id?: string,
 
-//     label_seq_id: <QueryParamInfo>{ name: 'label_seq_id', type: QueryParamType.Integer, description: 'Residue seq. number. Corresponds to the \'_atom_site.label_seq_id\' field.' },
-//     auth_seq_id: <QueryParamInfo>{ name: 'auth_seq_id', type: QueryParamType.Integer, exampleValue: '200', description: 'Author residue seq. number. Corresponds to the \'_atom_site.auth_seq_id\' field.' },
-//     label_comp_id: <QueryParamInfo>{ name: 'label_comp_id', type: QueryParamType.String, description: 'Residue name. Corresponds to the \'_atom_site.label_comp_id\' field.' },
-//     auth_comp_id: <QueryParamInfo>{ name: 'auth_comp_id', type: QueryParamType.String, exampleValue: 'REA', description: 'Author residue name. Corresponds to the \'_atom_site.auth_comp_id\' field.' },
-//     pdbx_PDB_ins_code: <QueryParamInfo>{ name: 'pdbx_PDB_ins_code', type: QueryParamType.String, description: 'Corresponds to the \'_atom_site.pdbx_PDB_ins_code\' field.' },
-// };
+    label_comp_id?: string,
+    auth_comp_id?: string,
+    label_seq_id?: string,
+    auth_seq_id?: string,
+    pdbx_PDB_ins_code?: string,
+
+    label_atom_id?: string,
+    auth_atom_id?: string,
+    type_symbol?: string
+}
 
 const AtomSiteTestParams: QueryParamInfo = {
     name: 'atom_site',
@@ -67,15 +72,19 @@ const RadiusParam: QueryParamInfo = {
     }
 };
 
-const QueryMap: { [id: string]: Partial<QueryDefinition> } = {
-    'full': { niceName: 'Full Structure', query: () => Queries.generators.all, description: 'The full structure.' },
-    'atoms': {
+function Q<Params = any>(definition: Partial<QueryDefinition<Params>>) {
+    return definition;
+}
+
+const QueryMap = {
+    'full': Q<{} | undefined>({ niceName: 'Full Structure', query: () => Queries.generators.all, description: 'The full structure.' }),
+    'atoms': Q<{ atom_site: AtomSiteSchema }>({
         niceName: 'Atoms',
         description: 'Atoms satisfying the given criteria.',
         query: p => Queries.combinators.merge(getAtomsTests(p.atom_site).map(test => Queries.generators.atoms(test))),
         params: [ AtomSiteTestParams ]
-    },
-    'symmetryMates': {
+    }),
+    'symmetryMates': Q<{ radius: number }>({
         niceName: 'Symmetry Mates',
         description: 'Computes crystal symmetry mates within the specified radius.',
         query: () => Queries.generators.all,
@@ -83,8 +92,8 @@ const QueryMap: { [id: string]: Partial<QueryDefinition> } = {
             return StructureSymmetry.builderSymmetryMates(s, p.radius).run();
         },
         params: [ RadiusParam ]
-    },
-    'assembly': {
+    }),
+    'assembly': Q<{ name: string }>({
         niceName: 'Assembly',
         description: 'Computes structural assembly.',
         query: () => Queries.generators.all,
@@ -98,8 +107,8 @@ const QueryMap: { [id: string]: Partial<QueryDefinition> } = {
             exampleValues: ['1'],
             description: 'Assembly name.'
         }]
-    },
-    'residueInteraction': {
+    }),
+    'residueInteraction': Q<{ atom_site: AtomSiteSchema, radius: number }>({
         niceName: 'Residue Interaction',
         description: 'Identifies all residues within the given radius from the source residue. Takes crystal symmetry into account.',
         query(p) {
@@ -116,16 +125,19 @@ const QueryMap: { [id: string]: Partial<QueryDefinition> } = {
             return StructureSymmetry.builderSymmetryMates(s, p.radius).run();
         },
         params: [ AtomSiteTestParams, RadiusParam ]
-    },
+    }),
 };
 
-export function getQueryByName(name: string): QueryDefinition {
+export type QueryName = keyof typeof QueryMap
+export type QueryParams<Q extends QueryName> = Partial<(typeof QueryMap)[Q]['@params']>
+
+export function getQueryByName(name: QueryName): QueryDefinition {
     return QueryMap[name] as QueryDefinition;
 }
 
 export const QueryList = (function () {
     const list: { name: string, definition: QueryDefinition }[] = [];
-    for (const k of Object.keys(QueryMap)) list.push({ name: k, definition: <QueryDefinition>QueryMap[k] });
+    for (const k of Object.keys(QueryMap)) list.push({ name: k, definition: <QueryDefinition>QueryMap[k as QueryName] });
     list.sort(function (a, b) { return a.name < b.name ? -1 : a.name > b.name ? 1 : 0 });
     return list;
 })();

src/servers/model/server/jobs.ts

@@ -5,7 +5,7 @@
  */
 
 import { UUID } from 'mol-util';
-import { getQueryByName, normalizeQueryParams, QueryDefinition } from './api';
+import { getQueryByName, normalizeQueryParams, QueryDefinition, QueryName, QueryParams } from './api';
 import { LinkedList } from 'mol-data/generic';
 
 export interface ResponseFormat {
@@ -28,23 +28,31 @@ export interface Job {
     outputFilename?: string
 }
 
-export function createJob(sourceId: '_local_' | string, entryId: string, queryName: string, params: any, modelNums?: number[], outputFilename?: string): Job {
-    const queryDefinition = getQueryByName(queryName);
-    if (!queryDefinition) throw new Error(`Query '${queryName}' is not supported.`);
+export interface JobDefinition<Name extends QueryName> {
+    sourceId?: string, // = '_local_',
+    entryId: string,
+    queryName: Name,
+    queryParams: QueryParams<Name>,
+    options?: { modelNums?: number[], outputFilename?: string, binary?: boolean }
+}
 
-    const normalizedParams = normalizeQueryParams(queryDefinition, params);
+export function createJob<Name extends QueryName>(definition: JobDefinition<Name>): Job {
+    const queryDefinition = getQueryByName(definition.queryName);
+    if (!queryDefinition) throw new Error(`Query '${definition.queryName}' is not supported.`);
 
+    const normalizedParams = normalizeQueryParams(queryDefinition, definition.queryParams);
+    const sourceId = definition.sourceId || '_local_';
     return {
         id: UUID.create(),
         datetime_utc: `${new Date().toISOString().replace(/T/, ' ').replace(/\..+/, '')}`,
-        key: `${sourceId}/${entryId}`,
+        key: `${sourceId}/${definition.entryId}`,
         sourceId,
-        entryId,
+        entryId: definition.entryId,
         queryDefinition,
         normalizedParams,
-        responseFormat: { isBinary: !!params.binary },
-        modelNums,
-        outputFilename
+        responseFormat: { isBinary: !!(definition.options && definition.options.binary) },
+        modelNums: definition.options && definition.options.modelNums,
+        outputFilename: definition.options && definition.options.outputFilename
     };
 }
 
@@ -55,8 +63,8 @@ class _JobQueue {
         return this.list.count;
     }
 
-    add(sourceId: '_local_' | string, entryId: string, queryName: string, params: any, modelNums?: number[], outputFilename?: string) {
-        const job = createJob(sourceId, entryId, queryName, params, modelNums, outputFilename);
+    add<Name extends QueryName>(definition: JobDefinition<Name>) {
+        const job = createJob(definition);
         this.list.addLast(job);
         return job.id;
     }

src/servers/model/server/landing.ts

@@ -0,0 +1,93 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import Version from '../version'
+
+const examples = [{
+    name: 'Atoms',
+    params: {
+        id: '1cbs',
+        name: 'atoms',
+        params: { atom_site: { label_comp_id: 'ALA' } }
+    }
+}, {
+    name: 'Residue Interaction',
+    params: {
+        id: '1cbs',
+        name: 'residueInteraction',
+        params: {
+            radius: 5,
+            atom_site: { 'label_comp_id': 'REA' }
+        }
+    }
+}, {
+    name: 'Full',
+    params: {
+        id: '1tqn',
+        name: 'full'
+    }
+}, {
+    name: 'Full (binary)',
+    params: {
+        id: '1tqn',
+        name: 'full',
+        binary: true
+    }
+}, {
+    name: 'Full (specific models)',
+    params: {
+        id: '1grm',
+        name: 'full',
+        modelNums: [ 2, 3 ]
+    }
+}];
+
+function create() {
+    return `<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="utf-8" />
+        <meta name="viewport" content="width=device-width, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0">
+        <title>Mol* ModelServer ${Version}</title>
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/skeleton/2.0.4/skeleton.min.css" />
+    </head>
+    <body>
+        <h1>Mol* Model Server ${Version}</h1>
+        <select id='example'>
+            <option value='-1'>Select example...</option>
+            ${examples.map((e, i) => `<option value=${i}>${e.name}</option>`)}
+        </select>
+        <br/>
+        <textarea style="height: 280px; width: 600px; font-family: monospace" id="query-text"></textarea><br>
+        <button class="button button-primary" style="width: 600px" id="query">Query</button>
+        <div id='error' style='color: red; font-weight: blue'></div>
+        <div>Static input files available as CIF and BinaryCIF at <a href='/ModelServer/static/cif/1cbs' target='_blank'>static/cif/id</a> and <a href='/ModelServer/static/bcif/1cbs' target='_blank'>static/bcif/id</a> respectively.</div>
+        <script>
+            var Examples = ${JSON.stringify(examples)};
+            var err = document.getElementById('error');
+            var exampleEl = document.getElementById('example'), queryTextEl = document.getElementById('query-text');
+            exampleEl.onchange = function () {
+                var i = +exampleEl.value;
+                if (i < 0) return;
+                queryTextEl.value = JSON.stringify(Examples[i].params, null, 2);
+            };
+            document.getElementById('query').onclick = function () {
+                err.innerText = '';
+                try {
+                    var q = JSON.parse(queryTextEl.value);
+                    var path = '/ModelServer/api/v1?' + encodeURIComponent(JSON.stringify(q));
+                    console.log(path);
+                    window.open(path, '_blank');
+                } catch (e) {
+                    err.innerText = '' + e;
+                }
+            };
+        </script>
+    </body>
+</html>`;
+}
+
+export const LandingPage = create();

src/servers/model/server/query.ts

@@ -70,8 +70,8 @@ export async function resolveJob(job: Job): Promise<CifWriter.Encoder<any>> {
 
         perf.start('encode');
         encoder.startDataBlock(sourceStructures[0].models[0].label.toUpperCase());
-        encoder.writeCategory(_model_server_result, [job]);
-        encoder.writeCategory(_model_server_params, [job]);
+        encoder.writeCategory(_model_server_result, job);
+        encoder.writeCategory(_model_server_params, job);
 
         // encoder.setFilter(mmCIF_Export_Filters.onlyPositions);
         encode_mmCIF_categories(encoder, result);
@@ -84,7 +84,7 @@ export async function resolveJob(job: Job): Promise<CifWriter.Encoder<any>> {
             encodeTimeMs: perf.time('encode')
         };
 
-        encoder.writeCategory(_model_server_stats, [stats]);
+        encoder.writeCategory(_model_server_stats, stats);
         encoder.encode();
         ConsoleLogger.logId(job.id, 'Query', 'Encoded.');
         return encoder;
@@ -101,9 +101,9 @@ function getEncodingProvider(structure: StructureWrapper) {
 
 function doError(job: Job, e: any) {
     const encoder = CifWriter.createEncoder({ binary: job.responseFormat.isBinary, encoderName: `ModelServer ${Version}` });
-    encoder.writeCategory(_model_server_result, [job]);
-    encoder.writeCategory(_model_server_params, [job]);
-    encoder.writeCategory(_model_server_error, ['' + e]);
+    encoder.writeCategory(_model_server_result, job);
+    encoder.writeCategory(_model_server_params, job);
+    encoder.writeCategory(_model_server_error, '' + e);
     encoder.encode();
     return encoder;
 }
@@ -155,12 +155,12 @@ const _model_server_stats_fields: CifField<number, Stats>[] = [
 
 const _model_server_result: CifWriter.Category<Job> = {
     name: 'model_server_result',
-    instance: (job) => ({ data: job, fields: _model_server_result_fields, rowCount: 1 })
+    instance: (job) => CifWriter.categoryInstance(_model_server_result_fields,{ data: job, rowCount: 1 })
 };
 
 const _model_server_error: CifWriter.Category<string> = {
     name: 'model_server_error',
-    instance: (message) => ({ data: message, fields: _model_server_error_fields, rowCount: 1 })
+    instance: (message) => CifWriter.categoryInstance(_model_server_error_fields, { data: message, rowCount: 1 })
 };
 
 const _model_server_params: CifWriter.Category<Job> = {
@@ -170,17 +170,12 @@ const _model_server_params: CifWriter.Category<Job> = {
         for (const k of Object.keys(job.normalizedParams)) {
             params.push([k, JSON.stringify(job.normalizedParams[k])]);
         }
-        return {
-            data: params,
-
-            fields: _model_server_params_fields,
-            rowCount: params.length
-        }
+        return CifWriter.categoryInstance(_model_server_params_fields, { data: params, rowCount: params.length });
     }
 };
 
 
 const _model_server_stats: CifWriter.Category<Stats> = {
     name: 'model_server_stats',
-    instance: (stats) => ({ data: stats, fields: _model_server_stats_fields, rowCount: 1 })
+    instance: (stats) => CifWriter.categoryInstance(_model_server_stats_fields, { data: stats, rowCount: 1 })
 }

src/servers/model/test.ts

@@ -34,21 +34,46 @@ function wrapFile(fn: string) {
     return w;
 }
 
-const basePath = path.join(__dirname, '..', '..', '..', '..')
-const examplesPath = path.join(basePath, 'examples')
-const outPath = path.join(basePath, 'build', 'test')
+export const basePath = path.join(__dirname, '..', '..', '..', '..')
+export const examplesPath = path.join(basePath, 'examples')
+export const outPath = path.join(basePath, 'build', 'test')
 if (!fs.existsSync(outPath)) fs.mkdirSync(outPath);
 
 async function run() {
     try {
-        // const request = createJob('_local_', 'e:/test/quick/1cbs_updated.cif', 'residueInteraction', { label_comp_id: 'REA' });
-        // const encoder = await resolveJob(request);
-        // const writer = wrapFile('e:/test/mol-star/1cbs_full.cif');
         // const testFile = '1crn.cif'
-        const testFile = '1grm_updated.cif'
-        const request = createJob('_local_', path.join(examplesPath, testFile), 'full', {});
+        // const testFile = '1grm_updated.cif'
+        // const testFile = 'C:/Projects/mol-star/molstar-proto/build/test/in/1grm_updated.cif'
+        // const request = createJob({
+        //     entryId: testFile,
+        //     queryName: 'full',
+        //     queryParams: { },
+        // });
+        const testFile = '1cbs_updated.cif'
+        const request = createJob({
+            entryId: path.join(examplesPath, testFile),
+            queryName: 'full',
+            queryParams: { }
+        });
+
+        // const request = createJob({
+        //     entryId: path.join(examplesPath, testFile),
+        //     queryName: 'atoms',
+        //     queryParams: {
+        //         atom_site: { label_comp_id: 'ALA' }
+        //     }
+        // });
+        // const request = createJob({
+        //     entryId: path.join(examplesPath, testFile),
+        //     queryName: 'residueInteraction',
+        //     queryParams: {
+        //         atom_site: { label_comp_id: 'REA' },
+        //         radius: 5
+        //     }
+        // });
         const encoder = await resolveJob(request);
         const writer = wrapFile(path.join(outPath, testFile));
+        // const writer = wrapFile(path.join(outPath, '1grm_test.cif'));
         encoder.writeTo(writer);
         writer.end();
     } catch (e) {

src/servers/model/utils/fetch-props-pdbe.ts

@@ -0,0 +1,91 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import fetch from 'node-fetch';
+import * as fs from 'fs'
+import * as path from 'path'
+import * as argparse from 'argparse'
+import { makeDir } from 'mol-util/make-dir';
+import { now } from 'mol-task';
+import { PerformanceMonitor } from 'mol-util/performance-monitor';
+
+const cmdParser = new argparse.ArgumentParser({
+    addHelp: true,
+    description: 'Download JSON data from PDBe API'
+});
+
+cmdParser.addArgument(['--in'], { help: 'Input folder', required: true });
+cmdParser.addArgument(['--out'], { help: 'Output folder', required: true });
+
+interface CmdArgs {
+    in: string,
+    out: string
+}
+
+const cmdArgs = cmdParser.parseArgs() as CmdArgs;
+
+function getPDBid(name: string) {
+    let idx = name.indexOf('_');
+    if (idx < 0) idx = name.indexOf('.');
+    return name.substr(0, idx).toLowerCase();
+}
+
+function findEntries() {
+    const files = fs.readdirSync(cmdArgs.in);
+    const cifTest = /\.cif$/;
+    const groups = new Map<string, string[]>();
+    const keys: string[] = [];
+
+    for (const f of files) {
+        if (!cifTest.test(f)) continue;
+        const id = getPDBid(f);
+        const groupId = `${id[1]}${id[2]}`;
+
+        if (groups.has(groupId)) groups.get(groupId)!.push(id);
+        else {
+            keys.push(groupId);
+            groups.set(groupId, [id]);
+        }
+    }
+
+    const ret: { key: string, entries: string[] }[] = [];
+    for (const key of keys) {
+        ret.push({ key, entries: groups.get(key)! })
+    }
+
+    return ret;
+}
+
+async function process() {
+    const entries = findEntries();
+    makeDir(cmdArgs.out);
+
+    const started = now();
+    let prog = 0;
+    for (const e of entries) {
+        const ts = now();
+        console.log(`${prog}/${entries.length} ${e.entries.length} entries.`)
+        const data = Object.create(null);
+
+        for (let ee of e.entries) {
+             const query = await fetch(`https://www.ebi.ac.uk/pdbe/api/validation/residuewise_outlier_summary/entry/${ee}`);
+             try {
+                if (query.status === 200) data[ee] = (await query.json())[ee] || { };
+                else console.error(ee, query.status);
+             } catch (e) {
+                console.error(ee, '' + e);
+             }
+        }
+        //const query = await fetch(`https://www.ebi.ac.uk/pdbe/api/validation/residuewise_outlier_summary/entry`, { method: 'POST', body });
+        //console.log(query.status);
+        //const data = await query.text();
+        fs.writeFileSync(path.join(cmdArgs.out, e.key + '.json'), JSON.stringify(data));
+        const time = now() - started;
+        console.log(`${++prog}/${entries.length} in ${PerformanceMonitor.format(time)} (last ${PerformanceMonitor.format(now() - ts)}, avg ${PerformanceMonitor.format(time / prog)})`);
+    }
+}
+
+process();

src/servers/model/utils/fetch-retry.ts

@@ -24,6 +24,7 @@ export async function fetchRetry(url: string, timeout: number, retryCount: numbe
         retryCount
     });
 
-    if (result.status >= 200 && result.status < 300) return result;
-    throw new Error(result.statusText);
+    return result;
+    // if (result.status >= 200 && result.status < 300) return result;
+    // throw new Error(result.statusText);
 }

src/servers/model/version.ts

@@ -4,4 +4,4 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
-export default '0.1.0';
+export default '0.8.0';

src/servers/volume/server/query/encode.ts

@@ -102,7 +102,7 @@ const _volume_data_3d_info: CifWriter.Category<ResultContext> = {
             sampledValuesInfo: result.query.data.header.sampling[result.query.samplingInfo.sampling.index].valuesInfo[result.channelIndex]
         };
 
-        return { data: ctx, fields: _volume_data_3d_info_fields, rowCount: 1 }
+        return { fields: _volume_data_3d_info_fields, source: [{ data: ctx, rowCount: 1 }] }
     }
 };
 
@@ -136,7 +136,7 @@ const _volume_data_3d: CifWriter.Category<ResultContext> = {
         }
 
         const fields = [CifWriter.Field.float('values', _volume_data_3d_number, { encoder, typedArray, digitCount: 6 })];
-        return { data, fields, rowCount: data.length };
+        return CifWriter.categoryInstance(fields, { data, rowCount: data.length });
     }
 }
 
@@ -174,12 +174,12 @@ const _density_server_result_fields = [
 
 const _density_server_result: CifWriter.Category<Data.QueryContext> = {
     name: 'density_server_result',
-    instance: ctx => ({ data: ctx, fields: _density_server_result_fields, rowCount: 1 })
+    instance: ctx => CifWriter.categoryInstance(_density_server_result_fields, { data: ctx, rowCount: 1 })
 }
 
 function write(encoder: CifWriter.Encoder, query: Data.QueryContext) {
     encoder.startDataBlock('SERVER');
-    encoder.writeCategory(_density_server_result, [query]);
+    encoder.writeCategory(_density_server_result, query);
 
     switch (query.kind) {
         case 'Data':
@@ -189,7 +189,7 @@ function write(encoder: CifWriter.Encoder, query: Data.QueryContext) {
         const header = query.data.header;
         for (let i = 0; i < header.channels.length; i++) {
             encoder.startDataBlock(header.channels[i]);
-            const ctx: ResultContext[] = [{ query, channelIndex: i }];
+            const ctx: ResultContext = { query, channelIndex: i };
 
             encoder.writeCategory(_volume_data_3d_info, ctx);
             encoder.writeCategory(_volume_data_3d, ctx);

tsconfig.json

@@ -21,7 +21,7 @@
             "mol-io": ["./mol-io"],
             "mol-math": ["./mol-math"],
             "mol-model": ["./mol-model"],
-            "mol-model-props": ["./mol-model-props"],
+            "mol-model-props": ["./mol-model-props", "./mol-model-props/index.ts"],
             "mol-ql": ["./mol-ql"],
             "mol-script": ["./mol-script"],
             "mol-task": ["./mol-task", "./mol-task/index.ts"],