Merge branch 'master' into custom-repr

package-lock.json

@@ -3326,12 +3326,14 @@
         "balanced-match": {
           "version": "1.0.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "brace-expansion": {
           "version": "1.1.11",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "balanced-match": "^1.0.0",
             "concat-map": "0.0.1"
@@ -3346,17 +3348,20 @@
         "code-point-at": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "concat-map": {
           "version": "0.0.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "console-control-strings": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "core-util-is": {
           "version": "1.0.2",
@@ -3473,7 +3478,8 @@
         "inherits": {
           "version": "2.0.3",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "ini": {
           "version": "1.3.5",
@@ -3485,6 +3491,7 @@
           "version": "1.0.0",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "number-is-nan": "^1.0.0"
           }
@@ -3499,6 +3506,7 @@
           "version": "3.0.4",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "brace-expansion": "^1.1.7"
           }
@@ -3506,12 +3514,14 @@
         "minimist": {
           "version": "0.0.8",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "minipass": {
           "version": "2.2.4",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "safe-buffer": "^5.1.1",
             "yallist": "^3.0.0"
@@ -3530,6 +3540,7 @@
           "version": "0.5.1",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "minimist": "0.0.8"
           }
@@ -3610,7 +3621,8 @@
         "number-is-nan": {
           "version": "1.0.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "object-assign": {
           "version": "4.1.1",
@@ -3622,6 +3634,7 @@
           "version": "1.4.0",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "wrappy": "1"
           }
@@ -3743,6 +3756,7 @@
           "version": "1.0.2",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "code-point-at": "^1.0.0",
             "is-fullwidth-code-point": "^1.0.0",

src/mol-io/reader/cif/data-model.ts

@@ -117,7 +117,7 @@ export function getCifFieldType(field: CifField): Column.Schema.Int | Column.Sch
     let floatCount = 0, hasString = false;
     for (let i = 0, _i = field.rowCount; i < _i; i++) {
         const k = field.valueKind(i);
-        if (k !== Column.ValueKind.Present) continue
+        if (k !== Column.ValueKind.Present) continue;
         const type = getNumberType(field.str(i));
         if (type === NumberType.Int) continue;
         else if (type === NumberType.Float) floatCount++;

src/mol-io/reader/common/text/number-parser.ts

@@ -128,5 +128,5 @@ export function getNumberType(str: string): NumberType {
         }
         else break;
     }
-    return NumberType.Int;
+    return start === end ? NumberType.Int : NumberType.NaN;
 }

src/mol-math/linear-algebra/matrix/principal-axes.ts

@@ -4,10 +4,10 @@
  * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
-import Matrix from './matrix.js';
-import { Vec3 } from '../3d.js';
+import Matrix from './matrix';
+import { Vec3 } from '../3d';
 // import { Vec3, Mat4 } from '../3d.js';
-import { svd } from './svd.js';
+import { svd } from './svd';
 
 // const negateVector = Vec3.create(-1, -1, -1)
 // const tmpMatrix = Mat4.identity()

src/mol-model/structure/export/mmcif.ts

@@ -20,6 +20,12 @@ export interface CifExportContext {
     cache: any
 }
 
+export namespace CifExportContext {
+    export function create(structure: Structure, model: Model): CifExportContext {
+        return { structure, model, cache: Object.create(null) };
+    }
+}
+
 function copy_mmCif_category(name: keyof mmCIF_Schema): CifCategory<CifExportContext> {
     return {
         name,
@@ -87,14 +93,17 @@ export const mmCIF_Export_Filters = {
 }
 
 /** Doesn't start a data block */
-export function encode_mmCIF_categories(encoder: CifWriter.Encoder, structure: Structure) {
+export function encode_mmCIF_categories(encoder: CifWriter.Encoder, structure: Structure, params?: { skipCategoryNames?: Set<string>, exportCtx?: CifExportContext }) {
     const models = structure.models;
     if (models.length !== 1) throw 'Can\'t export stucture composed from multiple models.';
     const model = models[0];
 
-    const ctx: CifExportContext[] = [{ structure, model, cache: Object.create(null) }];
+    const _params = params || { };
+
+    const ctx: CifExportContext[] = [_params.exportCtx ? _params.exportCtx : CifExportContext.create(structure, model)];
 
     for (const cat of Categories) {
+        if (_params.skipCategoryNames && _params.skipCategoryNames.has(cat.name)) continue;
         encoder.writeCategory(cat, ctx);
     }
     for (const customProp of model.customProperties.all) {
@@ -103,6 +112,7 @@ export function encode_mmCIF_categories(encoder: CifWriter.Encoder, structure: S
         const prefix = customProp.cifExport.prefix;
         const cats = customProp.cifExport.categories;
         for (const cat of cats) {
+            if (_params.skipCategoryNames && _params.skipCategoryNames.has(cat.name)) continue;
             if (cat.name.indexOf(prefix) !== 0) throw new Error(`Custom category '${cat.name}' name must start with prefix '${prefix}.'`);
             encoder.writeCategory(cat, ctx);
         }

src/mol-model/structure/query/context.ts

@@ -30,7 +30,7 @@ export class QueryContext implements QueryContextView {
     currentStructure: Structure = void 0 as any;
 
     /** Current link between atoms */
-    readonly atomicLink: Link.Location<Unit.Atomic> = void 0 as any;
+    readonly atomicLink: Link.Location<Unit.Atomic> = Link.Location() as Link.Location<Unit.Atomic>;
 
     setElement(unit: Unit, e: ElementIndex) {
         this.element.unit = unit;

src/mol-model/structure/query/queries/filters.ts

@@ -12,6 +12,9 @@ import { StructureSelection } from '../selection';
 import { structureAreIntersecting } from '../utils/structure-set';
 import { Vec3 } from 'mol-math/linear-algebra';
 import { checkStructureMaxRadiusDistance, checkStructureMinMaxDistance } from '../utils/structure-distance';
+import Structure from '../../structure/structure';
+import StructureElement from '../../structure/element';
+import { SortedArray } from 'mol-data/int';
 
 export function pick(query: StructureQuery, pred: QueryPredicate): StructureQuery {
     return ctx => {
@@ -205,5 +208,101 @@ function withinMinMaxRadius({ queryCtx, selection, target, minRadius, maxRadius,
     return ret.getSelection();
 }
 
+interface IsConnectedToCtx {
+    queryCtx: QueryContext,
+    input: Structure,
+    target: Structure,
+    bondTest: QueryFn<boolean>,
+    tElement: StructureElement
+}
+
+function checkConnected(ctx: IsConnectedToCtx, structure: Structure) {
+    const { queryCtx, input, target, bondTest, tElement } = ctx;
+
+    const interLinks = input.links;
+    for (const unit of structure.units) {
+        if (!Unit.isAtomic(unit)) continue;
+
+        const inputUnit = input.unitMap.get(unit.id) as Unit.Atomic;
+
+        const { offset, b } = inputUnit.links;
+        const linkedUnits = interLinks.getLinkedUnits(unit);
+        const luCount = linkedUnits.length;
+
+        queryCtx.atomicLink.aUnit = inputUnit;
+
+        const srcElements = unit.elements;
+        const inputElements = inputUnit.elements;
+
+        for (let i = 0 as StructureElement.UnitIndex, _i = srcElements.length; i < _i; i++) {
+            const inputIndex = SortedArray.indexOf(inputElements, srcElements[i]) as StructureElement.UnitIndex;
+
+            queryCtx.atomicLink.aIndex = inputIndex;
+            queryCtx.atomicLink.bUnit = inputUnit;
+
+            tElement.unit = unit;
+            for (let l = offset[inputIndex], _l = offset[inputIndex + 1]; l < _l; l++) {
+                tElement.element = inputElements[b[l]];
+                if (!target.hasElement(tElement)) continue;
+                queryCtx.atomicLink.bIndex = b[l] as StructureElement.UnitIndex;
+                if (bondTest(queryCtx)) return true;
+            }
+
+            for (let li = 0; li < luCount; li++) {
+                const lu = linkedUnits[li];
+                tElement.unit = lu.unitB;
+                queryCtx.atomicLink.bUnit = lu.unitB;
+                const bElements = lu.unitB.elements;
+                const bonds = lu.getBonds(inputIndex);
+                for (let bi = 0, _bi = bonds.length; bi < _bi; bi++) {
+                    const bond = bonds[bi];
+                    tElement.element = bElements[bond.indexB];
+                    if (!target.hasElement(tElement)) continue;
+                    queryCtx.atomicLink.bIndex = bond.indexB;
+                    if (bondTest(queryCtx)) return true;
+                }
+            }
+        }
+    }
 
-// TODO: isConnectedTo
+    return false;
+}
+
+export interface IsConnectedToParams {
+    query: StructureQuery,
+    target: StructureQuery,
+    bondTest?: QueryFn<boolean>,
+    disjunct: boolean,
+    invert: boolean
+}
+
+function defaultBondTest(ctx: QueryContext) {
+    return true;
+}
+
+export function isConnectedTo({ query, target, disjunct, invert, bondTest }: IsConnectedToParams): StructureQuery {
+    return ctx => {
+        const targetSel = target(ctx);
+        if (StructureSelection.isEmpty(targetSel)) return targetSel;
+        const selection = query(ctx);
+        if (StructureSelection.isEmpty(selection)) return selection;
+
+        const connCtx: IsConnectedToCtx = {
+            queryCtx: ctx,
+            input: ctx.inputStructure,
+            target: StructureSelection.unionStructure(targetSel),
+            bondTest: bondTest || defaultBondTest,
+            tElement: StructureElement.create()
+        }
+
+        const ret = StructureSelection.LinearBuilder(ctx.inputStructure);
+        ctx.pushCurrentLink();
+        StructureSelection.forEach(selection, (s, sI) => {
+            if (checkConnected(connCtx, s)) ret.add(s);
+            if (sI % 5 === 0) ctx.throwIfTimedOut();
+        })
+        ctx.popCurrentLink();
+
+        return ret.getSelection();
+    }
+}

src/mol-model/structure/query/queries/modifiers.ts

@@ -14,6 +14,7 @@ import { QueryContext, QueryFn } from '../context';
 import { structureIntersect, structureSubtract } from '../utils/structure-set';
 import { UniqueArray } from 'mol-data/generic';
 import { StructureSubsetBuilder } from '../../structure/util/subset-builder';
+import StructureElement from '../../structure/element';
 
 function getWholeResidues(ctx: QueryContext, source: Structure, structure: Structure) {
     const builder = source.subsetBuilder(true);
@@ -59,8 +60,7 @@ export function wholeResidues(query: StructureQuery): StructureQuery {
 
 export interface IncludeSurroundingsParams {
     radius: number,
-    // TODO
-    // atomRadius?: Element.Property<number>,
+    elementRadius?: QueryFn<number>,
     wholeResidues?: boolean
 }
 
@@ -82,9 +82,88 @@ function getIncludeSurroundings(ctx: QueryContext, source: Structure, structure:
     return !!params.wholeResidues ? getWholeResidues(ctx, source, builder.getStructure()) : builder.getStructure();
 }
 
+interface IncludeSurroundingsParamsWithRadius extends IncludeSurroundingsParams {
+    elementRadius: QueryFn<number>,
+    elementRadiusClosure: StructureElement.Property<number>,
+    sourceMaxRadius: number
+}
+
+function getIncludeSurroundingsWithRadius(ctx: QueryContext, source: Structure, structure: Structure, params: IncludeSurroundingsParamsWithRadius) {
+    const builder = new StructureUniqueSubsetBuilder(source);
+    const lookup = source.lookup3d;
+    const { elementRadius, elementRadiusClosure, sourceMaxRadius, radius } = params;
+
+    ctx.pushCurrentElement();
+    for (const unit of structure.units) {
+        ctx.element.unit = unit;
+        const { x, y, z } = unit.conformation;
+        const elements = unit.elements;
+
+        for (let i = 0, _i = elements.length; i < _i; i++) {
+            const e = elements[i];
+            ctx.element.element = e;
+            const eRadius = elementRadius(ctx);
+            lookup.findIntoBuilderWithRadius(x(e), y(e), z(e), eRadius, sourceMaxRadius, radius, elementRadiusClosure, builder);
+        }
+
+        ctx.throwIfTimedOut();
+    }
+
+    ctx.popCurrentElement();
+    return !!params.wholeResidues ? getWholeResidues(ctx, source, builder.getStructure()) : builder.getStructure();
+}
+
+function createElementRadiusFn(ctx: QueryContext, eRadius: QueryFn<number>): StructureElement.Property<number> {
+    return e => {
+        ctx.element.unit = e.unit;
+        ctx.element.element = e.element;
+        return eRadius(ctx);
+    }
+}
+
+function findStructureRadius(ctx: QueryContext, eRadius: QueryFn<number>) {
+    let r = 0;
+    for (const unit of ctx.inputStructure.units) {
+        ctx.element.unit = unit;
+        const elements = unit.elements;
+
+        for (let i = 0, _i = elements.length; i < _i; i++) {
+            const e = elements[i];
+            ctx.element.element = e;
+            const eR = eRadius(ctx);
+            if (eR > r) r = eR;
+        }
+
+    }
+    ctx.throwIfTimedOut();
+    return r;
+}
+
 export function includeSurroundings(query: StructureQuery, params: IncludeSurroundingsParams): StructureQuery {
     return ctx => {
         const inner = query(ctx);
+
+        if (params.elementRadius) {
+            const prms: IncludeSurroundingsParamsWithRadius = {
+                ...params,
+                elementRadius: params.elementRadius,
+                elementRadiusClosure: createElementRadiusFn(ctx, params.elementRadius),
+                sourceMaxRadius: findStructureRadius(ctx, params.elementRadius)
+            };
+
+            if (StructureSelection.isSingleton(inner)) {
+                const surr = getIncludeSurroundingsWithRadius(ctx, ctx.inputStructure, inner.structure, prms);
+                const ret = StructureSelection.Singletons(ctx.inputStructure, surr);
+                return ret;
+            } else {
+                const builder = new UniqueStructuresBuilder(ctx.inputStructure);
+                for (const s of inner.structures) {
+                    builder.add(getIncludeSurroundingsWithRadius(ctx, ctx.inputStructure, s, prms));
+                }
+                return builder.getSelection();
+            }
+        }
+
         if (StructureSelection.isSingleton(inner)) {
             const surr = getIncludeSurroundings(ctx, ctx.inputStructure, inner.structure, params);
             const ret = StructureSelection.Singletons(ctx.inputStructure, surr);
@@ -218,4 +297,86 @@ export function expandProperty(query: StructureQuery, property: QueryFn): Struct
     };
 }
 
-// TODO: unionBy (skip this one?), cluster, includeConnected, includeSurroundings with "radii"
+export interface IncludeConnectedParams {
+    query: StructureQuery,
+    bondTest?: QueryFn<boolean>,
+    layerCount: number,
+    wholeResidues: boolean
+}
+
+export function includeConnected({ query, layerCount, wholeResidues, bondTest }: IncludeConnectedParams): StructureQuery {
+    return ctx => {
+        return 0 as any;
+    }
+}
+
+// function defaultBondTest(ctx: QueryContext) {
+//     return true;
+// }
+
+// interface IncludeConnectedCtx {
+//     queryCtx: QueryContext,
+//     input: Structure,
+//     bondTest: QueryFn<boolean>,
+//     wholeResidues: boolean
+// }
+
+// type FrontierSet = UniqueArray<StructureElement.UnitIndex, StructureElement.UnitIndex>
+// type Frontier = { unitIds: UniqueArray<number>, elements: Map<number /* unit id */, FrontierSet> }
+
+// namespace Frontier {
+//     export function has({ elements }: Frontier, unitId: number, element: StructureElement.UnitIndex) {
+//         if (!elements.has(unitId)) return false;
+//         const xs = elements.get(unitId)!;
+//         return xs.keys.has(element);
+//     }
+
+//     export function create(pivot: Structure, input: Structure) {
+//         const unitIds = UniqueArray.create<number>();
+//         const elements: Frontier['elements'] = new Map();
+//         for (const unit of pivot.units) {
+//             if (!Unit.isAtomic(unit)) continue;
+
+//             UniqueArray.add(unitIds, unit.id, unit.id);
+//             const xs: FrontierSet = UniqueArray.create();
+//             elements.set(unit.id, xs);
+
+//             const pivotElements = unit.elements;
+//             const inputElements = input.unitMap.get(unit.id).elements;
+//             for (let i = 0, _i = pivotElements.length; i < _i; i++) {
+//                 const idx = SortedArray.indexOf(inputElements, pivotElements[i]) as StructureElement.UnitIndex;
+//                 UniqueArray.add(xs, idx, idx);
+//             }
+//         }
+
+//         return { unitIds, elements };
+//     }
+
+//     export function addFrontier(target: Frontier, from: Frontier) {
+//         for (const unitId of from.unitIds.array) {
+//             let xs: FrontierSet;
+//             if (target.elements.has(unitId)) {
+//                 xs = target.elements.get(unitId)!;
+//             } else {
+//                 xs = UniqueArray.create();
+//                 target.elements.set(unitId, xs);
+//                 UniqueArray.add(target.unitIds, unitId, unitId);
+//             }
+
+//             for (const e of from.elements.get(unitId)!.array) {
+//                 UniqueArray.add(xs, e, e);
+//             }
+//         }
+//         return target;
+//     }
+
+//     export function includeWholeResidues(structure: Structure, frontier: Frontier) {
+//         // ...
+//     }
+// }
+
+// function expandFrontier(ctx: IncludeConnectedCtx, currentFrontier: Frontier, result: Frontier): Frontier {
+//     return 0 as any;
+// }
+
+// TODO: unionBy (skip this one?), cluster

src/mol-model/structure/structure/structure.ts

@@ -114,6 +114,11 @@ class Structure {
         return this._props.models;
     }
 
+    hasElement(e: StructureElement) {
+        if (!this.unitMap.has(e.unit.id)) return false;
+        return SortedArray.has(this.unitMap.get(e.unit.id).elements, e.element);
+    }
+
     constructor(units: ArrayLike<Unit>) {
         const map = IntMap.Mutable<Unit>();
         let elementCount = 0;

src/mol-model/structure/structure/unit/links/data.ts

@@ -73,11 +73,11 @@ class InterUnitBonds {
 
 namespace InterUnitBonds {
     export class UnitPairBonds {
-        hasBonds(indexA: number) {
+        hasBonds(indexA: StructureElement.UnitIndex) {
             return this.linkMap.has(indexA);
         }
 
-        getBonds(indexA: number): ReadonlyArray<InterUnitBonds.BondInfo> {
+        getBonds(indexA: StructureElement.UnitIndex): ReadonlyArray<InterUnitBonds.BondInfo> {
             if (!this.linkMap.has(indexA)) return emptyArray;
             return this.linkMap.get(indexA)!;
         }

src/mol-model/structure/structure/util/lookup3d.ts

@@ -10,6 +10,7 @@ import { Vec3 } from 'mol-math/linear-algebra';
 import { computeStructureBoundary } from './boundary';
 import { OrderedSet } from 'mol-data/int';
 import { StructureUniqueSubsetBuilder } from './unique-subset-builder';
+import StructureElement from '../element';
 
 export class StructureLookup3D {
     private unitLookup: Lookup3D;
@@ -66,6 +67,39 @@ export class StructureLookup3D {
         }
     }
 
+    findIntoBuilderWithRadius(x: number, y: number, z: number, pivotR: number, maxRadius: number, radius: number, eRadius: StructureElement.Property<number>, builder: StructureUniqueSubsetBuilder) {
+        const { units } = this.structure;
+        const closeUnits = this.unitLookup.find(x, y, z, radius);
+        if (closeUnits.count === 0) return;
+
+        const se = StructureElement.create();
+        const queryRadius = pivotR + maxRadius + radius;
+
+        for (let t = 0, _t = closeUnits.count; t < _t; t++) {
+            const unit = units[closeUnits.indices[t]];
+            Vec3.set(this.pivot, x, y, z);
+            if (!unit.conformation.operator.isIdentity) {
+                Vec3.transformMat4(this.pivot, this.pivot, unit.conformation.operator.inverse);
+            }
+            const unitLookup = unit.lookup3d;
+            const groupResult = unitLookup.find(this.pivot[0], this.pivot[1], this.pivot[2], queryRadius);
+            if (groupResult.count === 0) continue;
+
+            const elements = unit.elements;
+            se.unit = unit;
+            builder.beginUnit(unit.id);
+            for (let j = 0, _j = groupResult.count; j < _j; j++) {
+                se.element = elements[groupResult.indices[j]];
+                const rr = eRadius(se);
+                if (Math.sqrt(groupResult.squaredDistances[j]) - pivotR - rr > radius) continue;
+                builder.addElement(elements[groupResult.indices[j]]);
+            }
+            builder.commitUnit();
+        }
+    }
+
+
+
     check(x: number, y: number, z: number, radius: number): boolean {
         const { units } = this.structure;
         const closeUnits = this.unitLookup.find(x, y, z, radius);

src/servers/model/preprocess.ts

@@ -0,0 +1,31 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import * as argparse from 'argparse'
+import { preprocessFile } from './preprocess/preprocess';
+
+const cmdParser = new argparse.ArgumentParser({
+    addHelp: true,
+    description: 'Preprocess CIF files to include custom properties and convert them to BinaryCIF format.'
+});
+cmdParser.addArgument(['--input', '-i'], { help: 'Input filename', required: true });
+cmdParser.addArgument(['--outCIF', '-oc'], { help: 'Output CIF filename', required: false });
+cmdParser.addArgument(['--outBCIF', '-ob'], { help: 'Output BinaryCIF filename', required: false });
+
+// TODO: "bulk" mode
+
+interface CmdArgs {
+    input: string,
+    outCIF?: string,
+    outBCIF?: string
+}
+
+const cmdArgs = cmdParser.parseArgs() as CmdArgs;
+
+if (cmdArgs.input) preprocessFile(cmdArgs.input, cmdArgs.outCIF, cmdArgs.outBCIF);
+
+// example:
+// node build\node_modules\servers\model\preprocess -i e:\test\Quick\1cbs_updated.cif -oc e:\test\mol-star\model\1cbs.cif -ob e:\test\mol-star\model\1cbs.bcif

src/servers/model/preprocess/converter.ts

@@ -0,0 +1,51 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import { CifCategory, CifField, CifFrame, getCifFieldType } from 'mol-io/reader/cif';
+import { CifWriter } from 'mol-io/writer/cif';
+import { Task } from 'mol-task';
+import { showProgress } from './util';
+
+function getCategoryInstanceProvider(cat: CifCategory, fields: CifWriter.Field[]): CifWriter.Category {
+    return {
+        name: cat.name,
+        instance: () => ({ data: cat, fields, rowCount: cat.rowCount })
+    };
+}
+
+function classify(name: string, field: CifField): CifWriter.Field {
+    const type = getCifFieldType(field);
+    if (type['@type'] === 'str') {
+        return { name, type: CifWriter.Field.Type.Str, value: field.str, valueKind: field.valueKind };
+    } else if (type['@type'] === 'float') {
+        return CifWriter.Field.float(name, field.float, { valueKind: field.valueKind, typedArray: Float64Array });
+    } else {
+        return CifWriter.Field.int(name, field.int, { valueKind: field.valueKind, typedArray: Int32Array });
+    }
+}
+
+export function classifyCif(frame: CifFrame) {
+    return Task.create('Classify CIF Data', async ctx => {
+        let maxProgress = 0;
+        for (const c of frame.categoryNames) maxProgress += frame.categories[c].fieldNames.length;
+
+        const ret: CifWriter.Category[] = [];
+
+        let current = 0;
+        for (const c of frame.categoryNames) {
+            const cat = frame.categories[c];
+            const fields: CifWriter.Field[] = [];
+            for (const f of cat.fieldNames) {
+                const cifField = classify(f, cat.getField(f)!);
+                fields.push(cifField);
+                current++;
+                if (ctx.shouldUpdate) await ctx.update({ message: 'Classifying...', current, max: maxProgress });
+            }
+            ret.push(getCategoryInstanceProvider(cat, fields));
+        }
+        return ret;
+    }).run(showProgress, 250);
+}

src/servers/model/preprocess/preprocess.ts

@@ -0,0 +1,63 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import { readStructure } from '../server/structure-wrapper';
+import { classifyCif } from './converter';
+import { ConsoleLogger } from 'mol-util/console-logger';
+import { Structure } from 'mol-model/structure';
+import { CifWriter } from 'mol-io/writer/cif';
+import Writer from 'mol-io/writer/writer';
+import { wrapFileToWriter } from '../server/api-local';
+import { Task } from 'mol-task';
+import { showProgress, clearLine } from './util';
+import { encode_mmCIF_categories, CifExportContext } from 'mol-model/structure/export/mmcif';
+
+// TODO: error handling, bulk mode
+
+export async function preprocessFile(filename: string, outputCif?: string, outputBcif?: string) {
+    ConsoleLogger.log('ModelServer', `Reading ${filename}...`);
+    const input = await readStructure('entry', '_local_', filename);
+    ConsoleLogger.log('ModelServer', `Classifying CIF categories...`);
+    const categories = await classifyCif(input.cifFrame);
+    clearLine();
+
+    const exportCtx = CifExportContext.create(input.structure, input.structure.models[0]);
+
+    if (outputCif) {
+        ConsoleLogger.log('ModelServer', `Encoding CIF...`);
+        const writer = wrapFileToWriter(outputCif);
+        const encoder = CifWriter.createEncoder({ binary: false });
+        await encode(input.structure, input.cifFrame.header, categories, encoder, exportCtx, writer);
+        clearLine();
+        writer.end();
+    }
+
+    if (outputBcif) {
+        ConsoleLogger.log('ModelServer', `Encoding BinaryCIF...`);
+        const writer = wrapFileToWriter(outputBcif);
+        const encoder = CifWriter.createEncoder({ binary: true, binaryAutoClassifyEncoding: true });
+        await encode(input.structure, input.cifFrame.header, categories, encoder, exportCtx, writer);
+        clearLine();
+        writer.end();
+    }
+    ConsoleLogger.log('ModelServer', `Done.`);
+}
+
+function encode(structure: Structure, header: string, categories: CifWriter.Category[], encoder: CifWriter.Encoder, exportCtx: CifExportContext, writer: Writer) {
+    return Task.create('Encode', async ctx => {
+        const skipCategoryNames = new Set<string>(categories.map(c => c.name));
+        encoder.startDataBlock(header);
+        let current = 0;
+        for (const cat of categories){
+            encoder.writeCategory(cat);
+            current++;
+            if (ctx.shouldUpdate) await ctx.update({ message: 'Encoding...', current, max: categories.length });
+        }
+        encode_mmCIF_categories(encoder, structure, { skipCategoryNames, exportCtx });
+        encoder.encode();
+        encoder.writeTo(writer);
+    }).run(showProgress, 250);
+}

src/servers/model/preprocess/util.ts

@@ -0,0 +1,17 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import { Progress } from 'mol-task';
+
+export function showProgress(p: Progress) {
+    process.stdout.write(`\r${new Array(80).join(' ')}`);
+    process.stdout.write(`\r${Progress.format(p)}`);
+}
+
+export function clearLine() {
+    process.stdout.write(`\r${new Array(80).join(' ')}`);
+    process.stdout.write(`\r`);
+}

src/servers/model/server/api-local.ts

@@ -39,7 +39,7 @@ export async function runLocal(input: LocalInput) {
     while (job) {
         try {
             const encoder = await resolveJob(job);
-            const writer = wrapFile(job.outputFilename!);
+            const writer = wrapFileToWriter(job.outputFilename!);
             encoder.writeTo(writer);
             writer.end();
             ConsoleLogger.logId(job.id, 'Query', 'Written.');
@@ -61,7 +61,7 @@ export async function runLocal(input: LocalInput) {
     StructureCache.expireAll();
 }
 
-function wrapFile(fn: string) {
+export function wrapFileToWriter(fn: string) {
     const w = {
         open(this: any) {
             if (this.opened) return;
@@ -71,7 +71,7 @@ function wrapFile(fn: string) {
         },
         writeBinary(this: any, data: Uint8Array) {
             this.open();
-            fs.writeSync(this.file, new Buffer(data));
+            fs.writeSync(this.file, new Buffer(data.buffer));
             return true;
         },
         writeString(this: any, data: string) {

src/servers/model/server/structure-wrapper.ts

@@ -8,7 +8,7 @@ import { Structure, Model, Format } from 'mol-model/structure';
 import { PerformanceMonitor } from 'mol-util/performance-monitor';
 import { Cache } from './cache';
 import Config from '../config';
-import CIF from 'mol-io/reader/cif'
+import CIF, { CifFrame } from 'mol-io/reader/cif'
 import * as util from 'util'
 import * as fs from 'fs'
 import * as zlib from 'zlib'
@@ -34,21 +34,22 @@ export interface StructureInfo {
     entryId: string
 }
 
-export class StructureWrapper {
-    info: StructureInfo;
+export interface StructureWrapper {
+    info: StructureInfo,
 
-    key: string;
-    approximateSize: number;
-    structure: Structure;
+    key: string,
+    approximateSize: number,
+    structure: Structure,
+    cifFrame: CifFrame
 }
 
-export async function getStructure(job: Job): Promise<StructureWrapper> {
-    if (Config.cacheParams.useCache) {
+export async function getStructure(job: Job, allowCache = true): Promise<StructureWrapper> {
+    if (allowCache && Config.cacheParams.useCache) {
         const ret = StructureCache.get(job.key);
         if (ret) return ret;
     }
     const ret = await readStructure(job.key, job.sourceId, job.entryId);
-    if (Config.cacheParams.useCache) {
+    if (allowCache && Config.cacheParams.useCache) {
         StructureCache.add(ret);
     }
     return ret;
@@ -84,7 +85,7 @@ async function parseCif(data: string|Uint8Array) {
     return parsed.result;
 }
 
-async function readStructure(key: string, sourceId: string, entryId: string) {
+export async function readStructure(key: string, sourceId: string | '_local_', entryId: string) {
     const filename = sourceId === '_local_' ? entryId : Config.mapFile(sourceId, entryId);
     if (!filename) throw new Error(`Cound not map '${key}' to a valid filename.`);
     if (!fs.existsSync(filename)) throw new Error(`Could not find source file for '${key}'.`);
@@ -127,7 +128,8 @@ async function readStructure(key: string, sourceId: string, entryId: string) {
         },
         key,
         approximateSize: typeof data === 'string' ? 2 * data.length : data.length,
-        structure
+        structure,
+        cifFrame: frame
     };
 
     return ret;