refactoring of column and CIF data model

src/mol-base/collections/_spec/table.spec.ts

@@ -4,7 +4,7 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
-import Column, { isTypedArray } from '../column'
+import Column, { ColumnHelpers } from '../column'
 import Table from '../table'
 
 describe('column', () => {
@@ -32,7 +32,7 @@ describe('column', () => {
     it('typed', () => {
         expect(typedWindow.value(0)).toBe(2);
         expect(typedWindow.rowCount).toBe(2);
-        expect(isTypedArray(typedWindow.toArray())).toBe(true);
+        expect(ColumnHelpers.isTypedArray(typedWindow.toArray())).toBe(true);
     });
 
     it('numStr', () => {

src/mol-base/collections/column.ts

@@ -89,6 +89,11 @@ namespace Column {
         return columnPermutation(column, indices, checkIndentity);
     }
 
+    /** A map of the 1st occurence of each value. */
+    export function createFirstIndexMap<T>(column: Column<T>) {
+        return createFirstIndexMapOfColumn(column);
+    }
+
     /** Makes the column backned by an array. Useful for columns that accessed often. */
     export function asArrayColumn<T>(c: Column<T>, array?: ToArrayParams['array']): Column<T> {
         if (c['@array']) return c;
@@ -99,6 +104,15 @@ namespace Column {
 
 export default Column;
 
+function createFirstIndexMapOfColumn<T>(c: Column<T>): Map<T, number> | undefined {
+    const map = new Map<T, number>();
+    for (let i = 0, _i = c.rowCount; i < _i; i++) {
+        const v = c.value(i);
+        if (!map.has(v)) return map.set(c.value(i), i);
+    }
+    return map;
+}
+
 function constColumn<T extends Column.Type>(v: T['T'], rowCount: number, type: T, valueKind: Column.ValueKind): Column<T['T']> {
     const value: Column<T['T']>['value'] = row => v;
     return {
@@ -109,7 +123,7 @@ function constColumn<T extends Column.Type>(v: T['T'], rowCount: number, type: T
         value,
         valueKind: row => valueKind,
         toArray: params => {
-            const { array } = createArray(rowCount, params);
+            const { array } = ColumnHelpers.createArray(rowCount, params);
             for (let i = 0, _i = array.length; i < _i; i++) array[i] = v;
             return array;
         },
@@ -126,7 +140,7 @@ function lambdaColumn<T extends Column.Type>({ value, valueKind, rowCount, type
         value,
         valueKind: valueKind ? valueKind : row => Column.ValueKind.Present,
         toArray: params => {
-            const { array, start } = createArray(rowCount, params);
+            const { array, start } = ColumnHelpers.createArray(rowCount, params);
             for (let i = 0, _i = array.length; i < _i; i++) array[i] = value(i + start);
             return array;
         },
@@ -140,7 +154,7 @@ function arrayColumn<T extends Column.Type>({ array, type, valueKind }: Column.A
         ? row => { const v = array[row]; return typeof v === 'string' ? v : '' + v; }
         : row => array[row];
 
-    const isTyped = isTypedArray(array);
+    const isTyped = ColumnHelpers.isTypedArray(array);
     return {
         '@type': type,
         '@array': array,
@@ -150,7 +164,7 @@ function arrayColumn<T extends Column.Type>({ array, type, valueKind }: Column.A
         valueKind: valueKind ? valueKind : row => Column.ValueKind.Present,
         toArray: type.kind === 'str'
             ? params => {
-                const { start, end } = getArrayBounds(rowCount, params);
+                const { start, end } = ColumnHelpers.getArrayBounds(rowCount, params);
                 const ret = new (params && typeof params.array !== 'undefined' ? params.array : (array as any).constructor)(end - start) as any;
                 for (let i = 0, _i = end - start; i < _i; i++) {
                     const v = array[start + i];
@@ -159,9 +173,9 @@ function arrayColumn<T extends Column.Type>({ array, type, valueKind }: Column.A
                 return ret;
             }
             : isTyped
-            ? params => typedArrayWindow(array, params) as any as ReadonlyArray<T>
+            ? params => ColumnHelpers.typedArrayWindow(array, params) as any as ReadonlyArray<T>
             : params => {
-                const { start, end } = getArrayBounds(rowCount, params);
+                const { start, end } = ColumnHelpers.getArrayBounds(rowCount, params);
                 if (start === 0 && end === array.length) return array as ReadonlyArray<T['T']>;
                 const ret = new (params && typeof params.array !== 'undefined' ? params.array : (array as any).constructor)(end - start) as any;
                 for (let i = 0, _i = end - start; i < _i; i++) ret[i] = array[start + i];
@@ -173,12 +187,12 @@ function arrayColumn<T extends Column.Type>({ array, type, valueKind }: Column.A
 
 function windowColumn<T>(column: Column<T>, start: number, end: number) {
     if (!column.isDefined) return Column.Undefined(end - start, column['@type']);
-    if (column['@array'] && isTypedArray(column['@array'])) return windowTyped(column, start, end);
+    if (column['@array'] && ColumnHelpers.isTypedArray(column['@array'])) return windowTyped(column, start, end);
     return windowFull(column, start, end);
 }
 
 function windowTyped<T>(c: Column<T>, start: number, end: number): Column<T> {
-    const array = typedArrayWindow(c['@array'], { start, end });
+    const array = ColumnHelpers.typedArrayWindow(c['@array'], { start, end });
     return arrayColumn({ array, type: c['@type'], valueKind: c.valueKind }) as any;
 }
 
@@ -194,7 +208,7 @@ function windowFull<T>(c: Column<T>, start: number, end: number): Column<T> {
         value,
         valueKind: start === 0 ? vk : row => vk(row + start),
         toArray: params => {
-            const { array } = createArray(rowCount, params);
+            const { array } = ColumnHelpers.createArray(rowCount, params);
             for (let i = 0, _i = array.length; i < _i; i++) array[i] = v(i + start);
             return array;
         },
@@ -202,21 +216,27 @@ function windowFull<T>(c: Column<T>, start: number, end: number): Column<T> {
     };
 }
 
+function isIdentity(map: ArrayLike<number>) {
+    for (let i = 0, _i = map.length; i < _i; i++) {
+        if (map[i] !== i) return false;
+    }
+    return true;
+}
+
 function columnPermutation<T>(c: Column<T>, map: ArrayLike<number>, checkIdentity: boolean): Column<T> {
     if (!c.isDefined) return c;
-    if (checkIdentity) {
-        let isIdentity = true;
-        for (let i = 0, _i = map.length; i < _i; i++) {
-            if (map[i] !== i) {
-                isIdentity = false;
-                break;
-            }
-        }
-        if (isIdentity) return c;
-    }
+    if (checkIdentity && isIdentity(map)) return c;
+    if (!c['@array']) return permutationArray(c, map);
     return permutationFull(c, map);
 }
 
+function permutationArray<T>(c: Column<T>, map: ArrayLike<number>): Column<T> {
+    const array = c['@array']!;
+    const ret = new (array as any).constructor(c.rowCount);
+    for (let i = 0, _i = c.rowCount; i < _i; i++) ret[i] = array[map[i]];
+    return arrayColumn({ array: ret, type: c['@type'], valueKind: c.valueKind });
+}
+
 function permutationFull<T>(c: Column<T>, map: ArrayLike<number>): Column<T> {
     const v = c.value, vk = c.valueKind, ave = c.areValuesEqual;
     const value: Column<T>['value'] = row => v(map[row]);
@@ -229,7 +249,7 @@ function permutationFull<T>(c: Column<T>, map: ArrayLike<number>): Column<T> {
         value,
         valueKind: row => vk(map[row]),
         toArray: params => {
-            const { array } = createArray(rowCount, params);
+            const { array } = ColumnHelpers.createArray(rowCount, params);
             for (let i = 0, _i = array.length; i < _i; i++) array[i] = v(map[i]);
             return array;
         },
@@ -237,39 +257,37 @@ function permutationFull<T>(c: Column<T>, map: ArrayLike<number>): Column<T> {
     };
 }
 
-/** A helped function for Column.toArray */
-export function getArrayBounds(rowCount: number, params?: Column.ToArrayParams) {
-    const start = params && typeof params.start !== 'undefined' ? Math.max(Math.min(params.start, rowCount - 1), 0) : 0;
-    const end = params && typeof params.end !== 'undefined' ? Math.min(params.end, rowCount) : rowCount;
-    return { start, end };
-}
+export namespace ColumnHelpers {
+    export function getArrayBounds(rowCount: number, params?: Column.ToArrayParams) {
+        const start = params && typeof params.start !== 'undefined' ? Math.max(Math.min(params.start, rowCount - 1), 0) : 0;
+        const end = params && typeof params.end !== 'undefined' ? Math.min(params.end, rowCount) : rowCount;
+        return { start, end };
+    }
 
-/** A helped function for Column.toArray */
-export function createArray(rowCount: number, params?: Column.ToArrayParams) {
-    const c = params && typeof params.array !== 'undefined' ? params.array : Array;
-    const { start, end } = getArrayBounds(rowCount, params);
-    return { array: new c(end - start) as any[], start, end };
-}
+    export function createArray(rowCount: number, params?: Column.ToArrayParams) {
+        const c = params && typeof params.array !== 'undefined' ? params.array : Array;
+        const { start, end } = getArrayBounds(rowCount, params);
+        return { array: new c(end - start) as any[], start, end };
+    }
 
-/** A helped function for Column.toArray */
-export function fillArrayValues(value: (row: number) => any, target: any[], start: number) {
-    for (let i = 0, _e = target.length; i < _e; i++) target[i] = value(start + i);
-    return target;
-}
+    export function fillArrayValues(value: (row: number) => any, target: any[], start: number) {
+        for (let i = 0, _e = target.length; i < _e; i++) target[i] = value(start + i);
+        return target;
+    }
 
-/** A helped function for Column.toArray */
-export function createAndFillArray(rowCount: number, value: (row: number) => any, params?: Column.ToArrayParams) {
-    const { array, start } = createArray(rowCount, params);
-    return fillArrayValues(value, array, start);
-}
+    export function createAndFillArray(rowCount: number, value: (row: number) => any, params?: Column.ToArrayParams) {
+        const { array, start } = createArray(rowCount, params);
+        return fillArrayValues(value, array, start);
+    }
 
-export function isTypedArray(data: any): boolean {
-    return !!data.buffer && typeof data.byteLength === 'number' && typeof data.BYTES_PER_ELEMENT === 'number';
-}
+    export function isTypedArray(data: any): boolean {
+        return !!data.buffer && typeof data.byteLength === 'number' && typeof data.BYTES_PER_ELEMENT === 'number';
+    }
 
-export function typedArrayWindow(data: any, params?: Column.ToArrayParams): ReadonlyArray<number> {
-    const { constructor, buffer, length, byteOffset, BYTES_PER_ELEMENT } = data;
-    const { start, end } = getArrayBounds(length, params);
-    if (start === 0 && end === length) return data;
-    return new constructor(buffer, byteOffset + BYTES_PER_ELEMENT * start, Math.min(length, end - start));
+    export function typedArrayWindow(data: any, params?: Column.ToArrayParams): ReadonlyArray<number> {
+        const { constructor, buffer, length, byteOffset, BYTES_PER_ELEMENT } = data;
+        const { start, end } = getArrayBounds(length, params);
+        if (start === 0 && end === length) return data;
+        return new constructor(buffer, byteOffset + BYTES_PER_ELEMENT * start, Math.min(length, end - start));
+    }
 }

src/mol-io/reader/_spec/column.spec.ts

@@ -7,7 +7,7 @@
 
 import FixedColumn from '../common/text/column/fixed'
 import TokenColumn from '../common/text/column/token'
-import Column, { typedArrayWindow } from '../../../mol-base/collections/column'
+import Column, { ColumnHelpers } from '../../../mol-base/collections/column'
 
 const lines = [
     '1.123 abc',
@@ -64,8 +64,8 @@ describe('token text column', () => {
 describe('binary column', () => {
     it('window works', () => {
         const xs = new Float64Array([1, 2, 3, 4]);
-        const w1 = typedArrayWindow(xs, { start: 1 });
-        const w2 = typedArrayWindow(xs, { start: 2, end: 4 });
+        const w1 = ColumnHelpers.typedArrayWindow(xs, { start: 1 });
+        const w2 = ColumnHelpers.typedArrayWindow(xs, { start: 2, end: 4 });
 
         expect(w1.length).toBe(3);
         for (let i = 0; i < w1.length; i++) expect(w1[i]).toBe(xs[i + 1]);

src/mol-io/reader/cif/binary/field.ts

@@ -4,7 +4,7 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
-import Column, { isTypedArray, createAndFillArray, typedArrayWindow } from '../../../../mol-base/collections/column'
+import Column, { ColumnHelpers } from '../../../../mol-base/collections/column'
 import * as Data from '../data-model'
 import { EncodedColumn } from './encoding'
 import decode from './decoder'
@@ -13,7 +13,7 @@ import { parseInt as fastParseInt, parseFloat as fastParseFloat } from '../../co
 export default function Field(column: EncodedColumn): Data.Field {
     const mask = column.mask ? decode(column.mask) as number[] : void 0;
     const data = decode(column.data);
-    const isNumeric = isTypedArray(data);
+    const isNumeric = ColumnHelpers.isTypedArray(data);
 
     const str: Data.Field['str'] = isNumeric
         ? mask
@@ -46,13 +46,12 @@ export default function Field(column: EncodedColumn): Data.Field {
         float,
         valueKind,
         areValuesEqual: (rowA, rowB) => data[rowA] === data[rowB],
-        stringEquals: (row, v) => str(row) === v,
-        toStringArray: params => createAndFillArray(rowCount, str, params),
+        toStringArray: params => ColumnHelpers.createAndFillArray(rowCount, str, params),
         toIntArray: isNumeric
-            ? params => typedArrayWindow(data, params)
-            : params => createAndFillArray(rowCount, int, params),
+            ? params => ColumnHelpers.typedArrayWindow(data, params)
+            : params => ColumnHelpers.createAndFillArray(rowCount, int, params),
         toFloatArray: isNumeric
-            ? params => typedArrayWindow(data, params)
-            : params => createAndFillArray(rowCount, float, params)
+            ? params => ColumnHelpers.typedArrayWindow(data, params)
+            : params => ColumnHelpers.createAndFillArray(rowCount, float, params)
     };
 }

src/mol-io/reader/cif/binary/parser.ts

@@ -25,7 +25,7 @@ function Category(data: Encoding.EncodedCategory): Data.Category {
         rowCount: data.rowCount,
         getField(name) {
             const col = map[name];
-            return col ? Field(col) : Data.DefaultUndefinedField(data.rowCount);
+            return col ? Field(col) : void 0;
         }
     }
 }

src/mol-io/reader/cif/data-model.ts

@@ -4,7 +4,7 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
-import Column, { createArray } from '../../../mol-base/collections/column'
+import Column from '../../../mol-base/collections/column'
 
 export interface File {
     readonly name?: string,
@@ -68,32 +68,12 @@ export interface Field {
     valueKind(row: number): Column.ValueKind,
 
     areValuesEqual(rowA: number, rowB: number): boolean,
-    stringEquals(row: number, value: string): boolean,
 
     toStringArray(params?: Column.ToArrayParams): ReadonlyArray<string>,
     toIntArray(params?: Column.ToArrayParams): ReadonlyArray<number>,
     toFloatArray(params?: Column.ToArrayParams): ReadonlyArray<number>
 }
 
-export function DefaultUndefinedField(rowCount: number): Field {
-    return {
-        '@array': void 0,
-        isDefined: false,
-        rowCount,
-        str: row => '',
-        int: row => 0,
-        float: row => 0,
-
-        valueKind: row => Column.ValueKind.NotPresent,
-        areValuesEqual: (rowA, rowB) => true,
-        stringEquals: (row, value) => value === null,
-
-        toStringArray: (p) => createArray(rowCount, p).array,
-        toIntArray: (p) => createArray(rowCount, p).array,
-        toFloatArray: (p) => createArray(rowCount, p).array
-    };
-}
-
 export function getMatrix(category: Category, field: string, rows: number, cols: number, row: number) {
     const ret: number[][] = [];
     for (let i = 0; i < rows; i++) {

src/mol-io/reader/cif/schema.ts

@@ -5,7 +5,7 @@
  */
 
 import * as Data from './data-model'
-import Column, { createAndFillArray } from '../../../mol-base/collections/column'
+import Column, { ColumnHelpers } from '../../../mol-base/collections/column'
 import Table from '../../../mol-base/collections/table'
 
 export function toTypedFrame<Schema extends FrameSchema, Frame extends TypedFrame<Schema> = TypedFrame<Schema>>(schema: Schema, frame: Data.Frame): Frame {
@@ -34,12 +34,12 @@ function getColumnCtor(t: Column.Type): ColumnCtor {
         case 'vector': return (f, c, k) => {
             const dim = t.dim;
             const value = (row: number) => Data.getVector(c, k, dim, row);
-            return createColumn(t, f, value, params => createAndFillArray(f.rowCount, value, params));
+            return createColumn(t, f, value, params => ColumnHelpers.createAndFillArray(f.rowCount, value, params));
         }
         case 'matrix': return (f, c, k) => {
             const rows = t.rows, cols = t.cols;
             const value = (row: number) => Data.getMatrix(c, k, rows, cols, row);
-            return createColumn(t, f, value, params => createAndFillArray(f.rowCount, value, params));
+            return createColumn(t, f, value, params => ColumnHelpers.createAndFillArray(f.rowCount, value, params));
         }
     }
 }

src/mol-io/reader/cif/text/field.ts

@@ -4,7 +4,7 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
-import Column, { createAndFillArray } from '../../../../mol-base/collections/column'
+import Column, { ColumnHelpers } from '../../../../mol-base/collections/column'
 import * as TokenColumn from '../../common/text/column/token'
 import { Tokens } from '../../common/text/tokenizer'
 import * as Data from '../data-model'
@@ -45,19 +45,8 @@ export default function CifTextField(tokens: Tokens, rowCount: number): Data.Fie
         float,
         valueKind,
         areValuesEqual: TokenColumn.areValuesEqualProvider(tokens),
-        stringEquals: (row, v) => {
-            const s = indices[2 * row];
-            const value = v || '';
-            if (!value && valueKind(row) !== Column.ValueKind.Present) return true;
-            const len = value.length;
-            if (len !== indices[2 * row + 1] - s) return false;
-            for (let i = 0; i < len; i++) {
-                if (data.charCodeAt(i + s) !== value.charCodeAt(i)) return false;
-            }
-            return true;
-        },
-        toStringArray: params => createAndFillArray(rowCount, str, params),
-        toIntArray: params => createAndFillArray(rowCount, int, params),
-        toFloatArray: params => createAndFillArray(rowCount, float, params)
+        toStringArray: params => ColumnHelpers.createAndFillArray(rowCount, str, params),
+        toIntArray: params => ColumnHelpers.createAndFillArray(rowCount, int, params),
+        toFloatArray: params => ColumnHelpers.createAndFillArray(rowCount, float, params)
     }
 }

src/mol-io/reader/common/text/column/fixed.ts

@@ -4,7 +4,7 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
-import Column, { createAndFillArray } from '../../../../../mol-base/collections/column'
+import Column, { ColumnHelpers } from '../../../../../mol-base/collections/column'
 import { trimStr, Tokens } from '../tokenizer'
 import { parseIntSkipLeadingWhitespace, parseFloatSkipLeadingWhitespace } from '../number-parser'
 
@@ -40,7 +40,7 @@ export function FixedColumn<T extends Column.Type>(lines: Tokens, offset: number
         rowCount,
         value,
         valueKind: row => Column.ValueKind.Present,
-        toArray: params => createAndFillArray(rowCount, value, params),
+        toArray: params => ColumnHelpers.createAndFillArray(rowCount, value, params),
         areValuesEqual: (rowA, rowB) => value(rowA) === value(rowB)
     };
 }

src/mol-io/reader/common/text/column/token.ts

@@ -4,7 +4,7 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
-import Column, { createAndFillArray } from '../../../../../mol-base/collections/column'
+import Column, { ColumnHelpers } from '../../../../../mol-base/collections/column'
 import { Tokens } from '../tokenizer'
 import { parseInt as fastParseInt, parseFloat as fastParseFloat } from '../number-parser'
 
@@ -32,7 +32,7 @@ export function TokenColumn<T extends Column.Type>(tokens: Tokens, type: T): Col
         rowCount,
         value,
         valueKind: row => Column.ValueKind.Present,
-        toArray: params => createAndFillArray(rowCount, value, params),
+        toArray: params => ColumnHelpers.createAndFillArray(rowCount, value, params),
         areValuesEqual: areValuesEqualProvider(tokens)
     };
 }