improving & testing list column type

src/mol-data/db/_spec/table.spec.ts

@@ -12,8 +12,8 @@ import Table from '../table'
 describe('column', () => {
     const cc = Column.ofConst(10, 2, Column.Schema.int);
     const arr = Column.ofArray({ array: [1, 2, 3, 4], schema: Column.Schema.int });
-    const arrNumberList = Column.ofArray({ array: [[1, 2], [3, 4], [5, 6]], schema: Column.Schema.List<number>() });
-    const arrStringList = Column.ofArray({ array: [['a', 'b'], ['c', 'd'], ['e', 'f']], schema: Column.Schema.List<string>() });
+    const arrNumberList = Column.ofArray({ array: [[1, 2], [3, 4], [5, 6]], schema: Column.Schema.List(' ', x => parseInt(x, 10)) });
+    const arrStringList = Column.ofArray({ array: [['a', 'b'], ['c', 'd'], ['e', 'f']], schema: Column.Schema.List(',', x => x) });
     const arrWindow = Column.window(arr, 1, 3);
 
     const typed = Column.ofArray({ array: new Int32Array([1, 2, 3, 4]), schema: Column.Schema.int });

src/mol-data/db/column.ts

@@ -36,7 +36,7 @@ namespace Column {
 
         export type Tensor = { '@type': 'tensor', T: Tensors, space: Tensors.Space } & Base<'tensor'>
         export type Aliased<T> = { '@type': 'aliased', T: T } & Base<'str' | 'int'>
-        export type List<T extends number|string> = { '@type': 'list', T: T[] } & Base<'list'>
+        export type List<T extends number|string> = { '@type': 'list', T: T[], separator: string, itemParse: (x: string) => T } & Base<'list'>
 
         export const str: Str = { '@type': 'str', T: '', valueType: 'str' };
         export const int: Int = { '@type': 'int', T: 0, valueType: 'int' };
@@ -54,8 +54,8 @@ namespace Column {
             if (typeof defaultValue !== 'undefined') return { ...t, T: defaultValue } as any as Aliased<T>;
             return t as any as Aliased<T>;
         }
-        export function List<T extends number|string>(defaultValue: T[] = []): List<T> {
-            return { '@type': 'list', T: defaultValue, valueType: 'list' }
+        export function List<T extends number|string>(separator: string, itemParse: (x: string) => T, defaultValue: T[] = []): List<T> {
+            return { '@type': 'list', T: defaultValue, separator, itemParse, valueType: 'list' }
         }
     }
 

src/mol-io/reader/_spec/cif.spec.ts

@@ -1,7 +1,8 @@
 /**
- * Copyright (c) 2017 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2017-2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * @author David Sehnal <david.sehnal@gmail.com>
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
 import * as Data from '../cif/data-model'
@@ -9,33 +10,45 @@ import TextField from '../cif/text/field'
 import * as Schema from '../cif/schema'
 import { Column } from 'mol-data/db'
 
-const columnData = `123abc`;
+const columnData = `123abc d,e,f '4 5 6'`;
+// 123abc d,e,f '4 5 6'
 
 const intField = TextField({ data: columnData, indices: [0, 1, 1, 2, 2, 3], count: 3 }, 3);
 const strField = TextField({ data: columnData, indices: [3, 4, 4, 5, 5, 6], count: 3 }, 3);
+const strListField = TextField({ data: columnData, indices: [7, 12], count: 1 }, 1);
+const intListField = TextField({ data: columnData, indices: [14, 19], count: 1 }, 1);
 
-const testBlock = Data.Block(['atoms'], {
-    atoms: Data.Category('atoms', 3, ['x', 'name'], {
-        x: intField,
-        name: strField
+const testBlock = Data.Block(['test'], {
+    test: Data.Category('test', 3, ['int', 'str', 'strList', 'intList'], {
+        int: intField,
+        str: strField,
+        strList: strListField,
+        intList: intListField
     })
 }, 'test');
 
 namespace TestSchema {
-    export const atoms = { x: Column.Schema.int, name: Column.Schema.str }
-    export const schema = { atoms }
+    export const test = {
+        int: Column.Schema.int,
+        str: Column.Schema.str,
+        strList: Column.Schema.List(',', x => x),
+        intList: Column.Schema.List(' ', x => parseInt(x, 10))
+    }
+    export const schema = { test }
 }
 
 describe('schema', () => {
     const db = Schema.toDatabase(TestSchema.schema, testBlock);
     it('property access', () => {
-        const { x, name } = db.atoms;
-        expect(x.value(0)).toBe(1);
-        expect(name.value(1)).toBe('b');
+        const { int, str, strList, intList } = db.test;
+        expect(int.value(0)).toBe(1);
+        expect(str.value(1)).toBe('b');
+        expect(strList.value(0)).toEqual(['d', 'e', 'f']);
+        expect(intList.value(0)).toEqual([4, 5, 6]);
     });
 
     it('toArray', () => {
-        const ret = db.atoms.x.toArray({ array: Int32Array });
+        const ret = db.test.int.toArray({ array: Int32Array });
         expect(ret.length).toBe(3);
         expect(ret[0]).toBe(1);
         expect(ret[1]).toBe(2);

src/mol-io/reader/cif/binary/field.ts

@@ -31,10 +31,6 @@ export default function Field(column: EncodedColumn): Data.Field {
         ? row => data[row]
         : row => { const v = data[row]; return fastParseFloat(v, 0, v.length); };
 
-    const list: Data.Field['list'] = mask
-        ? row => mask[row] === Column.ValueKind.Present ? data[row] : []
-        : row => data[row];
-
     const valueKind: Data.Field['valueKind'] = mask
         ? row => mask[row]
         : row => Column.ValueKind.Present;
@@ -48,7 +44,6 @@ export default function Field(column: EncodedColumn): Data.Field {
         str,
         int,
         float,
-        list,
         valueKind,
         areValuesEqual: (rowA, rowB) => data[rowA] === data[rowB],
         toStringArray: params => ColumnHelpers.createAndFillArray(rowCount, str, params),
@@ -57,7 +52,6 @@ export default function Field(column: EncodedColumn): Data.Field {
             : params => ColumnHelpers.createAndFillArray(rowCount, int, params),
         toFloatArray: isNumeric
             ? params => ColumnHelpers.typedArrayWindow(data, params)
-            : params => ColumnHelpers.createAndFillArray(rowCount, float, params),
-        toListArray: params => ColumnHelpers.createAndFillArray(rowCount, list, params)
+            : params => ColumnHelpers.createAndFillArray(rowCount, float, params)
     };
 }

src/mol-io/reader/cif/data-model.ts

@@ -68,7 +68,6 @@ export interface Field {
     str(row: number): string,
     int(row: number): number,
     float(row: number): number,
-    list<T extends number|string>(row: number): T[],
 
     valueKind(row: number): Column.ValueKind,
 
@@ -77,7 +76,6 @@ export interface Field {
     toStringArray(params?: Column.ToArrayParams<string>): ReadonlyArray<string>,
     toIntArray(params?: Column.ToArrayParams<number>): ReadonlyArray<number>,
     toFloatArray(params?: Column.ToArrayParams<number>): ReadonlyArray<number>
-    toListArray<T extends number|string>(params?: Column.ToArrayParams<T[]>): ReadonlyArray<T[]>
 }
 
 export function getTensor(category: Category, field: string, space: Tensor.Space, row: number): Tensor {

src/mol-io/reader/cif/schema.ts

@@ -7,6 +7,7 @@
 
 import { Database, Table, Column, ColumnHelpers } from 'mol-data/db'
 import { Tensor } from 'mol-math/linear-algebra'
+import { arrayEqual } from 'mol-util'
 import * as Data from './data-model'
 
 export function toDatabase<Schema extends Database.Schema, Frame extends Database<Schema> = Database<Schema>>(schema: Schema, frame: Data.Frame): Frame {
@@ -24,7 +25,7 @@ function getColumnCtor(t: Column.Schema): ColumnCtor {
         case 'str': return (f, c, k) => createColumn(t, f, f.str, f.toStringArray);
         case 'int': return (f, c, k) => createColumn(t, f, f.int, f.toIntArray);
         case 'float': return (f, c, k) => createColumn(t, f, f.float, f.toFloatArray);
-        case 'list': return (f, c, k) => createColumn(t, f, f.list, f.toListArray);
+        case 'list': throw new Error('Use createListColumn instead.');
         case 'tensor': throw new Error('Use createTensorColumn instead.');
     }
 }
@@ -42,6 +43,26 @@ function createColumn<T>(schema: Column.Schema, field: Data.Field, value: (row:
     };
 }
 
+function createListColumn<T extends number|string>(schema: Column.Schema.List<T>, category: Data.Category, key: string): Column<(number|string)[]> {
+    const separator = schema.separator;
+    const itemParse = schema.itemParse;
+
+    const f = category.getField(key);
+    const value = f ? (row: number) => f.str(row).split(separator).map(x => itemParse(x.trim())).filter(x => !!x) : (row: number) => []
+    const toArray: Column<T[]>['toArray'] = params => ColumnHelpers.createAndFillArray(category.rowCount, value, params)
+
+    return {
+        schema,
+        '@array': void 0,
+        isDefined: !!f,
+        rowCount: category.rowCount,
+        value,
+        valueKind: f ? f.valueKind : () => Column.ValueKind.NotPresent,
+        areValuesEqual: (rowA, rowB) => arrayEqual(value(rowA), value(rowB)),
+        toArray
+    };
+}
+
 function createTensorColumn(schema: Column.Schema.Tensor, category: Data.Category, key: string): Column<Tensor> {
     const space = schema.space;
     let firstFieldName: string;
@@ -84,7 +105,9 @@ class CategoryTable implements Table<any> { // tslint:disable-line:class-name
                 get: function() {
                     if (cache[k]) return cache[k];
                     const fType = schema[k];
-                    if (fType.valueType === 'tensor') {
+                    if (fType.valueType === 'list') {
+                        cache[k] = createListColumn(fType, category, k);
+                    } else if (fType.valueType === 'tensor') {
                         cache[k] = createTensorColumn(fType, category, k);
                     } else {
                         const ctor = getColumnCtor(fType);

src/mol-io/reader/cif/text/field.ts

@@ -28,12 +28,6 @@ export default function CifTextField(tokens: Tokens, rowCount: number): Data.Fie
         return fastParseFloat(data, indices[2 * row], indices[2 * row + 1]) || 0;
     };
 
-    const list: Data.Field['list'] = <T extends string|number>(row: number) => {
-        const ret = data.substring(indices[2 * row], indices[2 * row + 1]);
-        if (ret === '.' || ret === '?') return [];
-        return ret.split(',') as T[];
-    };
-
     const valueKind: Data.Field['valueKind'] = row => {
         const s = indices[2 * row];
         if (indices[2 * row + 1] - s !== 1) return Column.ValueKind.Present;
@@ -50,12 +44,10 @@ export default function CifTextField(tokens: Tokens, rowCount: number): Data.Fie
         str,
         int,
         float,
-        list,
         valueKind,
         areValuesEqual: TokenColumn.areValuesEqualProvider(tokens),
         toStringArray: params => ColumnHelpers.createAndFillArray(rowCount, str, params),
         toIntArray: params => ColumnHelpers.createAndFillArray(rowCount, int, params),
-        toFloatArray: params => ColumnHelpers.createAndFillArray(rowCount, float, params),
-        toListArray: params => ColumnHelpers.createAndFillArray(rowCount, list, params)
+        toFloatArray: params => ColumnHelpers.createAndFillArray(rowCount, float, params)
     }
 }

src/mol-io/writer/cif/encoder.ts

@@ -80,6 +80,10 @@ function columnValue(k: string) {
     return (i: number, d: any) => d[k].value(i);
 }
 
+function columnListValue(k: string) {
+    return (i: number, d: any) => d[k].value(i).join(d[k].schema.separator);
+}
+
 function columnTensorValue(k: string, ...coords: number[]) {
     return (i: number, d: any) => d[k].schema.space.get(d[k].value(i), ...coords);
 }
@@ -134,7 +138,7 @@ export namespace FieldDefinitions {
             } else if (t.valueType === 'str') {
                 fields.push({ name: k, type: FieldType.Str, value: columnValue(k), valueKind: columnValueKind(k) });
             } else if (t.valueType === 'list') {
-                throw new Error('list not implemented');
+                fields.push({ name: k, type: FieldType.Str, value: columnListValue(k), valueKind: columnValueKind(k) })
             } else if (t.valueType === 'tensor') {
                 fields.push(...getTensorDefinitions(k, t.space))
             } else {

src/mol-model/structure/model/formats/mmcif/assembly.ts

@@ -40,7 +40,7 @@ function createAssembly(format: mmCIF_Format, index: number, matrices: Matrices)
         if (assembly_id.value(i) !== id) continue;
         generators[generators.length] = {
             expression: oper_expression.value(i),
-            asymIds: asym_id_list.value(i).split(',').map(x => x.trim()).filter(x => !!x)
+            asymIds: asym_id_list.value(i)
         };
     }
 

src/mol-util/index.ts

@@ -2,6 +2,7 @@
  * Copyright (c) 2017 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * @author David Sehnal <david.sehnal@gmail.com>
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
 import BitFlags from './bit-flags'
@@ -11,4 +12,15 @@ import StringBuilder from './string-builder'
 import Time from './time'
 import UUID from './uuid'
 
-export { BitFlags, Computation, Scheduler, StringBuilder, Time, UUID }
+export { BitFlags, Computation, Scheduler, StringBuilder, Time, UUID }
+
+export function arrayEqual<T>(arr1: T[], arr2: T[]) {
+    const length = arr1.length
+    if (length !== arr2.length) return false
+    for (let i = 0; i < length; i++) {
+        if (arr1[i] !== arr2[i]) {
+            return false
+        }
+    }
+    return true
+}