chunked-array update

src/mol-data/util/_spec/chunked-array.spec.ts

@@ -0,0 +1,24 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import { ChunkedArray } from '../chunked-array'
+
+describe('Chunked Array', () => {
+    it('creation', () => {
+        const arr  = ChunkedArray.create<number>(_ => [], 2, 2);
+        ChunkedArray.add2(arr, 1, 2);
+        ChunkedArray.add2(arr, 3, 4);
+        expect(ChunkedArray.compact(arr)).toEqual([1, 2, 3, 4]);
+    });
+
+    it('initial', () => {
+        const arr  = ChunkedArray.create<number>(s => new Int32Array(s), 2, 6, new Int32Array([1, 2, 3, 4]));
+        ChunkedArray.add2(arr, 4, 3);
+        ChunkedArray.add2(arr, 2, 1);
+        ChunkedArray.add2(arr, 5, 6);
+        expect(ChunkedArray.compact(arr)).toEqual(new Int32Array([4, 3, 2, 1, 5, 6]));
+    });
+});

src/mol-data/util/chunked-array.ts

@@ -16,9 +16,7 @@ interface ChunkedArray<T> {
     ctor: (size: number) => any,
     elementSize: number,
 
-    linearGrowth: boolean,
-
-    initialSize: number,
+    growBy: number,
     allocatedSize: number,
     elementCount: number,
 
@@ -26,20 +24,16 @@ interface ChunkedArray<T> {
     currentChunk: any,
     currentIndex: number,
 
-    chunks: any[]
+    chunks: any[][]
 }
 
-// TODO: better api, write tests
 namespace ChunkedArray {
     export function is(x: any): x is ChunkedArray<any> {
         return x.creator && x.chunkSize;
     }
 
     function allocateNext(array: ChunkedArray<any>) {
-        let nextSize = !array.allocatedSize || array.linearGrowth
-            ? array.initialSize * array.elementSize
-            : Math.max(Math.ceil(0.61 * array.allocatedSize), 1);
-        if (nextSize % array.elementSize !== 0) nextSize += nextSize % array.elementSize;
+        let nextSize = array.growBy * array.elementSize;
         array.currentSize = nextSize;
         array.currentIndex = 0;
         array.currentChunk = array.ctor(nextSize);
@@ -80,16 +74,25 @@ namespace ChunkedArray {
         return array.elementCount++;
     }
 
+    export function compact<T>(array: ChunkedArray<T>, doNotResizeSingleton = false): ArrayLike<T> {
+        return _compact(array, doNotResizeSingleton);
+    }
 
-    export function compact<T>(array: ChunkedArray<T>): ArrayLike<T> {
+    export function _compact<T>(array: ChunkedArray<T>, doNotResizeSingleton: boolean): ArrayLike<T> {
         const { ctor, chunks, currentIndex } = array;
 
         if (!chunks.length) return ctor(0);
-        if (chunks.length === 1 && currentIndex === array.allocatedSize) {
-            return chunks[0];
+        if (chunks.length === 1) {
+            if (doNotResizeSingleton || currentIndex === array.allocatedSize) {
+                return chunks[0];
+            }
         }
 
-        const ret = ctor(array.elementSize * array.elementCount);
+        let size = 0;
+        for (let i = 0, _i = chunks.length - 1; i < _i; i++) size += chunks[i].length;
+        size += array.currentIndex;
+
+        const ret = ctor(size);
         let offset = 0;
 
         if (ret.buffer) {
@@ -115,13 +118,12 @@ namespace ChunkedArray {
         return ret;
     }
 
-    export function create<T>(ctor: (size: number) => any, elementSize: number, initialSize: number, linearGrowth: boolean): ChunkedArray<T> {
-        return {
+    export function create<T>(ctor: (size: number) => any, elementSize: number, growBy: number, initialChunk?: ArrayLike<T>): ChunkedArray<T> {
+        const ret: ChunkedArray<T> = {
             ctor,
             elementSize,
-            linearGrowth,
 
-            initialSize,
+            growBy,
             allocatedSize: 0,
             elementCount: 0,
 
@@ -130,7 +132,17 @@ namespace ChunkedArray {
             currentIndex: 0,
 
             chunks: []
-        } as ChunkedArray<T>;
+        };
+
+        if (!initialChunk) return ret;
+
+        if (initialChunk.length % elementSize !== 0) throw new Error('initialChunk length must be a multiple of the element size.');
+        ret.currentChunk = initialChunk;
+        ret.allocatedSize = initialChunk.length;
+        ret.currentSize = initialChunk.length;
+        ret.chunks[0] = initialChunk as any;
+
+        return ret;
     }
 }
 

src/mol-io/common/binary-cif/array-encoder.ts

@@ -357,7 +357,8 @@ export namespace ArrayEncoding {
         let map: any = Object.create(null);
         let strings: string[] = [];
         let accLength = 0;
-        let offsets = ChunkedArray.create<number>(s => new Int32Array(s), 1, 1024, true)
+        let offsets = ChunkedArray.create<number>(s => new Int32Array(s), 1,
+            Math.min(1024, data.length < 32 ? data.length + 1 : Math.round(data.length / 8) + 1));
         let output = new Int32Array(data.length);
 
         ChunkedArray.add(offsets, 0);

src/perf-tests/chunked-array-vs-native.ts

@@ -7,14 +7,14 @@ function testNative(size: number) {
     return xs;
 }
 
-function testChunkedTyped(size: number, chunk: number, linear: boolean) {
-    const xs = ChunkedArray.create(s => new Int32Array(s), 1, chunk, linear);
+function testChunkedTyped(size: number, chunk: number) {
+    const xs = ChunkedArray.create(s => new Int32Array(s), 1, chunk);
     for (let i = 0; i < size; i++) ChunkedArray.add(xs, i * i);
     return ChunkedArray.compact(xs);
 }
 
 function testChunkedNative(size: number, chunk: number) {
-    const xs = ChunkedArray.create(s => [], 1, chunk, false);
+    const xs = ChunkedArray.create(s => [], 1, chunk);
     for (let i = 0; i < size; i++) ChunkedArray.add(xs, i * i);
     return ChunkedArray.compact(xs);
 }
@@ -25,12 +25,12 @@ const N = 70000;
 
 suite
     .add('native', () => testNative(N))
-    .add('chunkedT 0.1k', () => testChunkedTyped(N, 100, false))
-    .add('chunkedT 4k', () => testChunkedTyped(N, 4096, false))
-    .add('chunkedT 4k lin', () => testChunkedTyped(N, 4096, true))
-    .add('chunkedT N / 2', () => testChunkedTyped(N, N / 2, false))
-    .add('chunkedT N', () => testChunkedTyped(N, N, false))
-    .add('chunkedT 2 * N', () => testChunkedTyped(N, 2 * N, false))
+    // .add('chunkedT 0.1k', () => testChunkedTyped(N, 100, false))
+    // .add('chunkedT 4k', () => testChunkedTyped(N, 4096, false))
+    .add('chunkedT 4k lin', () => testChunkedTyped(N, 4096))
+    // .add('chunkedT N / 2', () => testChunkedTyped(N, N / 2, false))
+    // .add('chunkedT N', () => testChunkedTyped(N, N, false))
+    // .add('chunkedT 2 * N', () => testChunkedTyped(N, 2 * N, false))
 
     .add('chunkedN N', () => testChunkedNative(N, N))
     .add('chunkedN 0.1k', () => testChunkedNative(N, 100))