wip, factoring out typed-array helpers

src/mol-io/common/simple-buffer.ts

@@ -2,6 +2,7 @@
  * Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * @author Alexander Rose <alexander.rose@weirdbyte.de>
+ * @author David Sehnal <david.sehnal@gmail.com>
  */
 
 import { defaults } from 'mol-util';
@@ -95,6 +96,8 @@ export namespace SimpleBuffer {
         return buffer
     }
 
+    export const IsNativeEndianLittle = new Uint16Array(new Uint8Array([0x12, 0x34]).buffer)[0] === 0x3412;
+
     /** source and target can't be the same */
     export function flipByteOrder(source: SimpleBuffer, target: Uint8Array, byteCount: number, elementByteSize: number, offset: number) {
         for (let i = 0, n = byteCount; i < n; i += elementByteSize) {
@@ -111,4 +114,10 @@ export namespace SimpleBuffer {
             intView[i] = ((val & 0xff) << 8) | ((val >> 8) & 0xff)
         }
     }
+
+    export function ensureLittleEndian(source: SimpleBuffer, target: SimpleBuffer, byteCount: number, elementByteSize: number, offset: number) {
+        if (IsNativeEndianLittle) return;
+        if (!byteCount || elementByteSize <= 1) return;
+        flipByteOrder(source, target, byteCount, elementByteSize, offset);
+    }
 }

src/mol-io/common/typed-array.ts

@@ -0,0 +1,76 @@
+/**
+ * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * Taken/adapted from DensityServer (https://github.com/dsehnal/DensityServer)
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
+ */
+
+import { FileHandle } from 'mol-io/common/file-handle';
+import { SimpleBuffer } from 'mol-io/common/simple-buffer';
+
+export type TypedArrayValueType = 'float32' | 'int8' | 'int16'
+
+export namespace TypedArrayValueType {
+    export const Float32: TypedArrayValueType = 'float32';
+    export const Int8: TypedArrayValueType = 'int8';
+    export const Int16: TypedArrayValueType = 'int16';
+}
+
+export type TypedArrayValueArray = Float32Array | Int8Array | Int16Array
+
+export interface TypedArrayBufferContext {
+    type: TypedArrayValueType,
+    elementByteSize: number,
+    readBuffer: Buffer,
+    valuesBuffer: Uint8Array,
+    values: TypedArrayValueArray
+}
+
+export function getElementByteSize(type: TypedArrayValueType) {
+    if (type === TypedArrayValueType.Float32) return 4;
+    if (type === TypedArrayValueType.Int16) return 2;
+    return 1;
+}
+
+export function makeTypedArray(type: TypedArrayValueType, buffer: ArrayBuffer, byteOffset = 0, length?: number): TypedArrayValueArray {
+    if (type === TypedArrayValueType.Float32) return new Float32Array(buffer, byteOffset, length);
+    if (type === TypedArrayValueType.Int16) return new Int16Array(buffer, byteOffset, length);
+    return new Int8Array(buffer, byteOffset, length);
+}
+
+export function createTypedArray(type: TypedArrayValueType, size: number) {
+    switch (type) {
+        case TypedArrayValueType.Float32: return new Float32Array(new ArrayBuffer(4 * size));
+        case TypedArrayValueType.Int8: return new Int8Array(new ArrayBuffer(1 * size));
+        case TypedArrayValueType.Int16: return new Int16Array(new ArrayBuffer(2 * size));
+    }
+    throw Error(`${type} is not a supported value format.`);
+}
+
+export function createTypedArrayBufferContext(size: number, type: TypedArrayValueType): TypedArrayBufferContext {
+    let elementByteSize = getElementByteSize(type);
+    let arrayBuffer = new ArrayBuffer(elementByteSize * size);
+    let readBuffer = new Buffer(arrayBuffer);
+    let valuesBuffer = SimpleBuffer.IsNativeEndianLittle ? arrayBuffer : new ArrayBuffer(elementByteSize * size);
+    return {
+        type,
+        elementByteSize,
+        readBuffer,
+        valuesBuffer: new Uint8Array(valuesBuffer),
+        values: makeTypedArray(type, valuesBuffer)
+    };
+}
+
+export async function readTypedArray(ctx: TypedArrayBufferContext, file: FileHandle, position: number, count: number, valueOffset: number, littleEndian?: boolean) {
+    let byteCount = ctx.elementByteSize * count;
+    let byteOffset = ctx.elementByteSize * valueOffset;
+
+    await file.readBuffer(position, ctx.readBuffer, byteCount, byteOffset);
+    if (ctx.elementByteSize > 1 && ((littleEndian !== void 0 && littleEndian !== SimpleBuffer.IsNativeEndianLittle) || !SimpleBuffer.IsNativeEndianLittle)) {
+        // fix the endian
+        SimpleBuffer.flipByteOrder(ctx.readBuffer, ctx.valuesBuffer, byteCount, ctx.elementByteSize, byteOffset);
+    }
+    return ctx.values;
+}

src/mol-io/reader/ccp4/parser.ts

@@ -9,6 +9,7 @@ import { Ccp4File, Ccp4Header } from './schema'
 import { ReaderResult as Result } from '../result'
 import { FileHandle } from '../../common/file-handle';
 import { SimpleBuffer } from 'mol-io/common/simple-buffer';
+import { TypedArrayValueType, getElementByteSize, makeTypedArray } from 'mol-io/common/typed-array';
 
 export async function readCcp4Header(file: FileHandle): Promise<{ header: Ccp4Header, littleEndian: boolean }> {
     const headerSize = 1024;
@@ -95,11 +96,11 @@ export async function readCcp4Header(file: FileHandle): Promise<{ header: Ccp4He
     return { header, littleEndian }
 }
 
-function getElementByteSize(mode: number) {
+function getTypedArrayValueType(mode: number) {
     switch (mode) {
-        case 2: return 4
-        case 1: return 2
-        case 0: return 1
+        case 2: return TypedArrayValueType.Float32
+        case 1: return TypedArrayValueType.Int16
+        case 0: return TypedArrayValueType.Int8
     }
     throw new Error(`ccp4 mode '${mode}' unsupported`);
 }
@@ -110,28 +111,20 @@ async function parseInternal(file: FileHandle, size: number, ctx: RuntimeContext
     const { header, littleEndian } = await readCcp4Header(file)
 
     const offset = 256 * 4 + header.NSYMBT
+    const valueType = getTypedArrayValueType(header.MODE)
     const { buffer, bytesRead } = await file.readBuffer(offset, size - offset)
 
     const count = header.NC * header.NR * header.NS
-    const elementByteSize = getElementByteSize(header.MODE)
+    const elementByteSize = getElementByteSize(valueType)
     const byteCount = count * elementByteSize
 
     if (byteCount !== bytesRead) {
         console.warn(`byteCount ${byteCount} and bytesRead ${bytesRead} differ`)
     }
 
-    let values
-    if (header.MODE === 2) {
-        values = new Float32Array(buffer.buffer, offset, count)
-    } else if (header.MODE === 1) {
-        values = new Int16Array(buffer.buffer, offset, count)
-    } else if (header.MODE === 0) {
-        values = new Int8Array(buffer.buffer, offset, count)
-    } else {
-        throw new Error(`ccp4 mode '${header.MODE}' unsupported`);
-    }
+    let values = makeTypedArray(valueType, buffer.buffer, offset, count)
 
-    if (!littleEndian) {
+    if (!littleEndian && valueType !== TypedArrayValueType.Int8) {
         SimpleBuffer.flipByteOrder(buffer, new Uint8Array(values.buffer), byteCount, elementByteSize, 0)
     }
 

src/servers/volume/common/data-format.ts

@@ -8,16 +8,7 @@
 
 import * as Schema from './binary-schema'
 import { FileHandle } from 'mol-io/common/file-handle';
-
-export type ValueType = 'float32' | 'int8' | 'int16'
-
-export namespace ValueType {
-    export const Float32: ValueType = 'float32';
-    export const Int8: ValueType = 'int8';
-    export const Int16: ValueType = 'int16';
-}
-
-export type ValueArray = Float32Array | Int8Array | Int16Array
+import { TypedArrayValueType } from 'mol-io/common/typed-array';
 
 export interface Spacegroup {
     number: number,
@@ -62,7 +53,7 @@ export interface Header {
     channels: string[],
 
     /** Determines the data type of the values */
-    valueType: ValueType,
+    valueType: TypedArrayValueType,
 
     /** The value are stored in blockSize^3 cubes */
     blockSize: number,
@@ -102,21 +93,6 @@ namespace _schema {
 
 const headerSchema = _schema.schema;
 
-export function getValueByteSize(type: ValueType) {
-    if (type === ValueType.Float32) return 4;
-    if (type === ValueType.Int16) return 2;
-    return 1;
-}
-
-export function createValueArray(type: ValueType, size: number) {
-    switch (type) {
-        case ValueType.Float32: return new Float32Array(new ArrayBuffer(4 * size));
-        case ValueType.Int8: return new Int8Array(new ArrayBuffer(1 * size));
-        case ValueType.Int16: return new Int16Array(new ArrayBuffer(2 * size));
-    }
-    throw Error(`${type} is not a supported value format.`);
-}
-
 export function encodeHeader(header: Header) {
     return Schema.encode(headerSchema, header);
 }

src/servers/volume/common/file.ts

@@ -8,12 +8,9 @@
 
 import * as fs from 'fs'
 import * as path from 'path'
-import * as DataFormat from './data-format'
 import { FileHandle } from 'mol-io/common/file-handle';
 import { SimpleBuffer } from 'mol-io/common/simple-buffer';
 
-export const IsNativeEndianLittle = new Uint16Array(new Uint8Array([0x12, 0x34]).buffer)[0] === 0x3412;
-
 export async function openRead(filename: string) {
     return new Promise<number>((res, rej) => {
         fs.open(filename, 'r', async (err, file) => {
@@ -64,64 +61,4 @@ const smallBuffer = SimpleBuffer.fromBuffer(new Buffer(8));
 export async function writeInt(file: FileHandle, value: number, position: number) {
     smallBuffer.writeInt32LE(value, 0);
     await file.writeBuffer(position, smallBuffer, 4);
-}
-
-export interface TypedArrayBufferContext {
-    type: DataFormat.ValueType,
-    elementByteSize: number,
-    readBuffer: Buffer,
-    valuesBuffer: Uint8Array,
-    values: DataFormat.ValueArray
-}
-
-function getElementByteSize(type: DataFormat.ValueType) {
-    if (type === DataFormat.ValueType.Float32) return 4;
-    if (type === DataFormat.ValueType.Int16) return 2;
-    return 1;
-}
-
-function makeTypedArray(type: DataFormat.ValueType, buffer: ArrayBuffer): DataFormat.ValueArray {
-    if (type === DataFormat.ValueType.Float32) return new Float32Array(buffer);
-    if (type === DataFormat.ValueType.Int16) return new Int16Array(buffer);
-    return new Int8Array(buffer);
-}
-
-export function createTypedArrayBufferContext(size: number, type: DataFormat.ValueType): TypedArrayBufferContext {
-    let elementByteSize = getElementByteSize(type);
-    let arrayBuffer = new ArrayBuffer(elementByteSize * size);
-    let readBuffer = new Buffer(arrayBuffer);
-    let valuesBuffer = IsNativeEndianLittle ? arrayBuffer : new ArrayBuffer(elementByteSize * size);
-    return {
-        type,
-        elementByteSize,
-        readBuffer,
-        valuesBuffer: new Uint8Array(valuesBuffer),
-        values: makeTypedArray(type, valuesBuffer)
-    };
-}
-
-function flipByteOrder(source: SimpleBuffer, target: Uint8Array, byteCount: number, elementByteSize: number, offset: number) {
-    for (let i = 0, n = byteCount; i < n; i += elementByteSize) {
-        for (let j = 0; j < elementByteSize; j++) {
-            target[offset + i + elementByteSize - j - 1] = source[offset + i + j];
-        }
-    }
-}
-
-export async function readTypedArray(ctx: TypedArrayBufferContext, file: FileHandle, position: number, count: number, valueOffset: number, littleEndian?: boolean) {
-    let byteCount = ctx.elementByteSize * count;
-    let byteOffset = ctx.elementByteSize * valueOffset;
-
-    await file.readBuffer(position, ctx.readBuffer, byteCount, byteOffset);
-    if (ctx.elementByteSize > 1 && ((littleEndian !== void 0 && littleEndian !== IsNativeEndianLittle) || !IsNativeEndianLittle)) {
-        // fix the endian
-        flipByteOrder(ctx.readBuffer, ctx.valuesBuffer, byteCount, ctx.elementByteSize, byteOffset);
-    }
-    return ctx.values;
-}
-
-export function ensureLittleEndian(source: SimpleBuffer, target: SimpleBuffer, byteCount: number, elementByteSize: number, offset: number) {
-    if (IsNativeEndianLittle) return;
-    if (!byteCount || elementByteSize <= 1) return;
-    flipByteOrder(source, target, byteCount, elementByteSize, offset);
 }

src/servers/volume/pack/data-model.ts

@@ -9,6 +9,7 @@ import * as Format from './format'
 import * as DataFormat from '../common/data-format'
 import { FileHandle } from 'mol-io/common/file-handle';
 import { SimpleBuffer } from 'mol-io/common/simple-buffer';
+import { TypedArrayValueArray, TypedArrayValueType } from 'mol-io/common/typed-array';
 
 const FORMAT_VERSION = '1.0.0';
 
@@ -25,16 +26,16 @@ export interface ValuesInfo {
 }
 
 export interface BlockBuffer {
-    values: DataFormat.ValueArray[],
+    values: TypedArrayValueArray[],
     buffers: SimpleBuffer[],
     slicesWritten: number
 }
 
 export interface DownsamplingBuffer {
     /** dimensions (sampleCount[1], sampleCount[0] / 2, 1), axis order (K, H, L) */
-    downsampleH: DataFormat.ValueArray,
+    downsampleH: TypedArrayValueArray,
     /** "Cyclic" (in the 1st dimensions) buffer with dimensions (5, sampleCount[0] / 2, sampleCount[1] / 2), axis order (L, H, K),  */
-    downsampleHK: DataFormat.ValueArray,
+    downsampleHK: TypedArrayValueArray,
 
     slicesWritten: number,
     startSliceIndex: number
@@ -76,7 +77,7 @@ export interface Context {
     isPeriodic: boolean,
 
     channels: Format.Context[],
-    valueType: DataFormat.ValueType,
+    valueType: TypedArrayValueType,
     blockSize: number,
     /** Able to store channels.length * blockSize^3 values. */
     cubeBuffer: SimpleBuffer,
@@ -101,7 +102,7 @@ export function createHeader(ctx: Context): DataFormat.Header {
 
     return {
         formatVersion: FORMAT_VERSION,
-        valueType: Format.getValueType(header),
+        valueType: header.valueType,
         blockSize: ctx.blockSize,
         axisOrder: header.axisOrder,
         origin: normalize(header.origin),

src/servers/volume/pack/downsampling.ts

@@ -7,10 +7,10 @@
  */
 
 import * as Data from './data-model'
-import * as DataFormat from '../common/data-format'
+import { TypedArrayValueArray } from 'mol-io/common/typed-array';
 
-/** 
- * Downsamples each slice of input data and checks if there is enough data to perform 
+/**
+ * Downsamples each slice of input data and checks if there is enough data to perform
  * higher rate downsampling.
  */
 export function downsampleLayer(ctx: Data.Context) {
@@ -25,8 +25,8 @@ export function downsampleLayer(ctx: Data.Context) {
     }
 }
 
-/** 
- * When the "native" (rate = 1) sampling is finished, there might still 
+/**
+ * When the "native" (rate = 1) sampling is finished, there might still
  * be some data left to be processed for the higher rate samplings.
  */
 export function finalize(ctx: Data.Context) {
@@ -46,7 +46,7 @@ export function finalize(ctx: Data.Context) {
 /**
  * The functions downsampleH and downsampleHK both essentially do the
  * same thing: downsample along H (1st axis in axis order) and K (2nd axis in axis order) axes respectively.
- * 
+ *
  * The reason there are two copies of almost the same code is performance:
  * Both functions use a different memory layout to improve cache coherency
  *  - downsampleU uses the H axis as the fastest moving one
@@ -54,7 +54,7 @@ export function finalize(ctx: Data.Context) {
  */
 
 
-function conv(w: number, c: number[], src: DataFormat.ValueArray, b: number, i0: number, i1: number, i2: number, i3: number, i4: number) {
+function conv(w: number, c: number[], src: TypedArrayValueArray, b: number, i0: number, i1: number, i2: number, i3: number, i4: number) {
     return w * (c[0] * src[b + i0] + c[1] * src[b + i1] + c[2] * src[b + i2] + c[3] * src[b + i3] + c[4] * src[b + i4]);
 }
 
@@ -63,7 +63,7 @@ function conv(w: number, c: number[], src: DataFormat.ValueArray, b: number, i0:
  * flipping the 1st and 2nd axis in the process to optimize cache coherency for downsampleUV call
  * (i.e. use (K, H, L) axis order).
  */
-function downsampleH(kernel: Data.Kernel, srcDims: number[], src: DataFormat.ValueArray, srcLOffset: number, buffer: Data.DownsamplingBuffer) {
+function downsampleH(kernel: Data.Kernel, srcDims: number[], src: TypedArrayValueArray, srcLOffset: number, buffer: Data.DownsamplingBuffer) {
     const target = buffer.downsampleH;
     const sizeH = srcDims[0], sizeK = srcDims[1], srcBaseOffset = srcLOffset * sizeH * sizeK;
     const targetH = Math.floor((sizeH + 1) / 2);
@@ -87,9 +87,9 @@ function downsampleH(kernel: Data.Kernel, srcDims: number[], src: DataFormat.Val
     }
 }
 
-/** 
- * Downsample first axis in the slice present in buffer.downsampleH 
- * The result is written into the "cyclical" downsampleHk buffer 
+/**
+ * Downsample first axis in the slice present in buffer.downsampleH
+ * The result is written into the "cyclical" downsampleHk buffer
  * in the (L, H, K) axis order.
  */
 function downsampleHK(kernel: Data.Kernel, dimsX: number[], buffer: Data.DownsamplingBuffer) {

src/servers/volume/pack/format.ts

@@ -6,15 +6,13 @@
  */
 
 import * as File from '../common/file'
-import * as DataFormat from '../common/data-format'
 import { FileHandle } from 'mol-io/common/file-handle';
 import { Ccp4Provider } from './format/ccp4';
-
-export const enum Mode { Int8 = 0, Int16 = 1, Float32 = 2 }
+import { TypedArrayBufferContext, TypedArrayValueArray, TypedArrayValueType, getElementByteSize, createTypedArrayBufferContext } from 'mol-io/common/typed-array';
 
 export interface Header {
     name: string,
-    mode: Mode,
+    valueType: TypedArrayValueType,
     grid: number[], // grid is converted to the axis order!!
     axisOrder: number[],
     extent: number[],
@@ -28,11 +26,11 @@ export interface Header {
 
 /** Represents a circular buffer for 2 * blockSize layers */
 export interface SliceBuffer {
-    buffer: File.TypedArrayBufferContext,
+    buffer: TypedArrayBufferContext,
     sliceCapacity: number,
     slicesRead: number,
 
-    values: DataFormat.ValueArray,
+    values: TypedArrayValueArray,
     sliceCount: number,
 
     /** Have all the input slice been read? */
@@ -55,18 +53,11 @@ export interface Context {
     provider: Provider
 }
 
-export function getValueType(header: Header) {
-    if (header.mode === Mode.Float32) return DataFormat.ValueType.Float32;
-    if (header.mode === Mode.Int16) return DataFormat.ValueType.Int16;
-    return DataFormat.ValueType.Int8;
-}
-
 export function assignSliceBuffer(data: Data, blockSize: number) {
-    const { extent } = data.header;
-    const valueType = getValueType(data.header);
-    const sliceSize = extent[0] * extent[1] * DataFormat.getValueByteSize(valueType);
-    const sliceCapacity = Math.max(1, Math.floor(Math.min(64 * 1024 * 1024, sliceSize * extent[2]) / sliceSize));
-    const buffer = File.createTypedArrayBufferContext(sliceCapacity * extent[0] * extent[1], valueType);
+    const { extent, valueType } = data.header;
+    const sliceSize = extent[0] * extent[1] * getElementByteSize(valueType);
+    const sliceCapacity = Math.max(1, Math.floor(Math.min(1 * 1024 * 1024, sliceSize * extent[2]) / sliceSize));
+    const buffer = createTypedArrayBufferContext(sliceCapacity * extent[0] * extent[1], valueType);
     data.slices = {
         buffer,
         sliceCapacity,

src/servers/volume/pack/format/ccp4.ts

@@ -9,6 +9,16 @@ import { FileHandle } from 'mol-io/common/file-handle';
 import { readCcp4Header } from 'mol-io/reader/ccp4/parser';
 import { Header, Provider } from '../format';
 import { readSlices } from './common';
+import { TypedArrayValueType } from 'mol-io/common/typed-array';
+
+function getTypedArrayValueType(mode: number) {
+    switch (mode) {
+        case 2: return TypedArrayValueType.Float32
+        case 1: return TypedArrayValueType.Int16
+        case 0: return TypedArrayValueType.Int8
+    }
+    throw new Error(`ccp4 mode '${mode}' unsupported`);
+}
 
 async function readHeader(name: string, file: FileHandle) {
     const { header: ccp4Header, littleEndian } = await readCcp4Header(file)
@@ -17,7 +27,7 @@ async function readHeader(name: string, file: FileHandle) {
     const nxyzStart = [ccp4Header.NCSTART, ccp4Header.NRSTART, ccp4Header.NSSTART];
     const header: Header = {
         name,
-        mode: ccp4Header.MODE,
+        valueType: getTypedArrayValueType(ccp4Header.MODE),
         grid: [ccp4Header.NX, ccp4Header.NY, ccp4Header.NZ],
         axisOrder: [ccp4Header.MAPC, ccp4Header.MAPR, ccp4Header.MAPS].map(i => i - 1),
         extent: [ccp4Header.NC, ccp4Header.NR, ccp4Header.NS],

src/servers/volume/pack/format/common.ts

@@ -6,7 +6,7 @@
  */
 
 import { Data } from '../format';
-import * as File from '../../common/file'
+import { readTypedArray } from 'mol-io/common/typed-array';
 
 export async function readSlices(data: Data) {
     const { slices, header } = data;
@@ -19,8 +19,9 @@ export async function readSlices(data: Data) {
     const sliceByteOffset = slices.buffer.elementByteSize * sliceSize * slices.slicesRead;
     const sliceCount = Math.min(slices.sliceCapacity, extent[2] - slices.slicesRead);
     const sliceByteCount = sliceCount * sliceSize;
+    console.log('sliceByteOffset', sliceByteOffset, 'sliceSize', sliceSize, 'sliceCount', sliceCount)
 
-    await File.readTypedArray(slices.buffer, data.file, header.dataOffset + sliceByteOffset, sliceByteCount, 0, header.littleEndian);
+    await readTypedArray(slices.buffer, data.file, header.dataOffset + sliceByteOffset, sliceByteCount, 0, header.littleEndian);
     slices.slicesRead += sliceCount;
     slices.sliceCount = sliceCount;
 

src/servers/volume/pack/format/dsn6.ts

@@ -5,16 +5,17 @@
  */
 
 import { FileHandle } from 'mol-io/common/file-handle';
-import { Header, Provider, Mode } from '../format';
+import { Header, Provider } from '../format';
 import { readSlices } from './common';
 import { readDsn6Header, dsn6HeaderSize } from 'mol-io/reader/dsn6/parser';
+import { TypedArrayValueType } from 'mol-io/common/typed-array';
 
 async function readHeader(name: string, file: FileHandle) {
     const { header: dsn6Header, littleEndian } = await readDsn6Header(file)
 
     const header: Header = {
         name,
-        mode: Mode.Int16,
+        valueType: TypedArrayValueType.Int16,
         grid: [dsn6Header.xRate, dsn6Header.yRate, dsn6Header.zRate],
         axisOrder: [0, 1, 2],
         extent: [dsn6Header.xExtent, dsn6Header.yExtent, dsn6Header.zExtent],

src/servers/volume/pack/sampling.ts

@@ -13,19 +13,21 @@ import * as Downsampling from './downsampling'
 import * as Writer from './writer'
 import * as DataFormat from '../common/data-format'
 import { FileHandle } from 'mol-io/common/file-handle';
+import { getElementByteSize, createTypedArray, TypedArrayValueType } from 'mol-io/common/typed-array';
+import { SimpleBuffer } from 'mol-io/common/simple-buffer';
 
 export async function createContext(filename: string, channels: Format.Context[], blockSize: number, isPeriodic: boolean): Promise<Data.Context> {
-    const header = channels[0].data.header;
-    const samplingCounts = getSamplingCounts(channels[0].data.header.extent, blockSize);
-    const valueType = Format.getValueType(header);
-    const cubeBuffer = new Buffer(new ArrayBuffer(channels.length * blockSize * blockSize * blockSize * DataFormat.getValueByteSize(valueType)));
+    const { extent, valueType, grid, origin } = channels[0].data.header;
 
-    const litteEndianCubeBuffer = File.IsNativeEndianLittle
+    const samplingCounts = getSamplingCounts(extent, blockSize);
+    const cubeBuffer = new Buffer(new ArrayBuffer(channels.length * blockSize * blockSize * blockSize * getElementByteSize(valueType)));
+
+    const litteEndianCubeBuffer = SimpleBuffer.IsNativeEndianLittle
         ? cubeBuffer
-        : new Buffer(new ArrayBuffer(channels.length * blockSize * blockSize * blockSize * DataFormat.getValueByteSize(valueType)));
+        : new Buffer(new ArrayBuffer(channels.length * blockSize * blockSize * blockSize * getElementByteSize(valueType)));
 
     // The data can be periodic iff the extent is the same as the grid and origin is 0.
-    if (header.grid.some((v, i) => v !== header.extent[i]) || header.origin.some(v => v !== 0)) {
+    if (grid.some((v, i) => v !== extent[i]) || origin.some(v => v !== 0)) {
         isPeriodic = false;
     }
 
@@ -93,9 +95,9 @@ function getSamplingCounts(baseSampleCount: number[], blockSize: number) {
     }
 }
 
-function createBlockBuffer(sampleCount: number[], blockSize: number, valueType: DataFormat.ValueType, numChannels: number): Data.BlockBuffer {
+function createBlockBuffer(sampleCount: number[], blockSize: number, valueType: TypedArrayValueType, numChannels: number): Data.BlockBuffer {
     const values = [];
-    for (let i = 0; i < numChannels; i++) values[i] = DataFormat.createValueArray(valueType, sampleCount[0] * sampleCount[1] * blockSize);
+    for (let i = 0; i < numChannels; i++) values[i] = createTypedArray(valueType, sampleCount[0] * sampleCount[1] * blockSize);
     return {
         values,
         buffers: values.map(xs => new Buffer(xs.buffer)),
@@ -103,12 +105,12 @@ function createBlockBuffer(sampleCount: number[], blockSize: number, valueType:
     };
 }
 
-function createDownsamplingBuffer(valueType: DataFormat.ValueType, sourceSampleCount: number[], targetSampleCount: number[], numChannels: number): Data.DownsamplingBuffer[] {
+function createDownsamplingBuffer(valueType: TypedArrayValueType, sourceSampleCount: number[], targetSampleCount: number[], numChannels: number): Data.DownsamplingBuffer[] {
     const ret = [];
     for (let i = 0; i < numChannels; i++) {
         ret[ret.length] = {
-            downsampleH: DataFormat.createValueArray(valueType, sourceSampleCount[1] * targetSampleCount[0]),
-            downsampleHK: DataFormat.createValueArray(valueType, 5 * targetSampleCount[0] * targetSampleCount[1]),
+            downsampleH: createTypedArray(valueType, sourceSampleCount[1] * targetSampleCount[0]),
+            downsampleHK: createTypedArray(valueType, 5 * targetSampleCount[0] * targetSampleCount[1]),
             slicesWritten: 0,
             startSliceIndex: 0
         }
@@ -116,7 +118,7 @@ function createDownsamplingBuffer(valueType: DataFormat.ValueType, sourceSampleC
     return ret;
 }
 
-function createSampling(index: number, valueType: DataFormat.ValueType, numChannels: number, sampleCounts: number[][], blockSize: number): Data.Sampling {
+function createSampling(index: number, valueType: TypedArrayValueType, numChannels: number, sampleCounts: number[][], blockSize: number): Data.Sampling {
     const sampleCount = sampleCounts[index];
     const valuesInfo: Data.ValuesInfo[] = [];
     for (let i = 0; i < numChannels; i++) {
@@ -135,7 +137,7 @@ function createSampling(index: number, valueType: DataFormat.ValueType, numChann
         downsampling: index < sampleCounts.length - 1 ? createDownsamplingBuffer(valueType, sampleCount, sampleCounts[index + 1], numChannels) : void 0,
 
         byteOffset: 0,
-        byteSize: numChannels * sampleCount[0] * sampleCount[1] * sampleCount[2] * DataFormat.getValueByteSize(valueType),
+        byteSize: numChannels * sampleCount[0] * sampleCount[1] * sampleCount[2] * getElementByteSize(valueType),
         writeByteOffset: 0
     }
 }

src/servers/volume/pack/writer.ts

@@ -7,8 +7,8 @@
  */
 
 import * as Data from './data-model'
-import * as File from '../common/file'
-import * as DataFormat from '../common/data-format'
+import { getElementByteSize } from 'mol-io/common/typed-array';
+import { SimpleBuffer } from 'mol-io/common/simple-buffer';
 
 /** Converts a layer to blocks and writes them to the output file. */
 export async function writeBlockLayer(ctx: Data.Context, sampling: Data.Sampling) {
@@ -32,7 +32,7 @@ function fillCubeBuffer(ctx: Data.Context, sampling: Data.Sampling, u: number, v
     const { blockSize, cubeBuffer } = ctx;
     const { sampleCount } = sampling;
     const { buffers, slicesWritten } = sampling.blocks;
-    const elementSize = DataFormat.getValueByteSize(ctx.valueType);
+    const elementSize = getElementByteSize(ctx.valueType);
     const sizeH = sampleCount[0], sizeHK = sampleCount[0] * sampleCount[1];
     const offsetH = u * blockSize,
         offsetK = v * blockSize;
@@ -52,7 +52,7 @@ function fillCubeBuffer(ctx: Data.Context, sampling: Data.Sampling, u: number, v
         }
     }
     // flip the byte order if needed.
-    File.ensureLittleEndian(ctx.cubeBuffer, ctx.litteEndianCubeBuffer, writeOffset, elementSize, 0);
+    SimpleBuffer.ensureLittleEndian(ctx.cubeBuffer, ctx.litteEndianCubeBuffer, writeOffset, elementSize, 0);
     return writeOffset;
 }
 

src/servers/volume/server/query/compose.ts

@@ -6,11 +6,10 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
-import * as DataFormat from '../../common/data-format'
 import * as Data from './data-model'
 import * as Box from '../algebra/box'
 import * as Coords from '../algebra/coordinate'
-import * as File from '../../common/file'
+import { createTypedArrayBufferContext, getElementByteSize, readTypedArray } from 'mol-io/common/typed-array';
 
 export default async function compose(query: Data.QueryContext.Data) {
     for (const block of query.samplingInfo.blocks) {
@@ -24,14 +23,14 @@ async function readBlock(query: Data.QueryContext.Data, coord: Coords.Grid<'Bloc
     const size = numChannels * blockSampleCount[0] * blockSampleCount[1] * blockSampleCount[2];
     const { valueType, blockSize } = query.data.header;
     const dataSampleCount = query.data.header.sampling[query.samplingInfo.sampling.index].sampleCount;
-    const buffer = File.createTypedArrayBufferContext(size, valueType);
+    const buffer = createTypedArrayBufferContext(size, valueType);
     const byteOffset = query.samplingInfo.sampling.byteOffset
-        + DataFormat.getValueByteSize(valueType) * numChannels * blockSize
+        + getElementByteSize(valueType) * numChannels * blockSize
         * (blockSampleCount[1] * blockSampleCount[2] * coord[0]
             + dataSampleCount[0] * blockSampleCount[2] * coord[1]
             + dataSampleCount[0] * dataSampleCount[1] * coord[2]);
 
-    const values = await File.readTypedArray(buffer, query.data.file, byteOffset, size, 0);
+    const values = await readTypedArray(buffer, query.data.file, byteOffset, size, 0);
     return {
         sampleCount: blockSampleCount,
         values

src/servers/volume/server/query/data-model.ts

@@ -12,6 +12,7 @@ import * as Box from '../algebra/box'
 import Writer from 'mol-io/writer/writer'
 import { SpacegroupCell } from 'mol-math/geometry';
 import { FileHandle } from 'mol-io/common/file-handle';
+import { TypedArrayValueArray } from 'mol-io/common/typed-array';
 
 //////////////////////////////////////
 // DATA
@@ -35,7 +36,7 @@ export interface DataContext {
 
 export interface BlockData {
     sampleCount: number[],
-    values: DataFormat.ValueArray
+    values: TypedArrayValueArray
 }
 
 //////////////////////////////////////
@@ -75,5 +76,5 @@ export namespace QueryContext {
     type Base = { guid: string, params: QueryParams }
     export type Error = { kind: 'Error', message: string } & Base
     export type Empty = { kind: 'Empty', data: DataContext } & Base
-    export type Data = { kind: 'Data', data: DataContext, samplingInfo: QuerySamplingInfo, values: DataFormat.ValueArray[] } & Base
+    export type Data = { kind: 'Data', data: DataContext, samplingInfo: QuerySamplingInfo, values: TypedArrayValueArray[] } & Base
 }

src/servers/volume/server/query/encode.ts

@@ -13,6 +13,7 @@ import VERSION from '../version'
 import * as DataFormat from '../../common/data-format'
 import { Column } from 'mol-data/db';
 import { ArrayEncoding, ArrayEncoder } from 'mol-io/common/binary-cif';
+import { TypedArrayValueType, TypedArrayValueArray } from 'mol-io/common/typed-array';
 
 export default function encode(query: Data.QueryContext, output: Data.QueryOutputStream) {
     let w = CifWriter.createEncoder({ binary: query.params.asBinary, encoderName: `VolumeServer ${VERSION}` });
@@ -106,7 +107,7 @@ const _volume_data_3d_info: CifWriter.Category<ResultContext> = {
     }
 };
 
-function _volume_data_3d_number(i: number, ctx: DataFormat.ValueArray): number {
+function _volume_data_3d_number(i: number, ctx: TypedArrayValueArray): number {
     return ctx[i];
 }
 
@@ -118,7 +119,7 @@ const _volume_data_3d: CifWriter.Category<ResultContext> = {
         const E = ArrayEncoding;
         let encoder: ArrayEncoder;
         let typedArray: any;
-        if (ctx.query.data.header.valueType === DataFormat.ValueType.Float32 || ctx.query.data.header.valueType === DataFormat.ValueType.Int16) {
+        if (ctx.query.data.header.valueType === TypedArrayValueType.Float32 || ctx.query.data.header.valueType === TypedArrayValueType.Int16) {
             let min: number, max: number;
             min = data[0], max = data[0];
             for (let i = 0, n = data.length; i < n; i++) {

src/servers/volume/server/query/execute.ts

@@ -22,6 +22,7 @@ import { SpacegroupCell } from 'mol-math/geometry';
 import { Vec3 } from 'mol-math/linear-algebra';
 import { UUID } from 'mol-util';
 import { FileHandle } from 'mol-io/common/file-handle';
+import { createTypedArray, TypedArrayValueType } from 'mol-io/common/typed-array';
 
 export default async function execute(params: Data.QueryParams, outputProvider: () => Data.QueryOutputStream) {
     const start = getTime();
@@ -142,10 +143,10 @@ function getQueryBox(data: Data.DataContext, queryBox: Data.QueryParamsBox) {
     }
 }
 
-function allocateValues(domain: Coords.GridDomain<'Query'>, numChannels: number, valueType: DataFormat.ValueType) {
+function allocateValues(domain: Coords.GridDomain<'Query'>, numChannels: number, valueType: TypedArrayValueType) {
     const values = [];
     for (let i = 0; i < numChannels; i++) {
-        values[values.length] = DataFormat.createValueArray(valueType, domain.sampleVolume);
+        values[values.length] = createTypedArray(valueType, domain.sampleVolume);
     }
     return values;
 }