volume server refactoring

src/servers/volume/pack/data-model.ts

@@ -5,7 +5,7 @@
  *
  * @author David Sehnal <david.sehnal@gmail.com>
  */
-import * as CCP4 from './ccp4'
+import * as Format from './format'
 import * as DataFormat from '../common/data-format'
 import { FileHandle } from 'mol-io/common/file-handle';
 import { SimpleBuffer } from 'mol-io/common/simple-buffer';
@@ -75,7 +75,7 @@ export interface Context {
     /** Periodic are x-ray density files that cover the entire grid and have [0,0,0] origin */
     isPeriodic: boolean,
 
-    channels: CCP4.Data[],
+    channels: Format.Context[],
     valueType: DataFormat.ValueType,
     blockSize: number,
     /** Able to store channels.length * blockSize^3 values. */
@@ -92,7 +92,7 @@ export interface Context {
 }
 
 export function createHeader(ctx: Context): DataFormat.Header {
-    const header = ctx.channels[0].header;
+    const header = ctx.channels[0].data.header;
     const grid = header.grid;
 
     function normalize(data: number[]) {
@@ -101,13 +101,13 @@ export function createHeader(ctx: Context): DataFormat.Header {
 
     return {
         formatVersion: FORMAT_VERSION,
-        valueType: CCP4.getValueType(header),
+        valueType: Format.getValueType(header),
         blockSize: ctx.blockSize,
         axisOrder: header.axisOrder,
         origin: normalize(header.origin),
         dimensions: normalize(header.extent),
         spacegroup: { number: header.spacegroupNumber, size: header.cellSize, angles: header.cellAngles, isPeriodic: ctx.isPeriodic },
-        channels: ctx.channels.map(c => c.header.name),
+        channels: ctx.channels.map(c => c.data.header.name),
         sampling: ctx.sampling.map(s => {
             const N = s.sampleCount[0] * s.sampleCount[1] * s.sampleCount[2];
             const valuesInfo = [];

src/servers/volume/pack/ccp4.ts → src/servers/volume/pack/format.ts

@@ -1,7 +1,5 @@
 /**
- * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
- *
- * Taken/adapted from DensityServer (https://github.com/dsehnal/DensityServer)
+ * Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * @author David Sehnal <david.sehnal@gmail.com>
  * @author Alexander Rose <alexander.rose@weirdbyte.de>
@@ -10,7 +8,7 @@
 import * as File from '../common/file'
 import * as DataFormat from '../common/data-format'
 import { FileHandle } from 'mol-io/common/file-handle';
-import { readCcp4Header } from 'mol-io/reader/ccp4/parser';
+import { Ccp4Provider } from './format/ccp4';
 
 export const enum Mode { Int8 = 0, Int16 = 1, Float32 = 2 }
 
@@ -47,6 +45,16 @@ export interface Data {
     slices: SliceBuffer
 }
 
+export interface Provider {
+    readHeader: (name: string, file: FileHandle) => Promise<Header>,
+    readSlices: (data: Data) => Promise<void>
+}
+
+export interface Context {
+    data: Data,
+    provider: Provider
+}
+
 export function getValueType(header: Header) {
     if (header.mode === Mode.Float32) return DataFormat.ValueType.Float32;
     if (header.mode === Mode.Int16) return DataFormat.ValueType.Int16;
@@ -87,58 +95,20 @@ export function compareHeaders(a: Header, b: Header) {
     return true;
 }
 
-async function readHeader(name: string, file: FileHandle) {
-    const { header: ccp4Header, littleEndian } = await readCcp4Header(file)
-
-    const origin2k = [ccp4Header.originX, ccp4Header.originY, ccp4Header.originZ];
-    const nxyzStart = [ccp4Header.NCSTART, ccp4Header.NRSTART, ccp4Header.NSSTART];
-    const header: Header = {
-        name,
-        mode: ccp4Header.MODE,
-        grid: [ccp4Header.NX, ccp4Header.NY, ccp4Header.NZ],
-        axisOrder: [ccp4Header.MAPC, ccp4Header.MAPR, ccp4Header.MAPS].map(i => i - 1),
-        extent: [ccp4Header.NC, ccp4Header.NR, ccp4Header.NS],
-        origin: origin2k[0] === 0.0 && origin2k[1] === 0.0 && origin2k[2] === 0.0 ? nxyzStart : origin2k,
-        spacegroupNumber: ccp4Header.ISPG,
-        cellSize: [ccp4Header.xLength, ccp4Header.yLength, ccp4Header.zLength],
-        cellAngles: [ccp4Header.alpha, ccp4Header.beta, ccp4Header.gamma],
-        // mean: readFloat(21),
-        littleEndian,
-        dataOffset: 256 * 4 + ccp4Header.NSYMBT /* symBytes */
-    };
-    // "normalize" the grid axis order
-    header.grid = [header.grid[header.axisOrder[0]], header.grid[header.axisOrder[1]], header.grid[header.axisOrder[2]]];
-    return header;
-}
-
-export async function readSlices(data: Data) {
-    const { slices, header } = data;
-    if (slices.isFinished) {
-        return;
-    }
-
-    const { extent } = header;
-    const sliceSize = extent[0] * extent[1];
-    const sliceByteOffset = slices.buffer.elementByteSize * sliceSize * slices.slicesRead;
-    const sliceCount = Math.min(slices.sliceCapacity, extent[2] - slices.slicesRead);
-    const sliceByteCount = sliceCount * sliceSize;
-
-    await File.readTypedArray(slices.buffer, data.file, header.dataOffset + sliceByteOffset, sliceByteCount, 0, header.littleEndian);
-    slices.slicesRead += sliceCount;
-    slices.sliceCount = sliceCount;
+export type Type = 'ccp4' // | 'dsn6'
 
-    if (slices.slicesRead >= extent[2]) {
-        slices.isFinished = true;
+export function getProviderFromType(type: Type): Provider {
+    switch (type) {
+        case 'ccp4': return Ccp4Provider
+        // case 'dsn6': return Dsn6Provider
     }
 }
 
-export async function open(name: string, filename: string): Promise<Data> {
+export async function open(name: string, filename: string, type: Type = 'ccp4'): Promise<Context> {
+    const provider = getProviderFromType(type)
     const descriptor = await File.openRead(filename);
     const file = FileHandle.fromDescriptor(descriptor)
-    const header = await readHeader(name, file);
-    return {
-        header,
-        file,
-        slices: void 0 as any
-    };
+    const header = await provider.readHeader(name, file);
+    const data = { header, file, slices: void 0 as any }
+    return { data, provider };
 }

src/servers/volume/pack/format/ccp4.ts

@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import { FileHandle } from 'mol-io/common/file-handle';
+import { readCcp4Header } from 'mol-io/reader/ccp4/parser';
+import { Header, Provider } from '../format';
+import { readSlices } from './common';
+
+async function readHeader(name: string, file: FileHandle) {
+    const { header: ccp4Header, littleEndian } = await readCcp4Header(file)
+
+    const origin2k = [ccp4Header.originX, ccp4Header.originY, ccp4Header.originZ];
+    const nxyzStart = [ccp4Header.NCSTART, ccp4Header.NRSTART, ccp4Header.NSSTART];
+    const header: Header = {
+        name,
+        mode: ccp4Header.MODE,
+        grid: [ccp4Header.NX, ccp4Header.NY, ccp4Header.NZ],
+        axisOrder: [ccp4Header.MAPC, ccp4Header.MAPR, ccp4Header.MAPS].map(i => i - 1),
+        extent: [ccp4Header.NC, ccp4Header.NR, ccp4Header.NS],
+        origin: origin2k[0] === 0.0 && origin2k[1] === 0.0 && origin2k[2] === 0.0 ? nxyzStart : origin2k,
+        spacegroupNumber: ccp4Header.ISPG,
+        cellSize: [ccp4Header.xLength, ccp4Header.yLength, ccp4Header.zLength],
+        cellAngles: [ccp4Header.alpha, ccp4Header.beta, ccp4Header.gamma],
+        littleEndian,
+        dataOffset: 256 * 4 + ccp4Header.NSYMBT /* symBytes */
+    };
+    // "normalize" the grid axis order
+    header.grid = [header.grid[header.axisOrder[0]], header.grid[header.axisOrder[1]], header.grid[header.axisOrder[2]]];
+    return header;
+}
+
+export const Ccp4Provider: Provider = { readHeader, readSlices }

src/servers/volume/pack/format/common.ts

@@ -0,0 +1,30 @@
+/**
+ * Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
+ */
+
+import { Data } from '../format';
+import * as File from '../../common/file'
+
+export async function readSlices(data: Data) {
+    const { slices, header } = data;
+    if (slices.isFinished) {
+        return;
+    }
+
+    const { extent } = header;
+    const sliceSize = extent[0] * extent[1];
+    const sliceByteOffset = slices.buffer.elementByteSize * sliceSize * slices.slicesRead;
+    const sliceCount = Math.min(slices.sliceCapacity, extent[2] - slices.slicesRead);
+    const sliceByteCount = sliceCount * sliceSize;
+
+    await File.readTypedArray(slices.buffer, data.file, header.dataOffset + sliceByteOffset, sliceByteCount, 0, header.littleEndian);
+    slices.slicesRead += sliceCount;
+    slices.sliceCount = sliceCount;
+
+    if (slices.slicesRead >= extent[2]) {
+        slices.isFinished = true;
+    }
+}

src/servers/volume/pack/format/dsn6.ts

@@ -0,0 +1,31 @@
+/**
+ * Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
+ */
+
+import { FileHandle } from 'mol-io/common/file-handle';
+import { Header, Provider, Mode } from '../format';
+import { readSlices } from './common';
+import { readDsn6Header, dsn6HeaderSize } from 'mol-io/reader/dsn6/parser';
+
+async function readHeader(name: string, file: FileHandle) {
+    const { header: dsn6Header, littleEndian } = await readDsn6Header(file)
+
+    const header: Header = {
+        name,
+        mode: Mode.Int16,
+        grid: [dsn6Header.xRate, dsn6Header.yRate, dsn6Header.zRate],
+        axisOrder: [0, 1, 2],
+        extent: [dsn6Header.xExtent, dsn6Header.yExtent, dsn6Header.zExtent],
+        origin: [dsn6Header.xStart, dsn6Header.yStart, dsn6Header.zStart],
+        spacegroupNumber: 0, // P 1
+        cellSize: [dsn6Header.xlen, dsn6Header.ylen, dsn6Header.zlen],
+        cellAngles: [dsn6Header.alpha, dsn6Header.beta, dsn6Header.gamma],
+        littleEndian,
+        dataOffset: dsn6HeaderSize
+    };
+    return header;
+}
+
+export const Dsn6Provider: Provider = { readHeader, readSlices }

src/servers/volume/pack/main.ts

@@ -6,7 +6,7 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
-import * as CCP4 from './ccp4'
+import * as Format from './format'
 import * as File from '../common/file'
 import * as Data from './data-model'
 import * as Sampling from './sampling'
@@ -49,7 +49,7 @@ async function allocateFile(ctx: Data.Context) {
     }
 }
 
-function determineBlockSize(data: CCP4.Data, blockSize: number) {
+function determineBlockSize(data: Format.Data, blockSize: number) {
     const { extent } = data.header;
     const maxLayerSize = 1024 * 1024 * 1024;
     const valueCount = extent[0] * extent[1];
@@ -83,20 +83,22 @@ async function create(filename: string, sourceDensities: { name: string, filenam
     process.stdout.write('Initializing... ');
     const files: FileHandle[] = [];
     try {
-        // Step 1a: Read the CCP4 headers
-        const channels: CCP4.Data[] = [];
-        for (const s of sourceDensities) channels.push(await CCP4.open(s.name, s.filename));
-        // Step 1b: Check if the CCP4 headers are compatible.
-        const isOk = channels.reduce((ok, s) => ok && CCP4.compareHeaders(channels[0].header, s.header), true);
+        // Step 1a: Read the Format headers
+        const channels: Format.Context[] = [];
+        for (const s of sourceDensities) {
+            channels.push(await Format.open(s.name, s.filename));
+        }
+        // Step 1b: Check if the Format headers are compatible.
+        const isOk = channels.reduce((ok, s) => ok && Format.compareHeaders(channels[0].data.header, s.data.header), true);
         if (!isOk) {
             throw new Error('Input file headers are not compatible (different grid, etc.).');
         }
-        const blockSize = determineBlockSize(channels[0], sourceBlockSize);
-        for (const ch of channels) CCP4.assignSliceBuffer(ch, blockSize);
+        const blockSize = determineBlockSize(channels[0].data, sourceBlockSize);
+        for (const ch of channels) Format.assignSliceBuffer(ch.data, blockSize);
 
         // Step 1c: Create data context.
         const context = await Sampling.createContext(filename, channels, blockSize, isPeriodic);
-        for (const s of channels) files.push(s.file);
+        for (const s of channels) files.push(s.data.file);
         files.push(context.file);
         process.stdout.write('   done.\n');
 

src/servers/volume/pack/sampling.ts

@@ -6,7 +6,7 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
-import * as CCP4 from './ccp4'
+import * as Format from './format'
 import * as Data from './data-model'
 import * as File from '../common/file'
 import * as Downsampling from './downsampling'
@@ -14,10 +14,10 @@ import * as Writer from './writer'
 import * as DataFormat from '../common/data-format'
 import { FileHandle } from 'mol-io/common/file-handle';
 
-export async function createContext(filename: string, channels: CCP4.Data[], blockSize: number, isPeriodic: boolean): Promise<Data.Context> {
-    const header = channels[0].header;
-    const samplingCounts = getSamplingCounts(channels[0].header.extent, blockSize);
-    const valueType = CCP4.getValueType(header);
+export async function createContext(filename: string, channels: Format.Context[], blockSize: number, isPeriodic: boolean): Promise<Data.Context> {
+    const header = channels[0].data.header;
+    const samplingCounts = getSamplingCounts(channels[0].data.header.extent, blockSize);
+    const valueType = Format.getValueType(header);
     const cubeBuffer = new Buffer(new ArrayBuffer(channels.length * blockSize * blockSize * blockSize * DataFormat.getValueByteSize(valueType)));
 
     const litteEndianCubeBuffer = File.IsNativeEndianLittle
@@ -61,9 +61,9 @@ export async function createContext(filename: string, channels: CCP4.Data[], blo
 
 export async function processData(ctx: Data.Context) {
     const channel = ctx.channels[0];
-    while (!channel.slices.isFinished) {
+    while (!channel.data.slices.isFinished) {
         for (const src of ctx.channels) {
-            await CCP4.readSlices(src);
+            await src.provider.readSlices(src.data);
         }
         await processSlices(ctx);
     }
@@ -149,7 +149,7 @@ function copyLayer(ctx: Data.Context, sliceIndex: number) {
     const targetOffset = blocks.slicesWritten * size;
 
     for (let channelIndex = 0; channelIndex < channels.length; channelIndex++) {
-        const src = channels[channelIndex].slices.values;
+        const src = channels[channelIndex].data.slices.values;
         const target = blocks.values[channelIndex];
         for (let i = 0; i < size; i++) {
             const v = src[srcOffset + i];
@@ -198,14 +198,14 @@ async function writeBlocks(ctx: Data.Context, isDataFinished: boolean) {
 
 async function processSlices(ctx: Data.Context) {
     const channel = ctx.channels[0];
-    const sliceCount = channel.slices.sliceCount;
+    const sliceCount = channel.data.slices.sliceCount;
     for (let i = 0; i < sliceCount; i++) {
         copyLayer(ctx, i);
         Downsampling.downsampleLayer(ctx);
 
         await writeBlocks(ctx, false);
 
-        const isDataFinished = i === sliceCount - 1 && channel.slices.isFinished;
+        const isDataFinished = i === sliceCount - 1 && channel.data.slices.isFinished;
         if (isDataFinished) {
             Downsampling.finalize(ctx);
             await writeBlocks(ctx, true);