wip, dsn6 support for volume server packing

src/mol-io/reader/ccp4/parser.ts

@@ -9,7 +9,7 @@ import { Ccp4File, Ccp4Header } from './schema'
 import { ReaderResult as Result } from '../result'
 import { FileHandle } from '../../common/file-handle';
 import { SimpleBuffer } from 'mol-io/common/simple-buffer';
-import { TypedArrayValueType, getElementByteSize, makeTypedArray } from 'mol-io/common/typed-array';
+import { TypedArrayValueType, getElementByteSize, makeTypedArray, TypedArrayBufferContext, readTypedArray } from 'mol-io/common/typed-array';
 
 export async function readCcp4Header(file: FileHandle): Promise<{ header: Ccp4Header, littleEndian: boolean }> {
     const headerSize = 1024;
@@ -96,6 +96,11 @@ export async function readCcp4Header(file: FileHandle): Promise<{ header: Ccp4He
     return { header, littleEndian }
 }
 
+export async function readCcp4Slices(buffer: TypedArrayBufferContext, file: FileHandle, byteOffset: number, length: number, littleEndian: boolean) {
+    // TODO support data from mapmode2to0, see below
+    await readTypedArray(buffer, file, byteOffset, length, 0, littleEndian);
+}
+
 function getTypedArrayValueType(mode: number) {
     switch (mode) {
         case 2: return TypedArrayValueType.Float32

src/mol-io/reader/dsn6/parser.ts

@@ -69,33 +69,12 @@ export async function readDsn6Header(file: FileHandle): Promise<{ header: Dsn6He
     return { header, littleEndian }
 }
 
-async function parseInternal(file: FileHandle, size: number, ctx: RuntimeContext): Promise<Dsn6File> {
-    await ctx.update({ message: 'Parsing DSN6/BRIX file...' });
-
-    const { header, littleEndian } = await readDsn6Header(file)
-    const { divisor, summand } = header
-
-    const { buffer, bytesRead } = await file.readBuffer(dsn6HeaderSize, size - dsn6HeaderSize)
+export async function parseDsn6Values(header: Dsn6Header, source: Uint8Array, target: Float32Array) {
+    const { divisor, summand, xExtent, yExtent, zExtent } = header
 
-    const xBlocks = Math.ceil(header.xExtent / 8)
-    const yBlocks = Math.ceil(header.yExtent / 8)
-    const zBlocks = Math.ceil(header.zExtent / 8)
-    const valueCount = header.xExtent * header.yExtent * header.zExtent
-
-    const count = xBlocks * 8 * yBlocks * 8 * zBlocks * 8
-    const elementByteSize = 1
-    const byteCount = count * elementByteSize
-
-    if (byteCount !== bytesRead) {
-        console.warn(`byteCount ${byteCount} and bytesRead ${bytesRead} differ`)
-    }
-
-    const values = new Float32Array(valueCount)
-
-    if (!littleEndian) {
-        // even though the values are one byte they need to be swapped like they are 2
-        SimpleBuffer.flipByteOrderInPlace2(buffer.buffer)
-    }
+    const xBlocks = Math.ceil(xExtent / 8)
+    const yBlocks = Math.ceil(yExtent / 8)
+    const zBlocks = Math.ceil(zExtent / 8)
 
     let offset = 0
     // loop over blocks
@@ -110,9 +89,9 @@ async function parseInternal(file: FileHandle, size: number, ctx: RuntimeContext
                         for (let i = 0; i < 8; ++i) {
                             const x = 8 * xx + i
                             // check if remaining slice-part contains values
-                            if (x < header.xExtent && y < header.yExtent && z < header.zExtent) {
-                                const idx = ((((x * header.yExtent) + y) * header.zExtent) + z)
-                                values[idx] = (buffer[offset] - summand) / divisor
+                            if (x < xExtent && y < yExtent && z < zExtent) {
+                                const idx = ((((x * yExtent) + y) * zExtent) + z)
+                                target[idx] = (source[offset] - summand) / divisor
                                 ++offset
                             } else {
                                 offset += 8 - i
@@ -124,6 +103,37 @@ async function parseInternal(file: FileHandle, size: number, ctx: RuntimeContext
             }
         }
     }
+}
+
+async function parseInternal(file: FileHandle, size: number, ctx: RuntimeContext): Promise<Dsn6File> {
+    await ctx.update({ message: 'Parsing DSN6/BRIX file...' });
+
+    const { header, littleEndian } = await readDsn6Header(file)
+    const { xExtent, yExtent, zExtent } = header
+
+    const { buffer, bytesRead } = await file.readBuffer(dsn6HeaderSize, size - dsn6HeaderSize)
+
+    const xBlocks = Math.ceil(xExtent / 8)
+    const yBlocks = Math.ceil(yExtent / 8)
+    const zBlocks = Math.ceil(zExtent / 8)
+    const valueCount = xExtent * yExtent * zExtent
+
+    const count = xBlocks * 8 * yBlocks * 8 * zBlocks * 8
+    const elementByteSize = 1
+    const byteCount = count * elementByteSize
+
+    if (byteCount !== bytesRead) {
+        console.warn(`byteCount ${byteCount} and bytesRead ${bytesRead} differ`)
+    }
+
+    const values = new Float32Array(valueCount)
+
+    if (!littleEndian) {
+        // even though the values are one byte they need to be swapped like they are 2
+        SimpleBuffer.flipByteOrderInPlace2(buffer.buffer)
+    }
+
+    await parseDsn6Values(header, buffer, values)
 
     const result: Dsn6File = { header, values };
     return result;

src/servers/volume/pack.ts

@@ -9,13 +9,30 @@
 import pack from './pack/main'
 import VERSION from './pack/version'
 
-let config = {
-    input: <{ name: string, filename: string }[]>[],
+interface Config {
+    input: { name: string, filename: string }[],
+    format: 'ccp4' | 'dsn6',
+    isPeriodic: boolean,
+    outputFilename: string,
+    blockSize: number
+}
+
+let config: Config = {
+    input: [],
+    format: 'ccp4',
     isPeriodic: false,
     outputFilename: '',
     blockSize: 96
 };
 
+function getFormat(format: string): Config['format'] {
+    switch (format.toLowerCase()) {
+        case 'ccp4': return 'ccp4'
+        case 'dsn6': return 'dsn6'
+    }
+    throw new Error(`unsupported format '${format}'`)
+}
+
 function printHelp() {
     let help = [
         `VolumeServer Packer ${VERSION}, (c) 2016 - now, David Sehnal`,
@@ -52,6 +69,9 @@ function parseInput() {
             case '-blocksize':
                 config.blockSize = +process.argv[++i];
                 break;
+            case '-format':
+                config.format = getFormat(process.argv[++i]);
+                break;
             case '-xray':
                 input = true;
                 config.input = [
@@ -82,5 +102,5 @@ function parseInput() {
 }
 
 if (parseInput()) {
-    pack(config.input, config.blockSize, config.isPeriodic, config.outputFilename);
+    pack(config.input, config.blockSize, config.isPeriodic, config.outputFilename, config.format);
 }

src/servers/volume/pack/format.ts

@@ -9,6 +9,7 @@ import * as File from '../common/file'
 import { FileHandle } from 'mol-io/common/file-handle';
 import { Ccp4Provider } from './format/ccp4';
 import { TypedArrayBufferContext, TypedArrayValueArray, TypedArrayValueType, getElementByteSize, createTypedArrayBufferContext } from 'mol-io/common/typed-array';
+import { Dsn6Provider } from './format/dsn6';
 
 export interface Header {
     name: string,
@@ -22,6 +23,7 @@ export interface Header {
     cellAngles: number[],
     littleEndian: boolean,
     dataOffset: number
+    originalHeader: unknown // TODO
 }
 
 /** Represents a circular buffer for 2 * blockSize layers */
@@ -86,16 +88,16 @@ export function compareHeaders(a: Header, b: Header) {
     return true;
 }
 
-export type Type = 'ccp4' // | 'dsn6'
+export type Type = 'ccp4' | 'dsn6'
 
 export function getProviderFromType(type: Type): Provider {
     switch (type) {
         case 'ccp4': return Ccp4Provider
-        // case 'dsn6': return Dsn6Provider
+        case 'dsn6': return Dsn6Provider
     }
 }
 
-export async function open(name: string, filename: string, type: Type = 'ccp4'): Promise<Context> {
+export async function open(name: string, filename: string, type: Type): Promise<Context> {
     const provider = getProviderFromType(type)
     const descriptor = await File.openRead(filename);
     const file = FileHandle.fromDescriptor(descriptor)

src/servers/volume/pack/format/ccp4.ts

@@ -6,9 +6,8 @@
  */
 
 import { FileHandle } from 'mol-io/common/file-handle';
-import { readCcp4Header } from 'mol-io/reader/ccp4/parser';
-import { Header, Provider } from '../format';
-import { readSlices } from './common';
+import { readCcp4Header, readCcp4Slices } from 'mol-io/reader/ccp4/parser';
+import { Header, Provider, Data } from '../format';
 import { TypedArrayValueType } from 'mol-io/common/typed-array';
 
 function getTypedArrayValueType(mode: number) {
@@ -36,11 +35,33 @@ async function readHeader(name: string, file: FileHandle) {
         cellSize: [ccp4Header.xLength, ccp4Header.yLength, ccp4Header.zLength],
         cellAngles: [ccp4Header.alpha, ccp4Header.beta, ccp4Header.gamma],
         littleEndian,
-        dataOffset: 256 * 4 + ccp4Header.NSYMBT /* symBytes */
+        dataOffset: 256 * 4 + ccp4Header.NSYMBT, /* symBytes */
+        originalHeader: ccp4Header
     };
     // "normalize" the grid axis order
     header.grid = [header.grid[header.axisOrder[0]], header.grid[header.axisOrder[1]], header.grid[header.axisOrder[2]]];
     return header;
 }
 
+export async function readSlices(data: Data) {
+    const { slices, header } = data;
+    if (slices.isFinished) {
+        return;
+    }
+
+    const { extent } = header;
+    const sliceSize = extent[0] * extent[1];
+    const sliceByteOffset = slices.buffer.elementByteSize * sliceSize * slices.slicesRead;
+    const sliceCount = Math.min(slices.sliceCapacity, extent[2] - slices.slicesRead);
+    const sliceByteCount = sliceCount * sliceSize;
+
+    await readCcp4Slices(slices.buffer, data.file, header.dataOffset + sliceByteOffset, sliceByteCount, header.littleEndian);
+    slices.slicesRead += sliceCount;
+    slices.sliceCount = sliceCount;
+
+    if (slices.slicesRead >= extent[2]) {
+        slices.isFinished = true;
+    }
+}
+
 export const Ccp4Provider: Provider = { readHeader, readSlices }

src/servers/volume/pack/format/dsn6.ts

@@ -5,10 +5,10 @@
  */
 
 import { FileHandle } from 'mol-io/common/file-handle';
-import { Header, Provider } from '../format';
-import { readSlices } from './common';
-import { readDsn6Header, dsn6HeaderSize } from 'mol-io/reader/dsn6/parser';
+import { Header, Provider, Data } from '../format';
+import { readDsn6Header, dsn6HeaderSize, parseDsn6Values } from 'mol-io/reader/dsn6/parser';
 import { TypedArrayValueType } from 'mol-io/common/typed-array';
+import { Dsn6Header } from 'mol-io/reader/dsn6/schema';
 
 async function readHeader(name: string, file: FileHandle) {
     const { header: dsn6Header, littleEndian } = await readDsn6Header(file)
@@ -20,13 +20,44 @@ async function readHeader(name: string, file: FileHandle) {
         axisOrder: [0, 1, 2],
         extent: [dsn6Header.xExtent, dsn6Header.yExtent, dsn6Header.zExtent],
         origin: [dsn6Header.xStart, dsn6Header.yStart, dsn6Header.zStart],
-        spacegroupNumber: 0, // P 1
+        spacegroupNumber: 1, // P 1
         cellSize: [dsn6Header.xlen, dsn6Header.ylen, dsn6Header.zlen],
         cellAngles: [dsn6Header.alpha, dsn6Header.beta, dsn6Header.gamma],
         littleEndian,
-        dataOffset: dsn6HeaderSize
+        dataOffset: dsn6HeaderSize,
+        originalHeader: dsn6Header
     };
     return header;
 }
 
+export async function readSlices(data: Data) {
+    // TODO due to the dsn6 data layout, the file must be read a a whole, need check if the file is too big for that
+
+    const { slices, header, file } = data;
+    if (slices.isFinished) {
+        return;
+    }
+
+    const { extent, originalHeader } = header;
+    const sliceCount = extent[2]
+
+    const { xExtent, yExtent, zExtent } = originalHeader as Dsn6Header
+    const xBlocks = Math.ceil(xExtent / 8)
+    const yBlocks = Math.ceil(yExtent / 8)
+    const zBlocks = Math.ceil(zExtent / 8)
+    const count = xBlocks * 8 * yBlocks * 8 * zBlocks * 8
+    const elementByteSize = 1
+    const byteCount = count * elementByteSize
+
+    const { buffer } = await file.readBuffer(dsn6HeaderSize, byteCount)
+    await parseDsn6Values(originalHeader as Dsn6Header, buffer, slices.values as Float32Array) // TODO fix cast
+
+    slices.slicesRead += sliceCount;
+    slices.sliceCount = sliceCount;
+
+    if (slices.slicesRead >= extent[2]) {
+        slices.isFinished = true;
+    }
+}
+
 export const Dsn6Provider: Provider = { readHeader, readSlices }

src/servers/volume/pack/main.ts

@@ -13,9 +13,9 @@ import * as Sampling from './sampling'
 import * as DataFormat from '../common/data-format'
 import { FileHandle } from 'mol-io/common/file-handle';
 
-export default async function pack(input: { name: string, filename: string }[], blockSize: number, isPeriodic: boolean, outputFilename: string) {
+export default async function pack(input: { name: string, filename: string }[], blockSize: number, isPeriodic: boolean, outputFilename: string, format: Format.Type) {
     try {
-        await create(outputFilename, input, blockSize, isPeriodic);
+        await create(outputFilename, input, blockSize, isPeriodic, format);
     } catch (e) {
         console.error('[Error] ' + e);
     }
@@ -69,7 +69,7 @@ async function writeHeader(ctx: Data.Context) {
     await ctx.file.writeBuffer(4, header);
 }
 
-async function create(filename: string, sourceDensities: { name: string, filename: string }[], sourceBlockSize: number, isPeriodic: boolean) {
+async function create(filename: string, sourceDensities: { name: string, filename: string }[], sourceBlockSize: number, isPeriodic: boolean, format: Format.Type) {
     const startedTime = getTime();
 
     if (sourceBlockSize % 4 !== 0 || sourceBlockSize < 4) {
@@ -80,13 +80,13 @@ async function create(filename: string, sourceDensities: { name: string, filenam
         throw Error('Specify at least one source density.');
     }
 
-    process.stdout.write('Initializing... ');
+    process.stdout.write(`Initializing using ${format} format...`);
     const files: FileHandle[] = [];
     try {
         // Step 1a: Read the Format headers
         const channels: Format.Context[] = [];
         for (const s of sourceDensities) {
-            channels.push(await Format.open(s.name, s.filename));
+            channels.push(await Format.open(s.name, s.filename, format));
         }
         // Step 1b: Check if the Format headers are compatible.
         const isOk = channels.reduce((ok, s) => ok && Format.compareHeaders(channels[0].data.header, s.data.header), true);