extended FileHandle API and use SimpleBuffer

src/mol-io/common/file-handle.ts

@@ -1,35 +1,56 @@
 /**
- * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * @author Alexander Rose <alexander.rose@weirdbyte.de>
+ * @author David Sehnal <david.sehnal@gmail.com>
  */
 
-import { defaults } from 'mol-util';
+import { defaults, noop } from 'mol-util';
+import { SimpleBuffer } from './simple-buffer';
+// only import 'fs' in node.js
+const fs = typeof document === 'undefined' ? require('fs') as typeof import('fs') : void 0;
 
 export interface FileHandle {
-    /** The number of bytes in the file */
-    length: number
     /**
+     * Asynchronously reads data, returning buffer and number of bytes read
+     *
      * @param position The offset from the beginning of the file from which data should be read.
-     * @param sizeOrBuffer The buffer the data will be written to. If a number a buffer of that size will be created.
-     * @param size The number of bytes to read.
+     * @param sizeOrBuffer The buffer the data will be read from.
+     * @param length The number of bytes to read.
      * @param byteOffset The offset in the buffer at which to start writing.
      */
-    readBuffer(position: number, sizeOrBuffer: Uint8Array | number, size?: number, byteOffset?: number): Promise<{ bytesRead: number, buffer: Uint8Array }>
+    readBuffer(position: number, sizeOrBuffer: SimpleBuffer | number, length?: number, byteOffset?: number): Promise<{ bytesRead: number, buffer: SimpleBuffer }>
+
+    /**
+     * Asynchronously writes buffer, returning the number of bytes written.
+     *
+     * @param position — The offset from the beginning of the file where this data should be written.
+     * @param buffer - The buffer data to be written.
+     * @param length — The number of bytes to write. If not supplied, defaults to buffer.length - offset.
+     */
+    writeBuffer(position: number, buffer: SimpleBuffer, length?: number): Promise<number>
+
+    /**
+     * Synchronously writes buffer, returning the number of bytes written.
+     *
+     * @param position — The offset from the beginning of the file where this data should be written.
+     * @param buffer - The buffer data to be written.
+     * @param length — The number of bytes to write. If not supplied, defaults to buffer.length - offset.
+     */
+    writeBufferSync(position: number, buffer: SimpleBuffer, length?: number): number
+
+    /** Closes a file handle */
+    close(): void
 }
 
 export namespace FileHandle {
-    export function fromBuffer(buffer: Uint8Array): FileHandle {
+    export function fromBuffer(buffer: SimpleBuffer): FileHandle {
         return {
-            length: buffer.length,
-            readBuffer: (position: number, sizeOrBuffer: Uint8Array | number, size?: number, byteOffset?: number) => {
+            readBuffer: (position: number, sizeOrBuffer: SimpleBuffer | number, size?: number, byteOffset?: number) => {
                 if (typeof sizeOrBuffer === 'number') {
                     const start = position
                     const end = Math.min(buffer.length, start + (defaults(size, sizeOrBuffer)))
-                    return Promise.resolve({
-                        bytesRead: end - start,
-                        buffer: buffer.subarray(start, end),
-                    })
+                    return Promise.resolve({ bytesRead: end - start, buffer: SimpleBuffer.fromUint8Array(buffer.subarray(start, end)) })
                 } else {
                     if (size === void 0) {
                         return Promise.reject('readBuffer: Specify size.');
@@ -37,10 +58,68 @@ export namespace FileHandle {
                     const start = position
                     const end = Math.min(buffer.length, start + defaults(size, sizeOrBuffer.length))
                     sizeOrBuffer.set(buffer.subarray(start, end), byteOffset)
-                    return Promise.resolve({
-                        bytesRead: end - start,
-                        buffer: sizeOrBuffer,
+                    return Promise.resolve({ bytesRead: end - start, buffer: sizeOrBuffer })
+                }
+            },
+            writeBuffer: (position: number, buffer: SimpleBuffer, length?: number) => {
+                length = defaults(length, buffer.length)
+                console.warn('FileHandle.writeBuffer not implemented')
+                return Promise.resolve(0)
+            },
+            writeBufferSync: (position: number, buffer: SimpleBuffer, length?: number, ) => {
+                length = defaults(length, buffer.length)
+                console.warn('FileHandle.writeSync not implemented')
+                return 0
+            },
+            close: noop
+        }
+    }
+
+    export function fromDescriptor(file: number): FileHandle {
+        if (fs === undefined) throw new Error('fs module not available')
+        return {
+            readBuffer: (position: number, sizeOrBuffer: SimpleBuffer | number, length?: number, byteOffset?: number) => {
+                return new Promise((res, rej) => {
+                    if (typeof sizeOrBuffer === 'number') {
+                        let buff = new Buffer(new ArrayBuffer(sizeOrBuffer));
+                        fs.read(file, buff, 0, sizeOrBuffer, position, (err, bytesRead, buffer) => {
+                            if (err) {
+                                rej(err);
+                                return;
+                            }
+                            res({ bytesRead, buffer });
+                        });
+                    } else {
+                        if (length === void 0) {
+                            rej('readBuffer: Specify size.');
+                            return;
+                        }
+                        fs.read(file, sizeOrBuffer, byteOffset ? +byteOffset : 0, length, position, (err, bytesRead, buffer) => {
+                            if (err) {
+                                rej(err);
+                                return;
+                            }
+                            res({ bytesRead, buffer });
+                        });
+                    }
+                })
+            },
+            writeBuffer: (position: number, buffer: Buffer, length?: number) => {
+                return new Promise<number>((res, rej) => {
+                    fs.write(file, buffer, 0, length !== void 0 ? length : buffer.length, position, (err, written) => {
+                        if (err) rej(err);
+                        else res(written);
                     })
+                })
+            },
+            writeBufferSync: (position: number, buffer: Uint8Array, length?: number) => {
+                return fs.writeSync(file, buffer, 0, length, position);
+            },
+            close: () => {
+                try {
+                    if (file !== void 0) fs.close(file, noop);
+                } catch (e) {
+
                 }
             }
         }

src/mol-io/reader/ccp4/parser.ts

@@ -8,15 +8,12 @@ import { Task, RuntimeContext } from 'mol-task';
 import { Ccp4File, Ccp4Header } from './schema'
 import { ReaderResult as Result } from '../result'
 import { FileHandle } from '../../common/file-handle';
+import { SimpleBuffer } from 'mol-io/common/simple-buffer';
 
-async function parseInternal(file: FileHandle, ctx: RuntimeContext): Promise<Result<Ccp4File>> {
-    await ctx.update({ message: 'Parsing CCP4/MRC file...' });
-
-    const { buffer } = await file.readBuffer(0, file.length)
+export async function readCcp4Header(file: FileHandle) {
+    const headerSize = 1024;
+    const { buffer } = await file.readBuffer(0, headerSize)
     const bin = buffer.buffer
-
-    const intView = new Int32Array(bin, 0, 56)
-    const floatView = new Float32Array(bin, 0, 56)
     const dv = new DataView(bin)
 
     // 53  MAP         Character string 'MAP ' to identify file type
@@ -25,88 +22,121 @@ async function parseInternal(file: FileHandle, ctx: RuntimeContext): Promise<Res
         dv.getUint8(52 * 4 + 2), dv.getUint8(52 * 4 + 3)
     )
     if (MAP !== 'MAP ') {
-        return Result.error('ccp4 format error, missing "MAP " string');
+        throw new Error('ccp4 format error, missing "MAP " string');
     }
 
     // 54  MACHST      Machine stamp indicating machine type which wrote file
     //                 17 and 17 for big-endian or 68 and 65 for little-endian
     const MACHST = [ dv.getUint8(53 * 4), dv.getUint8(53 * 4 + 1) ]
+    let littleEndian = true
     // found MRC files that don't have the MACHST stamp set and are big-endian
     if (MACHST[0] !== 68 && MACHST[1] !== 65) {
-        // flip byte order in-place
-        for (let i = 0, il = bin.byteLength; i < il; i += 4) {
-            dv.setFloat32(i, dv.getFloat32(i), true)
-        }
+        littleEndian = false;
     }
 
+    const readInt = (o: number) => dv.getInt32(o * 4, littleEndian)
+    const readFloat = (o: number) => dv.getFloat32(o * 4, littleEndian)
+
     const header: Ccp4Header = {
-        NC: intView[0],
-        NR: intView[1],
-        NS: intView[2],
+        NC: readInt(0),
+        NR: readInt(1),
+        NS: readInt(2),
 
-        MODE: intView[3],
+        MODE: readInt(3),
 
-        NCSTART: intView[4],
-        NRSTART: intView[5],
-        NSSTART: intView[6],
+        NCSTART: readInt(4),
+        NRSTART: readInt(5),
+        NSSTART: readInt(6),
 
-        NX: intView[7],
-        NY: intView[8],
-        NZ: intView[9],
+        NX: readInt(7),
+        NY: readInt(8),
+        NZ: readInt(9),
 
-        xLength: floatView[10],
-        yLength: floatView[11],
-        zLength: floatView[12],
+        xLength: readFloat(10),
+        yLength: readFloat(11),
+        zLength: readFloat(12),
 
-        alpha: floatView[13],
-        beta: floatView[14],
-        gamma: floatView[15],
+        alpha: readFloat(13),
+        beta: readFloat(14),
+        gamma: readFloat(15),
 
-        MAPC: intView[16],
-        MAPR: intView[17],
-        MAPS: intView[18],
+        MAPC: readInt(16),
+        MAPR: readInt(17),
+        MAPS: readInt(18),
 
-        AMIN: floatView[19],
-        AMAX: floatView[20],
-        AMEAN: floatView[21],
+        AMIN: readFloat(19),
+        AMAX: readFloat(20),
+        AMEAN: readFloat(21),
 
-        ISPG: intView[22],
+        ISPG: readInt(22),
 
-        NSYMBT: intView[23],
+        NSYMBT: readInt(23),
 
-        LSKFLG: intView[24],
+        LSKFLG: readInt(24),
 
         SKWMAT: [], // TODO bytes 26-34
         SKWTRN: [], // TODO bytes 35-37
 
+        userFlag1: readInt(39),
+        userFlag2: readInt(40),
+
         // bytes 50-52 origin in X,Y,Z used for transforms
-        originX: floatView[49],
-        originY: floatView[50],
-        originZ: floatView[51],
+        originX: readFloat(49),
+        originY: readFloat(50),
+        originZ: readFloat(51),
 
         MAP, // bytes 53 MAP
         MACHST, // bytes 54 MACHST
 
-        ARMS: floatView[54],
+        ARMS: readFloat(54),
 
         // TODO bytes 56 NLABL
         // TODO bytes 57-256 LABEL
     }
 
+    return { header, littleEndian }
+}
+
+function getElementByteSize(mode: number) {
+    if (mode === 2) {
+        return 4
+    } else if (mode === 1) {
+        return 2
+    } else if (mode === 0) {
+        return 1
+    } else {
+        throw new Error(`ccp4 mode '${mode}' unsupported`);
+    }
+}
+
+async function parseInternal(file: FileHandle, size: number, ctx: RuntimeContext): Promise<Ccp4File> {
+    await ctx.update({ message: 'Parsing CCP4/MRC file...' });
+
+    const { header, littleEndian } = await readCcp4Header(file)
+
     const offset = 256 * 4 + header.NSYMBT
     const count = header.NC * header.NR * header.NS
+    const elementByteSize = getElementByteSize(header.MODE)
+    const byteCount = count * elementByteSize
+
+    const { buffer } = await file.readBuffer(offset, size)
+
     let values
     if (header.MODE === 2) {
-        values = new Float32Array(bin, offset, count)
+        values = new Float32Array(buffer, offset, count)
     } else if (header.MODE === 0) {
-        values = new Int8Array(bin, offset, count)
+        values = new Int8Array(buffer, offset, count)
     } else {
-        return Result.error(`ccp4 mode '${header.MODE}' unsupported`);
+        throw new Error(`ccp4 mode '${header.MODE}' unsupported`);
+    }
+
+    if (!littleEndian) {
+        SimpleBuffer.flipByteOrder(buffer, new Uint8Array(values.buffer), byteCount, elementByteSize, 0)
     }
 
     // if the file was converted by mapmode2to0 - scale the data
     // based on uglymol (https://github.com/uglymol/uglymol) by Marcin Wojdyr (wojdyr)
-    if (intView[39] === -128 && intView[40] === 127) {
+    if (header.userFlag1 -128 && header.userFlag2 === 127) {
         values = new Float32Array(values)
         // scaling f(x)=b1*x+b0 such that f(-128)=min and f(127)=max
         const b1 = (header.AMAX - header.AMIN) / 255.0
@@ -117,13 +147,19 @@ async function parseInternal(file: FileHandle, ctx: RuntimeContext): Promise<Res
     }
 
     const result: Ccp4File = { header, values };
-    return Result.success(result);
+    return result
 }
 
-export function parseFile(file: FileHandle) {
-    return Task.create<Result<Ccp4File>>('Parse CCP4/MRC', ctx => parseInternal(file, ctx));
+export function parseFile(file: FileHandle, size: number) {
+    return Task.create<Result<Ccp4File>>('Parse CCP4/MRC', async ctx => {
+        try {
+            return Result.success(await parseInternal(file, size, ctx));
+        } catch (e) {
+            return Result.error(e);
+        }
+    })
 }
 
 export function parse(buffer: Uint8Array) {
-    return parseFile(FileHandle.fromBuffer(buffer))
+    return parseFile(FileHandle.fromBuffer(SimpleBuffer.fromUint8Array(buffer)), buffer.length)
 }

src/mol-io/reader/ccp4/schema.ts

@@ -81,6 +81,9 @@ export interface Ccp4Header {
      * May be used in CCP4 but not in MRC
      */
     SKWTRN: number[]
+    /** see https://github.com/uglymol/uglymol/blob/master/tools/mapmode2to0#L69 */
+    userFlag1: number,
+    userFlag2: number,
     /** x axis origin transformation (not used in CCP4) */
     originX: number
     /** y axis origin transformation (not used in CCP4) */

src/mol-io/reader/dsn6/parser.ts

@@ -8,6 +8,7 @@ import { Task, RuntimeContext } from 'mol-task';
 import { Dsn6File, Dsn6Header } from './schema'
 import { ReaderResult as Result } from '../result'
 import { FileHandle } from '../../common/file-handle';
+import { SimpleBuffer } from 'mol-io/common/simple-buffer';
 
 function parseBrixHeader(str: string): Dsn6Header {
     return {
@@ -56,10 +57,10 @@ function parseDsn6Header(int: Int16Array): Dsn6Header {
     }
 }
 
-async function parseInternal(file: FileHandle, ctx: RuntimeContext): Promise<Result<Dsn6File>> {
+async function parseInternal(file: FileHandle, size: number, ctx: RuntimeContext): Promise<Dsn6File> {
     await ctx.update({ message: 'Parsing DSN6/BRIX file...' });
 
-    const { buffer } = await file.readBuffer(0, file.length)
+    const { buffer } = await file.readBuffer(0, size)
     const bin = buffer.buffer
 
     const intView = new Int16Array(bin)
@@ -115,13 +116,19 @@ async function parseInternal(file: FileHandle, ctx: RuntimeContext): Promise<Res
     }
 
     const result: Dsn6File = { header, values };
-    return Result.success(result);
+    return result;
 }
 
-export function parseFile(file: FileHandle) {
-    return Task.create<Result<Dsn6File>>('Parse DSN6/BRIX', ctx => parseInternal(file, ctx));
+export function parseFile(file: FileHandle, size: number) {
+    return Task.create<Result<Dsn6File>>('Parse DSN6/BRIX', async ctx => {
+        try {
+            return Result.success(await parseInternal(file, size, ctx));
+        } catch (e) {
+            return Result.error(e);
+        }
+    })
 }
 
 export function parse(buffer: Uint8Array) {
-    return parseFile(FileHandle.fromBuffer(buffer))
+    return parseFile(FileHandle.fromBuffer(SimpleBuffer.fromUint8Array(buffer)), buffer.length)
 }

webpack.config.js

@@ -62,6 +62,7 @@ const sharedConfig = {
 
 function createEntryPoint(name, dir, out) {
     return {
+        node: { fs: 'empty' }, // TODO find better solution? Currently used in file-handle.ts
         entry: path.resolve(__dirname, `build/src/${dir}/${name}.js`),
         output: { filename: `${name}.js`, path: path.resolve(__dirname, `build/${out}`) },
         ...sharedConfig