Merge branch 'master' of https://github.com/molstar/molstar-proto into plugin

data/rcsb-graphql/codegen.js

@@ -5,17 +5,16 @@ const basePath = path.join(__dirname, '..', '..', 'src', 'mol-model-props', 'rcs
 
 generate({
     schema: 'http://rest-dev.rcsb.org/graphql',
-    documents: [
-        path.join(basePath, 'symmetry.gql.ts')
-    ],
+    documents: {
+        [path.join(basePath, 'symmetry.gql.ts')]: {
+            loader: path.join(__dirname, 'loader.js')
+        },
+    },
     generates: {
         [path.join(basePath, 'types.ts')]: {
             plugins: ['time', 'typescript-common', 'typescript-client']
         }
     },
-    // template: 'graphql-codegen-typescript-template',
-    // out: path.join(basePath),
-    // skipSchema: true,
     overwrite: true,
     config: path.join(__dirname, 'codegen.json')
 }, true).then(

data/rcsb-graphql/loader.js

@@ -0,0 +1,14 @@
+const { parse } = require('graphql');
+const { readFileSync } = require('fs');
+
+module.exports = function(docString, config) {
+    const str = readFileSync(docString, { encoding: 'utf-8' }).trim()
+                    .replace(/^export default `/, '')
+                    .replace(/`$/, '')
+    return [
+        {
+            filePath: docString,
+            content: parse(str)
+        }
+    ];
+};

package.json

@@ -75,16 +75,17 @@
   "author": "",
   "license": "MIT",
   "devDependencies": {
-    "@types/argparse": "^1.0.35",
+    "@types/argparse": "^1.0.36",
     "@types/benchmark": "^1.0.31",
     "@types/compression": "0.0.36",
     "@types/express": "^4.16.1",
-    "@types/jest": "^24.0.6",
-    "@types/node": "^11.9.4",
+    "@types/jest": "^24.0.9",
+    "@types/node": "^11.9.6",
     "@types/node-fetch": "^2.1.6",
-    "@types/react": "^16.8.4",
+    "@types/react": "^16.8.6",
     "@types/react-dom": "^16.8.2",
     "@types/webgl2": "0.0.4",
+    "@types/swagger-ui-dist": "3.0.0",
     "benchmark": "^2.1.4",
     "circular-dependency-plugin": "^5.0.2",
     "concurrently": "^4.1.0",
@@ -95,9 +96,9 @@
     "glslify": "^7.0.0",
     "glslify-import": "^3.1.0",
     "glslify-loader": "^2.0.0",
-    "graphql-code-generator": "^0.16.1",
-    "graphql-codegen-time": "^0.16.1",
-    "graphql-codegen-typescript-template": "^0.16.1",
+    "graphql-code-generator": "^0.18.0",
+    "graphql-codegen-time": "^0.18.0",
+    "graphql-codegen-typescript-template": "^0.18.0",
     "jest": "^24.1.0",
     "jest-raw-loader": "^1.0.1",
     "mini-css-extract-plugin": "^0.5.0",
@@ -107,11 +108,11 @@
     "sass-loader": "^7.1.0",
     "style-loader": "^0.23.1",
     "ts-jest": "^24.0.0",
-    "tslint": "^5.12.1",
+    "tslint": "^5.13.1",
     "typescript": "^3.3.3",
     "uglify-js": "^3.4.9",
     "util.promisify": "^1.0.0",
-    "webpack": "^4.29.5",
+    "webpack": "^4.29.6",
     "webpack-cli": "^3.2.3"
   },
   "dependencies": {
@@ -121,8 +122,9 @@
     "graphql": "^14.1.1",
     "immutable": "^3.8.2",
     "node-fetch": "^2.3.0",
-    "react": "^16.8.2",
-    "react-dom": "^16.8.2",
-    "rxjs": "^6.4.0"
+    "react": "^16.8.3",
+    "react-dom": "^16.8.3",
+    "rxjs": "^6.4.0",
+    "swagger-ui-dist": "^3.20.9"
   }
 }

src/mol-io/reader/cif/schema/bird.ts

@@ -1,7 +1,7 @@
 /**
  * Copyright (c) 2017-2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
- * Code-generated 'BIRD' schema file. Dictionary versions: mmCIF 5.304, IHM 0.139, CARB draft.
+ * Code-generated 'BIRD' schema file. Dictionary versions: mmCIF 5.305, IHM 0.139, CARB draft.
  *
  * @author mol-star package (src/apps/schema-generator/generate)
  */

src/mol-io/reader/cif/schema/ccd.ts

@@ -1,7 +1,7 @@
 /**
  * Copyright (c) 2017-2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
- * Code-generated 'CCD' schema file. Dictionary versions: mmCIF 5.304, IHM 0.139, CARB draft.
+ * Code-generated 'CCD' schema file. Dictionary versions: mmCIF 5.305, IHM 0.139, CARB draft.
  *
  * @author mol-star package (src/apps/schema-generator/generate)
  */

src/mol-io/reader/cif/schema/mmcif.ts

@@ -1,7 +1,7 @@
 /**
  * Copyright (c) 2017-2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
- * Code-generated 'mmCIF' schema file. Dictionary versions: mmCIF 5.304, IHM 0.139, CARB draft.
+ * Code-generated 'mmCIF' schema file. Dictionary versions: mmCIF 5.305, IHM 0.139, CARB draft.
  *
  * @author mol-star package (src/apps/schema-generator/generate)
  */
@@ -850,7 +850,7 @@ export const mmCIF_Schema = {
          * This data item is a pointer to _struct_conn_type.id in the
          * STRUCT_CONN_TYPE category.
          */
-        conn_type_id: Aliased<'covale' | 'disulf' | 'hydrog' | 'metalc' | 'mismat' | 'saltbr' | 'modres' | 'covale_base' | 'covale_sugar' | 'covale_phosphate'>(str),
+        conn_type_id: Aliased<'covale' | 'disulf' | 'metalc' | 'hydrog'>(str),
         /**
          * A description of special aspects of the connection.
          */

src/mol-model-formats/structure/mmcif/bonds/struct_conn.ts

@@ -138,8 +138,6 @@ export namespace StructConn {
         partners: { residueIndex: ResidueIndex, atomIndex: ElementIndex, symmetry: string }[]
     }
 
-    type StructConnType = typeof mmCIF_Schema.struct_conn.conn_type_id.T
-
     export function attachFromMmCif(model: Model): boolean {
         if (model.customProperties.has(Descriptor)) return true;
         if (model.sourceData.kind !== 'mmCIF') return false;
@@ -213,7 +211,7 @@ export namespace StructConn {
             const partners = _ps(i);
             if (partners.length < 2) continue;
 
-            const type = conn_type_id.value(i)! as StructConnType;
+            const type = conn_type_id.value(i) as typeof mmCIF_Schema.struct_conn_type.id.T; // TODO workaround for dictionary inconsistency
             const orderType = (pdbx_value_order.value(i) || '').toLowerCase();
             let flags = LinkType.Flag.None;
             let order = 1;
@@ -234,7 +232,10 @@ export namespace StructConn {
                     flags = LinkType.Flag.Covalent;
                     break;
                 case 'disulf': flags = LinkType.Flag.Covalent | LinkType.Flag.Sulfide; break;
-                case 'hydrog': flags = LinkType.Flag.Hydrogen; break;
+                case 'hydrog':
+                case 'mismat':
+                    flags = LinkType.Flag.Hydrogen;
+                    break;
                 case 'metalc': flags = LinkType.Flag.MetallicCoordination; break;
                 case 'saltbr': flags = LinkType.Flag.Ionic; break;
             }

src/mol-model-props/rcsb/graphql/symmetry.gql.ts

@@ -1,8 +1,4 @@
- // workaround so the query gets found by the codegen
-function gql (strs: TemplateStringsArray) { return strs.raw.join('') }
-
-export default
-gql`query AssemblySymmetry($pdbId: String!) {
+export default `query AssemblySymmetry($pdbId: String!) {
     assemblies(pdbId: $pdbId) {
         pdbx_struct_assembly {
             id

src/mol-model-props/rcsb/graphql/types.ts

@@ -1,4 +1,4 @@
-// Generated in 2019-01-30T16:38:09-08:00
+// Generated in 2019-03-01T14:48:33-08:00
 export type Maybe<T> = T | null;
 
 /** Built-in scalar representing an instant in time */

src/mol-util/string.ts

@@ -1,5 +1,5 @@
 /**
- * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
@@ -46,4 +46,10 @@ export function substringStartsWith(str: string, start: number, end: number, tar
         if (str.charCodeAt(start + i) !== target.charCodeAt(i)) return false;
     }
     return true;
+}
+
+export function interpolate(str: string, params: { [k: string]: any }) {
+    const names = Object.keys(params);
+    const values = Object.values(params);
+    return new Function(...names, `return \`${str}\`;`)(...values);
 }

src/servers/common/swagger-ui/index.ts

@@ -0,0 +1,37 @@
+/**
+ * Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
+ */
+
+import * as express from 'express'
+import * as fs from 'fs'
+import { getAbsoluteFSPath } from 'swagger-ui-dist'
+import { ServeStaticOptions } from 'serve-static';
+import { interpolate } from 'mol-util/string';
+
+export function swaggerUiAssetsHandler(options?: ServeStaticOptions) {
+    const opts = options || {}
+    opts.index = false
+    return express.static(getAbsoluteFSPath(), opts)
+}
+
+export interface SwaggerUIOptions {
+    openapiJsonUrl: string
+    apiPrefix: string
+    title: string
+    shortcutIconLink: string
+}
+
+function createHTML(options: SwaggerUIOptions) {
+    const htmlTemplate = fs.readFileSync(`${__dirname}/indexTemplate.html`).toString()
+    return interpolate(htmlTemplate, options)
+}
+
+export function swaggerUiIndexHandler(options: SwaggerUIOptions): express.Handler {
+    const html = createHTML(options)
+    return (req: express.Request, res: express.Response) => {
+        res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
+        res.end(html);
+    }
+}

src/servers/common/swagger-ui/indexTemplate.html

@@ -0,0 +1,66 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="UTF-8">
+        <title>${title}</title>
+        <link rel="stylesheet" type="text/css" href="${apiPrefix}/swagger-ui.css" >
+        ${shortcutIconLink}
+
+        <style>
+            html
+            {
+                box-sizing: border-box;
+                overflow: -moz-scrollbars-vertical;
+                overflow-y: scroll;
+            }
+            *,
+            *:before,
+            *:after
+            {
+                box-sizing: inherit;
+            }
+            body
+            {
+                margin:0;
+                background: #fafafa;
+            }
+        </style>
+    </head>
+
+    <body>
+        <div id="swagger-ui"></div>
+
+        <script src="${apiPrefix}/swagger-ui-bundle.js"> </script>
+        <script src="${apiPrefix}/swagger-ui-standalone-preset.js"> </script>
+        <script>
+            function HidePlugin() {
+                // this plugin overrides some components to return nothing
+                return {
+                    components: {
+                        Topbar: function () { return null },
+                        Models: function () { return null },
+                    }
+                }
+            }
+            window.onload = function () {
+                var ui = SwaggerUIBundle({
+                    url: '${openapiJsonUrl}',
+                    validatorUrl: null,
+                    docExpansion: 'list',
+                    dom_id: '#swagger-ui',
+                    deepLinking: true,
+                    presets: [
+                        SwaggerUIBundle.presets.apis,
+                        SwaggerUIStandalonePreset
+                    ],
+                    plugins: [
+                        SwaggerUIBundle.plugins.DownloadUrl,
+                        HidePlugin
+                    ],
+                    layout: 'StandaloneLayout'
+                })
+                window.ui = ui
+            }
+        </script>
+    </body>
+</html>

src/servers/volume/common/data-format.ts

@@ -32,7 +32,7 @@ export interface Sampling {
     rate: number,
     valuesInfo: ValuesInfo[],
 
-    /** Number of samples along each axis, in axisOrder  */
+    /** Number of samples along each axis, in axisOrder */
     sampleCount: number[]
 }
 

src/servers/volume/config.ts

@@ -0,0 +1,107 @@
+/**
+ * Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
+ */
+
+import * as argparse from 'argparse'
+
+export function addLimitsArgs(parser: argparse.ArgumentParser) {
+    parser.addArgument([ '--maxRequestBlockCount' ], {
+        defaultValue: DefaultLimitsConfig.maxRequestBlockCount,
+        metavar: 'COUNT',
+        help: `Maximum number of blocks that could be read in 1 query.
+This is somewhat tied to the maxOutputSizeInVoxelCountByPrecisionLevel
+in that the <maximum number of voxel> = maxRequestBlockCount * <block size>^3.
+The default block size is 96 which corresponds to 28,311,552 voxels with 32 max blocks.`
+    });
+    parser.addArgument([ '--maxFractionalBoxVolume' ], {
+        defaultValue: DefaultLimitsConfig.maxFractionalBoxVolume,
+        metavar: 'VOLUME',
+        help: `The maximum fractional volume of the query box (to prevent queries that are too big).`
+    });
+    parser.addArgument([ '--maxOutputSizeInVoxelCountByPrecisionLevel' ], {
+        nargs: '+',
+        defaultValue: DefaultLimitsConfig.maxOutputSizeInVoxelCountByPrecisionLevel,
+        metavar: 'LEVEL',
+        help: `What is the (approximate) maximum desired size in voxel count by precision level
+Rule of thumb: <response gzipped size> in [<voxel count> / 8, <voxel count> / 4].
+The maximum number of voxels is tied to maxRequestBlockCount.`
+    });
+}
+
+export function addServerArgs(parser: argparse.ArgumentParser) {
+    parser.addArgument([ '--apiPrefix' ], {
+        defaultValue: DefaultServerConfig.apiPrefix,
+        metavar: 'PREFIX',
+        help: `Specify the prefix of the API, i.e. <host>/<apiPrefix>/<API queries>`
+    });
+    parser.addArgument([ '--defaultPort' ], {
+        defaultValue: DefaultServerConfig.defaultPort,
+        metavar: 'PORT',
+        help: `Specify the prefix of the API, i.e. <host>/<apiPrefix>/<API queries>`
+    });
+
+    parser.addArgument([ '--shutdownTimeoutMinutes' ], {
+        defaultValue: DefaultServerConfig.shutdownTimeoutMinutes,
+        metavar: 'TIME',
+        help: `0 for off, server will shut down after this amount of minutes.`
+    });
+    parser.addArgument([ '--shutdownTimeoutVarianceMinutes' ], {
+        defaultValue: DefaultServerConfig.shutdownTimeoutVarianceMinutes,
+        metavar: 'VARIANCE',
+        help: `modifies the shutdown timer by +/- timeoutVarianceMinutes (to avoid multiple instances shutting at the same time)`
+    });
+    parser.addArgument([ '--idMap' ], {
+        nargs: 2,
+        action: 'append',
+        metavar: ['TYPE', 'PATH'] as any,
+        help: [
+            'Map `id`s for a `type` to a file path.',
+            'Example: x-ray \'../../data/mdb/xray/${id}-ccp4.mdb\'',
+            'Note: Can be specified multiple times.'
+        ].join('\n'),
+    });
+}
+
+const DefaultServerConfig = {
+    apiPrefix: '/VolumeServer',
+    defaultPort: 1337,
+    shutdownTimeoutMinutes: 24 * 60, /* a day */
+    shutdownTimeoutVarianceMinutes: 60,
+    idMap: [] as [string, string][]
+}
+export type ServerConfig = typeof DefaultServerConfig
+export const ServerConfig = { ...DefaultServerConfig }
+export function setServerConfig(config: ServerConfig) {
+    for (const name in DefaultServerConfig) {
+        ServerConfig[name as keyof ServerConfig] = config[name as keyof ServerConfig]
+    }
+}
+
+const DefaultLimitsConfig = {
+    maxRequestBlockCount: 32,
+    maxFractionalBoxVolume: 1024,
+    maxOutputSizeInVoxelCountByPrecisionLevel: [
+        0.5 * 1024 * 1024, // ~ 80*80*80
+        1 * 1024 * 1024,
+        2 * 1024 * 1024,
+        4 * 1024 * 1024,
+        8 * 1024 * 1024,
+        16 * 1024 * 1024, // ~ 256*256*256
+        24 * 1024 * 1024
+    ]
+}
+export type LimitsConfig = typeof DefaultLimitsConfig
+export const LimitsConfig = { ...DefaultLimitsConfig }
+export function setLimitsConfig(config: LimitsConfig) {
+    for (const name in DefaultLimitsConfig) {
+        LimitsConfig[name as keyof LimitsConfig] = config[name as keyof LimitsConfig]
+    }
+}
+
+export function setConfig(config: ServerConfig & LimitsConfig) {
+    setServerConfig(config)
+    setLimitsConfig(config)
+}

src/servers/volume/local.ts

@@ -1,17 +1,19 @@
 /**
- * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * Taken/adapted from DensityServer (https://github.com/dsehnal/DensityServer)
  *
  * @author David Sehnal <david.sehnal@gmail.com>
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
+import * as argparse from 'argparse'
 import * as LocalApi from './server/local-api'
 import VERSION from './server/version'
-
 import * as fs from 'fs'
+import { LimitsConfig, addLimitsArgs, setLimitsConfig } from './config';
 
-console.log(`VolumeServer ${VERSION}, (c) 2016 - now, David Sehnal`);
+console.log(`VolumeServer Local ${VERSION}, (c) 2018-2019, Mol* contributors`);
 console.log();
 
 function help() {
@@ -48,21 +50,25 @@ function help() {
         outputFolder: 'g:/test/local-test'
     }];
 
-    console.log('Usage: node local jobs.json');
-    console.log();
-    console.log('Example jobs.json:');
-    console.log(JSON.stringify(exampleJobs, null, 2));
+    return `Usage: node local jobs.json\n\nExample jobs.json: ${JSON.stringify(exampleJobs, null, 2)}`
 }
 
-async function run() {
-    if (process.argv.length !== 3) {
-        help();
-        return;
-    }
+const parser = new argparse.ArgumentParser({
+    addHelp: true,
+    description: help()
+});
+addLimitsArgs(parser)
+parser.addArgument(['jobs'], {
+    help: `Path to jobs JSON file.`
+})
+
+const config: LimitsConfig & { jobs: string } = parser.parseArgs()
+setLimitsConfig(config) // sets the config for global use
 
+async function run() {
     let jobs: LocalApi.JobEntry[];
     try {
-        jobs = JSON.parse(fs.readFileSync(process.argv[2], 'utf-8'));
+        jobs = JSON.parse(fs.readFileSync(config.jobs, 'utf-8'));
     } catch (e) {
         console.log('Error:');
         console.error(e);

src/servers/volume/pack.ts

@@ -1,106 +1,92 @@
 /**
- * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * Taken/adapted from DensityServer (https://github.com/dsehnal/DensityServer)
  *
  * @author David Sehnal <david.sehnal@gmail.com>
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
+import * as argparse from 'argparse'
 import pack from './pack/main'
 import VERSION from './pack/version'
 
+type FileFormat = 'ccp4' | 'dsn6'
+
 interface Config {
     input: { name: string, filename: string }[],
-    format: 'ccp4' | 'dsn6',
+    format: FileFormat,
     isPeriodic: boolean,
     outputFilename: string,
     blockSizeInMB: number
 }
 
-let config: Config = {
-    input: [],
-    format: 'ccp4',
-    isPeriodic: false,
-    outputFilename: '',
-    blockSizeInMB: 96
-};
-
-function getFormat(format: string): Config['format'] {
-    switch (format.toLowerCase()) {
-        case 'ccp4': return 'ccp4'
-        case 'dsn6': return 'dsn6'
+function getConfig(args: Args) {
+    const config: Partial<Config> = {
+        blockSizeInMB: args.blockSizeInMB,
+        format: args.format,
+        outputFilename: args.output
+    }
+    switch (args.mode) {
+        case 'em':
+            config.input = [
+                { name: 'em', filename: args.inputEm }
+            ];
+            config.isPeriodic = false;
+            break
+        case 'xray':
+            config.input = [
+                { name: '2Fo-Fc', filename: args.input2fofc },
+                { name: 'Fo-Fc', filename: args.inputFofc }
+            ];
+            config.isPeriodic = true;
+            break
     }
-    throw new Error(`unsupported format '${format}'`)
+    return config as Config
 }
 
-function printHelp() {
-    let help = [
-        `VolumeServer Packer ${VERSION}, (c) 2016 - now, David Sehnal`,
-        ``,
-        `The input data must be CCP4/MAP mode 2 (32-bit floats) files.`,
-        ``,
-        `Usage: `,
-        ``,
-        `  node pack -v`,
-        `    Print version.`,
-        ``,
-        `  node pack -xray main.ccp4 diff.ccp4 output.mdb [-blockSize 96]`,
-        `    Pack main and diff density into a single block file.`,
-        `    Optionally specify maximum block size.`,
-        ``,
-        `  node pack -em density.map output.mdb [-blockSize 96]`,
-        `    Pack single density into a block file.`,
-        `    Optionally specify maximum block size.`
-    ];
-    console.log(help.join('\n'));
+interface GeneralArgs {
+    blockSizeInMB: number
+    format: FileFormat
+    output: string
+}
+interface XrayArgs extends GeneralArgs {
+    mode: 'xray'
+    input2fofc: string
+    inputFofc: string
+}
+interface EmArgs extends GeneralArgs {
+    mode: 'em'
+    inputEm: string
 }
+type Args = XrayArgs | EmArgs
 
-function parseInput() {
-    let input = false;
+const parser = new argparse.ArgumentParser({
+    addHelp: true,
+    description: `VolumeServer Packer ${VERSION}, (c) 2018-2019, Mol* contributors`
+});
 
-    if (process.argv.length <= 2) {
-        printHelp();
-        process.exit();
-        return false;
-    }
+const subparsers = parser.addSubparsers({
+    title: 'Packing modes',
+    dest: 'mode'
+});
 
-    for (let i = 2; i < process.argv.length; i++) {
-        switch (process.argv[i].toLowerCase()) {
-            case '-blocksize':
-                config.blockSizeInMB = +process.argv[++i];
-                break;
-            case '-format':
-                config.format = getFormat(process.argv[++i]);
-                break;
-            case '-xray':
-                input = true;
-                config.input = [
-                    { name: '2Fo-Fc', filename: process.argv[++i] },
-                    { name: 'Fo-Fc', filename: process.argv[++i] }
-                ];
-                config.isPeriodic = true;
-                config.outputFilename = process.argv[++i];
-                break;
-            case '-em':
-                input = true;
-                config.input = [
-                    { name: 'em', filename: process.argv[++i] }
-                ];
-                config.outputFilename = process.argv[++i];
-                break;
-            case '-v':
-                console.log(VERSION);
-                process.exit();
-                return false;
-            default:
-                printHelp();
-                process.exit();
-                return false;
-        }
-    }
-    return input;
+function addGeneralArgs(parser: argparse.ArgumentParser) {
+    parser.addArgument(['output'], { help: `Output path.` })
+    parser.addArgument(['--blockSizeInMB'], { defaultValue: 96, help: `Maximum block size.`, metavar: 'SIZE' })
+    parser.addArgument(['--format'], { defaultValue: 'ccp4', help: `Input file format.` })
 }
 
-if (parseInput()) {
-    pack(config.input, config.blockSizeInMB, config.isPeriodic, config.outputFilename, config.format);
-}
+const xrayParser = subparsers.addParser('xray', { addHelp: true })
+xrayParser.addArgument(['input2fofc'], { help: `Path to 2fofc file.`, metavar: '2FOFC' })
+xrayParser.addArgument(['inputFofc'], { help: `Path to fofc file.`, metavar: 'FOFC' })
+addGeneralArgs(xrayParser)
+
+const emParser = subparsers.addParser('em', { addHelp: true })
+emParser.addArgument(['inputEm'], { help: `Path to EM density file.`, metavar: 'EM' })
+addGeneralArgs(emParser)
+
+const args: Args = parser.parseArgs();
+const config = getConfig(args)
+
+pack(config.input, config.blockSizeInMB, config.isPeriodic, config.outputFilename, config.format);

src/servers/volume/server-config.ts

@@ -1,77 +0,0 @@
-
-const Config = {
-    limits: {
-        /**
-         * Maximum number of blocks that could be read in 1 query.
-         * This is somewhat tied to the maxOutputSizeInVoxelCountByPrecisionLevel
-         * in that the <maximum number of voxel> = maxRequestBlockCount * <block size>^3.
-         * The default block size is 96 which corresponds to 28,311,552 voxels with 32 max blocks.
-         */
-        maxRequestBlockCount: 32,
-
-        /**
-         * The maximum fractional volume of the query box (to prevent queries that are too big).
-         */
-        maxFractionalBoxVolume: 1024,
-
-        /**
-         * What is the (approximate) maximum desired size in voxel count by precision level
-         * Rule of thumb: <response gzipped size> \in [<voxel count> / 8, <voxel count> / 4];
-         *
-         * The maximum number of voxels is tied to maxRequestBlockCount.
-         */
-        maxOutputSizeInVoxelCountByPrecisionLevel: [
-            0.5 * 1024 * 1024, // ~ 80*80*80
-            1 * 1024 * 1024,
-            2 * 1024 * 1024,
-            4 * 1024 * 1024,
-            8 * 1024 * 1024,
-            16 * 1024 * 1024, // ~ 256*256*256
-            24 * 1024 * 1024
-        ]
-    },
-
-    /**
-     * Specify the prefix of the API, i.e.
-     * <host>/<apiPrefix>/<API queries>
-     */
-    apiPrefix: '/VolumeServer',
-
-    /**
-     * If not specified otherwise by the 'port' environment variable, use this port.
-     */
-    defaultPort: 1337,
-
-    /**
-     * Node (V8) sometimes exhibits GC related issues  that significantly slow down the execution
-     * (https://github.com/nodejs/node/issues/8670).
-     * 
-     * Therefore an option is provided that automatically shuts down the server.
-     * For this to work, the server must be run using a deamon (i.e. forever.js on Linux
-     * or IISnode on Windows) so that the server is automatically restarted when the shutdown happens.
-     */
-    shutdownParams: {
-        // 0 for off, server will shut down after this amount of minutes.
-        timeoutMinutes: 24 * 60 /* a day */,
-        // modifies the shutdown timer by +/- timeoutVarianceMinutes (to avoid multiple instances shutting at the same time)
-        timeoutVarianceMinutes: 60
-    },
-
-    /**
-     * Maps a request identifier to a filename.
-     * 
-     * @param source 
-     *   Source of the data.
-     * @param id
-     *   Id provided in the request. For xray, PDB id, for emd, EMDB id number. 
-     */
-    mapFile(source: string, id: string) {
-        switch (source.toLowerCase()) {
-            case 'x-ray': return `g:/test/mdb/xray-${id.toLowerCase()}.mdb`;
-            case 'emd': return `g:/test/mdb/${id.toLowerCase()}.mdb`;
-            default: return void 0;
-        }
-    }
-}
-
-export default Config;

src/servers/volume/server.ts

@@ -1,9 +1,10 @@
 /**
- * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * Taken/adapted from DensityServer (https://github.com/dsehnal/DensityServer)
  *
  * @author David Sehnal <david.sehnal@gmail.com>
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
 import * as express from 'express'
@@ -11,19 +12,20 @@ import * as compression from 'compression'
 
 import init from './server/web-api'
 import VERSION from './server/version'
-import ServerConfig from './server-config'
 import { ConsoleLogger } from 'mol-util/console-logger'
 import { State } from './server/state'
+import { addServerArgs, addLimitsArgs, LimitsConfig, setConfig, ServerConfig } from './config';
+import * as argparse from 'argparse'
 
 function setupShutdown() {
-    if (ServerConfig.shutdownParams.timeoutVarianceMinutes > ServerConfig.shutdownParams.timeoutMinutes) {
+    if (ServerConfig.shutdownTimeoutVarianceMinutes > ServerConfig.shutdownTimeoutMinutes) {
         ConsoleLogger.log('Server', 'Shutdown timeout variance is greater than the timer itself, ignoring.');
     } else {
         let tVar = 0;
-        if (ServerConfig.shutdownParams.timeoutVarianceMinutes > 0) {
-            tVar = 2 * (Math.random() - 0.5) * ServerConfig.shutdownParams.timeoutVarianceMinutes;
+        if (ServerConfig.shutdownTimeoutVarianceMinutes > 0) {
+            tVar = 2 * (Math.random() - 0.5) * ServerConfig.shutdownTimeoutVarianceMinutes;
         }
-        let tMs = (ServerConfig.shutdownParams.timeoutMinutes + tVar) * 60 * 1000;
+        let tMs = (ServerConfig.shutdownTimeoutMinutes + tVar) * 60 * 1000;
 
         console.log(`----------------------------------------------------------------------------`);
         console.log(`  The server will shut down in ${ConsoleLogger.formatTime(tMs)} to prevent slow performance.`);
@@ -42,20 +44,29 @@ function setupShutdown() {
     }
 }
 
+const parser = new argparse.ArgumentParser({
+    addHelp: true,
+    description: `VolumeServer ${VERSION}, (c) 2018-2019, Mol* contributors`
+});
+addServerArgs(parser)
+addLimitsArgs(parser)
 
-let port = process.env.port || ServerConfig.defaultPort;
+const config: ServerConfig & LimitsConfig = parser.parseArgs()
+setConfig(config) // sets the config for global use
 
-let app = express();
+const port = process.env.port || ServerConfig.defaultPort;
+
+const app = express();
 app.use(compression({ level: 6, memLevel: 9, chunkSize: 16 * 16384, filter: () => true }));
 init(app);
 
 app.listen(port);
 
-console.log(`VolumeServer ${VERSION}, (c) 2016 - now, David Sehnal`);
+console.log(`VolumeServer ${VERSION}, (c) 2018-2019, Mol* contributors`);
 console.log(``);
 console.log(`The server is running on port ${port}.`);
 console.log(``);
 
-if (ServerConfig.shutdownParams && ServerConfig.shutdownParams.timeoutMinutes > 0) {
+if (config.shutdownTimeoutMinutes > 0) {
     setupShutdown();
 }

src/servers/volume/server/api.ts

@@ -11,41 +11,46 @@ import execute from './query/execute'
 import * as Data from './query/data-model'
 import { ConsoleLogger } from 'mol-util/console-logger'
 import * as DataFormat from '../common/data-format'
-import ServerConfig from '../server-config'
 import { FileHandle } from 'mol-io/common/file-handle';
+import { LimitsConfig } from '../config';
 
 export function getOutputFilename(source: string, id: string, { asBinary, box, detail, forcedSamplingLevel }: Data.QueryParams) {
     function n(s: string) { return (s || '').replace(/[ \n\t]/g, '').toLowerCase() }
     function r(v: number) { return Math.round(10 * v) / 10; }
     const det = forcedSamplingLevel !== void 0
         ? `l${forcedSamplingLevel}`
-        : `d${Math.min(Math.max(0, detail | 0), ServerConfig.limits.maxOutputSizeInVoxelCountByPrecisionLevel.length - 1)}`;
+        : `d${Math.min(Math.max(0, detail | 0), LimitsConfig.maxOutputSizeInVoxelCountByPrecisionLevel.length - 1)}`;
     const boxInfo = box.kind === 'Cell'
         ? 'cell'
         : `${box.kind === 'Cartesian' ? 'cartn' : 'frac'}_${r(box.a[0])}_${r(box.a[1])}_${r(box.a[2])}_${r(box.b[0])}_${r(box.b[1])}_${r(box.b[2])}`;
     return `${n(source)}_${n(id)}-${boxInfo}_${det}.${asBinary ? 'bcif' : 'cif'}`;
 }
 
+export interface ExtendedHeader extends DataFormat.Header {
+    availablePrecisions: { precision: number, maxVoxels: number }[]
+    isAvailable: boolean
+}
+
 /** Reads the header and includes information about available detail levels */
-export async function getHeaderJson(filename: string | undefined, sourceId: string) {
+export async function getExtendedHeaderJson(filename: string | undefined, sourceId: string) {
     ConsoleLogger.log('Header', sourceId);
     try {
         if (!filename || !File.exists(filename)) {
             ConsoleLogger.error(`Header ${sourceId}`, 'File not found.');
             return void 0;
         }
-        const header = { ...await readHeader(filename, sourceId) } as DataFormat.Header;
-        const { sampleCount } = header!.sampling[0];
+        const header: Partial<ExtendedHeader> = { ...await readHeader(filename, sourceId) };
+        const { sampleCount } = header.sampling![0];
         const maxVoxelCount = sampleCount[0] * sampleCount[1] * sampleCount[2];
-        const precisions = ServerConfig.limits.maxOutputSizeInVoxelCountByPrecisionLevel
+        const precisions = LimitsConfig.maxOutputSizeInVoxelCountByPrecisionLevel
             .map((maxVoxels, precision) => ({ precision, maxVoxels }));
         const availablePrecisions = [];
         for (const p of precisions) {
             availablePrecisions.push(p);
             if (p.maxVoxels > maxVoxelCount) break;
         }
-        (header as any).availablePrecisions = availablePrecisions;
-        (header as any).isAvailable = true;
+        header.availablePrecisions = availablePrecisions;
+        header.isAvailable = true;
         return JSON.stringify(header, null, 2);
     } catch (e) {
         ConsoleLogger.error(`Header ${sourceId}`, e);

src/servers/volume/server/documentation.ts

@@ -1,174 +0,0 @@
-/**
- * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
- *
- * Taken/adapted from DensityServer (https://github.com/dsehnal/DensityServer)
- *
- * @author David Sehnal <david.sehnal@gmail.com>
- */
-
-import VERSION from './version'
-import ServerConfig from '../server-config'
-
-function detail(i: number) {
-     return `<span class='id'>${i}</span><small> (${Math.round(100 * ServerConfig.limits.maxOutputSizeInVoxelCountByPrecisionLevel[i] / 1000 / 1000) / 100 }M voxels)</small>`;
-}
-const detailMax = ServerConfig.limits.maxOutputSizeInVoxelCountByPrecisionLevel.length - 1;
-const dataSource = `Specifies the data source (determined by the experiment method). Currently, <span class='id'>x-ray</span> and <span class='id'>em</span> sources are supported.`;
-const entryId = `Id of the entry. For <span class='id'>x-ray</span>, use PDB ID (i.e. <span class='id'>1cbs</span>) and for <span class='id'>em</span> use EMDB id (i.e. <span class='id'>emd-8116</span>).`;
-
-export default `
-<!DOCTYPE html>
-<html xmlns="http://www.w3.org/1999/xhtml">
-<head>
-<meta charset="utf-8" />
-<link rel='shortcut icon' href='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAMAAABEpIrGAAAABGdBTUEAALGPC/xhBQAAAAFzUkdCAK7OHOkAAAAnUExURQAAAMIrHrspHr0oH7soILonHrwqH7onILsoHrsoH7soH7woILwpIKgVokoAAAAMdFJOUwAQHzNxWmBHS5XO6jdtAmoAAACZSURBVDjLxZNRCsQgDAVNXmwb9f7nXZEaLRgXloXOhwQdjMYYwpOLw55fBT46KhbOKhmRR2zLcFJQj8UR+HxFgArIF5BKJbEncC6NDEdI5SatBRSDJwGAoiFDONrEJXWYhGMIcRJGCrb1TOtDahfUuQXd10jkFYq0ViIrbUpNcVT6redeC1+b9tH2WLR93Sx2VCzkv/7NjfABxjQHksGB7lAAAAAASUVORK5CYII=' />
-<title>VolumeServer (${VERSION})</title>
-<style>
-html { -ms-text-size-adjust: 100%; -webkit-text-size-adjust: 100%; }
-body { margin: 0; font-family: "Helvetica Neue",Helvetica,Arial,sans-serif; font-weight: 300; color: #333; line-height: 1.42857143; font-size: 14px }
-.container { padding: 0 15px; max-width: 970px; margin: 0 auto; }
-small { font-size: 80% }
-h2, h4 { font-weight: 500; line-height: 1.1; }
-h2 { color: black; font-size: 24px; }
-h4 { font-size: 18px; margin: 20px 0 10px 0 }
-h2 small { color: #777; font-weight: 300 }
-hr { box-sizing: content-box; height: 0; overflow: visible; }
-a { background-color: transparent; -webkit-text-decoration-skip: objects; text-decoration: none }
-a:active, a:hover { outline-width: 0; }
-a:focus, a:hover { text-decoration: underline; color: #23527c }
-.list-unstyled { padding: 0; list-style: none; margin: 0 0 10px 0 }
-.cs-docs-query-wrap { padding: 24px 0; border-bottom: 1px solid #eee }
-.cs-docs-query-wrap > h2 { margin: 0; color: black; }
-.cs-docs-query-wrap > h2 > span { color: #DE4D4E; font-family: Menlo,Monaco,Consolas,"Courier New",monospace; font-size: 90% }
-.cs-docs-param-name, .cs-docs-template-link { color: #DE4D4E; font-family: Menlo,Monaco,Consolas,"Courier New",monospace }
-table {margin: 0; padding: 0; }
-table th { font-weight: bold; border-bottom: none; text-align: left; padding: 6px 12px }
-td { padding: 6px 12px }
-td:not(:last-child), th:not(:last-child) { border-right: 1px dotted #ccc }
-tr:nth-child(even) { background: #f9f9f9 }
-span.id  { color: #DE4D4E; font-family: Menlo,Monaco,Consolas,"Courier New",monospace; }
-</style>
-</head>
-<body>
-<div class="container">
-<div style='text-align: center; margin-top: 24px;'><span style='font-weight: bold; font-size: 16pt'>VolumeServer</span> <span>${VERSION}</span></div>
-
-<div style='text-align: justify; padding: 24px 0; border-bottom: 1px solid #eee'>
-  <p>
-    <b>VolumeServer</b> is a service for accessing subsets of volumetric density data. It automatically downsamples the data
-    depending on the volume of the requested region to reduce the bandwidth requirements and provide near-instant access to even the
-    largest data sets.
-  </p>
-  <p>
-    It uses the text based <a href='https://en.wikipedia.org/wiki/Crystallographic_Information_File'>CIF</a> and binary
-    <a href='https://github.com/dsehnal/BinaryCIF' style='font-weight: bold'>BinaryCIF</a>
-    formats to deliver the data to the client.
-    The server support is integrated into the <a href='https://github.com/dsehnal/LiteMol' style='font-weight: bold'>LiteMol Viewer</a>.
-  </p>
-</div>
-
-<div class="cs-docs-query-wrap">
-  <h2>Data Header / Check Availability <span>/&lt;source&gt;/&lt;id&gt;</span><br>
-  <small>Returns a JSON response specifying if data is available and the maximum region that can be queried.</small></h2>
-  <div id="coordserver-documentation-ambientResidues-body" style="margin: 24px 24px 0 24px">
-    <h4>Examples</h4>
-    <a href="/VolumeServer/x-ray/1cbs" class="cs-docs-template-link" target="_blank" rel="nofollow">/x-ray/1cbs</a><br>
-    <a href="/VolumeServer/em/emd-8116" class="cs-docs-template-link" target="_blank" rel="nofollow">/em/emd-8116</a>
-    <h4>Parameters</h4>
-    <table cellpadding="0" cellspacing="0" style='width: 100%'>
-    <tbody><tr><th style='width: 80px'>Name</th><th>Description</th></tr>
-    <tr>
-    <td class="cs-docs-param-name">source</td>
-    <td>${dataSource}</td>
-    </tr>
-    <tr>
-    <td class="cs-docs-param-name">id</td>
-    <td>${entryId}</td>
-    </tr>
-    </tbody></table>
-  </div>
-</div>
-
-<div class="cs-docs-query-wrap">
-  <h2>Box <span>/&lt;source&gt;/&lt;id&gt;/box/&lt;a,b,c&gt;/&lt;u,v,w&gt;?&lt;optional parameters&gt;</span><br>
-  <small>Returns density data inside the specified box for the given entry. For X-ray data, returns 2Fo-Fc and Fo-Fc volumes in a single response.</small></h2>
-  <div style="margin: 24px 24px 0 24px">
-    <h4>Examples</h4>
-    <a href="/VolumeServer/em/emd-8003/box/-2,7,10/4,10,15.5?encoding=cif&space=cartesian" class="cs-docs-template-link" target="_blank" rel="nofollow">/em/emd-8003/box/-2,7,10/4,10,15.5?excoding=cif&space=cartesian</a><br>
-    <a href="/VolumeServer/x-ray/1cbs/box/0.1,0.1,0.1/0.23,0.31,0.18?space=fractional" class="cs-docs-template-link" target="_blank" rel="nofollow">/x-ray/1cbs/box/0.1,0.1,0.1/0.23,0.31,0.18?space=fractional</a>
-    <h4>Parameters</h4>
-    <table cellpadding="0" cellspacing="0" style='width: 100%'>
-    <tbody><tr><th style='width: 80px'>Name</th><th>Description</th></tr>
-    <tr>
-    <td class="cs-docs-param-name">source</td>
-    <td>${dataSource}</td>
-    </tr>
-    <tr>
-    <td class="cs-docs-param-name">id</td>
-    <td>${entryId}</td>
-    </tr>
-    <tr>
-    <td class="cs-docs-param-name">a,b,c</td>
-    <td>Bottom left corner of the query region in Cartesian or fractional coordinates (determined by the <span class='id'>&amp;space</span> query parameter).</td>
-    </tr>
-    <tr>
-    <td class="cs-docs-param-name">u,v,w</td>
-    <td>Top right corner of the query region in Cartesian or fractional coordinates (determined by the <span class='id'>&amp;space</span> query parameter).</td>
-    </tr>
-    <tr>
-    <td class="cs-docs-param-name">encoding</td>
-    <td>Determines if text based <span class='id'>CIF</span> or binary <span class='id'>BinaryCIF</span> encoding is used. An optional argument, default is <span class='id'>BinaryCIF</span> encoding.</td>
-    </tr>
-    <tr>
-    <td class="cs-docs-param-name">space</td>
-    <td>Determines the coordinate space the query is in. Can be <span class='id'>cartesian</span> or <span class='id'>fractional</span>. An optional argument, default values is <span class='id'>cartesian</span>.</td>
-    </tr>
-    <tr>
-      <td class="cs-docs-param-name">detail</td>
-      <td>
-        Determines the maximum number of voxels the query can return. Possible values are in the range from ${detail(0)} to ${detail(detailMax)}.
-        Default value is <span class='id'>0</span>. Note: different detail levels might lead to the same result.
-      </td>
-    </tr>
-    </tbody></table>
-  </div>
-</div>
-
-<div class="cs-docs-query-wrap">
-  <h2>Cell <span>/&lt;source&gt;/&lt;id&gt;/cell?&lt;optional parameters&gt;</span><br>
-  <small>Returns (downsampled) volume data for the entire "data cell". For X-ray data, returns unit cell of 2Fo-Fc and Fo-Fc volumes, for EM data returns everything.</small></h2>
-  <div style="margin: 24px 24px 0 24px">
-    <h4>Example</h4>
-    <a href="/VolumeServer/em/emd-8116/cell?detail=1" class="cs-docs-template-link" target="_blank" rel="nofollow">/em/emd-8116/cell?detail=1</a><br>
-    <h4>Parameters</h4>
-    <table cellpadding="0" cellspacing="0" style='width: 100%'>
-    <tbody><tr><th style='width: 80px'>Name</th><th>Description</th></tr>
-    <tr>
-    <td class="cs-docs-param-name">source</td>
-    <td>${dataSource}</td>
-    </tr>
-    <tr>
-    <td class="cs-docs-param-name">id</td>
-    <td>${entryId}</td>
-    </tr>
-    <tr>
-    <td class="cs-docs-param-name">encoding</td>
-    <td>Determines if text based <span class='id'>CIF</span> or binary <span class='id'>BinaryCIF</span> encoding is used. An optional argument, default is <span class='id'>BinaryCIF</span> encoding.</td>
-    </tr>
-    <tr>
-      <td class="cs-docs-param-name">detail</td>
-      <td>
-        Determines the maximum number of voxels the query can return. Possible values are in the range from ${detail(0)} to ${detail(detailMax)}.
-        Default value is <span class='id'>0</span>. Note: different detail levels might lead to the same result.
-      </td>
-    </tr>
-    </tbody></table>
-  </div>
-</div>
-
-
-<div style="color: #999;font-size:smaller;margin: 20px 0; text-align: right">&copy; 2016 &ndash; now, David Sehnal | Node ${process.version}</div>
-
-</body>
-</html>
-`;

src/servers/volume/server/query/execute.ts

@@ -13,7 +13,6 @@ import * as Coords from '../algebra/coordinate'
 import * as Box from '../algebra/box'
 import { ConsoleLogger } from 'mol-util/console-logger'
 import { State } from '../state'
-import ServerConfig from '../../server-config'
 
 import identify from './identify'
 import compose from './compose'
@@ -23,13 +22,14 @@ import { Vec3 } from 'mol-math/linear-algebra';
 import { UUID } from 'mol-util';
 import { FileHandle } from 'mol-io/common/file-handle';
 import { createTypedArray, TypedArrayValueType } from 'mol-io/common/typed-array';
+import { LimitsConfig } from 'servers/volume/config';
 
 export default async function execute(params: Data.QueryParams, outputProvider: () => Data.QueryOutputStream) {
     const start = getTime();
     State.pendingQueries++;
 
     const guid = UUID.create22() as any as string;
-    params.detail = Math.min(Math.max(0, params.detail | 0), ServerConfig.limits.maxOutputSizeInVoxelCountByPrecisionLevel.length - 1);
+    params.detail = Math.min(Math.max(0, params.detail | 0), LimitsConfig.maxOutputSizeInVoxelCountByPrecisionLevel.length - 1);
     ConsoleLogger.logId(guid, 'Info', `id=${params.sourceId},encoding=${params.asBinary ? 'binary' : 'text'},detail=${params.detail},${queryBoxToString(params.box)}`);
 
     let sourceFile: FileHandle | undefined;
@@ -114,7 +114,7 @@ function pickSampling(data: Data.DataContext, queryBox: Box.Fractional, forcedLe
         return createQuerySampling(data, data.sampling[Math.min(data.sampling.length, forcedLevel) - 1], queryBox);
     }
 
-    const sizeLimit = ServerConfig.limits.maxOutputSizeInVoxelCountByPrecisionLevel[precision] || (2 * 1024 * 1024);
+    const sizeLimit = LimitsConfig.maxOutputSizeInVoxelCountByPrecisionLevel[precision] || (2 * 1024 * 1024);
 
     for (const s of data.sampling) {
         const gridBox = Box.fractionalToGrid(queryBox, s.dataDomain);
@@ -122,7 +122,7 @@ function pickSampling(data: Data.DataContext, queryBox: Box.Fractional, forcedLe
 
         if (approxSize <= sizeLimit) {
             const sampling = createQuerySampling(data, s, queryBox);
-            if (sampling.blocks.length <= ServerConfig.limits.maxRequestBlockCount) {
+            if (sampling.blocks.length <= LimitsConfig.maxRequestBlockCount) {
                 return sampling;
             }
         }
@@ -168,7 +168,7 @@ function createQueryContext(data: Data.DataContext, params: Data.QueryParams, gu
         throw `The query box is not defined.`;
     }
 
-    if (dimensions[0] * dimensions[1] * dimensions[2] > ServerConfig.limits.maxFractionalBoxVolume) {
+    if (dimensions[0] * dimensions[1] * dimensions[2] > LimitsConfig.maxFractionalBoxVolume) {
         throw `The query box volume is too big.`;
     }
 

src/servers/volume/server/web-api.ts

@@ -1,25 +1,28 @@
 /**
- * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * Taken/adapted from DensityServer (https://github.com/dsehnal/DensityServer)
  *
  * @author David Sehnal <david.sehnal@gmail.com>
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
 import * as express from 'express'
 
 import * as Api from './api'
-
 import * as Data from './query/data-model'
 import * as Coords from './algebra/coordinate'
-import Docs from './documentation'
-import ServerConfig from '../server-config'
 import { ConsoleLogger } from 'mol-util/console-logger'
 import { State } from './state'
+import { LimitsConfig, ServerConfig } from '../config';
+import { interpolate } from 'mol-util/string';
+import { getSchema, shortcutIconLink } from './web-schema';
+import { swaggerUiIndexHandler, swaggerUiAssetsHandler } from 'servers/common/swagger-ui';
 
 export default function init(app: express.Express) {
+    app.locals.mapFile = getMapFileFn()
     function makePath(p: string) {
-        return ServerConfig.apiPrefix + '/' + p;
+        return `${ServerConfig.apiPrefix}/${p}`;
     }
 
     // Header
@@ -29,18 +32,40 @@ export default function init(app: express.Express) {
     // Cell /:src/:id/cell/?text=0|1&space=cartesian|fractional
     app.get(makePath(':source/:id/cell/?'), (req, res) => queryBox(req, res, getQueryParams(req, true)));
 
-    app.get('*', (req, res) => {
-        res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
-        res.end(Docs);
+    app.get(makePath('openapi.json'), (req, res) => {
+        res.writeHead(200, {
+            'Content-Type': 'application/json; charset=utf-8',
+            'Access-Control-Allow-Origin': '*',
+            'Access-Control-Allow-Headers': 'X-Requested-With'
+        });
+        res.end(JSON.stringify(getSchema()));
     });
+
+    app.use(makePath(''), swaggerUiAssetsHandler());
+    app.get(makePath(''), swaggerUiIndexHandler({
+        openapiJsonUrl: makePath('openapi.json'),
+        apiPrefix: ServerConfig.apiPrefix,
+        title: 'VolumeServer API',
+        shortcutIconLink
+    }));
 }
 
-function mapFile(type: string, id: string) {
-    return ServerConfig.mapFile(type || '', id || '');
+function getMapFileFn() {
+    const map = new Function('type', 'id', 'interpolate', [
+        'id = id.toLowerCase()',
+        'switch (type.toLowerCase()) {',
+            ...ServerConfig.idMap.map(mapping => {
+                const [type, path] = mapping
+                return `    case '${type}': return interpolate('${path}', { id });`
+            }),
+        '    default: return void 0;',
+        '}'
+    ].join('\n'))
+    return (type: string, id: string) => map(type, id, interpolate)
 }
 
 function wrapResponse(fn: string, res: express.Response) {
-    const w = {
+    return {
         do404(this: any) {
             if (!this.headerWritten) {
                 res.writeHead(404);
@@ -74,13 +99,11 @@ function wrapResponse(fn: string, res: express.Response) {
         ended: false,
         headerWritten: false
     };
-
-    return w;
 }
 
 function getSourceInfo(req: express.Request) {
     return {
-        filename: mapFile(req.params.source, req.params.id),
+        filename: req.app.locals.mapFile(req.params.source, req.params.id),
         id: `${req.params.source}/${req.params.id}`
     };
 }
@@ -104,7 +127,7 @@ async function getHeader(req: express.Request, res: express.Response) {
 
     try {
         const { filename, id } = getSourceInfo(req);
-        const header = await Api.getHeaderJson(filename, id);
+        const header = await Api.getExtendedHeaderJson(filename, id);
         if (!header) {
             res.writeHead(404);
             return;
@@ -130,7 +153,7 @@ function getQueryParams(req: express.Request, isCell: boolean): Data.QueryParams
     const a = [+req.params.a1, +req.params.a2, +req.params.a3];
     const b = [+req.params.b1, +req.params.b2, +req.params.b3];
 
-    const detail = Math.min(Math.max(0, (+req.query.detail) | 0), ServerConfig.limits.maxOutputSizeInVoxelCountByPrecisionLevel.length - 1)
+    const detail = Math.min(Math.max(0, (+req.query.detail) | 0), LimitsConfig.maxOutputSizeInVoxelCountByPrecisionLevel.length - 1)
     const isCartesian = (req.query.space || '').toLowerCase() !== 'fractional';
 
     const box: Data.QueryParamsBox = isCell
@@ -140,7 +163,7 @@ function getQueryParams(req: express.Request, isCell: boolean): Data.QueryParams
             : { kind: 'Fractional', a: Coords.fractional(a[0], a[1], a[2]), b: Coords.fractional(b[0], b[1], b[2]) });
 
     const asBinary = (req.query.encoding || '').toLowerCase() !== 'cif';
-    const sourceFilename = mapFile(req.params.source, req.params.id)!;
+    const sourceFilename = req.app.locals.mapFile(req.params.source, req.params.id)!;
 
     return {
         sourceFilename,

src/servers/volume/server/web-schema.ts

@@ -0,0 +1,261 @@
+/**
+ * Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import VERSION from './version'
+import { LimitsConfig, ServerConfig } from '../config';
+
+export function getSchema() {
+    function detail(i: number) {
+       return `${i} (${Math.round(100 * LimitsConfig.maxOutputSizeInVoxelCountByPrecisionLevel[i] / 1000 / 1000) / 100 }M voxels)`;
+    }
+    const detailMax = LimitsConfig.maxOutputSizeInVoxelCountByPrecisionLevel.length - 1;
+    const sources = ServerConfig.idMap.map(m => m[0])
+
+    return {
+        openapi: '3.0.0',
+        info: {
+            version: VERSION,
+            title: 'Volume Server',
+            description: 'The VolumeServer is a service for accessing subsets of volumetric data. It automatically downsamples the data depending on the volume of the requested region to reduce the bandwidth requirements and provide near-instant access to even the largest data sets.',
+        },
+        tags: [
+            {
+                name: 'General',
+            }
+        ],
+        paths: {
+            [`${ServerConfig.apiPrefix}/{source}/{id}/`]: {
+                get: {
+                    tags: ['General'],
+                    summary: 'Returns a JSON response specifying if data is available and the maximum region that can be queried.',
+                    operationId: 'getInfo',
+                    parameters: [
+                        { $ref: '#/components/parameters/source' },
+                        { $ref: '#/components/parameters/id' },
+                    ],
+                    responses: {
+                        200: {
+                            description: 'Volume availability and info',
+                            content: {
+                                'application/json': {
+                                    schema: { $ref: '#/components/schemas/info' }
+                                }
+                            }
+                        },
+                    },
+                }
+            },
+            [`${ServerConfig.apiPrefix}/{source}/{id}/box/{a1,a2,a3}/{b1,b2,b3}/`]: {
+                get: {
+                    tags: ['General'],
+                    summary: 'Returns density data inside the specified box for the given entry. For X-ray data, returns 2Fo-Fc and Fo-Fc volumes in a single response.',
+                    operationId: 'getBox',
+                    parameters: [
+                        { $ref: '#/components/parameters/source' },
+                        { $ref: '#/components/parameters/id' },
+                        {
+                            name: 'bottomLeftCorner',
+                            in: 'path',
+                            description: 'Bottom left corner of the query region in Cartesian or fractional coordinates (determined by the `space` query parameter).',
+                            required: true,
+                            schema: {
+                                type: 'list',
+                                items: {
+                                    type: 'float',
+                                }
+                            },
+                            style: 'simple'
+                        },
+                        {
+                            name: 'topRightCorner',
+                            in: 'path',
+                            description: 'Top right corner of the query region in Cartesian or fractional coordinates (determined by the `space` query parameter).',
+                            required: true,
+                            schema: {
+                                type: 'list',
+                                items: {
+                                    type: 'float',
+                                }
+                            },
+                            style: 'simple'
+                        },
+                        { $ref: '#/components/parameters/encoding' },
+                        { $ref: '#/components/parameters/detail' },
+                        {
+                            name: 'space',
+                            in: 'query',
+                            description: 'Determines the coordinate space the query is in. Can be cartesian or fractional. An optional argument, default values is cartesian.',
+                            schema: {
+                                type: 'string',
+                                enum: ['cartesian', 'fractional']
+                            },
+                            style: 'form'
+                        }
+                    ],
+                    responses: {
+                        200: {
+                            description: 'Volume box',
+                            content: {
+                                'text/plain': {},
+                                'application/octet-stream': {},
+                            }
+                        },
+                    },
+                }
+            },
+            [`${ServerConfig.apiPrefix}/{source}/{id}/cell/`]: {
+                get: {
+                    tags: ['General'],
+                    summary: 'Returns (downsampled) volume data for the entire "data cell". For X-ray data, returns unit cell of 2Fo-Fc and Fo-Fc volumes, for EM data returns everything.',
+                    operationId: 'getCell',
+                    parameters: [
+                        { $ref: '#/components/parameters/source' },
+                        { $ref: '#/components/parameters/id' },
+                        { $ref: '#/components/parameters/encoding' },
+                        { $ref: '#/components/parameters/detail' },
+                    ],
+                    responses: {
+                        200: {
+                            description: 'Volume cell',
+                            content: {
+                                'text/plain': {},
+                                'application/octet-stream': {},
+                            }
+                        },
+                    },
+                }
+            }
+        },
+        components: {
+            schemas: {
+                // TODO how to keep in sync with (or derive from) `api.ts/ExtendedHeader`
+                info: {
+                    properties: {
+                        formatVersion: {
+                            type: 'string',
+                            description: 'Format version number'
+                        },
+                        axisOrder: {
+                            type: 'array',
+                            items: { type: 'number' },
+                            description: 'Axis order from the slowest to fastest moving, same as in CCP4'
+                        },
+                        origin: {
+                            type: 'array',
+                            items: { type: 'number' },
+                            description: 'Origin in fractional coordinates, in axisOrder'
+                        },
+                        dimensions: {
+                            type: 'array',
+                            items: { type: 'number' },
+                            description: 'Dimensions in fractional coordinates, in axisOrder'
+                        },
+                        spacegroup: {
+                            properties: {
+                                number: { type: 'number' },
+                                size: {
+                                    type: 'array',
+                                    items: { type: 'number' }
+                                },
+                                angles: {
+                                    type: 'array',
+                                    items: { type: 'number' }
+                                },
+                                isPeriodic: {
+                                    type: 'boolean',
+                                    description: 'Determine if the data should be treated as periodic or not. (e.g. X-ray = periodic, EM = not periodic)'
+                                },
+                            }
+                        },
+                        channels: {
+                            type: 'array',
+                            items: { type: 'string' }
+                        },
+                        valueType: {
+                            type: 'string',
+                            enum: ['float32', 'int16', 'int8'],
+                            description: 'Determines the data type of the values'
+                        },
+                        blockSize: {
+                            type: 'number',
+                            description: 'The value are stored in blockSize^3 cubes'
+                        },
+                        sampling: {
+                            type: 'array',
+                            items: {
+                                properties: {
+                                    byteOffset: { type: 'number' },
+                                    rate: {
+                                        type: 'number',
+                                        description: 'How many values along each axis were collapsed into 1'
+                                    },
+                                    valuesInfo: {
+                                        properties: {
+                                            mean: { type: 'number' },
+                                            sigma: { type: 'number' },
+                                            min: { type: 'number' },
+                                            max: { type: 'number' },
+                                        }
+                                    },
+                                    sampleCount: {
+                                        type: 'array',
+                                        items: { type: 'number' },
+                                        description: 'Number of samples along each axis, in axisOrder'
+                                    },
+                                }
+                            }
+                        }
+                    }
+                }
+            },
+            parameters: {
+                source: {
+                    name: 'source',
+                    in: 'path',
+                    description: `Specifies the data source (determined by the experiment method). Currently supported sources are: ${sources.join(', ')}.`,
+                    required: true,
+                    schema: {
+                        type: 'string',
+                        enum: sources
+                    },
+                    style: 'simple'
+                },
+                id: {
+                    name: 'id',
+                    in: 'path',
+                    description: 'Id of the entry. For x-ray, use PDB ID (i.e. 1cbs) and for em use EMDB id (i.e. emd-8116).',
+                    required: true,
+                    schema: {
+                        type: 'string',
+                    },
+                    style: 'simple'
+                },
+                encoding: {
+                    name: 'encoding',
+                    in: 'query',
+                    description: 'Determines if text based CIF or binary BinaryCIF encoding is used. An optional argument, default is BinaryCIF encoding.',
+                    schema: {
+                        type: 'string',
+                        enum: ['cif', 'bcif']
+                    },
+                    style: 'form'
+                },
+                detail: {
+                    name: 'detail',
+                    in: 'query',
+                    description: `Determines the maximum number of voxels the query can return. Possible values are in the range from ${detail(0)} to ${detail(detailMax)}. Default value is 0. Note: different detail levels might lead to the same result.`,
+                    schema: {
+                        type: 'integer',
+                    },
+                    style: 'form'
+                }
+            }
+        }
+    }
+}
+
+export const shortcutIconLink = `<link rel='shortcut icon' href='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAMAAABEpIrGAAAABGdBTUEAALGPC/xhBQAAAAFzUkdCAK7OHOkAAAAnUExURQAAAMIrHrspHr0oH7soILonHrwqH7onILsoHrsoH7soH7woILwpIKgVokoAAAAMdFJOUwAQHzNxWmBHS5XO6jdtAmoAAACZSURBVDjLxZNRCsQgDAVNXmwb9f7nXZEaLRgXloXOhwQdjMYYwpOLw55fBT46KhbOKhmRR2zLcFJQj8UR+HxFgArIF5BKJbEncC6NDEdI5SatBRSDJwGAoiFDONrEJXWYhGMIcRJGCrb1TOtDahfUuQXd10jkFYq0ViIrbUpNcVT6redeC1+b9tH2WLR93Sx2VCzkv/7NjfABxjQHksGB7lAAAAAASUVORK5CYII=' />`