wip, server/model/preprocess

package.json

@@ -85,7 +85,6 @@
     "tslint": "^5.17.0",
     "typescript": "^3.5.1",
     "uglify-js": "^3.6.0",
-    "util.promisify": "^1.0.0",
     "webpack": "^4.32.2",
     "webpack-cli": "^3.3.2"
   },
@@ -111,6 +110,7 @@
     "react-dom": "^16.8.6",
     "rxjs": "^6.5.2",
     "swagger-ui-dist": "^3.22.2",
+    "util.promisify": "^1.0.0",
     "xhr2": "^0.2.0"
   }
 }

src/mol-model-props/rcsb/assembly-symmetry.ts

@@ -1,5 +1,5 @@
 /**
- * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
@@ -18,7 +18,6 @@ import { CifCategory } from '../../mol-io/reader/cif';
 import { PropertyWrapper } from '../../mol-model-props/common/wrapper';
 import { Task, RuntimeContext } from '../../mol-task';
 import { GraphQLClient } from '../../mol-util/graphql-client';
-import { ajaxGet } from '../../mol-util/data-source';
 
 const { str, int, float, Aliased, Vector, List } = Column.Schema;
 
@@ -183,13 +182,12 @@ export function AssemblySymmetry(db: AssemblySymmetry.Database): AssemblySymmetr
 type SymmetryKind = 'GLOBAL' | 'LOCAL' | 'PSEUDO'
 type SymmetryType = 'ASYMMETRIC' | 'CYCLIC' | 'DIHEDRAL' | 'HELICAL' | 'ICOSAHEDRAL' | 'OCTAHEDRAL' | 'TETRAHEDRAL'
 
-const Client = new GraphQLClient(AssemblySymmetry.GraphQLEndpointURL, ajaxGet)
-
 export namespace AssemblySymmetry {
     export function is(x: any): x is AssemblySymmetry {
         return x['@type'] === 'rcsb_assembly_symmetry'
     }
-    export const GraphQLEndpointURL = '//rest-dev.rcsb.org/graphql'
+    export const GraphQLEndpointURL = '//rest-staging.rcsb.org/graphql'
+
     export const Schema = {
         rcsb_assembly_symmetry_info: {
             updated_datetime_utc: Column.Schema.str
@@ -257,7 +255,7 @@ export namespace AssemblySymmetry {
 
     export const Descriptor = _Descriptor;
 
-    export async function attachFromCifOrAPI(model: Model, client: GraphQLClient = Client, ctx?: RuntimeContext) {
+    export async function attachFromCifOrAPI(model: Model, client: GraphQLClient, ctx?: RuntimeContext) {
         if (model.customProperties.has(Descriptor)) return true;
 
         let db: Database
@@ -266,8 +264,10 @@ export namespace AssemblySymmetry {
             db = createDatabaseFromCif(model)
         } else {
             let result: AssemblySymmetryGraphQL.Query
+            console.log('model.label.toLowerCase()', model.label.toLowerCase())
             const variables: AssemblySymmetryGraphQL.Variables = { pdbId: model.label.toLowerCase() };
             try {
+                console.log('foo', client)
                 result = await client.request<AssemblySymmetryGraphQL.Query>(ctx || RuntimeContext.Synchronous, query, variables);
             } catch (e) {
                 console.error(e)

src/servers/common/util.ts

@@ -0,0 +1,21 @@
+/**
+ * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
+ */
+
+import { ConsoleLogger } from '../../mol-util/console-logger';
+
+export function getParam<T>(params: any, ...path: string[]): T | undefined {
+    try {
+        let current = params;
+        for (const p of path) {
+            if (typeof current === 'undefined') return;
+            current = current[p];
+        }
+        return current;
+    } catch (e) {
+        ConsoleLogger.error('Config', `Unable to retrieve property ${path.join('.')} from ${JSON.stringify(params)}`);
+    }
+}

src/servers/model/config.ts

@@ -54,8 +54,9 @@ const config = {
      */
     customProperties: <ModelPropertyProviderConfig | string>{
         sources: [
-            './properties/pdbe',
-            './properties/rcsb'
+            'pdbe',
+            'rcsb',
+            'wwpdb'
         ],
         params: {
             PDBe: {
@@ -68,6 +69,14 @@ const config = {
                 File: {
                     residuewise_outlier_summary: 'e:/test/mol-star/model/props/'
                 }
+            },
+            RCSB: {
+                API: {
+                    assembly_symmetry: 'https://rest-staging.rcsb.org/graphql'
+                }
+            },
+            wwPDB: {
+                chemCompBondTablePath: ''
             }
         }
     },

src/servers/model/preprocess/master.ts

@@ -10,9 +10,27 @@ import * as argparse from 'argparse'
 import { runMaster, PreprocessEntry } from './parallel';
 import { ModelPropertyProviderConfig } from '../property-provider';
 
+function description() {
+    const exampleCfg = {
+        numProcesses: 1,
+        customProperties: {
+            sources: [
+                'wwpdb'
+            ],
+            params: {
+                wwPDB: {
+                    chemCompBondTablePath: './build/data/ccb.bcif'
+                }
+            }
+        }
+    }
+
+    return `Preprocess CIF files to include custom properties and convert them to BinaryCIF format.\n\nExample cfg.json: ${JSON.stringify(exampleCfg, null, 2)}`
+}
+
 const cmdParser = new argparse.ArgumentParser({
     addHelp: true,
-    description: 'Preprocess CIF files to include custom properties and convert them to BinaryCIF format.'
+    description: description()
 });
 cmdParser.addArgument(['--input', '-i'], { help: 'Input filename', required: false });
 cmdParser.addArgument(['--outCIF', '-oc'], { help: 'Output CIF filename', required: false });

src/servers/model/preprocess/preprocess.ts

@@ -21,7 +21,6 @@ export function preprocessFile(filename: string, propertyProvider?: ModelPropert
         : convert(filename, outputCif, outputBcif);
 }
 
-
 async function preprocess(filename: string, propertyProvider?: ModelPropertiesProvider, outputCif?: string, outputBcif?: string) {
     const input = await readStructureWrapper('entry', '_local_', filename, propertyProvider);
     const categories = await classifyCif(input.cifFrame);

src/servers/model/properties/providers/pdbe.ts

@@ -14,6 +14,7 @@ import { PDBePreferredAssembly } from '../../../../mol-model-props/pdbe/preferre
 import { PDBeStructRefDomain } from '../../../../mol-model-props/pdbe/struct-ref-domain';
 import { AttachModelProperty } from '../../property-provider';
 import { ConsoleLogger } from '../../../../mol-util/console-logger';
+import { getParam } from '../../../common/util';
 
 export const PDBe_structureQualityReport: AttachModelProperty = ({ model, params, cache }) => {
     const PDBe_apiSourceJson = useFileSource(params)
@@ -68,20 +69,6 @@ function useFileSource(params: any) {
     return !!getParam<boolean>(params, 'PDBe', 'UseFileSource')
 }
 
-function getParam<T>(params: any, ...path: string[]): T | undefined {
-    try {
-        let current = params;
-        for (const p of path) {
-            if (typeof current === 'undefined') return;
-            current = current[p];
-        }
-        return current;
-    } catch (e) {
-        ConsoleLogger.error('Config', `Unable to retrieve property ${path.join('.')} from ${JSON.stringify(params)}`);
-    }
-}
-
-
 function apiQueryProvider(urlPrefix: string, cache: any) {
     const cacheKey = UUID.create22();
     return async (model: Model) => {

src/servers/model/properties/providers/rcsb.ts

@@ -1,12 +1,23 @@
 /**
- * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2018-2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
 import { AssemblySymmetry } from '../../../../mol-model-props/rcsb/assembly-symmetry';
 import { AttachModelProperty } from '../../property-provider';
+import { ajaxGet } from '../../../../mol-util/data-source';
+import { GraphQLClient } from '../../../../mol-util/graphql-client';
+import { getParam } from '../../../common/util';
 
-export const RCSB_assemblySymmetry: AttachModelProperty = ({ model }) => {
-    return AssemblySymmetry.attachFromCifOrAPI(model)
+export const RCSB_assemblySymmetry: AttachModelProperty = ({ model, params }) => {
+    const url = getApiUrl(params, 'assembly_symmetry', `https:${AssemblySymmetry.GraphQLEndpointURL}`)
+    const client = new GraphQLClient(url, ajaxGet)
+    return AssemblySymmetry.attachFromCifOrAPI(model, client)
+}
+
+function getApiUrl(params: any, name: string, fallback: string) {
+    const path = getParam<string>(params, 'RCSB', 'API', name);
+    if (!path) return fallback;
+    return path;
 }

src/servers/model/properties/providers/wwpdb.ts

@@ -0,0 +1,44 @@
+/**
+ * Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
+ */
+
+import * as fs from 'fs'
+import * as util from 'util'
+import { AttachModelProperty } from '../../property-provider';
+import { ChemCompBond } from '../../../../mol-model-props/wwpdb/chem-comp-bond';
+import { Table } from '../../../../mol-data/db';
+import { CIF } from '../../../../mol-io/reader/cif';
+import { getParam } from '../../../common/util';
+
+require('util.promisify').shim()
+const readFile = util.promisify(fs.readFile)
+
+export const wwPDB_chemCompBond: AttachModelProperty = ({ model, params }) => {
+    const wwPDB_apiSourceTable = getChemCompBondTableProvider(getTablePath(params))
+    return ChemCompBond.attachFromCifOrTable(model, { wwPDB_apiSourceTable });
+}
+
+async function read(path: string) {
+    return path.endsWith('.bcif') ? new Uint8Array(await readFile(path)) : readFile(path, 'utf8');
+}
+
+function getChemCompBondTableProvider(path: string): () => Promise<Table<ChemCompBond.Schema['chem_comp_bond']>> {
+    let chemCompBondTable: Table<ChemCompBond.Schema['chem_comp_bond']>
+    return async function() {
+        if (chemCompBondTable === undefined) {
+            const parsed = await CIF.parse(await read(path)).run()
+            if (parsed.isError) throw new Error(parsed.toString())
+            const table = CIF.toDatabase(ChemCompBond.Schema, parsed.result.blocks[0])
+            chemCompBondTable = table.chem_comp_bond
+        }
+        return chemCompBondTable
+    }
+}
+
+function getTablePath(params: any) {
+    const path = getParam<string>(params, 'wwPDB', 'chemCompBondTablePath');
+    if (!path) throw new Error(`wwPDB 'chemCompBondTablePath' not set!`);
+    return path;
+}

src/servers/model/properties/wwpdb.ts

@@ -0,0 +1,16 @@
+/**
+ * Copyright (c) 2019 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
+ */
+
+import { AttachModelProperties } from '../property-provider';
+import { wwPDB_chemCompBond } from './providers/wwpdb';
+
+export const attachModelProperties: AttachModelProperties = (args) => {
+    // return a list of promises that start attaching the props in parallel
+    // (if there are downloads etc.)
+    return [
+        wwPDB_chemCompBond(args)
+    ];
+}

src/servers/model/property-provider.ts

@@ -9,6 +9,17 @@ import { Model } from '../../mol-model/structure';
 import Config from './config';
 import { ConsoleLogger } from '../../mol-util/console-logger';
 
+// TODO enable dynamic imports again
+import * as pdbeProps from './properties/pdbe'
+import * as rcsbProps from './properties/rcsb'
+import * as wwpdbProps from './properties/wwpdb'
+
+const attachModelProperties: { [k: string]: AttachModelProperties } = {
+    pdbe: pdbeProps.attachModelProperties,
+    rcsb: rcsbProps.attachModelProperties,
+    wwpdb: wwpdbProps.attachModelProperties
+}
+
 export interface ModelPropertyProviderConfig {
     sources: string[],
     params?: { [name: string]: any }
@@ -39,7 +50,11 @@ export function createModelPropertiesProvider(configOrPath: ModelPropertyProvide
 
     const ps: AttachModelProperties[] = [];
     for (const p of config.sources) {
-        ps.push(require(p).attachModelProperties);
+        if (p in attachModelProperties) {
+            ps.push(attachModelProperties[p]);
+        } else {
+            ConsoleLogger.error('Config', `Could not find property provider '${p}', ignoring.`);
+        }
     }
 
     return (model, cache) => {

src/servers/volume/local.ts

@@ -16,7 +16,7 @@ import { LimitsConfig, addLimitsArgs, setLimitsConfig } from './config';
 console.log(`VolumeServer Local ${VERSION}, (c) 2018-2019, Mol* contributors`);
 console.log();
 
-function help() {
+function description() {
     const exampleJobs: LocalApi.JobEntry[] = [{
         source: {
             filename: `g:/test/mdb/xray-1tqn.mdb`,
@@ -55,7 +55,7 @@ function help() {
 
 const parser = new argparse.ArgumentParser({
     addHelp: true,
-    description: help()
+    description: description()
 });
 addLimitsArgs(parser)
 parser.addArgument(['jobs'], {

webpack.config.js

@@ -73,6 +73,12 @@ function createNodeEntryPoint(name, dir, out) {
         target: 'node',
         entry: path.resolve(__dirname, `lib/${dir}/${name}.js`),
         output: { filename: `${name}.js`, path: path.resolve(__dirname, `build/${out}`) },
+        externals: {
+            argparse: 'require("argparse")',
+            'node-fetch': 'require("node-fetch")',
+            'util.promisify': 'require("util.promisify")',
+            xhr2: 'require("xhr2")',
+        },
         ...sharedConfig
     }
 }