model-server: Parametize custom property definitions

src/mol-util/console-logger.ts

@@ -34,6 +34,9 @@ export namespace ConsoleLogger {
         if (e.stack) console.error(e.stack);
     }
 
+    export function warn(ctx: string, e: any) {
+        console.error(`[Warn] (${ctx}) ${e}`);
+    }
 
     export function errorId(guid: string | String, e: any) {
         console.error(`[${guid}][Error] ${e}`);

src/servers/model/config.ts

@@ -48,12 +48,27 @@ const config = {
     maxQueueLength: 30,
 
     /**
-     * Paths (relative to the root directory of the model server) to JavaScript files that specify custom properties
+     * Provide a property config or a path a JSON file with the config.
      */
-    customPropertyProviders: [
-        './properties/pdbe',
-        // './properties/rcsb'
-    ],
+    customProperties: <import('./property-provider').ModelPropertyProviderConfig | string>{
+        sources: [
+            './properties/pdbe',
+            // './properties/rcsb'
+        ],
+        params: {
+            PDBe: {
+                UseFileSource: false,
+                API: {
+                    residuewise_outlier_summary: 'https://www.ebi.ac.uk/pdbe/api/validation/residuewise_outlier_summary/entry',
+                    preferred_assembly: 'https://www.ebi.ac.uk/pdbe/api/pdb/entry/summary',
+                    struct_ref_domain: 'https://www.ebi.ac.uk/pdbe/api/mappings/sequence_domains'
+                },
+                File: {
+                    residuewise_outlier_summary: 'e:/test/mol-star/model/props/'
+                }
+            }
+        }
+    },
 
     /**
      * Maps a request identifier to a filename.

src/servers/model/preprocess/master.ts

@@ -8,6 +8,7 @@ import * as fs from 'fs'
 import * as path from 'path'
 import * as argparse from 'argparse'
 import { runMaster, PreprocessEntry } from './parallel';
+import { ModelPropertyProviderConfig } from '../property-provider';
 
 const cmdParser = new argparse.ArgumentParser({
     addHelp: true,
@@ -37,13 +38,13 @@ interface CmdArgs {
 
 export interface PreprocessConfig {
     numProcesses?: number,
-    customPropertyProviders?: string[]
+    customProperties?: ModelPropertyProviderConfig | string
 }
 
 const cmdArgs = cmdParser.parseArgs() as CmdArgs;
 
 let entries: PreprocessEntry[] = []
-let config: PreprocessConfig = { numProcesses: 1, customPropertyProviders: [] }
+let config: PreprocessConfig = { numProcesses: 1, customProperties: void 0 }
 
 if (cmdArgs.input) entries.push({ source: cmdArgs.input, cif: cmdArgs.outCIF, bcif: cmdArgs.outBCIF });
 // else if (cmdArgs.bulk) runBulk(cmdArgs.bulk);

src/servers/model/preprocess/parallel.ts

@@ -9,7 +9,7 @@ import * as cluster from 'cluster'
 import { now } from 'mol-task';
 import { PerformanceMonitor } from 'mol-util/performance-monitor';
 import { preprocessFile } from './preprocess';
-import { createModelPropertiesProviderFromSources } from '../property-provider';
+import { createModelPropertiesProvider } from '../property-provider';
 
 type PreprocessConfig = import('./master').PreprocessConfig
 
@@ -50,7 +50,7 @@ export function runMaster(config: PreprocessConfig, entries: PreprocessEntry[])
 
 export function runChild() {
     process.on('message', async ({ entries, config }: { entries: PreprocessEntry[], config: PreprocessConfig }) => {
-        const props = createModelPropertiesProviderFromSources(config.customPropertyProviders || []);
+        const props = createModelPropertiesProvider(config.customProperties);
         for (const entry of entries) {
             try {
                 await preprocessFile(entry.source, props, entry.cif, entry.bcif);
@@ -64,7 +64,7 @@ export function runChild() {
 }
 
 async function runSingle(entry: PreprocessEntry, config: PreprocessConfig, onMessage: (msg: any) => void) {
-    const props = createModelPropertiesProviderFromSources(config.customPropertyProviders || []);
+    const props = createModelPropertiesProvider(config.customProperties);
     try {
         await preprocessFile(entry.source, props, entry.cif, entry.bcif);
     } catch (e) {

src/servers/model/properties/pdbe.ts

@@ -5,15 +5,15 @@
  * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
-import { Model } from 'mol-model/structure';
 import { PDBe_structureQualityReport, PDBe_preferredAssembly, PDBe_structRefDomain } from './providers/pdbe';
+import { AttachModelProperties } from '../property-provider';
 
-export function attachModelProperties(model: Model, cache: object): Promise<any>[] {
+export const attachModelProperties: AttachModelProperties = (args) => {
     // return a list of promises that start attaching the props in parallel
     // (if there are downloads etc.)
     return [
-        PDBe_structureQualityReport(model, cache),
-        PDBe_preferredAssembly(model, cache),
-        PDBe_structRefDomain(model, cache)
+        PDBe_structureQualityReport(args),
+        PDBe_preferredAssembly(args),
+        PDBe_structRefDomain(args)
     ];
 }

src/servers/model/properties/providers/pdbe.ts

@@ -5,73 +5,98 @@
  */
 
 import * as fs from 'fs'
+import * as path from 'path'
 import { Model } from 'mol-model/structure';
 import { StructureQualityReport } from 'mol-model-props/pdbe/structure-quality-report';
 import { fetchRetry } from '../../utils/fetch-retry';
 import { UUID } from 'mol-util';
 import { PDBePreferredAssembly } from 'mol-model-props/pdbe/preferred-assembly';
 import { PDBeStructRefDomain } from 'mol-model-props/pdbe/struct-ref-domain';
+import { AttachModelProperty } from '../../property-provider';
+import { ConsoleLogger } from 'mol-util/console-logger';
 
-const USE_FILE_SOURCE = false;
+export const PDBe_structureQualityReport: AttachModelProperty = ({ model, params, cache }) => {
+    const PDBe_apiSourceJson = useFileSource(params)
+        ? residuewise_outlier_summary.getDataFromAggregateFile(getFilePrefix(params, 'residuewise_outlier_summary'))
+        : apiQueryProvider(getApiUrl(params, 'residuewise_outlier_summary', 'https://www.ebi.ac.uk/pdbe/api/validation/residuewise_outlier_summary/entry'), cache);
 
-export function PDBe_structureQualityReport(model: Model, cache: any) {
-    return StructureQualityReport.attachFromCifOrApi(model, {
-        PDBe_apiSourceJson: USE_FILE_SOURCE
-            ? residuewise_outlier_summary.getDataFromAggregateFile
-            : residuewise_outlier_summary.getDataFromApiProvider(cache)
-    });
+    return StructureQualityReport.attachFromCifOrApi(model, { PDBe_apiSourceJson });
 }
 
-export function PDBe_preferredAssembly(model: Model, cache: any) {
-    return PDBePreferredAssembly.attachFromCifOrApi(model, {
-        PDBe_apiSourceJson: USE_FILE_SOURCE
-            ? void 0
-            : preferred_assembly.getDataFromApiProvider(cache)
-    });
+export const PDBe_preferredAssembly: AttachModelProperty = ({ model, params, cache }) => {
+    const PDBe_apiSourceJson = apiQueryProvider(getApiUrl(params, 'preferred_assembly', 'https://www.ebi.ac.uk/pdbe/api/pdb/entry/summary'), cache);
+    return PDBePreferredAssembly.attachFromCifOrApi(model, { PDBe_apiSourceJson });
 }
 
-export function PDBe_structRefDomain(model: Model, cache: any) {
-    return PDBeStructRefDomain.attachFromCifOrApi(model, {
-        PDBe_apiSourceJson: USE_FILE_SOURCE
-            ? void 0
-            : struct_ref_domain.getDataFromApiProvider(cache)
-    });
+export const PDBe_structRefDomain: AttachModelProperty = ({ model, params, cache }) => {
+    const PDBe_apiSourceJson = apiQueryProvider(getApiUrl(params, 'struct_ref_domain', 'https://www.ebi.ac.uk/pdbe/api/mappings/sequence_domains'), cache);
+    return PDBeStructRefDomain.attachFromCifOrApi(model, { PDBe_apiSourceJson });
 }
 
-namespace preferred_assembly {
-    export const getDataFromApiProvider = apiQueryProvider('https://www.ebi.ac.uk/pdbe/api/pdb/entry/summary');
+namespace residuewise_outlier_summary {
+    const json = new Map<string, any>();
+    export function getDataFromAggregateFile(pathPrefix: string) {
+        // This is for "testing" purposes and should probably only read
+        // a single file with the appropriate prop in the "production" version.
+        return async (model: Model) => {
+            const key = `${model.label[1]}${model.label[2]}`;
+            if (!json.has(key)) {
+                const fn = path.join(pathPrefix, `${key}.json`);
+                if (!fs.existsSync(fn)) json.set(key, { });
+                // TODO: use async readFile?
+                else json.set(key, JSON.parse(fs.readFileSync(fn, 'utf8')));
+            }
+            return json.get(key)![model.label.toLowerCase()] || { };
+        }
+    }
 }
 
-namespace struct_ref_domain {
-    export const getDataFromApiProvider = apiQueryProvider('https://www.ebi.ac.uk/pdbe/api/mappings/sequence_domains');
+function getApiUrl(params: any, name: string, fallback: string) {
+    const url = getParam<string>(params, 'PDBe', 'API', name);
+    if (!url) return fallback;
+    if (url[url.length - 1] === '/') return url.substring(0, url.length - 1);
+    return url;
 }
 
-namespace residuewise_outlier_summary {
-    const json = new Map<string, any>();
+function getFilePrefix(params: any, name: string) {
+    const ret = getParam<string>(params, 'PDBe', 'File', name);
+    if (!ret) throw new Error(`PDBe file prefix '${name}' not set!`);
+    return ret;
+}
+
+function useFileSource(params: any) {
+    return !!getParam<boolean>(params, 'PDBe', 'UseFileSource')
+}
 
-    export async function getDataFromAggregateFile(model: Model) {
-        const key = `${model.label[1]}${model.label[2]}`;
-        if (!json.has(key)) {
-            const fn = `e:/test/mol-star/model/props/${key}.json`;
-            if (!fs.existsSync(fn)) json.set(key, { });
-            // TODO: use async readFile?
-            else json.set(key, JSON.parse(fs.readFileSync(fn, 'utf8')));
+function getParam<T>(params: any, ...path: string[]): T | undefined {
+    try {
+        let current = params;
+        for (const p of path) {
+            if (typeof current === 'undefined') return;
+            current = current[p];
         }
-        return json.get(key)![model.label.toLowerCase()] || { };
+        return current;
+    } catch (e) {
+        ConsoleLogger.error('Config', `Unable to retrieve property ${path.join('.')} from ${JSON.stringify(params)}`);
     }
-
-    export const getDataFromApiProvider = apiQueryProvider('https://www.ebi.ac.uk/pdbe/api/validation/residuewise_outlier_summary/entry');
 }
 
-function apiQueryProvider(urlPrefix: string) {
-    return (cache: any) => {
-        const cacheKey = UUID.create();
-        return async (model: Model) => {
+
+function apiQueryProvider(urlPrefix: string, cache: any) {
+    const cacheKey = UUID.create();
+    return async (model: Model) => {
+        try {
             if (cache[cacheKey]) return cache[cacheKey];
             const rawData = await fetchRetry(`${urlPrefix}/${model.label.toLowerCase()}`, 1500, 5);
+            // TODO: is this ok?
+            if (rawData.status !== 200) return { };
             const json = (await rawData.json())[model.label.toLowerCase()] || { };
             cache[cacheKey] = json;
             return json;
+        } catch (e) {
+            // TODO: handle better
+            ConsoleLogger.warn('Props', `Count not retrieve prop @${`${urlPrefix}/${model.label.toLowerCase()}`}`);
+            return { };
         }
     }
 }

src/servers/model/property-provider.ts

@@ -4,27 +4,48 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
+import * as fs from 'fs'
 import { Model } from 'mol-model/structure';
 import Config from './config';
+import { ConsoleLogger } from 'mol-util/console-logger';
 
-export type ModelPropertiesProvider = (model: Model, cache: object) => Promise<any>[]
+export interface ModelPropertyProviderConfig {
+    sources: string[],
+    params?: { [name: string]: any }
+}
+
+export type AttachModelProperty = (args: { model: Model, params: any, cache: any }) => Promise<any>
+export type AttachModelProperties = (args: { model: Model, params: any, cache: any }) => Promise<any>[]
+export type ModelPropertiesProvider = (model: Model, cache: any) => Promise<any>[]
 
 export function createModelPropertiesProviderFromConfig(): ModelPropertiesProvider {
-    return createModelPropertiesProviderFromSources(Config.customPropertyProviders);
+    return createModelPropertiesProvider(Config.customProperties);
 }
 
-export function createModelPropertiesProviderFromSources(sources: string[]): ModelPropertiesProvider {
-    if (!sources || sources.length === 0) return () => [];
+export function createModelPropertiesProvider(configOrPath: ModelPropertyProviderConfig | string | undefined): ModelPropertiesProvider {
+    let config: ModelPropertyProviderConfig;
+    if (typeof configOrPath === 'string') {
+        try {
+            config = JSON.parse(fs.readFileSync(configOrPath, 'utf8'));
+        } catch {
+            ConsoleLogger.error('Config', `Could not read property provider config file '${configOrPath}', ignoring.`);
+            return () => [];
+        }
+    } else {
+        config = configOrPath!;
+    }
+
+    if (!config || !config.sources || config.sources.length === 0) return () => [];
 
-    const ps: ModelPropertiesProvider[] = [];
-    for (const p of sources) {
+    const ps: AttachModelProperties[] = [];
+    for (const p of config.sources) {
         ps.push(require(p).attachModelProperties);
     }
 
     return (model, cache) => {
         const ret: Promise<any>[] = [];
         for (const p of ps) {
-            for (const e of p(model, cache)) ret.push(e);
+            for (const e of p({ model, cache, params: config.params })) ret.push(e);
         }
         return ret;
     }

src/servers/model/utils/fetch-retry.ts

@@ -24,6 +24,7 @@ export async function fetchRetry(url: string, timeout: number, retryCount: numbe
         retryCount
     });
 
-    if (result.status >= 200 && result.status < 300) return result;
-    throw new Error(result.statusText);
+    return result;
+    // if (result.status >= 200 && result.status < 300) return result;
+    // throw new Error(result.statusText);
 }