model-server download custom properties

src/mol-model-props/pdbe/structure-quality-report.ts

@@ -27,7 +27,9 @@ const _Descriptor = ModelPropertyDescriptor({
         prefix: 'pdbe',
         categories: [{
             name: 'pdbe_structure_quality_report',
-            instance() {
+            instance(ctx) {
+                if (ctx.globalCache.pdbe_structure_quality_report) return CifWriter.Category.Empty;
+                ctx.globalCache.pdbe_structure_quality_report = true;
                 return { fields: _structure_quality_report_fields, rowCount: 1 }
             }
         }, {
@@ -122,9 +124,7 @@ export namespace StructureQualityReport {
             const data = toTable(Schema.pdbe_structure_quality_report_issues, model.sourceData.frame.categories.pdbe_structure_quality_report);
             issueMap = createIssueMapFromCif(model, data);
         } else if (params.PDBe_apiSourceJson) {
-            const id = model.label.toLowerCase();
-            const json = await params.PDBe_apiSourceJson(model);
-            const data = json[id];
+            const data = await params.PDBe_apiSourceJson(model);
             if (!data) return false;
             issueMap = createIssueMapFromJson(model, data);
         } else {

src/mol-model/structure/export/mmcif.ts

@@ -17,13 +17,16 @@ import { _pdbx_struct_mod_residue } from './categories/modified-residues';
 export interface CifExportContext {
     structure: Structure,
     model: Model,
-    cache: any
+    localCache: any,
+    /** useful when exporting multiple models at the same time */
+    globalCache: any
 }
 
 export namespace CifExportContext {
     export function create(structures: Structure | Structure[]): CifExportContext[] {
-        if (Array.isArray(structures)) return structures.map(structure => ({ structure, model: structure.models[0], cache: Object.create(null) }));
-        return [{ structure: structures, model: structures.models[0], cache: Object.create(null) }];
+        const globalCache = Object.create(null);
+        if (Array.isArray(structures)) return structures.map(structure => ({ structure, model: structure.models[0], localCache: Object.create(null), globalCache }));
+        return [{ structure: structures, model: structures.models[0], localCache: Object.create(null), globalCache }];
     }
 }
 

src/mol-model/structure/model/properties/custom/residue.ts

@@ -26,7 +26,7 @@ export namespace ResidueCustomProperty {
     };
 
     function getExportCtx<T>(exportCtx: CifExportContext, prop: ResidueCustomProperty<T>): ExportCtx<T> {
-        if (exportCtx.cache[prop.id]) return exportCtx.cache[prop.id];
+        if (exportCtx.localCache[prop.id]) return exportCtx.localCache[prop.id];
         const residueIndex = exportCtx.model.atomicHierarchy.residueAtomSegments.index;
         const elements = getStructureElements(exportCtx.structure, prop);
         return {

src/mol-util/make-dir.ts

@@ -0,0 +1,21 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import * as fs from 'fs';
+
+export function makeDir(path: string, root?: string): boolean {
+    let dirs = path.split(/\/|\\/g),
+        dir = dirs.shift();
+
+    root = (root || '') + dir + '/';
+
+    try { fs.mkdirSync(root); }
+    catch (e) {
+        if (!fs.statSync(root).isDirectory()) throw new Error(e);
+    }
+
+    return !dirs.length || makeDir(dirs.join('/'), root);
+}

src/servers/model/properties/providers/pdbe.ts

@@ -4,20 +4,39 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
- import { Model } from 'mol-model/structure';
+import * as fs from 'fs'
+import { Model } from 'mol-model/structure';
 import { StructureQualityReport } from 'mol-model-props/pdbe/structure-quality-report';
 import { fetchRetry } from '../../utils/fetch-retry';
 import { UUID } from 'mol-util';
 
-const cacheKey = UUID.create();
 export function PDBe_structureQualityReport(model: Model, cache: any) {
     return StructureQualityReport.attachFromCifOrApi(model, {
-        PDBe_apiSourceJson: async model => {
+        PDBe_apiSourceJson: residuewise_outlier_summary.getDataFromFile
+    });
+}
+
+namespace residuewise_outlier_summary {
+    const json = new Map<string, any>();
+
+    export async function getDataFromFile(model: Model) {
+        const key = `${model.label[1]}${model.label[2]}`;
+        if (!json.has(key)) {
+            const fn = `e:/test/mol-star/model/props/${key}.json`;
+            if (!fs.existsSync(fn)) json.set(key, { });
+            else json.set(key, JSON.parse(fs.readFileSync(fn, 'utf8')));
+        }
+        return json.get(key)![model.label.toLowerCase()] || { };
+    }
+
+    export function getDataFromApiProvider(cache: any) {
+        const cacheKey = UUID.create();
+        return async (model: Model) => {
             if (cache[cacheKey]) return cache[cacheKey];
             const rawData = await fetchRetry(`https://www.ebi.ac.uk/pdbe/api/validation/residuewise_outlier_summary/entry/${model.label.toLowerCase()}`, 1500, 5);
-            const json = await rawData.json();
+            const json = (await rawData.json())[model.label.toLowerCase()] || { };
             cache[cacheKey] = json;
             return json;
         }
-    });
+    }
 }

src/servers/model/utils/fetch-props-pdbe.ts

@@ -0,0 +1,80 @@
+/**
+ * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import fetch from 'node-fetch';
+import * as fs from 'fs'
+import * as path from 'path'
+import * as argparse from 'argparse'
+import { makeDir } from 'mol-util/make-dir';
+import { now } from 'mol-task';
+import { PerformanceMonitor } from 'mol-util/performance-monitor';
+
+const cmdParser = new argparse.ArgumentParser({
+    addHelp: true,
+    description: 'Download JSON data from PDBe API'
+});
+
+cmdParser.addArgument(['--in'], { help: 'Input folder', required: true });
+cmdParser.addArgument(['--out'], { help: 'Output folder', required: true });
+
+interface CmdArgs {
+    in: string,
+    out: string
+}
+
+const cmdArgs = cmdParser.parseArgs() as CmdArgs;
+
+function getPDBid(name: string) {
+    let idx = name.indexOf('_');
+    if (idx < 0) idx = name.indexOf('.');
+    return name.substr(0, idx).toLowerCase();
+}
+
+function findEntries() {
+    const files = fs.readdirSync(cmdArgs.in);
+    const cifTest = /\.cif$/;
+    const groups = new Map<string, string[]>();
+    const keys: string[] = [];
+
+    for (const f of files) {
+        if (!cifTest.test(f)) continue;
+        const id = getPDBid(f);
+        const groupId = `${id[1]}${id[2]}`;
+
+        if (groups.has(groupId)) groups.get(groupId)!.push(id);
+        else {
+            keys.push(groupId);
+            groups.set(groupId, [id]);
+        }
+    }
+
+    const ret: { key: string, entries: string[] }[] = [];
+    for (const key of keys) {
+        ret.push({ key, entries: groups.get(key)! })
+    }
+
+    return ret;
+}
+
+async function process() {
+    const entries = findEntries();
+    makeDir(cmdArgs.out);
+
+    const started = now();
+    let prog = 0;
+    for (const e of entries) {
+        const ts = now();
+        console.log(`${prog}/${entries.length} ${e.entries.length} entries.`)
+        const body = e.entries.join(',');
+        const query = await fetch(`https://www.ebi.ac.uk/pdbe/api/validation/residuewise_outlier_summary/entry`, { method: 'POST', body });
+        const data = await query.text();
+        fs.writeFileSync(path.join(cmdArgs.out, e.key + '.json'), data);
+        const time = now() - started;
+        console.log(`${++prog}/${entries.length} in ${PerformanceMonitor.format(time)} (last ${PerformanceMonitor.format(now() - ts)}, avg ${PerformanceMonitor.format(time / prog)})`);
+    }
+}
+
+process();