local-api.ts 4.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144
  1. /**
  2. * Copyright (c) 2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
  3. *
  4. * Taken/adapted from DensityServer (https://github.com/dsehnal/DensityServer)
  5. *
  6. * @author David Sehnal <david.sehnal@gmail.com>
  7. */
  8. import * as Api from './api'
  9. import * as Data from './query/data-model'
  10. import * as Coordinate from './algebra/coordinate'
  11. import * as fs from 'fs'
  12. import * as path from 'path'
  13. export interface JobEntry {
  14. source: {
  15. filename: string,
  16. name: string,
  17. id: string
  18. },
  19. query: {
  20. kind: 'box' | 'cell',
  21. space?: 'fractional' | 'cartesian',
  22. bottomLeft?: number[],
  23. topRight?: number[],
  24. }
  25. params: {
  26. /** Determines the detail level as specified in server-config */
  27. detail?: number,
  28. /**
  29. * Determines the sampling level:
  30. * 1: Original data
  31. * 2: Downsampled by factor 1/2
  32. * ...
  33. * N: downsampled 1/2^(N-1)
  34. */
  35. forcedSamplingLevel?: number,
  36. asBinary: boolean,
  37. },
  38. outputFolder: string
  39. }
  40. export async function run(jobs: JobEntry[]) {
  41. let progress = 0;
  42. let started = getTime();
  43. for (const job of jobs) {
  44. try {
  45. await query(job);
  46. } catch (e) {
  47. console.error(e);
  48. }
  49. progress++;
  50. const elapsed = (getTime() - started) / 1000;
  51. console.log(`[Progress] ${progress}/${jobs.length} in ${elapsed.toFixed(2)}s`);
  52. }
  53. }
  54. function getTime() {
  55. let t = process.hrtime();
  56. return t[0] * 1000 + t[1] / 1000000;
  57. }
  58. async function query(job: JobEntry) {
  59. let box: Data.QueryParamsBox;
  60. if (job.query.kind.toLocaleLowerCase() === 'cell') {
  61. box = { kind: 'Cell' };
  62. } else if (job.query.space === 'fractional') {
  63. box = {
  64. kind: 'Fractional',
  65. a: Coordinate.fractional(job.query.bottomLeft![0], job.query.bottomLeft![1], job.query.bottomLeft![2]),
  66. b: Coordinate.fractional(job.query.topRight![0], job.query.topRight![1], job.query.topRight![2]),
  67. }
  68. } else {
  69. box = {
  70. kind: 'Cartesian',
  71. a: Coordinate.cartesian(job.query.bottomLeft![0], job.query.bottomLeft![1], job.query.bottomLeft![2]),
  72. b: Coordinate.cartesian(job.query.topRight![0], job.query.topRight![1], job.query.topRight![2]),
  73. }
  74. }
  75. const params: Data.QueryParams = {
  76. sourceFilename: job.source.filename,
  77. sourceId: job.source.id,
  78. asBinary: job.params.asBinary,
  79. box,
  80. detail: !job.params.detail ? 0 : job.params.detail,
  81. forcedSamplingLevel: job.params.forcedSamplingLevel
  82. };
  83. if (!fs.existsSync(job.outputFolder)) {
  84. makeDir(job.outputFolder);
  85. }
  86. const filename = path.join(job.outputFolder, Api.getOutputFilename(job.source.name, job.source.id, params))
  87. const res = () => wrapFile(filename);
  88. await Api.queryBox(params, res)
  89. }
  90. function makeDir(path: string, root?: string): boolean {
  91. let dirs = path.split(/\/|\\/g),
  92. dir = dirs.shift();
  93. root = (root || '') + dir + '/';
  94. try {
  95. fs.mkdirSync(root);
  96. } catch (e) {
  97. if (!fs.statSync(root).isDirectory()) throw new Error(e);
  98. }
  99. return !dirs.length || makeDir(dirs.join('/'), root);
  100. }
  101. function wrapFile(fn: string) {
  102. const w = {
  103. open(this: any) {
  104. if (this.opened) return;
  105. this.file = fs.openSync(fn, 'w');
  106. this.opened = true;
  107. },
  108. writeBinary(this: any, data: Uint8Array) {
  109. this.open();
  110. fs.writeSync(this.file, Buffer.from(data));
  111. return true;
  112. },
  113. writeString(this: any, data: string) {
  114. this.open();
  115. fs.writeSync(this.file, data);
  116. return true;
  117. },
  118. end(this: any) {
  119. if (!this.opened || this.ended) return;
  120. fs.close(this.file, function () { });
  121. this.ended = true;
  122. },
  123. file: 0,
  124. ended: false,
  125. opened: false
  126. };
  127. return w;
  128. }