feat!: make parser async and use native zlib decompression

cheminfo · Dec 5, 2024 · 17f7bcd · 17f7bcd
1 parent 4273aaa
commit 17f7bcd
Show file tree

Hide file tree

Showing 10 changed files with 111 additions and 28 deletions.
diff --git a/README.md b/README.md
@@ -45,7 +45,7 @@ import { parseMZ } from 'mzdata';
 
 // mzData files
 const mzDataFile = readFileSync(__dirname + '/tiny.mzData.xml');
-var response = parseMZ(mzDataFile);
+var response = await parseMZ(mzDataFile);
 ```
 
 ## Ontology
@@ -58,8 +58,6 @@ You can find various examples files at:
 
 http://www.psidev.info/mzML
 
-## [API Documentation](https://cheminfo-js.github.io/mzData/)
-
 ## License
 
 [MIT](./LICENSE)

diff --git a/src/Options.js b/src/Options.js
@@ -0,0 +1,6 @@
+/**
+ * @typedef {Object} Options
+ * @property {import('cheminfo-types').Logger} [logger] - A potential logger
+ */
+
+export {};
diff --git a/src/index.js b/src/index.js
@@ -7,9 +7,10 @@ const decoder = new TextDecoder();
 /**
  * Reads a mzData v1.05 file
  * @param {ArrayBuffer|string} xml - ArrayBuffer or String or any Typed Array (including Node.js' Buffer from v4) with the data
- * @return {{times: Array<number>, series: { ms: { data:Array<Array<number>>}}}}
+ * @param {import('./Options.js').Options} [options={}]
+ * @return Promise<{{times: Array<number>, series: { ms: { data:Array<Array<number>>}}}}>
  */
-export function parseMZ(xml) {
+export async function parseMZ(xml, options = {}) {
   if (typeof xml === 'string') {
     const encoder = new TextEncoder();
     xml = encoder.encode(xml);
@@ -24,11 +25,11 @@ export function parseMZ(xml) {
     : xml.substring(0, 200);
 
   if (header.includes('mzData')) {
-    return parseMzData(xml);
+    return parseMzData(xml, options);
   } else if (header.includes('mzML')) {
-    return parseMzML(xml);
+    return parseMzML(xml, options);
   } else if (header.includes('mzXML')) {
-    return parseMzXML(xml);
+    return parseMzXML(xml, options);
   } else {
     throw new Error(`MZ parser: unknown format`);
   }

diff --git a/src/mzdata/parseMzData.js b/src/mzdata/parseMzData.js
@@ -1,4 +1,5 @@
 import { parse } from 'arraybuffer-xml-parser';
+import { recursiveResolve } from 'ml-spectra-processing';
 
 import { decodeBase64 } from '../util/decodeBase64';
 
@@ -7,7 +8,14 @@ import { processSpectrumList } from './processSpectrumList';
 
 const decoder = new TextDecoder();
 
-export function parseMzData(arrayBuffer) {
+/**
+ *
+ * @param {*} arrayBuffer
+ * @param {import('../Options.js').Options} [options={}]
+ * @returns
+ */
+export async function parseMzData(arrayBuffer, options = {}) {
+  const { logger = console } = options;
   const result = {
     metadata: {},
     times: [],
@@ -19,14 +27,20 @@ export function parseMzData(arrayBuffer) {
   };
 
   let parsed = parse(arrayBuffer, {
-    attributeNamePrefix: '',
     attributesNodeName: 'attributes',
+    attributeNameProcessor: (attributeName) => attributeName,
     tagValueProcessor: (value, node) => {
       if (node.tagName !== 'data') return decoder.decode(value);
-      return decodeBase64(node.value, node.attributes);
+      const promise = decodeBase64(node.bytes, node.attributes);
+      // avoid unhandled promise rejection and swallow the error
+      promise.catch((error) => {
+        logger.error('error decoding base64', error);
+        return [];
+      });
+      return promise;
     },
   });
-
+  await recursiveResolve(parsed);
   processMetadata(parsed.mzData, result.metadata);
   processSpectrumList(parsed.mzData, result.times, result.series.ms.data);
 

diff --git a/src/mzml/parseMzML.js b/src/mzml/parseMzML.js
@@ -1,4 +1,5 @@
 import { parse } from 'arraybuffer-xml-parser';
+import { recursiveResolve } from 'ml-spectra-processing';
 
 import { decodeBase64 } from '../util/decodeBase64';
 
@@ -8,7 +9,8 @@ const decoder = new TextDecoder();
 
 // https://www.psidev.info/mzml
 // CV = Controlled vocabulary
-export function parseMzML(arrayBuffer) {
+export async function parseMzML(arrayBuffer, options = {}) {
+  const { logger = console } = options;
   const result = {
     metadata: {},
     times: [],
@@ -20,17 +22,24 @@ export function parseMzML(arrayBuffer) {
   };
 
   let parsed = parse(arrayBuffer, {
-    attributeNamePrefix: '',
     attributesNodeName: 'attributes',
+    attributeNameProcessor: (attributeName) => attributeName,
     tagValueProcessor: (value, node) => {
       if (node.tagName !== 'binary') return decoder.decode(value);
       const ontologies = node.parent.children.cvParam.map(
         (entry) => entry.attributes.accession,
       );
-
-      return decodeBase64(node.value, { ontologies });
+      const promise = decodeBase64(node.bytes, { ontologies });
+      // avoid unhandled promise rejection and swallow the error
+      promise.catch((error) => {
+        logger.error('error decoding base64', error);
+        return [];
+      });
+      return promise;
     },
   });
+  // parsed file still contains promises
+  await recursiveResolve(parsed);
 
   const mzML = parsed.mzML || parsed.indexedmzML.mzML;
 

diff --git a/src/mzml/utils.js b/src/mzml/utils.js
@@ -1,12 +1,13 @@
 import { decode } from 'uint8-base64';
-import { inflate } from 'pako';
 
-export function decoder(base64Encoded, options = {}) {
+import { inflate } from '../util/inflate.js';
+
+export async function decoder(base64Encoded, options = {}) {
   const { compressionAlgorithm } = options;
   let decoded;
   switch (compressionAlgorithm) {
     case 'zlib':
-      decoded = inflate(decode(base64Encoded));
+      decoded = await inflate(decode(base64Encoded));
       break;
     case undefined:
     case '':

diff --git a/src/mzxml/parseMzXML.js b/src/mzxml/parseMzXML.js
@@ -1,12 +1,20 @@
 import { parse } from 'arraybuffer-xml-parser';
+import { recursiveResolve } from 'ml-spectra-processing';
 
 import { decodeBase64 } from '../util/decodeBase64';
 
 import { processSpectrumList } from './processSpectrumList';
 
 const decoder = new TextDecoder();
 
-export function parseMzXML(arrayBuffer) {
+/**
+ *
+ * @param {*} arrayBuffer
+ * @param {import('../Options.js').Options} [options]
+ * @returns
+ */
+export async function parseMzXML(arrayBuffer, options = {}) {
+  const { logger = console } = options;
   const result = {
     metadata: {},
     times: [],
@@ -17,17 +25,25 @@ export function parseMzXML(arrayBuffer) {
     },
   };
   let parsed = parse(arrayBuffer, {
-    attributeNamePrefix: '',
     attributesNodeName: 'attributes',
+    attributeNameProcessor: (attributeName) => attributeName,
     tagValueProcessor: (value, node) => {
       if (node.tagName !== 'peaks') return decoder.decode(value);
-      return decodeBase64(node.value, {
+
+      const promise = decodeBase64(node.bytes, {
         precision: node.attributes.precision,
         endian: node.attributes.byteOrder,
         compression: node.attributes.compressionType,
       });
+      // avoid unhandled promise rejection and swallow the error
+      promise.catch((error) => {
+        logger.error('error decoding base64', error);
+        return [];
+      });
+      return promise;
     },
   });
+  await recursiveResolve(parsed);
 
   processSpectrumList(parsed.mzXML, result.times, result.series.ms);
 

diff --git a/src/mzxml/utils.js b/src/mzxml/utils.js
@@ -1,13 +1,13 @@
 import { decode } from 'uint8-base64';
 
-import { inflate } from 'pako';
+import { inflate } from '../util/inflate.js';
 
-export function decoder(base64Encoded, options = {}) {
+export async function decoder(base64Encoded, options = {}) {
   const { compressionAlgorithm } = options;
   let decoded;
   switch (compressionAlgorithm) {
     case 'zlib':
-      decoded = inflate(decode(base64Encoded));
+      decoded = await inflate(decode(base64Encoded));
       break;
     case undefined:
     case '':

diff --git a/src/util/decodeBase64.js b/src/util/decodeBase64.js
@@ -1,7 +1,8 @@
-import { inflate } from 'pako';
 import { decode } from 'uint8-base64';
 
-export function decodeBase64(base64, options = {}) {
+import { inflate } from './inflate.js';
+
+export async function decodeBase64(base64, options = {}) {
   let {
     endian = 'little',
     precision,
@@ -28,7 +29,7 @@ export function decodeBase64(base64, options = {}) {
   let uint8Array = decode(base64);
   switch (compression.toLowerCase()) {
     case 'zlib':
-      uint8Array = inflate(uint8Array);
+      uint8Array = await inflate(uint8Array);
       break;
     case '':
     case 'none':

diff --git a/src/util/inflate.js b/src/util/inflate.js
@@ -0,0 +1,37 @@
+export async function inflate(zlibCompressedData) {
+  // Strip the zlib header and footer
+  const strippedData = zlibCompressedData.subarray(2, -4); // Remove 2-byte header and 4-byte Adler-32 footer
+
+  const inputStream = new ReadableStream({
+    start(controller) {
+      controller.enqueue(strippedData);
+      controller.close();
+    },
+  });
+
+  const decompressedStream = inputStream.pipeThrough(
+    new DecompressionStream('deflate-raw'),
+  );
+
+  const reader = decompressedStream.getReader();
+  const chunks = [];
+  let totalLength = 0;
+
+  while (true) {
+    // eslint-disable-next-line no-await-in-loop
+    const { value, done } = await reader.read();
+    if (done) break;
+    chunks.push(value);
+    totalLength += value.length;
+  }
+
+  // Combine chunks into a single Uint8Array
+  const decompressedData = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const chunk of chunks) {
+    decompressedData.set(chunk, offset);
+    offset += chunk.length;
+  }
+
+  return decompressedData;
+}