Fixes compatibility with the Node.js 18 fetch implementation

- The response body is a WebStream and not a NodeJS Stream - The "url" option is not allowed in Response constructor
rubensworks · Jul 15, 2022 · 8d5cd3b · 8d5cd3b
1 parent 424601b
commit 8d5cd3b
Show file tree

Hide file tree

Showing 18 changed files with 39 additions and 34 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -12,7 +12,7 @@ jobs:
           key: ${{ runner.os }}-lint-modules-${{ hashFiles('**/yarn.lock') }}
       - uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
       - run: yarn install
       - run: yarn run lint
 
@@ -25,6 +25,7 @@ jobs:
           - 12.x
           - 14.x
           - 16.x
+          - 18.x
     steps:
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v3

diff --git a/lib/Util.ts b/lib/Util.ts
@@ -1,12 +1,14 @@
-import {createReadStream, createWriteStream, existsSync, readFileSync, ReadStream, writeFileSync} from "fs";
+import {createReadStream, createWriteStream, existsSync, readFileSync, writeFileSync} from "fs";
 import {JsonLdParser} from "jsonld-streaming-parser";
 import * as RDF from "@rdfjs/types";
 import {RdfXmlParser} from "rdfxml-streaming-parser";
 import {PassThrough} from "stream";
 import {DocumentLoaderCached} from "./DocumentLoaderCached";
 import {GeneralizedN3StreamParser} from "./GeneralizedN3StreamParser";
-// tslint:disable:no-var-requires
-const streamifyString = require('streamify-string');
+import {ReadableWebToNodeStream} from 'readable-web-to-node-stream';
+
+// tslint:disable-next-line:no-var-requires
+const isStream = require('is-stream');
 
 /**
  * Utility functions
@@ -145,8 +147,12 @@ export class Util {
       if (!response.ok) {
         throw new Error(`Could not find ${url}`);
       }
-      const body1 = (<any> response.body).pipe(new PassThrough());
-      const body2 = (<any> response.body).pipe(new PassThrough());
+      /* istanbul ignore next */
+      const body: NodeJS.ReadableStream = isStream(response.body) || response.body === null ?
+          <any> response.body :
+          new ReadableWebToNodeStream(response.body);
+      const body1 = body.pipe(new PassThrough());
+      const body2 = body.pipe(new PassThrough());
 
       // Remove unneeded headers
       response.headers.delete('content-length');
@@ -161,7 +167,7 @@ export class Util {
           writeStream.on('close', resolve);
           writeStream.on('error', reject);
         });*/
-        writeFileSync(cachePathLocal + '.url', response.url);
+        writeFileSync(cachePathLocal + '.url', response.url || url);
         const headersRaw: any = {};
         response.headers.forEach((value: string, key: string) => headersRaw[key] = value);
         writeFileSync(cachePathLocal + '.headers', JSON.stringify(headersRaw));
@@ -170,7 +176,7 @@ export class Util {
       return {
         body: body2,
         headers: response.headers,
-        url: response.url,
+        url: response.url || url,
       };
     }
   }
@@ -214,7 +220,7 @@ export class Util {
  * A fetch response.
  */
 export interface IFetchResponse {
-  body: ReadStream;
+  body: NodeJS.ReadableStream;
   headers: Headers;
   url: string;
 }

diff --git a/package.json b/package.json
@@ -89,6 +89,7 @@
     "@types/sax": "^1.0.1",
     "arrayify-stream": "^2.0.0",
     "cross-fetch": "^3.0.6",
+    "is-stream": "^2.0.0",
     "json-stable-stringify": "^1.0.1",
     "jsonld-streaming-parser": "^3.0.0",
     "log-symbols": "^4.0.0",
@@ -102,6 +103,7 @@
     "rdf-string": "^1.6.0",
     "rdf-terms": "^1.7.0",
     "rdfxml-streaming-parser": "^2.0.0",
+    "readable-web-to-node-stream": "^3.0.2",
     "relative-to-absolute-iri": "^1.0.6",
     "sparqljson-parse": "^2.0.1",
     "sparqlxml-parse": "^2.0.1",

diff --git a/test/DocumentLoaderCached-test.ts b/test/DocumentLoaderCached-test.ts
@@ -14,9 +14,8 @@ const streamifyString = require('streamify-string');
     break;
   default:
     return Promise.reject(new Error('Fetch error'));
-    break;
   }
-  return Promise.resolve(new Response(body, <any> { headers: new Headers({ a: 'b' }), status: 200, url }));
+  return Promise.resolve(new Response(body, <any> { headers: new Headers({ a: 'b' }), status: 200 }));
 };
 
 describe('DocumentLoaderCached', () => {

diff --git a/test/ManifestLoader-test.ts b/test/ManifestLoader-test.ts
@@ -70,7 +70,7 @@ const streamifyString = require('streamify-string');
     break;
   }
   const headers = new Headers({ 'Content-Type': 'text/turtle' });
-  return Promise.resolve(new Response(body, <any> { headers, status: 200, url }));
+  return Promise.resolve(new Response(body, <any> { headers, status: 200 }));
 };
 
 describe('ManifestLoader', () => {

diff --git a/test/Util-test.ts b/test/Util-test.ts
@@ -27,7 +27,7 @@ const DF = new DataFactory<RDF.BaseQuad>();
     return Promise.reject(new Error('Fetch error'));
     break;
   }
-  return Promise.resolve(new Response(body, <any> { headers: new Headers({ a: 'b' }), status: 200, url }));
+  return Promise.resolve(new Response(body, <any> { headers: new Headers({ a: 'b' }), status: 200 }));
 };
 
 describe('Util', () => {

diff --git a/test/testcase/rdfsyntax/TestCaseEval-test.ts b/test/testcase/rdfsyntax/TestCaseEval-test.ts
@@ -38,9 +38,8 @@ const DF = new DataFactory();
     break;
   default:
     return Promise.reject(new Error('Fetch error'));
-    break;
   }
-  return Promise.resolve(new Response(body, <any> { headers: new Headers({ a: 'b' }), status: 200, url }));
+  return Promise.resolve(new Response(body, <any> { headers: new Headers({ a: 'b' }), status: 200 }));
 };
 
 describe('TestCaseEvalHandler', () => {

diff --git a/test/testcase/rdfsyntax/TestCaseSyntax-test.ts b/test/testcase/rdfsyntax/TestCaseSyntax-test.ts
@@ -38,9 +38,8 @@ const DF = new DataFactory();
     break;
   default:
     return Promise.reject(new Error('Fetch error'));
-    break;
   }
-  return Promise.resolve(new Response(body, <any> { headers: new Headers({ a: 'b' }), status: 200, url }));
+  return Promise.resolve(new Response(body, <any> { headers: new Headers({ a: 'b' }), status: 200 }));
 };
 
 describe('TestCaseSyntaxHandler positive', () => {

diff --git a/test/testcase/rdfsyntax/jsonld/TestCaseJsonLdFromRdf-test.ts b/test/testcase/rdfsyntax/jsonld/TestCaseJsonLdFromRdf-test.ts
@@ -60,9 +60,8 @@ const DF = new DataFactory();
     break;
   default:
     return Promise.reject(new Error('Fetch error'));
-    break;
   }
-  return Promise.resolve(new Response(body, <any> { headers: new Headers({ a: 'b' }), status: 200, url }));
+  return Promise.resolve(new Response(body, <any> { headers: new Headers({ a: 'b' }), status: 200 }));
 };
 
 describe('TestCaseJsonLdFromRdfHandler', () => {

diff --git a/test/testcase/rdfsyntax/jsonld/TestCaseJsonLdFromRdfNegative-test.ts b/test/testcase/rdfsyntax/jsonld/TestCaseJsonLdFromRdfNegative-test.ts
@@ -31,9 +31,8 @@ const streamifyString = require('streamify-string');
     break;
   default:
     return Promise.reject(new Error('Fetch error'));
-    break;
   }
-  return Promise.resolve(new Response(body, <any> { headers: new Headers({ a: 'b' }), status: 200, url }));
+  return Promise.resolve(new Response(body, <any> { headers: new Headers({ a: 'b' }), status: 200 }));
 };
 
 describe('TestCaseJsonLdFromRdfNegativeHandler', () => {

diff --git a/test/testcase/rdfsyntax/jsonld/TestCaseJsonLdSyntaxHandler-test.ts b/test/testcase/rdfsyntax/jsonld/TestCaseJsonLdSyntaxHandler-test.ts
@@ -32,9 +32,8 @@ const DF = new DataFactory();
     break;
   default:
     return Promise.reject(new Error('Fetch error'));
-    break;
   }
-  return Promise.resolve(new Response(body, <any> { headers: new Headers({ a: 'b' }), status: 200, url }));
+  return Promise.resolve(new Response(body, <any> { headers: new Headers({ a: 'b' }), status: 200 }));
 };
 
 describe('TestCaseJsonLdSyntaxHandler', () => {

diff --git a/test/testcase/rdfsyntax/jsonld/TestCaseJsonLdToRdf-test.ts b/test/testcase/rdfsyntax/jsonld/TestCaseJsonLdToRdf-test.ts
@@ -42,9 +42,8 @@ const DF = new DataFactory();
     break;
   default:
     return Promise.reject(new Error('Fetch error'));
-    break;
   }
-  return Promise.resolve(new Response(body, <any> { headers: new Headers({ a: 'b' }), status: 200, url }));
+  return Promise.resolve(new Response(body, <any> { headers: new Headers({ a: 'b' }), status: 200 }));
 };
 
 describe('TestCaseJsonLdToRdfHandler', () => {

diff --git a/test/testcase/rdfsyntax/jsonld/TestCaseJsonLdToRdfNegative-test.ts b/test/testcase/rdfsyntax/jsonld/TestCaseJsonLdToRdfNegative-test.ts
@@ -32,9 +32,8 @@ const DF = new DataFactory();
     break;
   default:
     return Promise.reject(new Error('Fetch error'));
-    break;
   }
-  return Promise.resolve(new Response(body, <any> { headers: new Headers({ a: 'b' }), status: 200, url }));
+  return Promise.resolve(new Response(body, <any> { headers: new Headers({ a: 'b' }), status: 200 }));
 };
 
 describe('TestCaseJsonLdToRdfNegativeHandler', () => {

diff --git a/test/testcase/sparql/TestCaseNegativeSyntax-test.ts b/test/testcase/sparql/TestCaseNegativeSyntax-test.ts
@@ -21,9 +21,8 @@ const DF = new DataFactory();
     break;
   default:
     return Promise.reject(new Error('Fetch error'));
-    break;
   }
-  return Promise.resolve(new Response(body, <any> { headers: new Headers({ a: 'b' }), status: 200, url }));
+  return Promise.resolve(new Response(body, <any> { headers: new Headers({ a: 'b' }), status: 200 }));
 };
 
 describe('TestCaseNegativeSyntaxHandler', () => {

diff --git a/test/testcase/sparql/TestCasePositiveSyntax-test.ts b/test/testcase/sparql/TestCasePositiveSyntax-test.ts
@@ -21,9 +21,8 @@ const DF = new DataFactory();
     break;
   default:
     return Promise.reject(new Error('Fetch error'));
-    break;
   }
-  return Promise.resolve(new Response(body, <any> { headers: new Headers({ a: 'b' }), status: 200, url }));
+  return Promise.resolve(new Response(body, <any> { headers: new Headers({ a: 'b' }), status: 200 }));
 };
 
 describe('TestCasePositiveSyntaxHandler', () => {

diff --git a/test/testcase/sparql/TestCaseQueryEvaluation-test.ts b/test/testcase/sparql/TestCaseQueryEvaluation-test.ts
@@ -47,9 +47,8 @@ const DF = new DataFactory();
     break;
   default:
     return Promise.reject(new Error('Fetch error for ' + url));
-    break;
   }
-  return Promise.resolve(new Response(body, <any> { headers, status: 200, url }));
+  return Promise.resolve(new Response(body, <any> { headers, status: 200 }));
 };
 
 describe('TestCaseQueryEvaluationHandler', () => {

diff --git a/test/testcase/sparql/TestCaseUpdateEvaluation-test.ts b/test/testcase/sparql/TestCaseUpdateEvaluation-test.ts
@@ -35,7 +35,7 @@ const DF = new DataFactory();
     return Promise.reject(new Error('Fetch error for ' + url));
     break;
   }
-  return Promise.resolve(new Response(body, <any> { headers, status: 200, url }));
+  return Promise.resolve(new Response(body, <any> { headers, status: 200 }));
 };
 
 describe('TestCaseUpdateEvaluationHandler', () => {

diff --git a/yarn.lock b/yarn.lock
@@ -2904,6 +2904,13 @@ readable-stream@^4.0.0:
   dependencies:
     abort-controller "^3.0.0"
 
+readable-web-to-node-stream@^3.0.2:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.2.tgz#5d52bb5df7b54861fd48d015e93a2cb87b3ee0bb"
+  integrity sha512-ePeK6cc1EcKLEhJFt/AebMCLL+GgSKhuygrZ/GLaKZYEecIgIECf4UaUuaByiGtzckwR4ain9VzUh95T1exYGw==
+  dependencies:
+    readable-stream "^3.6.0"
+
 redent@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/redent/-/redent-2.0.0.tgz#c1b2007b42d57eb1389079b3c8333639d5e1ccaa"