Remove reliance on requestTracker in loadGraphs (#9440)

* Change requestGraph cache key * Change bundleGraph cache key * Change assetGraph cache key * Change manner in which requestGraph is found * Remove assetGraph reliance on requestTracker * Remove bundleGraph reliance requestTracker * Clean up * Change throws to logs * Fix unit test * Ignore log statement * Fix lint comment * Better error logging * Make flow happy
parcel-bundler · Dec 15, 2023 · 9b66e25 · 9b66e25
1 parent ebe4e21
commit 9b66e25
Show file tree

Hide file tree

Showing 5 changed files with 136 additions and 115 deletions.
diff --git a/packages/core/core/src/RequestTracker.js b/packages/core/core/src/RequestTracker.js
@@ -1110,7 +1110,8 @@ export default class RequestTracker {
 
   async writeToCache() {
     let cacheKey = getCacheKey(this.options);
-    let requestGraphKey = hashString(`${cacheKey}:requestGraph`);
+    let requestGraphKey =
+      hashString(`${cacheKey}:requestGraph`) + '-RequestGraph';
     let snapshotKey = hashString(`${cacheKey}:snapshot`);
 
     if (this.options.shouldDisableCache) {
@@ -1206,7 +1207,8 @@ async function loadRequestGraph(options): Async<RequestGraph> {
   }
 
   let cacheKey = getCacheKey(options);
-  let requestGraphKey = hashString(`${cacheKey}:requestGraph`);
+  let requestGraphKey =
+    hashString(`${cacheKey}:requestGraph`) + '-RequestGraph';
   if (await options.cache.hasLargeBlob(requestGraphKey)) {
     let requestGraph: RequestGraph = deserialize(
       await options.cache.getLargeBlob(requestGraphKey),

diff --git a/packages/core/core/src/requests/AssetGraphRequest.js b/packages/core/core/src/requests/AssetGraphRequest.js
@@ -157,9 +157,12 @@ export class AssetGraphBuilder {
     this.shouldBuildLazily = shouldBuildLazily ?? false;
     this.lazyIncludes = lazyIncludes ?? [];
     this.lazyExcludes = lazyExcludes ?? [];
-    this.cacheKey = hashString(
-      `${PARCEL_VERSION}${name}${JSON.stringify(entries) ?? ''}${options.mode}`,
-    );
+    this.cacheKey =
+      hashString(
+        `${PARCEL_VERSION}${name}${JSON.stringify(entries) ?? ''}${
+          options.mode
+        }`,
+      ) + '-AssetGraph';
 
     this.isSingleChangeRebuild =
       api

diff --git a/packages/core/core/src/requests/BundleGraphRequest.js b/packages/core/core/src/requests/BundleGraphRequest.js
@@ -189,11 +189,12 @@ class BundlerRunner {
     this.pluginOptions = new PluginOptions(
       optionsProxy(this.options, api.invalidateOnOptionChange),
     );
-    this.cacheKey = hashString(
-      `${PARCEL_VERSION}:BundleGraph:${JSON.stringify(options.entries) ?? ''}${
-        options.mode
-      }`,
-    );
+    this.cacheKey =
+      hashString(
+        `${PARCEL_VERSION}:BundleGraph:${
+          JSON.stringify(options.entries) ?? ''
+        }${options.mode}`,
+      ) + '-BundleGraph';
   }
 
   async loadConfigs() {

diff --git a/packages/core/integration-tests/test/cache.js b/packages/core/integration-tests/test/cache.js
@@ -6214,11 +6214,12 @@ describe('cache', function () {
         getParcelOptions(entries, options),
       );
 
-      let bundleGraphCacheKey = hashString(
-        `${version}:BundleGraph:${
-          JSON.stringify(resolvedOptions.entries) ?? ''
-        }${resolvedOptions.mode}`,
-      );
+      let bundleGraphCacheKey =
+        hashString(
+          `${version}:BundleGraph:${
+            JSON.stringify(resolvedOptions.entries) ?? ''
+          }${resolvedOptions.mode}`,
+        ) + '-BundleGraph';
 
       assert(
         deserialize(

diff --git a/packages/dev/query/src/index.js b/packages/dev/query/src/index.js
@@ -1,5 +1,5 @@
 // @flow strict-local
-/* eslint-disable monorepo/no-internal-import */
+/* eslint-disable no-console, monorepo/no-internal-import */
 import type {ContentKey, NodeId} from '@parcel/graph';
 import type {PackagedBundleInfo} from '@parcel/core/src/types';
 
@@ -27,56 +27,113 @@ export async function loadGraphs(cacheDir: string): Promise<{|
   bundleInfo: ?Map<ContentKey, PackagedBundleInfo>,
   cacheInfo: ?Map<string, Array<string | number>>,
 |}> {
-  function filesBySizeAndModifiedTime() {
-    let files = fs.readdirSync(cacheDir).map(f => {
-      let stat = fs.statSync(path.join(cacheDir, f));
-      return [path.join(cacheDir, f), stat.size, stat.mtime];
+  function filesByTypeAndModifiedTime() {
+    let files = fs.readdirSync(cacheDir);
+
+    let requestGraphFiles = [];
+    let bundleGraphFiles = [];
+    let assetGraphFiles = [];
+
+    files.forEach(f => {
+      if (f.endsWith('-0')) {
+        let stat = fs.statSync(path.join(cacheDir, f));
+        let info = [path.join(cacheDir, f), stat.size, stat.mtime];
+
+        if (f.endsWith('RequestGraph-0')) {
+          requestGraphFiles.push(info);
+        } else if (f.endsWith('BundleGraph-0')) {
+          bundleGraphFiles.push(info);
+        } else if (f.endsWith('AssetGraph-0')) {
+          assetGraphFiles.push(info);
+        }
+      }
     });
 
-    files.sort(([, a], [, b]) => b - a);
-    files.sort(([, , a], [, , b]) => b - a);
+    requestGraphFiles.sort(([, , aTime], [, , bTime]) => bTime - aTime);
+    bundleGraphFiles.sort(([, , aTime], [, , bTime]) => bTime - aTime);
+    assetGraphFiles.sort(([, , aTime], [, , bTime]) => bTime - aTime);
 
-    return files.map(([f]) => f);
+    return {
+      requestGraphFiles: requestGraphFiles.map(([f]) => f),
+      bundleGraphFiles: bundleGraphFiles.map(([f]) => f),
+      assetGraphFiles: assetGraphFiles.map(([f]) => f),
+    };
   }
 
   let cacheInfo: Map<string, Array<string | number>> = new Map();
-  let timeToDeserialize = 0;
 
-  let requestTracker;
+  let {requestGraphFiles, bundleGraphFiles, assetGraphFiles} =
+    filesByTypeAndModifiedTime();
   const cache = new LMDBCache(cacheDir);
-  for (let f of filesBySizeAndModifiedTime()) {
-    // Empty filename or not the first chunk
-    if (path.extname(f) !== '' || !f.endsWith('-0')) continue;
+
+  // Get requestTracker
+  let requestTracker;
+  if (requestGraphFiles.length > 0) {
     try {
       let file = await cache.getLargeBlob(
-        path.basename(f).slice(0, -'-0'.length),
+        path.basename(requestGraphFiles[0]).slice(0, -'-0'.length),
       );
 
+      let timeToDeserialize = Date.now();
+      let obj = v8.deserialize(file);
+      timeToDeserialize = Date.now() - timeToDeserialize;
+
+      invariant(obj['$$type']?.endsWith('RequestGraph'));
+      let date = Date.now();
+      requestTracker = new RequestTracker({
+        graph: RequestGraph.deserialize(obj.value),
+        // $FlowFixMe
+        farm: null,
+        // $FlowFixMe
+        options: null,
+      });
+      timeToDeserialize += Date.now() - date;
       cacheInfo.set('RequestGraph', [Buffer.byteLength(file)]);
+      cacheInfo.get('RequestGraph')?.push(timeToDeserialize);
+    } catch (e) {
+      console.log('Error loading Request Graph\n', e);
+    }
+  }
 
-      timeToDeserialize = Date.now();
+  // Get bundleGraph
+  let bundleGraph;
+  if (bundleGraphFiles.length > 0) {
+    try {
+      let file = await cache.getLargeBlob(
+        path.basename(bundleGraphFiles[0]).slice(0, -'-0'.length),
+      );
+
+      let timeToDeserialize = Date.now();
       let obj = v8.deserialize(file);
+      invariant(obj.bundleGraph != null);
+      bundleGraph = BundleGraph.deserialize(obj.bundleGraph.value);
       timeToDeserialize = Date.now() - timeToDeserialize;
 
-      /* if (obj.assetGraph != null && obj.assetGraph.value.hash != null) {
-        assetGraph = AssetGraph.deserialize(obj.assetGraph.value);
-      } else if (obj.bundleGraph != null) {
-        bundleGraph = BundleGraph.deserialize(obj.bundleGraph.value);
-      } else */
-      if (obj['$$type']?.endsWith('RequestGraph')) {
-        let date = Date.now();
-        requestTracker = new RequestTracker({
-          graph: RequestGraph.deserialize(obj.value),
-          // $FlowFixMe
-          farm: null,
-          // $FlowFixMe
-          options: null,
-        });
-        timeToDeserialize += Date.now() - date;
-        break;
-      }
+      cacheInfo.set('BundleGraph', [Buffer.byteLength(file)]);
+      cacheInfo.get('BundleGraph')?.push(timeToDeserialize);
+    } catch (e) {
+      console.log('Error loading Bundle Graph\n', e);
+    }
+  }
+
+  // Get assetGraph
+  let assetGraph;
+  if (assetGraphFiles.length > 0) {
+    try {
+      let file = await cache.getLargeBlob(
+        path.basename(assetGraphFiles[0]).slice(0, -'-0'.length),
+      );
+
+      let timeToDeserialize = Date.now();
+      let obj = v8.deserialize(file);
+      invariant(obj.assetGraph != null);
+      assetGraph = AssetGraph.deserialize(obj.assetGraph.value);
+      timeToDeserialize = Date.now() - timeToDeserialize;
+
+      cacheInfo.set('AssetGraph', [Buffer.byteLength(file)]);
+      cacheInfo.get('AssetGraph')?.push(timeToDeserialize);
     } catch (e) {
-      // noop
+      console.log('Error loading Asset Graph\n', e);
     }
   }
 
@@ -87,77 +144,34 @@ export async function loadGraphs(cacheDir: string): Promise<{|
   }
 
   // Load graphs by finding the main subrequests and loading their results
-  let assetGraph, bundleGraph, bundleInfo;
-  cacheInfo.set('BundleGraph', []);
-  cacheInfo.set('AssetGraph', []);
-  invariant(requestTracker);
-  let buildRequestId = requestTracker.graph.getNodeIdByContentKey(
-    'parcel_build_request',
-  );
-  let buildRequestNode = nullthrows(
-    requestTracker.graph.getNode(buildRequestId),
-  );
-  invariant(buildRequestNode.type === 1 && buildRequestNode.requestType === 1);
-  let buildRequestSubRequests = getSubRequests(buildRequestId);
-
-  let bundleGraphRequestNode = buildRequestSubRequests.find(
-    n => n.type === 1 && n.requestType === 2,
-  );
-  if (bundleGraphRequestNode != null) {
-    bundleGraph = BundleGraph.deserialize(
-      (
-        await loadLargeBlobRequestRequest(
-          cache,
-          bundleGraphRequestNode,
-          cacheInfo,
-        )
-      ).bundleGraph.value,
+  let bundleInfo;
+  try {
+    invariant(requestTracker);
+    let buildRequestId = requestTracker.graph.getNodeIdByContentKey(
+      'parcel_build_request',
+    );
+    let buildRequestNode = nullthrows(
+      requestTracker.graph.getNode(buildRequestId),
+    );
+    invariant(
+      buildRequestNode.type === 1 && buildRequestNode.requestType === 1,
     );
+    let buildRequestSubRequests = getSubRequests(buildRequestId);
 
-    let assetGraphRequest = getSubRequests(
-      requestTracker.graph.getNodeIdByContentKey(bundleGraphRequestNode.id),
-    ).find(n => n.type === 1 && n.requestType === 3);
-    if (assetGraphRequest != null) {
-      assetGraph = AssetGraph.deserialize(
-        (await loadLargeBlobRequestRequest(cache, assetGraphRequest, cacheInfo))
-          .assetGraph.value,
-      );
+    let writeBundlesRequest = buildRequestSubRequests.find(
+      n => n.type === 1 && n.requestType === 11,
+    );
+    if (writeBundlesRequest != null) {
+      invariant(writeBundlesRequest.type === 1);
+      // $FlowFixMe[incompatible-cast]
+      bundleInfo = (nullthrows(writeBundlesRequest.result): Map<
+        ContentKey,
+        PackagedBundleInfo,
+      >);
     }
-  }
-  cacheInfo.get('RequestGraph')?.push(timeToDeserialize);
-  let writeBundlesRequest = buildRequestSubRequests.find(
-    n => n.type === 1 && n.requestType === 11,
-  );
-  if (writeBundlesRequest != null) {
-    invariant(writeBundlesRequest.type === 1);
-    // $FlowFixMe[incompatible-cast]
-    bundleInfo = (nullthrows(writeBundlesRequest.result): Map<
-      ContentKey,
-      PackagedBundleInfo,
-    >);
+  } catch (e) {
+    console.log('Error loading bundleInfo\n', e);
   }
 
   return {assetGraph, bundleGraph, requestTracker, bundleInfo, cacheInfo};
 }
-
-async function loadLargeBlobRequestRequest(cache, node, cacheInfo) {
-  invariant(node.type === 1);
-
-  let cachedFile = await cache.getLargeBlob(nullthrows(node.resultCacheKey));
-
-  let TTD = Date.now();
-  let result = v8.deserialize(cachedFile);
-  TTD = Date.now() - TTD;
-
-  if (node.requestType === 2) {
-    cacheInfo.get('BundleGraph')?.push(cachedFile.byteLength); //Add size
-    cacheInfo.get('BundleGraph')?.push(TTD);
-  }
-
-  if (node.requestType === 3) {
-    cacheInfo.get('AssetGraph')?.push(cachedFile.byteLength);
-    cacheInfo.get('AssetGraph')?.push(TTD);
-  }
-
-  return result;
-}