diff --git a/e2e/cases/cache/index.test.ts b/e2e/cases/cache/index.test.ts
index 82ee875d0f..b153f352f4 100644
--- a/e2e/cases/cache/index.test.ts
+++ b/e2e/cases/cache/index.test.ts
@@ -1,61 +1,11 @@
 import fs from 'node:fs';
 import path from 'node:path';
-import { build } from '@e2e/helper';
+import { build, webpackOnlyTest } from '@e2e/helper';
 import { expect, test } from '@playwright/test';
 import type { RsbuildConfig } from '@rsbuild/core';
 import fse from 'fs-extra';
 
-test('should save the buildDependencies to cache directory and hit cache', async () => {
-  const cacheDirectory = path.resolve(
-    __dirname,
-    './node_modules/.cache/test-build-dependencies',
-  );
-
-  process.env.TEST_ENV = undefined;
-
-  const buildConfig = {
-    cwd: __dirname,
-    rsbuildConfig: {
-      tools: {
-        bundlerChain: (chain) => {
-          if (process.env.TEST_ENV === 'a') {
-            chain.resolve.extensions.prepend('.test.js');
-          }
-        },
-      },
-      performance: {
-        buildCache: {
-          cacheDirectory,
-        },
-      },
-    } as RsbuildConfig,
-  };
-
-  const configFile = path.resolve(cacheDirectory, 'buildDependencies.json');
-
-  fs.rmSync(cacheDirectory, { recursive: true, force: true });
-
-  // first build without cache
-  let rsbuild = await build(buildConfig);
-
-  expect(
-    (await rsbuild.getIndexFile()).content.includes('222222'),
-  ).toBeTruthy();
-
-  const buildDependencies = await fse.readJSON(configFile);
-  expect(Object.keys(buildDependencies)).toEqual(['tsconfig']);
-
-  process.env.TEST_ENV = 'a';
-
-  // hit cache, so the extension '.test.js' will not work
-  rsbuild = await build(buildConfig);
-
-  expect(
-    (await rsbuild.getIndexFile()).content.includes('222222'),
-  ).toBeTruthy();
-});
-
-test('cacheDigest should work', async () => {
+test('`buildCache.cacheDigest` should work as expected', async () => {
   const cacheDirectory = path.resolve(
     __dirname,
     './node_modules/.cache/test-cache-digest',
@@ -96,3 +46,56 @@ test('cacheDigest should work', async () => {
     (await rsbuild.getIndexFile()).content.includes('111111'),
   ).toBeTruthy();
 });
+
+webpackOnlyTest(
+  'should save the buildDependencies to cache directory and hit cache',
+  async () => {
+    const cacheDirectory = path.resolve(
+      __dirname,
+      './node_modules/.cache/test-build-dependencies',
+    );
+
+    process.env.TEST_ENV = undefined;
+
+    const buildConfig = {
+      cwd: __dirname,
+      rsbuildConfig: {
+        tools: {
+          bundlerChain: (chain) => {
+            if (process.env.TEST_ENV === 'a') {
+              chain.resolve.extensions.prepend('.test.js');
+            }
+          },
+        },
+        performance: {
+          buildCache: {
+            cacheDirectory,
+          },
+        },
+      } as RsbuildConfig,
+    };
+
+    const configFile = path.resolve(cacheDirectory, 'buildDependencies.json');
+
+    fs.rmSync(cacheDirectory, { recursive: true, force: true });
+
+    // first build without cache
+    let rsbuild = await build(buildConfig);
+
+    expect(
+      (await rsbuild.getIndexFile()).content.includes('222222'),
+    ).toBeTruthy();
+
+    const buildDependencies = await fse.readJSON(configFile);
+    expect(Object.keys(buildDependencies)).toEqual(['tsconfig']);
+
+    process.env.TEST_ENV = 'a';
+
+    // hit cache, so the extension '.test.js' will not work
+    rsbuild = await build(buildConfig);
+
+    expect(
+      (await rsbuild.getIndexFile()).content.includes('222222'),
+    ).toBeTruthy();
+  },
+);
diff --git a/packages/core/src/plugins/cache.ts b/packages/core/src/plugins/cache.ts
index 994920ee85..cc7faca0d9 100644
--- a/packages/core/src/plugins/cache.ts
+++ b/packages/core/src/plugins/cache.ts
@@ -11,7 +11,11 @@ import type {
   RsbuildPlugin,
 } from '../types';
 
-async function validateCache(
+/**
+ * If the filenames in the buildDependencies are changed, webpack will not invalidate the previous cache.
+ * So we need to remove the cache directory to make sure the cache is invalidated.
+ */
+async function validateWebpackCache(
   cacheDirectory: string,
   buildDependencies: Record<string, string[]>,
 ) {
@@ -34,10 +38,6 @@ async function validateCache(
       return;
     }
 
-    /**
-     * If the filenames in the buildDependencies are changed, webpack will not invalidate the previous cache.
-     * So we need to remove the cache directory to make sure the cache is invalidated.
-     */
     await fs.promises.rm(cacheDirectory, { force: true, recursive: true });
   }
 
@@ -133,7 +133,9 @@ export const pluginCache = (): RsbuildPlugin => ({
         environment,
       );
 
-      await validateCache(cacheDirectory, buildDependencies);
+      if (bundlerType === 'webpack') {
+        await validateWebpackCache(cacheDirectory, buildDependencies);
+      }
 
       const useDigest =
         Array.isArray(cacheConfig.cacheDigest) &&