diff --git a/e2e/setup/fixtures/package-name/rstest.esm.config.ts b/e2e/setup/fixtures/package-name/rstest.esm.config.ts
new file mode 100644
index 000000000..3bcca88ce
--- /dev/null
+++ b/e2e/setup/fixtures/package-name/rstest.esm.config.ts
@@ -0,0 +1,15 @@
+import { join } from 'node:path';
+import { defineConfig } from '@rstest/core';
+
+import fse from 'fs-extra';
+
+fse.copySync(
+  join(__dirname, './test-setup-esm-fixtures'),
+  join(__dirname, './node_modules/test-setup-esm'),
+);
+
+export default defineConfig({
+  passWithNoTests: true,
+  setupFiles: ['test-setup-esm'],
+  exclude: ['**/node_modules/**', '**/dist/**'],
+});
diff --git a/e2e/setup/fixtures/package-name/rstest.fileProtocol.config.ts b/e2e/setup/fixtures/package-name/rstest.fileProtocol.config.ts
new file mode 100644
index 000000000..0b8febc3e
--- /dev/null
+++ b/e2e/setup/fixtures/package-name/rstest.fileProtocol.config.ts
@@ -0,0 +1,15 @@
+import { join } from 'node:path';
+import { defineConfig } from '@rstest/core';
+
+import fse from 'fs-extra';
+
+fse.copySync(
+  join(__dirname, './test-setup-fixtures'),
+  join(__dirname, './node_modules/test-setup'),
+);
+
+export default defineConfig({
+  passWithNoTests: true,
+  setupFiles: [import.meta.resolve('test-setup')],
+  exclude: ['**/node_modules/**', '**/dist/**'],
+});
diff --git a/e2e/setup/fixtures/package-name/test-setup-esm-fixtures/index.js b/e2e/setup/fixtures/package-name/test-setup-esm-fixtures/index.js
new file mode 100644
index 000000000..a778887b6
--- /dev/null
+++ b/e2e/setup/fixtures/package-name/test-setup-esm-fixtures/index.js
@@ -0,0 +1,2 @@
+process.env.RETEST_SETUP_FLAG = '1';
+process.env.NODE_ENV = 'rstest:production';
diff --git a/e2e/setup/fixtures/package-name/test-setup-esm-fixtures/package.json b/e2e/setup/fixtures/package-name/test-setup-esm-fixtures/package.json
new file mode 100644
index 000000000..6c5fdcc38
--- /dev/null
+++ b/e2e/setup/fixtures/package-name/test-setup-esm-fixtures/package.json
@@ -0,0 +1,10 @@
+{
+  "private": true,
+  "name": "@rstest/tests-setup-esm",
+  "exports": {
+    ".": {
+      "import": "./index.js"
+    }
+  },
+  "version": "1.0.0"
+}
diff --git a/e2e/setup/index.test.ts b/e2e/setup/index.test.ts
index d9be609fb..66a55f386 100644
--- a/e2e/setup/index.test.ts
+++ b/e2e/setup/index.test.ts
@@ -56,4 +56,30 @@ describe('test setup file', async () => {
     });
     await expectExecSuccess();
   });
+
+  it('should resolve setup file correctly when setupFiles path with file protocol', async () => {
+    const { expectExecSuccess } = await runRstestCli({
+      command: 'rstest',
+      args: ['run', '-c', 'rstest.fileProtocol.config.ts'],
+      options: {
+        nodeOptions: {
+          cwd: join(__dirname, 'fixtures/package-name'),
+        },
+      },
+    });
+    await expectExecSuccess();
+  });
+
+  it('should resolve setup file correctly when setupFile is pure es module', async () => {
+    const { expectExecSuccess } = await runRstestCli({
+      command: 'rstest',
+      args: ['run', '-c', 'rstest.esm.config.ts'],
+      options: {
+        nodeOptions: {
+          cwd: join(__dirname, 'fixtures/package-name'),
+        },
+      },
+    });
+    await expectExecSuccess();
+  });
 });
diff --git a/packages/core/src/core/listTests.ts b/packages/core/src/core/listTests.ts
index 99d91d30a..99e61132a 100644
--- a/packages/core/src/core/listTests.ts
+++ b/packages/core/src/core/listTests.ts
@@ -11,7 +11,6 @@ import type {
 import {
   bgColor,
   color,
-  getSetupFiles,
   getTaskNameWithPrefix,
   getTestEntries,
   logger,
@@ -28,6 +27,7 @@ const collectTests = async ({
   context: RstestContext;
   globTestSourceEntries: (name: string) => Promise<Record<string, string>>;
 }) => {
+  const { getSetupFiles } = await import('../utils/getSetupFiles');
   const setupFiles = Object.fromEntries(
     context.projects.map((project) => {
       const {
diff --git a/packages/core/src/core/runTests.ts b/packages/core/src/core/runTests.ts
index 8ddbdadf7..c20395f0a 100644
--- a/packages/core/src/core/runTests.ts
+++ b/packages/core/src/core/runTests.ts
@@ -1,13 +1,7 @@
 import { createCoverageProvider } from '../coverage';
 import { createPool } from '../pool';
 import type { EntryInfo } from '../types';
-import {
-  clearScreen,
-  color,
-  getSetupFiles,
-  getTestEntries,
-  logger,
-} from '../utils';
+import { clearScreen, color, getTestEntries, logger } from '../utils';
 import { isCliShortcutsEnabled, setupCliShortcuts } from './cliShortcuts';
 import { runGlobalSetup, runGlobalTeardown } from './globalSetup';
 import { createRsbuildServer, prepareRsbuild } from './rsbuild';
@@ -54,6 +48,8 @@ export async function runTests(context: Rstest): Promise<void> {
     return entries;
   };
 
+  const { getSetupFiles } = await import('../utils/getSetupFiles');
+
   const setupFiles = Object.fromEntries(
     context.projects.map((project) => {
       const {
diff --git a/packages/core/src/utils/getSetupFiles.ts b/packages/core/src/utils/getSetupFiles.ts
new file mode 100644
index 000000000..12bafb7ae
--- /dev/null
+++ b/packages/core/src/utils/getSetupFiles.ts
@@ -0,0 +1,50 @@
+import { existsSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { rspack } from '@rsbuild/core';
+import pathe from 'pathe';
+import { color, getAbsolutePath } from './helper';
+import { formatTestEntryName } from './testFiles';
+
+const tryResolve = (request: string, rootPath: string) => {
+  const { resolver } = rspack.experiments;
+  const esmFirstResolver = new resolver.ResolverFactory({
+    conditionNames: ['node', 'import', 'require'],
+  });
+  const { path: resolvedPath } = esmFirstResolver.sync(rootPath, request);
+  return resolvedPath;
+};
+
+export const getSetupFiles = (
+  setups: string[],
+  rootPath: string,
+): Record<string, string> => {
+  if (!setups.length) {
+    return {};
+  }
+  return Object.fromEntries(
+    setups.map((filePath) => {
+      const setupFile = filePath.startsWith('file://')
+        ? fileURLToPath(filePath)
+        : filePath;
+      const setupFilePath = getAbsolutePath(rootPath, setupFile);
+      try {
+        if (!existsSync(setupFilePath)) {
+          let errorMessage = `Setup file ${color.red(setupFile)} not found`;
+          if (setupFilePath !== setupFile) {
+            errorMessage += color.gray(` (resolved path: ${setupFilePath})`);
+          }
+          throw errorMessage;
+        }
+        const relativePath = pathe.relative(rootPath, setupFilePath);
+        return [formatTestEntryName(relativePath), setupFilePath];
+      } catch (err) {
+        const resolvedPath = tryResolve(setupFile, rootPath);
+        // support use package name as setupFiles value
+        if (resolvedPath) {
+          return [formatTestEntryName(setupFile), resolvedPath];
+        }
+        throw err;
+      }
+    }),
+  );
+};
diff --git a/packages/core/src/utils/testFiles.ts b/packages/core/src/utils/testFiles.ts
index 75ede92fa..67b0825f2 100644
--- a/packages/core/src/utils/testFiles.ts
+++ b/packages/core/src/utils/testFiles.ts
@@ -1,10 +1,8 @@
-import { existsSync } from 'node:fs';
 import fs from 'node:fs/promises';
-import { createRequire } from 'node:module';
 import pathe from 'pathe';
 import { glob } from 'tinyglobby';
 import type { Project } from '../types';
-import { castArray, color, getAbsolutePath, parsePosix } from './helper';
+import { castArray, color, parsePosix } from './helper';
 
 export const filterFiles = (
   testFiles: string[],
@@ -126,47 +124,6 @@ export const getTestEntries = async ({
   );
 };
 
-const tryResolve = (request: string, rootPath: string) => {
-  try {
-    const require = createRequire(rootPath);
-    return require.resolve(request, { paths: [rootPath] });
-  } catch (_err) {
-    return undefined;
-  }
-};
-
-export const getSetupFiles = (
-  setups: string[],
-  rootPath: string,
-): Record<string, string> => {
-  if (!setups.length) {
-    return {};
-  }
-  return Object.fromEntries(
-    setups.map((setupFile) => {
-      const setupFilePath = getAbsolutePath(rootPath, setupFile);
-      try {
-        if (!existsSync(setupFilePath)) {
-          let errorMessage = `Setup file ${color.red(setupFile)} not found`;
-          if (setupFilePath !== setupFile) {
-            errorMessage += color.gray(` (resolved path: ${setupFilePath})`);
-          }
-          throw errorMessage;
-        }
-        const relativePath = pathe.relative(rootPath, setupFilePath);
-        return [formatTestEntryName(relativePath), setupFilePath];
-      } catch (err) {
-        const resolvedPath = tryResolve(setupFile, rootPath);
-        // support use package name as setupFiles value
-        if (resolvedPath) {
-          return [formatTestEntryName(setupFile), resolvedPath];
-        }
-        throw err;
-      }
-    }),
-  );
-};
-
 export const prettyTestPath = (testPath: string): string => {
   const { dir, base } = parsePosix(testPath);
 
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 53d0fd6a2..cf50392fc 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -296,6 +296,8 @@ importers:
         specifier: workspace:*
         version: link:../../../../../packages/core
 
+  e2e/setup/fixtures/package-name/test-setup-esm-fixtures: {}
+
   e2e/setup/fixtures/package-name/test-setup-fixtures: {}
 
   e2e/ssr/fixtures: