diff --git a/packages/vite/src/node/__tests__/build.spec.ts b/packages/vite/src/node/__tests__/build.spec.ts
index 024bba0322a447..3a255f2f265129 100644
--- a/packages/vite/src/node/__tests__/build.spec.ts
+++ b/packages/vite/src/node/__tests__/build.spec.ts
@@ -1187,6 +1187,29 @@ test('watch rebuild manifest', async (ctx) => {
   `)
 })
 
+test('copies public directory after building same environment with write false first', async (ctx) => {
+  const root = resolve(dirname, 'fixtures/public-dir-write-false')
+  ctx.onTestFinished(() =>
+    fsp.rm(resolve(root, 'dist'), { recursive: true, force: true }),
+  )
+
+  const builder = await createBuilder({
+    root,
+    configFile: false,
+    logLevel: 'silent',
+  })
+
+  builder.environments.client.config.build.write = false
+  await builder.build(builder.environments.client)
+
+  builder.environments.client.config.build.write = true
+  await builder.build(builder.environments.client)
+
+  await expect(
+    fsp.readFile(resolve(root, 'dist/favicon.svg'), 'utf-8'),
+  ).resolves.toBe('<svg></svg>')
+})
+
 /**
  * for each chunks in output1, if there's a chunk in output2 with the same fileName,
  * ensure that the chunk code is the same. if not, the chunk hash should have changed.
diff --git a/packages/vite/src/node/__tests__/fixtures/public-dir-write-false/index.html b/packages/vite/src/node/__tests__/fixtures/public-dir-write-false/index.html
new file mode 100644
index 00000000000000..e69de29bb2d1d6
diff --git a/packages/vite/src/node/__tests__/fixtures/public-dir-write-false/public/favicon.svg b/packages/vite/src/node/__tests__/fixtures/public-dir-write-false/public/favicon.svg
new file mode 100644
index 00000000000000..dc1ced5b6b300c
--- /dev/null
+++ b/packages/vite/src/node/__tests__/fixtures/public-dir-write-false/public/favicon.svg
@@ -0,0 +1 @@
+<svg></svg>
\ No newline at end of file
diff --git a/packages/vite/src/node/plugins/prepareOutDir.ts b/packages/vite/src/node/plugins/prepareOutDir.ts
index eb8fd6554c6d21..6f5f16544cabdc 100644
--- a/packages/vite/src/node/plugins/prepareOutDir.ts
+++ b/packages/vite/src/node/plugins/prepareOutDir.ts
@@ -20,10 +20,10 @@ export function prepareOutDirPlugin(): Plugin {
         if (rendered.has(this.environment)) {
           return
         }
-        rendered.add(this.environment)
 
         const { config } = this.environment
         if (config.build.write) {
+          rendered.add(this.environment)
           const { root, build: options } = config
           const resolvedOutDirs = getResolvedOutDirs(
             root,