feat: Preserve UTF-8 BOM when editing files

docs/users/configuration/settings.md

@@ -51,14 +51,15 @@ Settings are organized into categories. All settings should be placed within the
 
 #### general
 
-| Setting                         | Type    | Description                                                                                                | Default     |
-| ------------------------------- | ------- | ---------------------------------------------------------------------------------------------------------- | ----------- |
-| `general.preferredEditor`       | string  | The preferred editor to open files in.                                                                     | `undefined` |
-| `general.vimMode`               | boolean | Enable Vim keybindings.                                                                                    | `false`     |
-| `general.disableAutoUpdate`     | boolean | Disable automatic updates.                                                                                 | `false`     |
-| `general.disableUpdateNag`      | boolean | Disable update notification prompts.                                                                       | `false`     |
-| `general.gitCoAuthor`           | boolean | Automatically add a Co-authored-by trailer to git commit messages when commits are made through Qwen Code. | `true`      |
-| `general.checkpointing.enabled` | boolean | Enable session checkpointing for recovery.                                                                 | `false`     |
+| Setting                         | Type    | Description                                                                                                                                                                     | Default     |
+| ------------------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------- |
+| `general.preferredEditor`       | string  | The preferred editor to open files in.                                                                                                                                          | `undefined` |
+| `general.vimMode`               | boolean | Enable Vim keybindings.                                                                                                                                                         | `false`     |
+| `general.disableAutoUpdate`     | boolean | Disable automatic updates.                                                                                                                                                      | `false`     |
+| `general.disableUpdateNag`      | boolean | Disable update notification prompts.                                                                                                                                            | `false`     |
+| `general.gitCoAuthor`           | boolean | Automatically add a Co-authored-by trailer to git commit messages when commits are made through Qwen Code.                                                                      | `true`      |
+| `general.checkpointing.enabled` | boolean | Enable session checkpointing for recovery.                                                                                                                                      | `false`     |
+| `general.defaultFileEncoding`   | string  | Default encoding for new files. Use `"utf-8"` (default) for UTF-8 without BOM, or `"utf-8-bom"` for UTF-8 with BOM. Only change this if your project specifically requires BOM. | `"utf-8"`   |
 
 #### output
 

integration-tests/utf-bom-encoding.test.ts

@@ -5,7 +5,7 @@
  */
 
 import { describe, it, expect, beforeAll, afterAll } from 'vitest';
-import { writeFileSync } from 'node:fs';
+import { writeFileSync, readFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { TestRig } from './test-helper.js';
 
@@ -121,4 +121,98 @@ d('BOM end-to-end integration', () => {
       'BOM_OK UTF-32BE',
     );
   });
+
+  it('should preserve UTF-8 BOM when editing existing file', async () => {
+    // Create a file with UTF-8 BOM and Chinese content
+    const originalContent =
+      '// 这是一个测试文件\n// 包含中文注释\nfunction test() {\n  return "hello";\n}\n';
+    const fileWithBOM = Buffer.concat([
+      Buffer.from([0xef, 0xbb, 0xbf]),
+      Buffer.from(originalContent, 'utf8'),
+    ]);
+
+    const filename = 'bom-test.js';
+    writeFileSync(join(dir, filename), fileWithBOM);
+
+    // Ask Qwen Code to edit the file
+    const prompt = `edit the file ${filename} to change the return value from "hello" to "world"`;
+    await rig.run(prompt);
+    await rig.waitForToolCall('edit_file');
+
+    // Read the modified file as raw bytes
+    const modifiedBuffer = readFileSync(join(dir, filename));
+
+    // Verify BOM is preserved (first 3 bytes should be EF BB BF)
+    expect(modifiedBuffer[0]).toBe(0xef);
+    expect(modifiedBuffer[1]).toBe(0xbb);
+    expect(modifiedBuffer[2]).toBe(0xbf);
+
+    // Verify the content was actually changed to include 'world'
+    const modifiedContent = modifiedBuffer.toString('utf8');
+    expect(modifiedContent).toContain('world');
+  });
+
+  it('should preserve UTF-8 BOM when overwriting file with write_file', async () => {
+    // Create a file with UTF-8 BOM
+    const originalContent = '// Original BOM file\nconst x = 1;\n';
+    const fileWithBOM = Buffer.concat([
+      Buffer.from([0xef, 0xbb, 0xbf]),
+      Buffer.from(originalContent, 'utf8'),
+    ]);
+
+    const filename = 'bom-overwrite.js';
+    writeFileSync(join(dir, filename), fileWithBOM);
+
+    // Ask Qwen Code to overwrite the file with new content
+    const prompt = `overwrite the file ${filename} with: const y = 2;\n// new content`;
+    await rig.run(prompt);
+    await rig.waitForToolCall('write_file');
+
+    // Read the modified file as raw bytes
+    const modifiedBuffer = readFileSync(join(dir, filename));
+
+    // Verify BOM is preserved (first 3 bytes should be EF BB BF)
+    expect(modifiedBuffer[0]).toBe(0xef);
+    expect(modifiedBuffer[1]).toBe(0xbb);
+    expect(modifiedBuffer[2]).toBe(0xbf);
+
+    // Verify the new content includes 'const y = 2'
+    const modifiedContent = modifiedBuffer.toString('utf8');
+    expect(modifiedContent).toContain('const y = 2');
+  });
+});
+
+describe('BOM with defaultFileEncoding configuration', () => {
+  it('should create new file with BOM when defaultFileEncoding is utf-8-bom', async () => {
+    const rigWithBOM = new TestRig();
+    await rigWithBOM.setup('bom-default-encoding', {
+      settings: {
+        general: {
+          defaultFileEncoding: 'utf-8-bom',
+        },
+      },
+    });
+
+    const filename = 'new-file-with-bom.js';
+
+    // Ask Qwen Code to create a new file
+    const prompt = `create a new file called ${filename} with content: const greeting = "hello";`;
+    await rigWithBOM.run(prompt);
+    await rigWithBOM.waitForToolCall('write_file');
+
+    // Read the created file as raw bytes
+    const filePath = join(rigWithBOM.testDir!, filename);
+    const fileBuffer = readFileSync(filePath);
+
+    // Verify BOM is present (first 3 bytes should be EF BB BF)
+    expect(fileBuffer[0]).toBe(0xef);
+    expect(fileBuffer[1]).toBe(0xbb);
+    expect(fileBuffer[2]).toBe(0xbf);
+
+    // Verify the content includes the expected string
+    const fileContent = fileBuffer.toString('utf8');
+    expect(fileContent).toContain('const greeting');
+
+    await rigWithBOM.cleanup();
+  });
 });

packages/cli/src/acp-integration/service/filesystem.test.ts

@@ -12,10 +12,98 @@ import { ACP_ERROR_CODES } from '../errorCodes.js';
 const createFallback = (): FileSystemService => ({
   readTextFile: vi.fn(),
   writeTextFile: vi.fn(),
+  detectFileBOM: vi.fn().mockResolvedValue(false),
   findFiles: vi.fn().mockReturnValue([]),
 });
 
 describe('AcpFileSystemService', () => {
+  describe('detectFileBOM', () => {
+    it('detects BOM through ACP client when content starts with U+FEFF', async () => {
+      const client = {
+        readTextFile: vi
+          .fn()
+          .mockResolvedValue({ content: '\ufeff// BOM file' }),
+      } as unknown as import('../acp.js').Client;
+
+      const svc = new AcpFileSystemService(
+        client,
+        'session-1',
+        { readTextFile: true, writeTextFile: true },
+        createFallback(),
+      );
+
+      const result = await svc.detectFileBOM('/test/file.txt');
+      expect(result).toBe(true);
+      expect(client.readTextFile).toHaveBeenCalledWith({
+        path: '/test/file.txt',
+        sessionId: 'session-1',
+        line: null,
+        limit: 1,
+      });
+    });
+
+    it('detects no BOM through ACP client when content does not start with U+FEFF', async () => {
+      const client = {
+        readTextFile: vi.fn().mockResolvedValue({ content: '// No BOM file' }),
+      } as unknown as import('../acp.js').Client;
+
+      const svc = new AcpFileSystemService(
+        client,
+        'session-2',
+        { readTextFile: true, writeTextFile: true },
+        createFallback(),
+      );
+
+      const result = await svc.detectFileBOM('/test/file.txt');
+      expect(result).toBe(false);
+    });
+
+    it('falls back to local filesystem when ACP client fails', async () => {
+      const client = {
+        readTextFile: vi.fn().mockRejectedValue(new Error('Network error')),
+      } as unknown as import('../acp.js').Client;
+
+      const fallback = createFallback();
+      (fallback.detectFileBOM as ReturnType<typeof vi.fn>).mockResolvedValue(
+        true,
+      );
+
+      const svc = new AcpFileSystemService(
+        client,
+        'session-3',
+        { readTextFile: true, writeTextFile: true },
+        fallback,
+      );
+
+      const result = await svc.detectFileBOM('/test/file.txt');
+      expect(result).toBe(true);
+      expect(fallback.detectFileBOM).toHaveBeenCalledWith('/test/file.txt');
+    });
+
+    it('falls back to local filesystem when readTextFile capability is disabled', async () => {
+      const client = {
+        readTextFile: vi.fn(),
+      } as unknown as import('../acp.js').Client;
+
+      const fallback = createFallback();
+      (fallback.detectFileBOM as ReturnType<typeof vi.fn>).mockResolvedValue(
+        false,
+      );
+
+      const svc = new AcpFileSystemService(
+        client,
+        'session-4',
+        { readTextFile: false, writeTextFile: true },
+        fallback,
+      );
+
+      const result = await svc.detectFileBOM('/test/file.txt');
+      expect(result).toBe(false);
+      expect(fallback.detectFileBOM).toHaveBeenCalledWith('/test/file.txt');
+      expect(client.readTextFile).not.toHaveBeenCalled();
+    });
+  });
+
   describe('readTextFile ENOENT handling', () => {
     it('converts RESOURCE_NOT_FOUND error to ENOENT', async () => {
       const resourceNotFoundError = {

packages/cli/src/acp-integration/service/filesystem.ts

@@ -54,17 +54,45 @@ export class AcpFileSystemService implements FileSystemService {
     return response.content;
   }
 
-  async writeTextFile(filePath: string, content: string): Promise<void> {
+  async writeTextFile(
+    filePath: string,
+    content: string,
+    options?: { bom?: boolean },
+  ): Promise<void> {
     if (!this.capabilities.writeTextFile) {
-      return this.fallback.writeTextFile(filePath, content);
+      return this.fallback.writeTextFile(filePath, content, options);
     }
 
+    // Prepend BOM character if requested
+    const finalContent = options?.bom ? '\uFEFF' + content : content;
+
     await this.client.writeTextFile({
       path: filePath,
-      content,
+      content: finalContent,
       sessionId: this.sessionId,
     });
   }
+
+  async detectFileBOM(filePath: string): Promise<boolean> {
+    // Try to detect BOM through ACP client first by reading first line
+    if (this.capabilities.readTextFile) {
+      try {
+        const response = await this.client.readTextFile({
+          path: filePath,
+          sessionId: this.sessionId,
+          line: null,
+          limit: 1,
+        });
+        // Check if content starts with BOM character (U+FEFF)
+        return response.content.charCodeAt(0) === 0xfeff;
+      } catch {
+        // Fall through to fallback if ACP read fails
+      }
+    }
+    // Fall back to local filesystem detection
+    return this.fallback.detectFileBOM(filePath);
+  }
+
   findFiles(fileName: string, searchPaths: readonly string[]): string[] {
     return this.fallback.findFiles(fileName, searchPaths);
   }

packages/cli/src/config/config.ts

@@ -10,6 +10,7 @@ import {
   Config,
   DEFAULT_QWEN_EMBEDDING_MODEL,
   FileDiscoveryService,
+  FileEncoding,
   getCurrentGeminiMdFilename,
   loadServerHierarchicalMemory,
   setGeminiMdFilename as setServerGeminiMdFilename,
@@ -1030,6 +1031,8 @@ export async function loadCliConfig(
     // always be true and the settings file can never disable recording.
     chatRecording:
       argv.chatRecording ?? settings.general?.chatRecording ?? true,
+    defaultFileEncoding:
+      settings.general?.defaultFileEncoding ?? FileEncoding.UTF8,
     lsp: {
       enabled: lspEnabled,
     },

packages/cli/src/config/settingsSchema.ts

@@ -244,6 +244,20 @@ const SETTINGS_SCHEMA = {
           'Enable saving chat history to disk. Disabling this will also prevent --continue and --resume from working.',
         showInDialog: false,
       },
+      defaultFileEncoding: {
+        type: 'enum',
+        label: 'Default File Encoding',
+        category: 'General',
+        requiresRestart: false,
+        default: 'utf-8',
+        description:
+          'Default encoding for new files. Use "utf-8" (default) for UTF-8 without BOM, or "utf-8-bom" for UTF-8 with BOM. Only change this if your project specifically requires BOM.',
+        showInDialog: false,
+        options: [
+          { value: 'utf-8', label: 'UTF-8 (without BOM)' },
+          { value: 'utf-8-bom', label: 'UTF-8 with BOM' },
+        ],
+      },
     },
   },
   output: {

packages/core/src/config/config.ts

@@ -36,6 +36,8 @@ import { FileDiscoveryService } from '../services/fileDiscoveryService.js';
 import {
   type FileSystemService,
   StandardFileSystemService,
+  type FileEncodingType,
+  FileEncoding,
 } from '../services/fileSystemService.js';
 import { GitService } from '../services/gitService.js';
 
@@ -350,6 +352,7 @@ export interface ConfigParameters {
   chatCompression?: ChatCompressionSettings;
   interactive?: boolean;
   trustedFolder?: boolean;
+  defaultFileEncoding?: FileEncodingType;
   useRipgrep?: boolean;
   useBuiltinRipgrep?: boolean;
   shouldUseNodePtyShell?: boolean;
@@ -512,6 +515,7 @@ export class Config {
   private readonly eventEmitter?: EventEmitter;
   private readonly useSmartEdit: boolean;
   private readonly channel: string | undefined;
+  private readonly defaultFileEncoding: FileEncodingType;
 
   constructor(params: ConfigParameters) {
     this.sessionId = params.sessionId ?? randomUUID();
@@ -625,6 +629,7 @@ export class Config {
     this.enableToolOutputTruncation = params.enableToolOutputTruncation ?? true;
     this.useSmartEdit = params.useSmartEdit ?? false;
     this.channel = params.channel;
+    this.defaultFileEncoding = params.defaultFileEncoding ?? FileEncoding.UTF8;
     this.storage = new Storage(this.targetDir);
     this.vlmSwitchMode = params.vlmSwitchMode;
     this.inputFormat = params.inputFormat ?? InputFormat.TEXT;
@@ -1432,6 +1437,14 @@ export class Config {
     return this.channel;
   }
 
+  /**
+   * Get the default file encoding for new files.
+   * @returns FileEncodingType
+   */
+  getDefaultFileEncoding(): FileEncodingType {
+    return this.defaultFileEncoding;
+  }
+
   /**
    * Get the current FileSystemService
    */