From 31c5539ea8c951d359d32bba8620a45dc66878a6 Mon Sep 17 00:00:00 2001
From: Tomas Matousek <tomat@microsoft.com>
Date: Thu, 8 Jan 2026 15:04:16 -0800
Subject: [PATCH 1/4] Use the compiler encoding for basline source files
 recovered from file system.

Hot Reload compares the current document snapshot with a baseline document snapshot to find out what the semantic difference is between them.

The the baseline solution is captured when the debugging session starts, but the document content at that point doesn't necessarily match the source code that the compiler used to compile the baseline assembly. We compare the checksum of the binary content of the document against the checksum that the compiler stored in the PDB.
If the baseline document checksum doesn't match we read the source file from disk, in case it hasn't been overwritten yet and still contains the content used by the compiler for baseline compilation. If the checksum matches the PDB we know that the decoded text of the document can be used as a baseline for change detection.

When reading the file content we need to use the exact encoding that the compiler used, otherwise we might interpret the binary content differently than the compiler did.
Previously we used the IDE encoding, but it turns out the IDE doesn't necessarily know what encoding was used by the compiler. E.g. LSP doesn't have a concept of encoding and thus the LSP server always uses UTF8 when creating SourceText.

We could try to make sure the encoding is always correctly set in the IDE. The LSP server could detect the encoding.
However, Hot Reload already has all the information that the compiler had when compiling the assembly.
The compiler auto-detects the encoding from the file content unless it's specified via CodePage project property.
If the property is specified the value is stored in the compiler options record in the PDB, which we can read.

This PR changes the code to always auto-detect the encoding from file content. Implementing support for CodePage property is tracked by a follow up issue: https://github.com/dotnet/roslyn/issues/81930
---
 .../EditAndContinue/CommittedSolution.cs      | 10 +--
 .../EditAndContinueWorkspaceServiceTests.cs   | 62 ++++++++++++++-----
 2 files changed, 49 insertions(+), 23 deletions(-)

diff --git a/src/Features/Core/Portable/EditAndContinue/CommittedSolution.cs b/src/Features/Core/Portable/EditAndContinue/CommittedSolution.cs
index 69083fbf5ef2f..8f30454c93287 100644
--- a/src/Features/Core/Portable/EditAndContinue/CommittedSolution.cs
+++ b/src/Features/Core/Portable/EditAndContinue/CommittedSolution.cs
@@ -358,7 +358,7 @@ public bool ContainsDocument(DocumentId documentId)
             return SourceText.From(text, sourceText.Encoding, checksumAlgorithm);
         }
 
-        return await Task.Run(() => TryGetPdbMatchingSourceTextFromDisk(log, filePath, sourceText.Encoding, requiredChecksum, checksumAlgorithm), cancellationToken).ConfigureAwait(false);
+        return await Task.Run(() => TryGetPdbMatchingSourceTextFromDisk(log, filePath, requiredChecksum, checksumAlgorithm), cancellationToken).ConfigureAwait(false);
     }
 
     private static DebugInformationReaderProvider? GetMethodDebugInfoReader(TraceLog log, CompilationOutputs compilationOutputs, string projectName)
@@ -403,7 +403,6 @@ private static bool IsMatchingSourceText(SourceText sourceText, ImmutableArray<b
     private static Optional<SourceText?> TryGetPdbMatchingSourceTextFromDisk(
         TraceLog log,
         string sourceFilePath,
-        Encoding? encoding,
         ImmutableArray<byte> requiredChecksum,
         SourceHashAlgorithm checksumAlgorithm)
     {
@@ -411,11 +410,8 @@ private static bool IsMatchingSourceText(SourceText sourceText, ImmutableArray<b
         {
             using var fileStream = new FileStream(sourceFilePath, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete);
 
-            // We must use the encoding of the document as determined by the IDE (the editor).
-            // This might differ from the encoding that the compiler chooses, so if we just relied on the compiler we 
-            // might end up updating the committed solution with a document that has a different encoding than 
-            // the one that's in the workspace, resulting in false document changes when we compare the two.
-            var sourceText = SourceText.From(fileStream, encoding, checksumAlgorithm);
+            // TODO: Consider CodePage compiler setting (https://github.com/dotnet/roslyn/issues/81930)
+            var sourceText = SourceText.From(fileStream, encoding: null, checksumAlgorithm);
 
             if (IsMatchingSourceText(sourceText, requiredChecksum, checksumAlgorithm))
             {
diff --git a/src/Features/Test/EditAndContinue/EditAndContinueWorkspaceServiceTests.cs b/src/Features/Test/EditAndContinue/EditAndContinueWorkspaceServiceTests.cs
index e1320aab6463d..1a613a0640e24 100644
--- a/src/Features/Test/EditAndContinue/EditAndContinueWorkspaceServiceTests.cs
+++ b/src/Features/Test/EditAndContinue/EditAndContinueWorkspaceServiceTests.cs
@@ -26,7 +26,6 @@
 using Microsoft.CodeAnalysis.UnitTests;
 using Roslyn.Test.Utilities;
 using Roslyn.Test.Utilities.TestGenerators;
-using Roslyn.Utilities;
 using Xunit;
 
 namespace Microsoft.CodeAnalysis.EditAndContinue.UnitTests;
@@ -1493,31 +1492,43 @@ void M()
         ], _telemetryLog);
     }
 
-    [Fact]
-    public async Task Encodings()
+    [Theory]
+    [CombinatorialData]
+    [WorkItem("https://github.com/dotnet/roslyn/issues/81930")]
+    [WorkItem("https://devdiv.visualstudio.com/DevDiv/_workitems/edit/2067885")]
+    public async Task Encodings(bool matchingContent)
     {
         Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
 
-        var source1 = "class C1 { void M() { System.Console.WriteLine(\"ã\"); } }";
+        var editorSource = "class C1 { public void こんにちは() {} }";
+        var fileSource = matchingContent ? editorSource : "class C1 { public virtual void こんにちは() {} }";
+
+        // encoding from the editor (e.g selected by the user in the IDE, or used implicitly by LSP):
+        var editorEncoding = Encoding.UTF8;
+
+        // The actual encoding used by the compiler is either detected from the file content itself (e.g. Unicode encodings)
+        // or it can also be set in the project via CodePage property.
 
-        var encoding = Encoding.GetEncoding(1252);
+        // TODO: https://github.com/dotnet/roslyn/issues/81930
+        // We do not currently account for CodePage property value and thus an encoding such as Shift-JIS that can't be detected
+        // from the file content does not work.
+        // var compilerEncoding = Encoding.GetEncoding("SJIS");
+        var compilerEncoding = Encoding.Unicode;
 
         var dir = Temp.CreateDirectory();
-        var sourceFile = dir.CreateFile("test.cs").WriteAllText(source1, encoding);
+        var sourceFile = dir.CreateFile("test.cs").WriteAllText(fileSource, compilerEncoding);
 
         using var workspace = CreateWorkspace(out var solution, out var service);
 
-        var projectId = ProjectId.CreateNewId();
-        var documentId = DocumentId.CreateNewId(projectId);
+        DocumentId documentId;
 
         solution = solution.
-            AddProject(projectId, "test", "test", LanguageNames.CSharp).
+            AddTestProject("test", out var projectId).Solution.
             WithProjectChecksumAlgorithm(projectId, SourceHashAlgorithm.Sha1).
-            AddMetadataReferences(projectId, TargetFrameworkUtil.GetReferences(TargetFramework.Mscorlib40)).
-            AddDocument(documentId, "test.cs", SourceText.From(source1, encoding, SourceHashAlgorithm.Sha1), filePath: sourceFile.Path);
+            AddDocument(documentId = DocumentId.CreateNewId(projectId), "test.cs", SourceText.From(editorSource, editorEncoding, SourceHashAlgorithm.Sha1), filePath: sourceFile.Path);
 
         // use different checksum alg to trigger PdbMatchingSourceTextProvider call:
-        var moduleId = EmitAndLoadLibraryToDebuggee(projectId, source1, sourceFilePath: sourceFile.Path, encoding: encoding, checksumAlgorithm: SourceHashAlgorithm.Sha256);
+        var moduleId = EmitAndLoadLibraryToDebuggee(projectId, fileSource, sourceFilePath: sourceFile.Path, encoding: compilerEncoding, checksumAlgorithm: SourceHashAlgorithm.Sha256);
 
         var sourceTextProviderCalled = false;
         var sourceTextProvider = new MockPdbMatchingSourceTextProvider()
@@ -1535,12 +1546,31 @@ public async Task Encodings()
 
         EnterBreakState(debuggingSession);
 
-        var (document, state) = await debuggingSession.LastCommittedSolution.GetDocumentAndStateAsync(solution.GetRequiredDocument(documentId), CancellationToken.None);
-        var text = await document.GetTextAsync();
-        Assert.Same(encoding, text.Encoding);
-        Assert.Equal(CommittedSolution.DocumentState.MatchesBuildOutput, state);
+        var document = solution.GetRequiredDocument(documentId);
+        var (committedDocument, state) = await debuggingSession.LastCommittedSolution.GetDocumentAndStateAsync(document, CancellationToken.None);
 
         Assert.True(sourceTextProviderCalled);
+        Assert.Equal(CommittedSolution.DocumentState.MatchesBuildOutput, state);
+
+        if (matchingContent)
+        {
+            // The file text content matches the document text, hence we reuse the existing document:
+            Assert.Same(document, committedDocument);
+        }
+        else
+        {
+            var committedText = await committedDocument.GetTextAsync(CancellationToken.None);
+
+            // We have now baseline document whose encoding differs from the current document.
+            // The content is the same though, so semantics is the same.
+            Assert.Same(compilerEncoding, committedText.Encoding);
+            Assert.Equal(fileSource, committedText.ToString());
+
+            var diagnostics = await debuggingSession.GetDocumentDiagnosticsAsync(document, s_noActiveSpans, CancellationToken.None);
+            AssertEx.Equal(
+                [$"{document.FilePath}: (0,11)-(0,30): Error ENC0004: {string.Format(FeaturesResources.Updating_the_modifiers_of_0_requires_restarting_the_application, FeaturesResources.method)}"],
+                InspectDiagnostics(diagnostics));
+        }
 
         EndDebuggingSession(debuggingSession);
     }

From 45eaff2f2f5422486b5dff65aa64be286d4ec9b2 Mon Sep 17 00:00:00 2001
From: Tomas Matousek <tomat@microsoft.com>
Date: Thu, 8 Jan 2026 17:09:57 -0800
Subject: [PATCH 2/4] test

---
 .../EditAndContinue/EditAndContinueWorkspaceServiceTests.cs  | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/Features/Test/EditAndContinue/EditAndContinueWorkspaceServiceTests.cs b/src/Features/Test/EditAndContinue/EditAndContinueWorkspaceServiceTests.cs
index 1a613a0640e24..c8034527ec1c6 100644
--- a/src/Features/Test/EditAndContinue/EditAndContinueWorkspaceServiceTests.cs
+++ b/src/Features/Test/EditAndContinue/EditAndContinueWorkspaceServiceTests.cs
@@ -1555,6 +1555,11 @@ public async Task Encodings(bool matchingContent)
         if (matchingContent)
         {
             // The file text content matches the document text, hence we reuse the existing document:
+            var documentText = await document.GetTextAsync(CancellationToken.None);
+            var committedText = await committedDocument.GetTextAsync(CancellationToken.None);
+
+            Assert.Equal(committedText.ToString(), documentText.ToString());
+            Assert.True(committedText.ContentEquals(documentText));
             Assert.Same(document, committedDocument);
         }
         else

From 3c03ea022d0404fc92bef43eae9e534a114c6274 Mon Sep 17 00:00:00 2001
From: Tomas Matousek <tomat@microsoft.com>
Date: Thu, 8 Jan 2026 17:13:04 -0800
Subject: [PATCH 3/4] test

---
 .../EditAndContinue/EditAndContinueWorkspaceServiceTests.cs     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Features/Test/EditAndContinue/EditAndContinueWorkspaceServiceTests.cs b/src/Features/Test/EditAndContinue/EditAndContinueWorkspaceServiceTests.cs
index c8034527ec1c6..e9711b607a566 100644
--- a/src/Features/Test/EditAndContinue/EditAndContinueWorkspaceServiceTests.cs
+++ b/src/Features/Test/EditAndContinue/EditAndContinueWorkspaceServiceTests.cs
@@ -1568,7 +1568,7 @@ public async Task Encodings(bool matchingContent)
 
             // We have now baseline document whose encoding differs from the current document.
             // The content is the same though, so semantics is the same.
-            Assert.Same(compilerEncoding, committedText.Encoding);
+            Assert.Equal(compilerEncoding.WebName, committedText.Encoding.WebName);
             Assert.Equal(fileSource, committedText.ToString());
 
             var diagnostics = await debuggingSession.GetDocumentDiagnosticsAsync(document, s_noActiveSpans, CancellationToken.None);

From 4f84fabfdbf47a958d6ee2b976f12a4365911ef0 Mon Sep 17 00:00:00 2001
From: Tomas Matousek <tomat@microsoft.com>
Date: Fri, 9 Jan 2026 08:41:23 -0800
Subject: [PATCH 4/4] More test cases

---
 .../EditAndContinueWorkspaceServiceTests.cs   | 34 ++++++++++++++-----
 1 file changed, 26 insertions(+), 8 deletions(-)

diff --git a/src/Features/Test/EditAndContinue/EditAndContinueWorkspaceServiceTests.cs b/src/Features/Test/EditAndContinue/EditAndContinueWorkspaceServiceTests.cs
index e9711b607a566..ef2610f08d125 100644
--- a/src/Features/Test/EditAndContinue/EditAndContinueWorkspaceServiceTests.cs
+++ b/src/Features/Test/EditAndContinue/EditAndContinueWorkspaceServiceTests.cs
@@ -5,6 +5,7 @@
 #nullable disable
 
 using System;
+using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.IO;
 using System.Linq;
@@ -1492,11 +1493,34 @@ void M()
         ], _telemetryLog);
     }
 
+    public static TheoryData<bool, Encoding> EncodingsTestCases()
+    {
+        var data = new TheoryData<bool, Encoding>();
+        foreach (var encoding in new[]
+        {
+            new UTF8Encoding(encoderShouldEmitUTF8Identifier: true),
+            new UTF8Encoding(encoderShouldEmitUTF8Identifier: false),
+            Encoding.Unicode,
+            Encoding.BigEndianUnicode,
+
+            // TODO: https://github.com/dotnet/roslyn/issues/81930
+            // We do not currently account for CodePage property value and thus an encoding such as Shift-JIS that can't be detected
+            // from the file content does not work.
+            // Encoding.GetEncoding("SJIS");
+        })
+        {
+            data.Add(true, encoding);
+            data.Add(false, encoding);
+        }
+
+        return data;
+    }
+
     [Theory]
-    [CombinatorialData]
+    [MemberData(nameof(EncodingsTestCases))]
     [WorkItem("https://github.com/dotnet/roslyn/issues/81930")]
     [WorkItem("https://devdiv.visualstudio.com/DevDiv/_workitems/edit/2067885")]
-    public async Task Encodings(bool matchingContent)
+    public async Task Encodings(bool matchingContent, Encoding compilerEncoding)
     {
         Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
 
@@ -1509,12 +1533,6 @@ public async Task Encodings(bool matchingContent)
         // The actual encoding used by the compiler is either detected from the file content itself (e.g. Unicode encodings)
         // or it can also be set in the project via CodePage property.
 
-        // TODO: https://github.com/dotnet/roslyn/issues/81930
-        // We do not currently account for CodePage property value and thus an encoding such as Shift-JIS that can't be detected
-        // from the file content does not work.
-        // var compilerEncoding = Encoding.GetEncoding("SJIS");
-        var compilerEncoding = Encoding.Unicode;
-
         var dir = Temp.CreateDirectory();
         var sourceFile = dir.CreateFile("test.cs").WriteAllText(fileSource, compilerEncoding);