Emit opted-in string literals into data section as UTF8

docs/features/string-literals-data-section.md

@@ -0,0 +1,18 @@
+# String literals in data section
+
+This opt-in Roslyn feature allows changing how string literals in C# programs are emitted into PE files (`.dll`/`.exe`).
+By default, string literals are emitted to the UserString heap which is limited to [2^24 bytes](https://github.com/dotnet/roslyn/issues/9852).
+When the limit is reached, the following compiler error is reported by Roslyn:
+
+```
+error CS8103: Combined length of user strings used by the program exceeds allowed limit. Try to decrease use of string literals.
+```
+
+By turning on the feature flag `utf8-string-encoding`, string literals (where possible) are instead emitted as UTF-8 data into a different section of the PE file
+which does not have the same limit. The emit format is similar to [explicit u8 string literals](https://learn.microsoft.com/en-us/dotnet/csharp/language-reference/proposals/csharp-11.0/utf8-string-literals).
+
+The feature flag can take a non-negative integer threshold. Only string literals whose length is over the threshold are emitted in the new way described above.
+By default, the threshold is 100. Specifying 0 means all string literals are considered for the feature. Specifying `off` turns off the feature (this is the default).
+
+The feature flag can be specified on the command line like `/features:utf8-string-encoding` or `/features:utf8-string-encoding=20`,
+or in a project file in a `<PropertyGroup>` like `<Features>$(Features);utf8-string-encoding</Features>` or `<Features>$(Features);utf8-string-encoding=20</Features>`.

src/Compilers/CSharp/Portable/CSharpResources.resx

@@ -5309,7 +5309,7 @@ To remove the warning, you can use /reference instead (set the Embed Interop Typ
     <value>Syntax tree should be created from a submission.</value>
   </data>
   <data name="ERR_TooManyUserStrings" xml:space="preserve">
-    <value>Combined length of user strings used by the program exceeds allowed limit. Try to decrease use of string literals.</value>
+    <value>Combined length of user strings used by the program exceeds allowed limit. Consider using feature flag 'utf8-string-encoding'.</value>
   </data>
   <data name="ERR_PatternNullableType" xml:space="preserve">
     <value>It is not legal to use nullable type '{0}?' in a pattern; use the underlying type '{0}' instead.</value>

src/Compilers/CSharp/Portable/CodeGen/CodeGenerator.cs

@@ -8,6 +8,7 @@
 using System.Collections.Immutable;
 using System.Diagnostics;
 using System.Reflection.Metadata;
+using System.Text;
 using Microsoft.CodeAnalysis.CodeGen;
 using Microsoft.CodeAnalysis.CSharp.Emit;
 using Microsoft.CodeAnalysis.CSharp.Symbols;
@@ -32,6 +33,7 @@ internal sealed partial class CodeGenerator
         private readonly BindingDiagnosticBag _diagnostics;
         private readonly ILEmitStyle _ilEmitStyle;
         private readonly bool _emitPdbSequencePoints;
+        private readonly int? _utf8StringEncodingThreshold;
 
         private readonly HashSet<LocalSymbol> _stackLocals;
 
@@ -87,7 +89,8 @@ public CodeGenerator(
             PEModuleBuilder moduleBuilder,
             BindingDiagnosticBag diagnostics,
             OptimizationLevel optimizations,
-            bool emittingPdb)
+            bool emittingPdb,
+            Compilation compilation)
         {
             Debug.Assert((object)method != null);
             Debug.Assert(boundBody != null);
@@ -101,6 +104,7 @@ public CodeGenerator(
             _builder = builder;
             _module = moduleBuilder;
             _diagnostics = diagnostics;
+            _utf8StringEncodingThreshold = compilation.Utf8StringEncodingThreshold;
 
             if (!method.GenerateDebugInfo)
             {

src/Compilers/CSharp/Portable/CodeGen/EmitExpression.cs

@@ -3481,13 +3481,35 @@ private void EmitConstantExpression(TypeSymbol type, ConstantValue constantValue
                 {
                     EmitInitObj(type, used, syntaxNode);
                 }
-                else
+                else if (!TryEmitStringLiteralAsUtf8Encoded(constantValue, syntaxNode))
                 {
                     _builder.EmitConstantValue(constantValue);
                 }
             }
         }
 
+        private bool TryEmitStringLiteralAsUtf8Encoded(ConstantValue constantValue, SyntaxNode syntaxNode)
+        {
+            // Emit long strings into data section so they don't overflow the UserString heap.
+            if (constantValue.IsString &&
+                constantValue.StringValue.Length > _utf8StringEncodingThreshold &&
+                LocalRewriter.TryGetUtf8ByteRepresentation(constantValue.StringValue, out byte[] utf8Bytes, out _))
+            {
+                var data = utf8Bytes.ToImmutableArray();
+                var field = _builder.module.GetFieldForDataString(data, syntaxNode, _diagnostics.DiagnosticBag);
+                if (field is null)
+                {
+                    return false;
+                }
+
+                _builder.EmitOpCode(ILOpCode.Ldsfld);
+                _builder.EmitToken(field, syntaxNode, _diagnostics.DiagnosticBag);
+                return true;
+            }
+
+            return false;
+        }
+
         private void EmitInitObj(TypeSymbol type, bool used, SyntaxNode syntaxNode)
         {
             if (used)

src/Compilers/CSharp/Portable/Compiler/MethodCompiler.cs

@@ -183,6 +183,8 @@ public static void CompileMethodBodies(
 
                 methodCompiler.WaitForWorkers();
 
+                moduleBeingBuiltOpt.FreezeDataStringHolders(diagnostics.DiagnosticBag);
+
                 // all threads that were adding methods must be finished now, we can freeze the class:
                 var privateImplClass = moduleBeingBuiltOpt.FreezePrivateImplementationDetails();
                 if (privateImplClass != null)
@@ -671,8 +673,11 @@ private void CompileSynthesizedMethods(PrivateImplementationDetails privateImplC
             foreach (Cci.IMethodDefinition definition in privateImplClass.GetMethods(context).Concat(privateImplClass.GetTopLevelAndNestedTypeMethods(context)))
             {
                 var method = (MethodSymbol)definition.GetInternalSymbol();
-                Debug.Assert(method.SynthesizesLoweredBoundBody);
-                method.GenerateMethodBody(compilationState, diagnostics);
+                if (method is not null)
+                {
+                    Debug.Assert(method.SynthesizesLoweredBoundBody);
+                    method.GenerateMethodBody(compilationState, diagnostics);
+                }
             }
 
             CompileSynthesizedMethods(compilationState);
@@ -1514,7 +1519,7 @@ private static MethodBody GenerateMethodBody(
             {
                 StateMachineMoveNextBodyDebugInfo moveNextBodyDebugInfoOpt = null;
 
-                var codeGen = new CodeGen.CodeGenerator(method, block, builder, moduleBuilder, diagnosticsForThisMethod, optimizations, emittingPdb);
+                var codeGen = new CodeGen.CodeGenerator(method, block, builder, moduleBuilder, diagnosticsForThisMethod, optimizations, emittingPdb, compilation);
 
                 if (diagnosticsForThisMethod.HasAnyErrors())
                 {

src/Compilers/CSharp/Portable/Errors/MessageProvider.cs

@@ -250,6 +250,7 @@ public override void ReportDuplicateMetadataReferenceWeak(DiagnosticBag diagnost
         public override int ERR_EncUpdateFailedMissingSymbol => (int)ErrorCode.ERR_EncUpdateFailedMissingSymbol;
         public override int ERR_InvalidDebugInfo => (int)ErrorCode.ERR_InvalidDebugInfo;
         public override int ERR_FunctionPointerTypesInAttributeNotSupported => (int)ErrorCode.ERR_FunctionPointerTypesInAttributeNotSupported;
+        public override int ERR_MissingPredefinedMember => (int)ErrorCode.ERR_MissingPredefinedMember;
 
         // Generators:
         public override int WRN_GeneratorFailedDuringInitialization => (int)ErrorCode.WRN_GeneratorFailedDuringInitialization;

src/Compilers/CSharp/Portable/Lowering/LocalRewriter/LocalRewriter.cs

@@ -8,6 +8,7 @@
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
 using System.Linq;
+using System.Text;
 using Microsoft.CodeAnalysis.CodeGen;
 using Microsoft.CodeAnalysis.CSharp.Emit;
 using Microsoft.CodeAnalysis.CSharp.Symbols;
@@ -21,6 +22,8 @@ namespace Microsoft.CodeAnalysis.CSharp
 {
     internal sealed partial class LocalRewriter : BoundTreeRewriterWithStackGuard
     {
+        private static readonly UTF8Encoding s_utf8 = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true);
+
         private readonly CSharpCompilation _compilation;
         private readonly SyntheticBoundNodeFactory _factory;
         private readonly SynthesizedSubmissionFields _previousSubmissionFields;

src/Compilers/CSharp/Portable/Lowering/LocalRewriter/LocalRewriter_Conversion.cs

@@ -6,6 +6,7 @@
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using Microsoft.CodeAnalysis.CSharp.Symbols;
 using Microsoft.CodeAnalysis.PooledObjects;
@@ -120,20 +121,37 @@ private BoundExpression MakeUtf8Span(BoundExpression node, IReadOnlyList<byte>?
             return result;
         }
 
-        private byte[]? GetUtf8ByteRepresentation(BoundUtf8String node)
+        internal static bool TryGetUtf8ByteRepresentation(
+            string s,
+            [NotNullWhen(returnValue: true)] out byte[]? result,
+            [NotNullWhen(returnValue: false)] out string? error)
         {
-            var utf8 = new System.Text.UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true);
-
             try
             {
-                return utf8.GetBytes(node.Value);
+                result = s_utf8.GetBytes(s);
+                error = null;
+                return true;
             }
             catch (Exception ex)
+            {
+                result = null;
+                error = ex.Message;
+                return false;
+            }
+        }
+
+        private byte[]? GetUtf8ByteRepresentation(BoundUtf8String node)
+        {
+            if (TryGetUtf8ByteRepresentation(node.Value, out byte[]? result, out string? error))
+            {
+                return result;
+            }
+            else
             {
                 _diagnostics.Add(
                     ErrorCode.ERR_CannotBeConvertedToUtf8,
                     node.Syntax.Location,
-                    ex.Message);
+                    error);
 
                 return null;
             }