Fix LibraryImportDiagnosticsAnalyzer and DownlevelLibraryImportDiagnosticsAnalyzer missing diagnostics when StringMarshalling is set

src/libraries/System.Runtime.InteropServices/gen/LibraryImportGenerator/Analyzers/LibraryImportDiagnosticsAnalyzer.cs

@@ -47,7 +47,7 @@ public class LibraryImportDiagnosticsAnalyzer : DiagnosticAnalyzer
 
         public override void Initialize(AnalysisContext context)
         {
-            context.ConfigureGeneratedCodeAnalysis(GeneratedCodeAnalysisFlags.None);
+            context.ConfigureGeneratedCodeAnalysis(GeneratedCodeAnalysisFlags.Analyze);
             context.EnableConcurrentExecution();
             context.RegisterCompilationStartAction(context =>
             {

src/libraries/System.Runtime.InteropServices/gen/LibraryImportGenerator/LibraryImportGenerator.cs

@@ -375,6 +375,38 @@ private static LocalFunctionStatementSyntax CreateTargetDllImportAsLocalStatemen
         {
             Debug.Assert(!options.GenerateForwarders, "GenerateForwarders should have already been handled to use a forwarder stub");
 
+            var dllImportArgs = new List<AttributeArgumentSyntax>
+            {
+                AttributeArgument(LiteralExpression(
+                    SyntaxKind.StringLiteralExpression,
+                    Literal(libraryImportData.ModuleName))),
+                AttributeArgument(
+                    NameEquals(nameof(DllImportAttribute.EntryPoint)),
+                    null,
+                    LiteralExpression(
+                        SyntaxKind.StringLiteralExpression,
+                        Literal(libraryImportData.EntryPoint ?? stubMethodName))),
+                AttributeArgument(
+                    NameEquals(nameof(DllImportAttribute.ExactSpelling)),
+                    null,
+                    LiteralExpression(SyntaxKind.TrueLiteralExpression))
+            };
+
+            // When StringMarshalling.Utf16 is specified, forward CharSet = Unicode to the inner
+            // DllImport. This ensures that any types forwarded to the runtime marshaller (e.g.
+            // StringBuilder) use the correct encoding instead of defaulting to Ansi.
+            if (libraryImportData.IsUserDefined.HasFlag(InteropAttributeMember.StringMarshalling)
+                && libraryImportData.StringMarshalling == StringMarshalling.Utf16)
+            {
+                dllImportArgs.Add(AttributeArgument(
+                    NameEquals(nameof(DllImportAttribute.CharSet)),
+                    null,
+                    MemberAccessExpression(
+                        SyntaxKind.SimpleMemberAccessExpression,
+                        AliasQualifiedName("global", IdentifierName(typeof(CharSet).FullName)),
+                        IdentifierName(nameof(CharSet.Unicode)))));
+            }
+
             (ParameterListSyntax parameterList, TypeSyntax returnType, AttributeListSyntax returnTypeAttributes) = stubGenerator.GenerateTargetMethodSignatureData();
             LocalFunctionStatementSyntax localDllImport = LocalFunctionStatement(returnType, stubTargetName)
                 .AddModifiers(
@@ -388,24 +420,7 @@ private static LocalFunctionStatementSyntax CreateTargetDllImportAsLocalStatemen
                                 Attribute(
                                     NameSyntaxes.DllImportAttribute,
                                     AttributeArgumentList(
-                                        SeparatedList(
-                                            new[]
-                                            {
-                                                AttributeArgument(LiteralExpression(
-                                                        SyntaxKind.StringLiteralExpression,
-                                                        Literal(libraryImportData.ModuleName))),
-                                                AttributeArgument(
-                                                    NameEquals(nameof(DllImportAttribute.EntryPoint)),
-                                                    null,
-                                                    LiteralExpression(
-                                                        SyntaxKind.StringLiteralExpression,
-                                                        Literal(libraryImportData.EntryPoint ?? stubMethodName))),
-                                                AttributeArgument(
-                                                    NameEquals(nameof(DllImportAttribute.ExactSpelling)),
-                                                    null,
-                                                    LiteralExpression(SyntaxKind.TrueLiteralExpression))
-                                            }
-                                            )))))))
+                                        SeparatedList(dllImportArgs)))))))
                 .WithParameterList(parameterList);
             if (returnTypeAttributes is not null)
             {

src/libraries/System.Runtime.InteropServices/tests/LibraryImportGenerator.UnitTests/Compiles.cs

@@ -686,6 +686,67 @@ protected override void VerifyFinalCompilation(Compilation compilation)
             }
         }
 
+        [Theory]
+        [InlineData("StringMarshalling.Utf16")]
+        [InlineData("StringMarshalling.Utf8")]
+        public async Task ForwardedTypesWithStringMarshalling_InnerDllImportHasCharSet(string stringMarshalling)
+        {
+            bool expectCharSetUnicode = stringMarshalling == "StringMarshalling.Utf16";
+            string source = $$"""
+                using System.Runtime.InteropServices;
+                partial class Test
+                {
+                    [LibraryImport("DoesNotExist", StringMarshalling = {{stringMarshalling}})]
+                    public static partial string Method(string s, int i);
+                }
+                """;
+
+            var test = new InnerDllImportCharSetTest(expectCharSetUnicode)
+            {
+                TestCode = source,
+                TestBehaviors = TestBehaviors.SkipGeneratedSourcesCheck
+            };
+
+            await test.RunAsync();
+        }
+
+        class InnerDllImportCharSetTest : VerifyCS.Test
+        {
+            private readonly bool _expectCharSetUnicode;
+
+            public InnerDllImportCharSetTest(bool expectCharSetUnicode)
+                : base(referenceAncillaryInterop: false)
+            {
+                _expectCharSetUnicode = expectCharSetUnicode;
+            }
+
+            protected override void VerifyFinalCompilation(Compilation compilation)
+            {
+                SyntaxTree generatedCode = compilation.SyntaxTrees.Last();
+                var localFunctions = generatedCode.GetRoot()
+                    .DescendantNodes().OfType<LocalFunctionStatementSyntax>()
+                    .ToList();
+
+                LocalFunctionStatementSyntax innerDllImport = Assert.Single(localFunctions);
+                AttributeSyntax dllImportAttr = innerDllImport.AttributeLists
+                    .SelectMany(al => al.Attributes)
+                    .Single(a => a.Name.ToString().Contains("DllImport"));
+
+                AttributeArgumentSyntax? charSetArgument = dllImportAttr.ArgumentList!.Arguments
+                    .SingleOrDefault(a => a.NameEquals?.Name.Identifier.Text == nameof(DllImportAttribute.CharSet));
+
+                if (_expectCharSetUnicode)
+                {
+                    Assert.NotNull(charSetArgument);
+                    Assert.Equal($"{nameof(CharSet)}.{nameof(CharSet.Unicode)}", charSetArgument.Expression.ToString());
+                }
+                else
+                {
+                    Assert.Null(charSetArgument);
+                }
+            }
+        }
+
         public static IEnumerable<object[]> CodeSnippetsToCompileWithMarshalType()
         {
             yield break;

src/libraries/System.Runtime.InteropServices/tests/LibraryImportGenerator.UnitTests/Diagnostics.cs

@@ -413,6 +413,60 @@ await VerifyCS.VerifyAnalyzerAsync(source,
                     .WithArguments("Method", "Test"));
         }
 
+        [Theory]
+        [InlineData("StringMarshalling = StringMarshalling.Utf16")]
+        [InlineData("StringMarshalling = StringMarshalling.Utf8")]
+        [InlineData("")]
+        public async Task StringBuilderNotSupported_ReportsDiagnostic(string stringMarshallingArg)
+        {
+            string marshallingPart = string.IsNullOrEmpty(stringMarshallingArg)
+                ? ""
+                : $", {stringMarshallingArg}";
+
+            // StringBuilder as a simple parameter
+            string source = $$"""
+
+                using System.Runtime.InteropServices;
+                using System.Text;
+                partial class Test
+                {
+                    [LibraryImport("DoesNotExist"{{marshallingPart}})]
+                    public static partial void Method(StringBuilder {|#0:sb|});
+                }
+                """;
+
+            await VerifyCS.VerifyAnalyzerAsync(source,
+                VerifyCS.Diagnostic(GeneratorDiagnostics.ParameterTypeNotSupported)
+                    .WithLocation(0)
+                    .WithArguments("System.Text.StringBuilder", "sb"));
+        }
+
+        [Theory]
+        [InlineData("StringMarshalling = StringMarshalling.Utf16")]
+        [InlineData("StringMarshalling = StringMarshalling.Utf8")]
+        public async Task StringBuilderNotSupported_WithStringParam_ReportsDiagnostic(string stringMarshallingArg)
+        {
+            // StringBuilder with [Out] alongside a string parameter
+            string source = $$"""
+
+                using System.Runtime.InteropServices;
+                using System.Text;
+                partial class Test
+                {
+                    [LibraryImport("DoesNotExist", {{stringMarshallingArg}})]
+                    internal static partial int Method(
+                        string volumeMountPoint,
+                        [Out] StringBuilder {|#0:volumeName|},
+                        int bufferLength);
+                }
+                """;
+
+            await VerifyCS.VerifyAnalyzerAsync(source,
+                VerifyCS.Diagnostic(GeneratorDiagnostics.ParameterTypeNotSupported)
+                    .WithLocation(0)
+                    .WithArguments("System.Text.StringBuilder", "volumeName"));
+        }
+
         private static void VerifyDiagnostics(DiagnosticResult[] expectedDiagnostics, Diagnostic[] actualDiagnostics)
         {
             Assert.True(expectedDiagnostics.Length == actualDiagnostics.Length,