Add callback-based Encode to AsnWriter

src/libraries/Common/src/System/Security/Cryptography/DSASecurityTransforms.macOS.cs

@@ -170,24 +170,17 @@ internal static SafeSecKeyRefHandle ImportKey(DSAParameters parameters)
                     hasPrivateKey = false;
                 }
 
-                byte[] rented = CryptoPool.Rent(keyWriter.GetEncodedLength());
-
-                if (!keyWriter.TryEncode(rented, out int written))
-                {
-                    Debug.Fail("TryEncode failed with a pre-allocated buffer");
-                    throw new InvalidOperationException();
-                }
-
-                // Explicitly clear the inner buffer
-                keyWriter.Reset();
-
                 try
                 {
-                    return Interop.AppleCrypto.ImportEphemeralKey(rented.AsSpan(0, written), hasPrivateKey);
+                    return keyWriter.Encode(hasPrivateKey, static (hasPrivateKey, encoded) =>
+                    {
+                        return Interop.AppleCrypto.ImportEphemeralKey(encoded, hasPrivateKey);
+                    });
                 }
                 finally
                 {
-                    CryptoPool.Return(rented, written);
+                    // Explicitly clear the inner buffer
+                    keyWriter.Reset();
                 }
             }
 

src/libraries/Common/src/System/Security/Cryptography/EccSecurityTransforms.macOS.cs

@@ -64,24 +64,17 @@ private static SafeSecKeyRefHandle ImportLegacyPrivateKey(ref ECParameters param
         {
             AsnWriter keyWriter = EccKeyFormatHelper.WriteECPrivateKey(parameters);
 
-            byte[] rented = CryptoPool.Rent(keyWriter.GetEncodedLength());
-
-            if (!keyWriter.TryEncode(rented, out int written))
-            {
-                Debug.Fail("TryEncode failed with a pre-allocated buffer");
-                throw new InvalidOperationException();
-            }
-
-            // Explicitly clear the inner buffer
-            keyWriter.Reset();
-
             try
             {
-                return Interop.AppleCrypto.ImportEphemeralKey(rented.AsSpan(0, written), true);
+                return keyWriter.Encode(static encoded =>
+                {
+                    return Interop.AppleCrypto.ImportEphemeralKey(encoded, true);
+                });
             }
             finally
             {
-                CryptoPool.Return(rented, written);
+                // Explicitly clear the inner buffer
+                keyWriter.Reset();
             }
         }
     }

src/libraries/Common/src/System/Security/Cryptography/RSAAndroid.cs

@@ -434,7 +434,11 @@ public override unsafe void ImportRSAPublicKey(ReadOnlySpan<byte> source, out in
                         AsnWriter writer = new AsnWriter(AsnEncodingRules.DER);
                         spki.Encode(writer);
 
-                        SafeRsaHandle key = Interop.AndroidCrypto.DecodeRsaSubjectPublicKeyInfo(writer.Encode());
+                        SafeRsaHandle key = writer.Encode(static (encoded) =>
+                        {
+                            return Interop.AndroidCrypto.DecodeRsaSubjectPublicKeyInfo(encoded);
+                        });
+
                         if (key is null || key.IsInvalid)
                         {
                             key?.Dispose();

src/libraries/System.Formats.Asn1/ref/System.Formats.Asn1.cs

@@ -158,6 +158,10 @@ public void CopyTo(System.Formats.Asn1.AsnWriter destination) { }
         public int Encode(System.Span<byte> destination) { throw null; }
         public bool EncodedValueEquals(System.Formats.Asn1.AsnWriter other) { throw null; }
         public bool EncodedValueEquals(System.ReadOnlySpan<byte> other) { throw null; }
+#if NET9_0_OR_GREATER
+        public TReturn Encode<TReturn>(System.Func<System.ReadOnlySpan<byte>, TReturn> encodeCallback) { throw null; }
+        public TReturn Encode<TState, TReturn>(TState state, System.Func<TState, System.ReadOnlySpan<byte>, TReturn> encodeCallback) where TState : allows ref struct { throw null; }
+#endif
         public int GetEncodedLength() { throw null; }
         public void PopOctetString(System.Formats.Asn1.Asn1Tag? tag = default(System.Formats.Asn1.Asn1Tag?)) { }
         public void PopSequence(System.Formats.Asn1.Asn1Tag? tag = default(System.Formats.Asn1.Asn1Tag?)) { }

src/libraries/System.Formats.Asn1/src/Resources/Strings.resx

@@ -105,6 +105,9 @@
   <data name="AsnWriter_EncodeUnbalancedStack" xml:space="preserve">
     <value>Encode cannot be called while a Sequence, Set-Of, or Octet String is still open.</value>
   </data>
+  <data name="AsnWriter_ModifyingWhileEncoding" xml:space="preserve">
+    <value>The AsnWriter cannot be written to while performing the Encode callback.</value>
+  </data>
   <data name="AsnWriter_PopWrongTag" xml:space="preserve">
     <value>Cannot pop the requested tag as it is not currently in progress.</value>
   </data>

src/libraries/System.Formats.Asn1/src/System/Formats/Asn1/AsnWriter.cs

@@ -20,6 +20,9 @@ public sealed partial class AsnWriter
         private byte[] _buffer = null!;
         private int _offset;
         private Stack<StackFrame>? _nestingStack;
+#if NET9_0_OR_GREATER
+        private int _encodeDepth;
+#endif
 
         /// <summary>
         ///   Gets the encoding rules in use by this writer.
@@ -78,6 +81,13 @@ public AsnWriter(AsnEncodingRules ruleSet, int initialCapacity) : this(ruleSet)
         /// </summary>
         public void Reset()
         {
+#if NET9_0_OR_GREATER
+            if (_encodeDepth != 0)
+            {
+                throw new InvalidOperationException(SR.AsnWriter_ModifyingWhileEncoding);
+            }
+#endif
+
             if (_offset > 0)
             {
                 Debug.Assert(_buffer != null);
@@ -199,6 +209,87 @@ public byte[] Encode()
             return _buffer.AsSpan(0, _offset).ToArray();
         }
 
+#if NET9_0_OR_GREATER
+        /// <summary>
+        ///   Provides the encoded representation of the data to the specified callback.
+        /// </summary>
+        /// <param name="encodeCallback">
+        ///   The callback that receives the encoded data.
+        /// </param>
+        /// <typeparam name="TReturn">
+        ///   The type of the return value.
+        /// </typeparam>
+        /// <returns>
+        ///   Returns the value returned from <paramref name="encodeCallback" />.
+        /// </returns>
+        /// <exception cref="ArgumentNullException">
+        ///   <paramref name="encodeCallback"/> is <see langword="null"/>.
+        /// </exception>
+        /// <exception cref="InvalidOperationException">
+        ///   A <see cref="PushSequence"/> or <see cref="PushSetOf"/> has not been closed via
+        ///   <see cref="PopSequence"/> or <see cref="PopSetOf"/>.
+        /// </exception>
+        public TReturn Encode<TReturn>(Func<ReadOnlySpan<byte>, TReturn> encodeCallback)
+        {
+            if (encodeCallback is null)
+                throw new ArgumentNullException(nameof(encodeCallback));
+
+            try
+            {
+                _encodeDepth = checked(_encodeDepth + 1);
+                ReadOnlySpan<byte> encoded = EncodeAsSpan();
+                return encodeCallback(encoded);
+            }
+            finally
+            {
+                _encodeDepth--;
+            }
+        }
+
+        /// <summary>
+        ///   Provides the encoded representation of the data to the specified callback.
+        /// </summary>
+        /// <param name="encodeCallback">
+        ///   The callback that receives the encoded data.
+        /// </param>
+        /// <param name="state">
+        ///   The state to pass to <paramref name="encodeCallback" />.
+        /// </param>
+        /// <typeparam name="TState">
+        ///   The type of the state.
+        /// </typeparam>
+        /// <typeparam name="TReturn">
+        ///   The type of the return value.
+        /// </typeparam>
+        /// <returns>
+        ///   Returns the value returned from <paramref name="encodeCallback" />.
+        /// </returns>
+        /// <exception cref="ArgumentNullException">
+        ///   <paramref name="encodeCallback"/> is <see langword="null"/>.
+        /// </exception>
+        /// <exception cref="InvalidOperationException">
+        ///   A <see cref="PushSequence"/> or <see cref="PushSetOf"/> has not been closed via
+        ///   <see cref="PopSequence"/> or <see cref="PopSetOf"/>.
+        /// </exception>
+        public TReturn Encode<TState, TReturn>(TState state, Func<TState, ReadOnlySpan<byte>, TReturn> encodeCallback)
+            where TState : allows ref struct
+        {
+            if (encodeCallback is null)
+                throw new ArgumentNullException(nameof(encodeCallback));
+
+            try
+            {
+                _encodeDepth = checked(_encodeDepth + 1);
+                ReadOnlySpan<byte> encoded = EncodeAsSpan();
+                return encodeCallback(state, encoded);
+            }
+            finally
+            {
+                _encodeDepth--;
+            }
+        }
+#endif
+
         private ReadOnlySpan<byte> EncodeAsSpan()
         {
             if ((_nestingStack?.Count ?? 0) != 0)
@@ -265,6 +356,13 @@ private void EnsureWriteCapacity(int pendingCount)
                 throw new OverflowException();
             }
 
+#if NET9_0_OR_GREATER
+            if (_encodeDepth != 0)
+            {
+                throw new InvalidOperationException(SR.AsnWriter_ModifyingWhileEncoding);
+            }
+#endif
+
             if (_buffer == null || _buffer.Length - _offset < pendingCount)
             {
 #if CHECK_ACCURATE_ENSURE

src/libraries/System.Formats.Asn1/tests/Writer/Asn1WriterTests.cs

@@ -16,6 +16,18 @@ internal static void Verify(AsnWriter writer, string expectedHex)
             Assert.Equal(expectedHex, encoded.ByteArrayToHex());
             Assert.Equal(expectedSize, encoded.Length);
 
+#if NET9_0_OR_GREATER
+            string hexEncoded = writer.Encode(Convert.ToHexString);
+            Assert.Equal(expectedHex, hexEncoded);
+
+            object state = new();
+            hexEncoded = writer.Encode(state, (object callbackState, ReadOnlySpan<byte> encoded) => {
+                Assert.Same(state, callbackState);
+                return Convert.ToHexString(encoded);
+            });
+            Assert.Equal(expectedHex, hexEncoded);
+#endif
+
             // Now verify TryEncode's boundary conditions.
             byte[] encoded2 = new byte[encoded.Length + 3];
             encoded2[0] = 255;

src/libraries/System.Formats.Asn1/tests/Writer/PushPopSetOf.cs

@@ -465,6 +465,18 @@ public static void CannotEncodeWhileUnbalanced(AsnEncodingRules ruleSet, bool cu
             Assert.Throws<InvalidOperationException>(() => writer.TryEncode(buf, out written));
             Assert.Throws<InvalidOperationException>(() => writer.EncodedValueEquals(buf));
             Assert.Equal(-5, written);
+
+#if NET9_0_OR_GREATER
+            Assert.Throws<InvalidOperationException>(() => writer.Encode<object>(_ => {
+                Assert.Fail("Callback should not have been called.");
+                return null;
+            }));
+
+            Assert.Throws<InvalidOperationException>(() => writer.Encode<object, object>(null, (_, _) => {
+                Assert.Fail("Callback should not have been called.");
+                return null;
+            }));
+#endif
         }
 
         [Theory]

src/libraries/System.Formats.Asn1/tests/Writer/SimpleWriterTests.cs

@@ -49,6 +49,18 @@ public static void EncodeEmpty(AsnEncodingRules ruleSet)
             Assert.Equal(0, written);
             Assert.True(writer.EncodedValueEquals(ReadOnlySpan<byte>.Empty));
 
+#if NET9_0_OR_GREATER
+            writer.Encode<object>(encoded => {
+                Assert.Equal(0, encoded.Length);
+                return null;
+            });
+
+            writer.Encode<object, object>(null, (_, encoded) => {
+                Assert.Equal(0, encoded.Length);
+                return null;
+            });
+#endif
+
             Span<byte> negativeTest = stackalloc byte[] { 5, 0 };
             Assert.False(writer.EncodedValueEquals(negativeTest));
         }
@@ -235,6 +247,38 @@ public static void InitialCapacity_ResizeBlockAligns()
             Assert.Equal(1024, buffer?.Length);
         }
 
+#if NET9_0_OR_GREATER
+        [Fact]
+        public static void Encode_Callback_NoModifications()
+        {
+            AsnWriter writer = new AsnWriter(AsnEncodingRules.DER);
+
+            writer.Encode(writer, static (writer, encoded) =>
+            {
+                writer.Encode(writer, static (writer, encoded) =>
+                {
+                    Assert.Throws<InvalidOperationException>(() => writer.WriteNull());
+                    return (object)null;
+                });
+
+                Assert.Throws<InvalidOperationException>(() => writer.WriteNull());
+                return (object)null;
+            });
+
+            writer.Encode(writer, static (writer, encoded) =>
+            {
+                writer.Encode(writer, static (writer, encoded) =>
+                {
+                    Assert.Throws<InvalidOperationException>(() => writer.Reset());
+                    return (object)null;
+                });
+
+                Assert.Throws<InvalidOperationException>(() => writer.Reset());
+                return (object)null;
+            });
+        }
+#endif
+
         private static byte[]? PeekRawBuffer(AsnWriter writer)
         {
             FieldInfo bufField = typeof(AsnWriter).GetField("_buffer", BindingFlags.Instance | BindingFlags.NonPublic);

src/libraries/System.Security.Cryptography.Pkcs/src/System/Security/Cryptography/Pkcs/SignerInfo.cs

@@ -653,7 +653,15 @@ private bool CheckHash(bool compatMode)
                     {
                         writer.PopSetOf();
 
+#if NET9_0_OR_GREATER
+                        writer.Encode(hasher, static (hasher, encoded) =>
+                        {
+                            hasher.AppendData(encoded);
+                            return (object?)null;
+                        });
+#else
                         hasher.AppendData(writer.Encode());
+#endif
                     }
                 }
             }

src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/X509Certificates/CertificateRequest.cs

@@ -890,7 +890,7 @@ public X509Certificate2 Create(
             };
 
             certificate.Encode(writer);
-            X509Certificate2 ret = X509CertificateLoader.LoadCertificate(writer.Encode());
+            X509Certificate2 ret = writer.Encode(X509CertificateLoader.LoadCertificate);
             CryptoPool.Return(normalizedSerial);
             return ret;
         }

src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/X509Certificates/CertificateRevocationListBuilder.CdpExtension.cs

@@ -108,8 +108,10 @@ public static X509Extension BuildCrlDistributionPointExtension(
             // CRLDistributionPoints
             writer.PopSequence();
 
-            byte[] encoded = writer.Encode();
-            return new X509Extension(Oids.CrlDistributionPointsOid, encoded, critical);
+            return writer.Encode(critical, static (critical, encoded) =>
+            {
+                return new X509Extension(Oids.CrlDistributionPointsOid, encoded, critical);
+            });
         }
     }
 }

src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/X509Certificates/OpenSslX509Encoder.cs

@@ -280,8 +280,11 @@ private static DSAOpenSsl BuildDsaPublicKey(byte[] encodedKeyValue, byte[] encod
             DSAOpenSsl dsa = new DSAOpenSsl();
             try
             {
-                dsa.ImportSubjectPublicKeyInfo(writer.Encode(), out _);
-                return dsa;
+                return writer.Encode(dsa, static (dsa, encoded) =>
+                {
+                    dsa.ImportSubjectPublicKeyInfo(encoded, out _);
+                    return dsa;
+                });
             }
             catch (Exception)
             {

src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/X509Certificates/SubjectAlternativeNameBuilder.cs

@@ -77,10 +77,10 @@ public X509Extension Build(bool critical = false)
                 }
             }
 
-            return new X509Extension(
-                Oids.SubjectAltName,
-                writer.Encode(),
-                critical);
+            return writer.Encode(critical, static (critical, encoded) =>
+            {
+                return new X509Extension(Oids.SubjectAltName, encoded, critical);
+            });
         }
 
         private void AddGeneralName(GeneralNameAsn generalName)

src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/X509Certificates/X500DistinguishedNameBuilder.cs

@@ -350,11 +350,7 @@ public X500DistinguishedName Build()
                 }
             }
 
-            byte[] rented = CryptoPool.Rent(_writer.GetEncodedLength());
-            int encoded = _writer.Encode(rented);
-            X500DistinguishedName name = new X500DistinguishedName(rented.AsSpan(0, encoded));
-            CryptoPool.Return(rented, clearSize: 0); // Distinguished Names do not contain sensitive information.
-            return name;
+            return _writer.Encode(static encoded => new X500DistinguishedName(encoded));
         }
 
         private void EncodeComponent(