AppDomain configuration is serialized without using BinFmt

src/Build.UnitTests/BackEnd/NodeConfiguration_Tests.cs

@@ -0,0 +1,59 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.Build.BackEnd;
+using Microsoft.Build.Execution;
+using Microsoft.Build.Logging;
+using Microsoft.Build.UnitTests.BackEnd;
+using Shouldly;
+using Xunit;
+
+namespace Microsoft.Build.Engine.UnitTests.BackEnd
+{
+    public class NodeConfiguration_Tests
+    {
+#if FEATURE_APPDOMAIN
+        /// <summary>
+        /// Test serialization / deserialization of the AppDomainSetup instance. 
+        /// </summary>
+        [Theory]
+        [InlineData(new byte[] { 1, 2, 3 })]
+        [InlineData(null)]
+        public void TestTranslationWithAppDomainSetup(byte[] configBytes)
+        {
+            AppDomainSetup setup = new AppDomainSetup();
+
+            NodeConfiguration config = new NodeConfiguration(
+                nodeId: 1,
+                buildParameters: new BuildParameters(),
+                forwardingLoggers: Array.Empty<LoggerDescription>(),
+                appDomainSetup: setup,
+                loggingNodeConfiguration: new LoggingNodeConfiguration());
+
+            setup.SetConfigurationBytes(configBytes);
+
+            ((ITranslatable)config).Translate(TranslationHelpers.GetWriteTranslator());
+            INodePacket packet = NodeConfiguration.FactoryForDeserialization(TranslationHelpers.GetReadTranslator());
+
+            packet.ShouldBeOfType<NodeConfiguration>();
+            NodeConfiguration deserializedConfig = (NodeConfiguration)packet;
+
+            deserializedConfig.AppDomainSetup.ShouldNotBeNull();
+
+            if (configBytes is null)
+            {
+                deserializedConfig.AppDomainSetup.GetConfigurationBytes().ShouldBeNull();
+            }
+            else
+            {
+                deserializedConfig.AppDomainSetup.GetConfigurationBytes().SequenceEqual(configBytes).ShouldBeTrue();
+            }
+        }
+#endif
+    }
+}

src/Build.UnitTests/BackEnd/TaskHostConfiguration_Tests.cs

@@ -6,6 +6,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text;
 using System.Threading;
 
 
@@ -376,6 +377,57 @@ public void TestTranslationWithNullDictionary()
             Assert.Equal(expectedGlobalProperties, deserializedConfig.GlobalProperties);
         }
 
+#if FEATURE_APPDOMAIN
+        /// <summary>
+        /// Test serialization / deserialization of the AppDomainSetup instance. 
+        /// </summary>
+        [Theory]
+        [InlineData(new byte[] { 1, 2, 3 })]
+        [InlineData(null)]
+        public void TestTranslationWithAppDomainSetup(byte[] configBytes)
+        {
+            AppDomainSetup setup = new AppDomainSetup();
+
+            TaskHostConfiguration config = new TaskHostConfiguration(
+                nodeId: 1,
+                startupDirectory: Directory.GetCurrentDirectory(),
+                buildProcessEnvironment: null,
+                culture: Thread.CurrentThread.CurrentCulture,
+                uiCulture: Thread.CurrentThread.CurrentUICulture,
+                appDomainSetup: setup,
+                lineNumberOfTask: 1,
+                columnNumberOfTask: 1,
+                projectFileOfTask: @"c:\my project\myproj.proj",
+                continueOnError: _continueOnErrorDefault,
+                taskName: "TaskName",
+                taskLocation: @"c:\MyTasks\MyTask.dll",
+                isTaskInputLoggingEnabled: false,
+                taskParameters: new Dictionary<string, object>(),
+                globalParameters: new Dictionary<string, string>(),
+                warningsAsErrors: null,
+                warningsNotAsErrors: null,
+                warningsAsMessages: null);
+
+            setup.SetConfigurationBytes(configBytes);
+
+            ((ITranslatable)config).Translate(TranslationHelpers.GetWriteTranslator());
+            INodePacket packet = TaskHostConfiguration.FactoryForDeserialization(TranslationHelpers.GetReadTranslator());
+
+            TaskHostConfiguration deserializedConfig = packet as TaskHostConfiguration;
+
+            deserializedConfig.AppDomainSetup.ShouldNotBeNull();
+
+            if (configBytes is null)
+            {
+                deserializedConfig.AppDomainSetup.GetConfigurationBytes().ShouldBeNull();
+            }
+            else
+            {
+                deserializedConfig.AppDomainSetup.GetConfigurationBytes().SequenceEqual(configBytes).ShouldBeTrue();
+            }
+        }
+#endif
+
         /// <summary>
         /// Test serialization / deserialization when the parameter dictionary is empty. 
         /// </summary>

src/Build/BackEnd/Node/NodeConfiguration.cs

@@ -28,6 +28,11 @@ internal class NodeConfiguration : INodePacket
         private BuildParameters _buildParameters;
 
 #if FEATURE_APPDOMAIN
+        /// <summary>
+        /// The app domain configuration bytes sent via RPC.
+        /// </summary>
+        private byte[] _appDomainConfigBytes;
+
         /// <summary>
         /// The app domain information needed for setting up AppDomain-isolated tasks.
         /// </summary>
@@ -161,7 +166,19 @@ public void Translate(ITranslator translator)
             translator.Translate(ref _buildParameters, BuildParameters.FactoryForDeserialization);
             translator.TranslateArray(ref _forwardingLoggers, LoggerDescription.FactoryForTranslation);
 #if FEATURE_APPDOMAIN
-            translator.TranslateDotNet(ref _appDomainSetup);
+            // Set the configuration bytes just before serialization in case the SetConfigurationBytes was invoked during lifetime of this instance.
+            if (translator.Mode == TranslationDirection.WriteToStream)
+            {
+                _appDomainConfigBytes = _appDomainSetup?.GetConfigurationBytes();
+            }
+
+            translator.Translate(ref _appDomainConfigBytes);
+
+            if (translator.Mode == TranslationDirection.ReadFromStream)
+            {
+                _appDomainSetup = new AppDomainSetup();
+                _appDomainSetup.SetConfigurationBytes(_appDomainConfigBytes);
+            }
 #endif
             translator.Translate(ref _loggingNodeConfiguration);
         }
@@ -173,6 +190,7 @@ internal static INodePacket FactoryForDeserialization(ITranslator translator)
         {
             NodeConfiguration configuration = new NodeConfiguration();
             configuration.Translate(translator);
+
             return configuration;
         }
         #endregion

src/Shared/TaskHostConfiguration.cs

@@ -44,6 +44,11 @@ internal class TaskHostConfiguration : INodePacket
         private CultureInfo _uiCulture = CultureInfo.CurrentUICulture;
 
 #if FEATURE_APPDOMAIN
+        /// <summary>
+        /// The app domain configuration bytes sent via RPC.
+        /// </summary>
+        private byte[] _appDomainConfigBytes;
+
         /// <summary>
         /// The AppDomainSetup that we may want to use on AppDomainIsolated tasks. 
         /// </summary>
@@ -417,7 +422,19 @@ public void Translate(ITranslator translator)
             translator.TranslateCulture(ref _culture);
             translator.TranslateCulture(ref _uiCulture);
 #if FEATURE_APPDOMAIN
-            translator.TranslateDotNet(ref _appDomainSetup);
+            // Set the configuration bytes just before serialization in case the SetConfigurationBytes was invoked during lifetime of this instance.
+            if (translator.Mode == TranslationDirection.WriteToStream)
+            {
+                _appDomainConfigBytes = _appDomainSetup?.GetConfigurationBytes();
+            }
+
+            translator.Translate(ref _appDomainConfigBytes);
+
+            if (translator.Mode == TranslationDirection.ReadFromStream)
+            {
+                _appDomainSetup = new AppDomainSetup();
+                _appDomainSetup.SetConfigurationBytes(_appDomainConfigBytes);
+            }
 #endif
             translator.Translate(ref _lineNumberOfTask);
             translator.Translate(ref _columnNumberOfTask);
@@ -458,6 +475,7 @@ internal static INodePacket FactoryForDeserialization(ITranslator translator)
         {
             TaskHostConfiguration configuration = new TaskHostConfiguration();
             configuration.Translate(translator);
+
             return configuration;
         }
     }