.NET: Foundry.Hosting IT - avoid MSB3026 in publish step

.github/workflows/dotnet-build-and-test.yml

@@ -379,6 +379,14 @@ jobs:
       # We rebuild and push the test container image on every IT run so framework code changes
       # are picked up; the image tag is content-hashed across the test container source AND its
       # framework project references, so identical content is a no-op push.
+      #
+      # `-UsePrebuiltProjectReferences` opts into the no-rebuild fast path: publish skips
+      # rebuilding ProjectReferences and consumes the DLLs the prior "Build Foundry hosted IT
+      # (and its deps)" step already produced. This avoids MSB3026 ("file is being used by
+      # another process") collisions caused by the previous build's shared-compilation server
+      # still holding file handles to those DLLs. Safe in CI because the prebuild step ran in
+      # the same job against the same source. Do not remove the prebuild step (the subsequent
+      # `dotnet test --no-build` step depends on it too).
       - name: Build and push Foundry Hosted Agents test container
         id: build-foundry-hosted-image
         shell: pwsh
@@ -388,7 +396,7 @@ jobs:
           if ([string]::IsNullOrWhiteSpace($registry)) {
             throw "IT_HOSTED_AGENT_REGISTRY not set in the integration environment."
           }
-          & "${{ github.workspace }}/dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-build-image.ps1" -Registry $registry | Tee-Object -FilePath $env:GITHUB_ENV -Append
+          & "${{ github.workspace }}/dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-build-image.ps1" -Registry $registry -UsePrebuiltProjectReferences | Tee-Object -FilePath $env:GITHUB_ENV -Append
 
       - name: Run Foundry Hosted Agents Integration Tests
         shell: pwsh

dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-build-image.ps1

@@ -41,7 +41,14 @@ param(
 
     [string] $Repository = "foundry-hosting-it",
 
-    [string] $TestContainerProject = "dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer"
+    [string] $TestContainerProject = "dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer",
+
+    # Explicit opt-in for the no-rebuild fast path. CI sets this after running the
+    # "Build Foundry hosted IT (and its deps)" step, which guarantees the prebuilt
+    # library DLLs match current source. Off by default so local invocations always
+    # let publish rebuild ProjectReferences and never produce an image whose tag is
+    # computed from current source while the contents come from a stale build.
+    [switch] $UsePrebuiltProjectReferences
 )
 
 $ErrorActionPreference = "Stop"
@@ -100,7 +107,60 @@ if (Test-Path $out) {
     Remove-Item -Recurse -Force $out
 }
 
-dotnet publish $TestContainerProject -c Release -f net10.0 -r linux-musl-x64 --self-contained false -o $out --tl:off | Out-Host
+# Conditionally tell publish to skip rebuilding ProjectReferences and consume the
+# prebuilt library DLLs in place. This avoids two failure modes that arise when
+# the CI workflow runs a `dotnet build` of the same library projects immediately
+# before this script:
+#   1) MSB3026 "file is being used by another process" when publish's MSBuild
+#      tries to overwrite src/<lib>/bin/Release/net10.0/<lib>.dll while the
+#      previous build's shared-compilation server still holds a file handle.
+#   2) Publish needlessly rebuilding identical managed (RID-agnostic) library
+#      DLLs that prebuild already produced.
+# Gated on -UsePrebuiltProjectReferences (a strict opt-in) instead of marker
+# detection, because a developer machine may have a stale Release build of the
+# libraries from days ago; using those would silently produce an image whose
+# content is older than the source the tag is computed from.
+$publishExtraArgs = @()
+if ($UsePrebuiltProjectReferences) {
+    Write-Host "-UsePrebuiltProjectReferences: skipping ProjectReference rebuild." -ForegroundColor DarkGray
+    $publishExtraArgs += "-p:BuildProjectReferences=false"
+} else {
+    # Preflight: in default (rebuild) mode, publish propagates RuntimeIdentifier=linux-musl-x64
+    # to library ProjectReferences and writes their intermediates to a RID-suffixed obj path
+    # (e.g. obj/Release/net10.0/linux-musl-x64/). DefaultItemExcludes follows the new
+    # IntermediateOutputPath, so any *.AssemblyInfo.cs left in obj/Release/net10.0/ from a
+    # prior `dotnet build` is no longer excluded and gets picked up by the **/*.cs Compile
+    # glob, producing CS0579 "duplicate attribute" errors. Detect that state up front and
+    # tell the user exactly how to recover.
+    $staleObjProbes = @(
+        "dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/obj/Release/net10.0",
+        "dotnet/src/Microsoft.Agents.AI.Foundry/obj/Release/net10.0",
+        "dotnet/src/Microsoft.Agents.AI/obj/Release/net10.0",
+        "dotnet/src/Microsoft.Agents.AI.Abstractions/obj/Release/net10.0"
+    )
+    $stale = @($staleObjProbes | Where-Object { Test-Path (Join-Path $_ "*.AssemblyInfo.cs") })
+    if ($stale.Count -gt 0) {
+        $msg = @(
+            "Detected prior Release/net10.0 build outputs in:"
+            ($stale | ForEach-Object { "  - $_" })
+            ""
+            "Publish would propagate -r linux-musl-x64 to those ProjectReferences and the"
+            "leftover obj/Release/net10.0/*.AssemblyInfo.cs files would cause CS0579 duplicate"
+            "attribute errors. Pick one:"
+            "  (a) Pass -UsePrebuiltProjectReferences (skips ProjectReference rebuild and"
+            "      uses the existing src/<lib>/bin/Release/net10.0/*.dll outputs in place)."
+            "      Only safe when you know those DLLs match current source - this is the path"
+            "      CI uses immediately after its 'Build Foundry hosted IT (and its deps)' step."
+            "  (b) Remove the stale obj/Release trees, e.g.:"
+            "        Remove-Item -Recurse -Force dotnet/src/Microsoft.Agents.AI*/obj/Release"
+            "      and re-run."
+        ) -join "`n"
+        throw $msg
+    }
+    Write-Host "Letting publish build ProjectReferences (pass -UsePrebuiltProjectReferences in CI to skip)." -ForegroundColor DarkGray
+}
+
+dotnet publish $TestContainerProject -c Release -f net10.0 -r linux-musl-x64 --self-contained false -o $out @publishExtraArgs --tl:off | Out-Host
 if ($LASTEXITCODE -ne 0) {
     throw "dotnet publish failed with exit code $LASTEXITCODE."
 }

dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentFrameworkResponseHandlerTelemetryTests.cs

@@ -37,7 +37,7 @@ public class AgentFrameworkResponseHandlerTelemetryTests
     public async Task CreateAsync_DefaultAgent_EmitsInvokeAgentSpanAsync()
     {
         // Arrange
-        var activities = new List<Activity>();
+        var activities = new ConcurrentActivityList();
         using var tracerProvider = Sdk.CreateTracerProviderBuilder()
             .AddSource(ResponsesSourceName)
             .AddInMemoryExporter(activities)
@@ -56,7 +56,7 @@ public async Task CreateAsync_DefaultAgent_EmitsInvokeAgentSpanAsync()
         await foreach (var _ in handler.CreateAsync(request, context, CancellationToken.None)) { }
 
         // Assert — filter by agent name to isolate this test's span from any parallel test spans
-        var mySpan = Assert.Single(activities.Where(a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name"))).ToList());
+        var mySpan = Assert.Single(activities.Snapshot().Where(a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name"))).ToList());
         Assert.Equal("invoke_agent", mySpan.GetTagItem("gen_ai.operation.name"));
         Assert.NotNull(mySpan.GetTagItem("gen_ai.agent.id"));
     }
@@ -65,7 +65,7 @@ public async Task CreateAsync_DefaultAgent_EmitsInvokeAgentSpanAsync()
     public async Task CreateAsync_KeyedAgent_EmitsInvokeAgentSpanAsync()
     {
         // Arrange
-        var activities = new List<Activity>();
+        var activities = new ConcurrentActivityList();
         using var tracerProvider = Sdk.CreateTracerProviderBuilder()
             .AddSource(ResponsesSourceName)
             .AddInMemoryExporter(activities)
@@ -84,7 +84,7 @@ public async Task CreateAsync_KeyedAgent_EmitsInvokeAgentSpanAsync()
         await foreach (var _ in handler.CreateAsync(request, context, CancellationToken.None)) { }
 
         // Assert — filter by agent name to isolate this test's span
-        var mySpan = Assert.Single(activities.Where(a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name"))).ToList());
+        var mySpan = Assert.Single(activities.Snapshot().Where(a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name"))).ToList());
         Assert.Equal("invoke_agent", mySpan.GetTagItem("gen_ai.operation.name"));
     }
 
@@ -95,8 +95,8 @@ public async Task CreateAsync_AlreadyInstrumentedAgent_EmitsSingleSpanPerRunAsyn
         // If ApplyOpenTelemetry double-wraps, an extra span would appear on ResponsesSourceName.
         // If it correctly skips wrapping, only the pre-wrap's unique source emits spans.
         var preWrapSource = Guid.NewGuid().ToString();
-        var preWrapActivities = new List<Activity>();
-        var responsesActivities = new List<Activity>();
+        var preWrapActivities = new ConcurrentActivityList();
+        var responsesActivities = new ConcurrentActivityList();
 
         using var preWrapProvider = Sdk.CreateTracerProviderBuilder()
             .AddSource(preWrapSource)
@@ -125,18 +125,19 @@ public async Task CreateAsync_AlreadyInstrumentedAgent_EmitsSingleSpanPerRunAsyn
         await foreach (var _ in handler.CreateAsync(request, context, CancellationToken.None)) { }
 
         // Assert — pre-wrap source emits exactly 1 span (agent ran)
-        Assert.Single(preWrapActivities);
-        Assert.Equal("invoke_agent", preWrapActivities[0].GetTagItem("gen_ai.operation.name"));
+        var preWrapSnapshot = preWrapActivities.Snapshot();
+        Assert.Single(preWrapSnapshot);
+        Assert.Equal("invoke_agent", preWrapSnapshot[0].GetTagItem("gen_ai.operation.name"));
 
         // ResponsesSourceName emits 0 spans — ApplyOpenTelemetry skipped wrapping the pre-instrumented agent
-        Assert.DoesNotContain(responsesActivities, a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name")));
+        Assert.DoesNotContain(responsesActivities.Snapshot(), a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name")));
     }
 
     [Fact]
     public async Task CreateAsync_DefaultAgent_SpanDisplayNameContainsAgentNameAsync()
     {
         // Arrange
-        var activities = new List<Activity>();
+        var activities = new ConcurrentActivityList();
         using var tracerProvider = Sdk.CreateTracerProviderBuilder()
             .AddSource(ResponsesSourceName)
             .AddInMemoryExporter(activities)
@@ -155,7 +156,7 @@ public async Task CreateAsync_DefaultAgent_SpanDisplayNameContainsAgentNameAsync
         await foreach (var _ in handler.CreateAsync(request, context, CancellationToken.None)) { }
 
         // Assert — display name follows "invoke_agent {Name}({Id})" convention; filter by agent name to isolate
-        var mySpan = Assert.Single(activities.Where(a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name"))).ToList());
+        var mySpan = Assert.Single(activities.Snapshot().Where(a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name"))).ToList());
         Assert.Contains("invoke_agent", mySpan.DisplayName, StringComparison.Ordinal);
         Assert.Contains(TelemetryTestAgent.AgentName, mySpan.DisplayName, StringComparison.Ordinal);
     }
@@ -231,4 +232,35 @@ private static async IAsyncEnumerable<AgentResponseUpdate> SingleUpdateAsync(
     }
 
     private sealed class TelemetryAgentSession : AgentSession;
+
+    /// <summary>
+    /// Thread-safe <see cref="ICollection{Activity}"/> used by OTel's InMemoryExporter to capture
+    /// activities emitted on globally-listened sources. Required because the exporter writes into
+    /// the supplied collection from background Activity completion callbacks while the test thread
+    /// may be enumerating it for assertions, and other tests in the same assembly may emit on the
+    /// same source concurrently. A plain <see cref="List{Activity}"/> trips
+    /// "Collection was modified; enumeration operation may not execute." in that scenario.
+    /// </summary>
+    private sealed class ConcurrentActivityList : ICollection<Activity>
+    {
+        private readonly List<Activity> _items = new();
+        private readonly object _gate = new();
+
+        public int Count { get { lock (this._gate) { return this._items.Count; } } }
+        public bool IsReadOnly => false;
+
+        public void Add(Activity item) { lock (this._gate) { this._items.Add(item); } }
+        public void Clear() { lock (this._gate) { this._items.Clear(); } }
+        public bool Contains(Activity item) { lock (this._gate) { return this._items.Contains(item); } }
+        public void CopyTo(Activity[] array, int arrayIndex) { lock (this._gate) { this._items.CopyTo(array, arrayIndex); } }
+        public bool Remove(Activity item) { lock (this._gate) { return this._items.Remove(item); } }
+
+        public Activity[] Snapshot()
+        {
+            lock (this._gate) { return this._items.ToArray(); }
+        }
+
+        public IEnumerator<Activity> GetEnumerator() => ((IEnumerable<Activity>)this.Snapshot()).GetEnumerator();
+        System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => this.GetEnumerator();
+    }
 }