Fix transient gRPC "Connection refused" errors in integration tests by adding HTTP reachability check to DaprdContainer startup

src/Dapr.Testcontainers/AssemblyInfo.cs

@@ -0,0 +1,16 @@
+// ------------------------------------------------------------------------
+// Copyright 2025 The Dapr Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//     http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//  ------------------------------------------------------------------------
+
+using System.Runtime.CompilerServices;
+
+[assembly: InternalsVisibleTo("Dapr.Testcontainers.Test, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b1f597635c44597fcecb493e2b1327033b29b1a98ac956a1a538664b68f87d45fbaada0438a15a6265e62864947cc067d8da3a7d93c5eb2fcbb850e396c8684dba74ea477d82a1bbb18932c0efb30b64ff1677f85ae833818707ac8b49ad8062ca01d2c89d8ab1843ae73e8ba9649cd28666b539444dcdee3639f95e2a099bb2")]

src/Dapr.Testcontainers/Common/ContainerReadinessProbe.cs

@@ -0,0 +1,214 @@
+// ------------------------------------------------------------------------
+// Copyright 2025 The Dapr Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//     http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//  ------------------------------------------------------------------------
+
+using System;
+using System.Net.Http;
+using System.Net.Sockets;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Dapr.Testcontainers.Common;
+
+/// <summary>
+/// Provides methods to poll for container readiness conditions such as TCP port
+/// reachability and HTTP health endpoint availability.
+/// </summary>
+internal static class ContainerReadinessProbe
+{
+    /// <summary>
+    /// Polls the given TCP host/port until a connection can be established or the
+    /// timeout elapses.
+    /// </summary>
+    /// <param name="host">The host to connect to (e.g. "127.0.0.1").</param>
+    /// <param name="port">The TCP port to connect to.</param>
+    /// <param name="timeout">Maximum time to wait before throwing <see cref="TimeoutException"/>.</param>
+    /// <param name="cancellationToken">Token used to cancel waiting.</param>
+    /// <exception cref="TimeoutException">Thrown when the port does not become reachable within <paramref name="timeout"/>.</exception>
+    internal static async Task WaitForTcpPortAsync(
+        string host,
+        int port,
+        TimeSpan timeout,
+        CancellationToken cancellationToken)
+    {
+        var start = DateTimeOffset.UtcNow;
+        Exception? lastError = null;
+
+        while (DateTimeOffset.UtcNow - start < timeout)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            try
+            {
+                using var client = new TcpClient();
+                var connectTask = client.ConnectAsync(host, port);
+
+                var completed = await Task.WhenAny(connectTask,
+                    Task.Delay(TimeSpan.FromMilliseconds(250), cancellationToken));
+                if (completed == connectTask)
+                {
+                    // Will throw if the connection failed
+                    await connectTask;
+                    return;
+                }
+            }
+            catch (Exception ex) when (ex is SocketException or InvalidOperationException)
+            {
+                lastError = ex;
+            }
+
+            await Task.Delay(TimeSpan.FromMilliseconds(200), cancellationToken);
+        }
+
+        throw new TimeoutException($"Timed out waiting for TCP port {host}:{port} to accept connections.", lastError);
+    }
+
+    /// <summary>
+    /// Polls the given HTTP <paramref name="url"/> until the HTTP server sends <em>any</em>
+    /// response — including error responses such as 5xx — or the timeout elapses. Only retries
+    /// when the underlying TCP connection is refused or the per-attempt timeout fires, meaning
+    /// the HTTP server is not yet listening.
+    /// </summary>
+    /// <remarks>
+    /// Use this method when you need to verify that an HTTP server has started and is processing
+    /// requests without caring about application-level health status. For Dapr specifically,
+    /// <c>/v1.0/healthz</c> may return 500 while Dapr is still initializing components or while
+    /// a connected app has not yet started, but the server is already accepting and routing
+    /// requests. A single successful HTTP round-trip (regardless of status code) guarantees that
+    /// the HTTP and gRPC servers are both active, which eliminates the transient
+    /// "Connection refused" window that can occur immediately after the TCP port first opens.
+    /// </remarks>
+    /// <param name="url">The URL to GET, e.g. "http://127.0.0.1:3500/v1.0/healthz".</param>
+    /// <param name="timeout">Maximum total time to wait before throwing <see cref="TimeoutException"/>.</param>
+    /// <param name="cancellationToken">Token used to cancel waiting.</param>
+    /// <param name="httpClient">
+    /// Optional <see cref="HttpClient"/> to use. When <c>null</c> a new instance is created and
+    /// disposed automatically. Supply a custom instance for testing purposes.
+    /// </param>
+    /// <exception cref="TimeoutException">Thrown when no HTTP response is received within <paramref name="timeout"/>.</exception>
+    internal static async Task WaitForHttpReachableAsync(
+        string url,
+        TimeSpan timeout,
+        CancellationToken cancellationToken,
+        HttpClient? httpClient = null)
+    {
+        var ownsClient = httpClient is null;
+        httpClient ??= new HttpClient();
+
+        try
+        {
+            var start = DateTimeOffset.UtcNow;
+            Exception? lastError = null;
+
+            while (DateTimeOffset.UtcNow - start < timeout)
+            {
+                cancellationToken.ThrowIfCancellationRequested();
+
+                try
+                {
+                    // Bound each individual attempt so a stalled connection does not exhaust the overall timeout.
+                    using var requestCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+                    requestCts.CancelAfter(TimeSpan.FromSeconds(5));
+
+                    // Any HTTP response (including 5xx) means the server is accepting connections
+                    // and actively processing requests.
+                    await httpClient.GetAsync(url, requestCts.Token);
+                    return;
+                }
+                catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or OperationCanceledException)
+                {
+                    if (cancellationToken.IsCancellationRequested)
+                        throw;
+
+                    lastError = ex;
+                }
+
+                await Task.Delay(TimeSpan.FromMilliseconds(500), cancellationToken);
+            }
+
+            throw new TimeoutException(
+                $"Timed out waiting for HTTP server at {url} to start accepting connections.", lastError);
+        }
+        finally
+        {
+            if (ownsClient)
+                httpClient.Dispose();
+        }
+    }
+
+    /// <summary>
+    /// Polls the given HTTP <paramref name="url"/> until a 2xx response is received or the
+    /// timeout elapses. Each individual HTTP attempt is bounded by a 5-second timeout to
+    /// avoid stalling when the endpoint is not yet accepting connections.
+    /// </summary>
+    /// <param name="url">The URL to GET, e.g. "http://127.0.0.1:3500/v1.0/healthz".</param>
+    /// <param name="timeout">Maximum total time to wait before throwing <see cref="TimeoutException"/>.</param>
+    /// <param name="cancellationToken">Token used to cancel waiting.</param>
+    /// <param name="httpClient">
+    /// Optional <see cref="HttpClient"/> to use. When <c>null</c> a new instance is created and
+    /// disposed automatically. Supply a custom instance for testing purposes.
+    /// </param>
+    /// <exception cref="TimeoutException">Thrown when the endpoint does not return a 2xx response within <paramref name="timeout"/>.</exception>
+    internal static async Task WaitForHttpHealthAsync(
+        string url,
+        TimeSpan timeout,
+        CancellationToken cancellationToken,
+        HttpClient? httpClient = null)
+    {
+        var ownsClient = httpClient is null;
+        httpClient ??= new HttpClient();
+
+        try
+        {
+            var start = DateTimeOffset.UtcNow;
+            Exception? lastError = null;
+
+            while (DateTimeOffset.UtcNow - start < timeout)
+            {
+                cancellationToken.ThrowIfCancellationRequested();
+
+                try
+                {
+                    // Bound each individual attempt so a stalled connection does not exhaust the overall timeout.
+                    using var requestCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+                    requestCts.CancelAfter(TimeSpan.FromSeconds(5));
+
+                    var response = await httpClient.GetAsync(url, requestCts.Token);
+                    var statusCode = (int)response.StatusCode;
+                    if (statusCode >= 200 && statusCode < 300)
+                    {
+                        return;
+                    }
+
+                    lastError = new HttpRequestException($"Health endpoint at {url} returned HTTP {statusCode}.");
+                }
+                catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or OperationCanceledException)
+                {
+                    if (cancellationToken.IsCancellationRequested)
+                        throw;
+
+                    lastError = ex;
+                }
+
+                await Task.Delay(TimeSpan.FromMilliseconds(500), cancellationToken);
+            }
+
+            throw new TimeoutException(
+                $"Timed out waiting for health endpoint {url} to return a successful response.", lastError);
+        }
+        finally
+        {
+            if (ownsClient)
+                httpClient.Dispose();
+        }
+    }
+}

src/Dapr.Testcontainers/Containers/Dapr/DaprdContainer.cs

@@ -14,7 +14,6 @@
 using System;
 using System.Collections.Generic;
 using System.Net;
-using System.Net.Sockets;
 using System.Threading;
 using System.Threading.Tasks;
 using Dapr.Testcontainers.Common;
@@ -167,76 +166,44 @@ public DaprdContainer(
     /// <inheritdoc />
 	public async Task StartAsync(CancellationToken cancellationToken = default)
 	{
-        try
-        {
-            await _container.StartAsync(cancellationToken);
-
-            var mappedHttpPort = _container.GetMappedPublicPort(InternalHttpPort);
-            var mappedGrpcPort = _container.GetMappedPublicPort(InternalGrpcPort);
-
-            if (_requestedHttpPort is not null && mappedHttpPort != _requestedHttpPort.Value)
-            {
-                throw new InvalidOperationException(
-                    $"Dapr HTTP port mapping mismatch. Requested {_requestedHttpPort.Value}, but Docker mapped {mappedHttpPort}");
-            }
+        await _container.StartAsync(cancellationToken);
 
-            if (_requestedGrpcPort is not null && mappedGrpcPort != _requestedGrpcPort.Value)
-            {
-                throw new InvalidOperationException(
-                    $"Dapr gRPC port mapping mismatch. Requested {_requestedGrpcPort.Value}, but Docker mapped {mappedGrpcPort}");
-            }
+        var mappedHttpPort = _container.GetMappedPublicPort(InternalHttpPort);
+        var mappedGrpcPort = _container.GetMappedPublicPort(InternalGrpcPort);
 
-            HttpPort = mappedHttpPort;
-            GrpcPort = mappedGrpcPort;
-
-            // The container log wait strategy can fire before the host port is actually accepting connections
-            // (especially on Windows). Ensure the ports are reachable from the test process.
-            await WaitForTcpPortAsync("127.0.0.1", HttpPort, TimeSpan.FromSeconds(30), cancellationToken);
-            await WaitForTcpPortAsync("127.0.0.1", GrpcPort, TimeSpan.FromSeconds(30), cancellationToken);
-        }
-        catch (Exception ex)
+        if (_requestedHttpPort is not null && mappedHttpPort != _requestedHttpPort.Value)
         {
-            var msg = ex.Message;
-            throw;
+            throw new InvalidOperationException(
+                $"Dapr HTTP port mapping mismatch. Requested {_requestedHttpPort.Value}, but Docker mapped {mappedHttpPort}");
         }
-    }
 
-    private static async Task WaitForTcpPortAsync(
-        string host,
-        int port,
-        TimeSpan timeout,
-        CancellationToken cancellationToken)
-    {
-        var start = DateTimeOffset.UtcNow;
-        Exception? lastError = null;
-
-        while (DateTimeOffset.UtcNow - start < timeout)
+        if (_requestedGrpcPort is not null && mappedGrpcPort != _requestedGrpcPort.Value)
         {
-            cancellationToken.ThrowIfCancellationRequested();
-
-            try
-            {
-                using var client = new TcpClient();
-                var connectTask = client.ConnectAsync(host, port);
-
-                var completed = await Task.WhenAny(connectTask,
-                    Task.Delay(TimeSpan.FromMilliseconds(250), cancellationToken));
-                if (completed == connectTask)
-                {
-                    // Will throw if connect failed
-                    await connectTask;
-                    return;
-                }
-            }
-            catch (Exception ex) when (ex is SocketException or InvalidOperationException)
-            {
-                lastError = ex;
-            }
-
-            await Task.Delay(TimeSpan.FromMilliseconds(200), cancellationToken);
+            throw new InvalidOperationException(
+                $"Dapr gRPC port mapping mismatch. Requested {_requestedGrpcPort.Value}, but Docker mapped {mappedGrpcPort}");
         }
 
-        throw new TimeoutException($"Timed out waiting for TCP port {host}:{port} to accept connections.", lastError);
+        HttpPort = mappedHttpPort;
+        GrpcPort = mappedGrpcPort;
+
+        // The container log wait strategy can fire before the host port is actually accepting connections
+        // (especially on Windows). Ensure the ports are reachable from the test process.
+        await ContainerReadinessProbe.WaitForTcpPortAsync("127.0.0.1", HttpPort, TimeSpan.FromSeconds(30), cancellationToken);
+        await ContainerReadinessProbe.WaitForTcpPortAsync("127.0.0.1", GrpcPort, TimeSpan.FromSeconds(30), cancellationToken);
+
+        // Even after the TCP ports start accepting connections the Dapr runtime may still be
+        // initializing (connecting to Placement/Scheduler, loading components, starting the
+        // workflow engine). Poll the HTTP port until the Dapr HTTP server starts processing
+        // requests. Any HTTP response (including 5xx) confirms that the HTTP server — and by
+        // extension the gRPC server — is actively routing requests, eliminating the brief window
+        // in which the gRPC port accepts TCP connections but the gRPC handlers are not yet
+        // installed. This prevents the transient "Error connecting to subchannel / Connection
+        // refused" errors that occur when the gRPC client first connects while the runtime is
+        // still completing its startup sequence.
+        await ContainerReadinessProbe.WaitForHttpReachableAsync(
+            $"http://127.0.0.1:{HttpPort}/v1.0/healthz",
+            TimeSpan.FromSeconds(30),
+            cancellationToken);
     }
 
     /// <inheritdoc />