diff --git a/.agents/routine-last-run.md b/.agents/routine-last-run.md index df9f46c8b..66b8538cb 100644 --- a/.agents/routine-last-run.md +++ b/.agents/routine-last-run.md @@ -71,3 +71,61 @@ so the next run can resume from the same line of work without re-discovering it. **Handoff:** none. No work that belongs in `qyl-unit-tests`, `qyl-integration-tests`, or `qyl-e2e-tests` was deferred from this run. + +## qyl-e2e-tests 2026-05-17 02:45 + +**Outcome:** bootstrap PR opened. + +First-ever run of `qyl-e2e-tests` on a workstation with the full toolchain +present (dotnet 10.0.300, Docker 29.4.0). The previous run (2026-05-16) +exited as a no-op because the remote container lacked `dotnet`. + +Per the skill's bootstrap section, this run produced the **infrastructure-only +PR** (project + topology fixture + Nuke target + central-package pins). **No +scenario tests** were added; the next routine run picks up from here and adds +the first scenario. + +**Changes shipped:** +- `tests/qyl.e2e.tests/qyl.e2e.tests.csproj` — new test project, `ANcpLua.NET.Sdk.Test`. +- `tests/qyl.e2e.tests/E2ECollection.cs` — `[CollectionDefinition("E2E", DisableParallelization = true)]`. +- `tests/qyl.e2e.tests/Topology/QylTopologyOptions.cs` — image tags + startup timeout. +- `tests/qyl.e2e.tests/Topology/QylTopologyFixture.cs` — programmatic Testcontainers + topology: bridge network → `qyl-collector:latest` → `qyl-mcp:latest`; WireMock + process-local; containers reach it via `host.docker.internal`. + `WithImagePullPolicy(_ => false)` enforces local-image-only. +- `tests/qyl.e2e.tests/Bootstrap/WireMockLlmSeamTests.cs` — `Category=E2EBootstrap` + smoke (no Docker) — proves the WireMock seam roundtrips a scripted + `/v1/chat/completions` and shows up in `LogEntries`. +- `eng/build/BuildTest.cs` — new `E2ETests` target depending on + `IDocker.DockerImageBuild`, filters `Category=E2E`. Default `Test` excludes + `Category=E2E` (bootstrap tests stay in). +- `qyl.slnx` — registered the project. +- `Version.props` + `Directory.Packages.props` — added `WireMock.Net 2.6.0` and + `Testcontainers 4.11.0`. Split-pinned `OpenTelemetry.Instrumentation.AspNetCore` + to 1.15.2 (forced by WireMock.Net 2.6.0 transitive); Http + Runtime stay at + the umbrella 1.15.1 (no 1.15.2 release exists for them). + +**Verification:** +- `dotnet build qyl.slnx` — 0 errors, 1454 warnings (baseline 1393; ~61 new + warnings are all `MultipleGlobalAnalyzerKeys` from the dual-`.globalconfig` + worktree setup — benign, present on every worktree build). +- `dotnet test tests/qyl.e2e.tests --filter-trait Category=E2EBootstrap` — 3 + consecutive runs, all green (~1s each, 0 flakes). +- `nuke E2ETests` target wired up but **not executed** this run (would require + rebuilding all four qyl Docker images — out of scope for the bootstrap PR). + +**Overlap with `Smoke`:** `eng/smoke/run.sh` (Nuke target `Smoke`) is the +PRD #173 quality gate using real Ollama + real qyl Compose stack. E2E uses +**WireMock** for deterministic LLM stubbing. The two don't overlap — Smoke +covers "does the stack work against a real model"; E2E covers "does the +stack route data correctly given a known-bad/redacted LLM response". + +**Gaps for the next run:** add the first real scenario. Highest-value candidates: +1. Agent submits chat → trace arrives at downstream sink with credentials redacted. +2. MCP HTTP session reconnect after transient collector failure. +3. Cost rollup updates after a single chat completion. + +Pick exactly one. Add a sink container (e.g. an OTel collector configured to +write to a file volume) to `QylTopologyFixture` for scenario 1. + +**Handoff:** bootstrap PR opened; next cycle picks up from here. diff --git a/Directory.Packages.props b/Directory.Packages.props index 13737d9ce..410f10603 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -20,8 +20,10 @@ - + SDK version (1.15.3). AspNetCore is bumped to 1.15.2 because + WireMock.Net 2.6.0 (consumed by tests/qyl.e2e.tests) transitively + requires it; Http and Runtime have no 1.15.2 release, hence the split. --> + @@ -108,6 +110,9 @@ + + + diff --git a/Version.props b/Version.props index 1286b83a9..5e68298a2 100644 --- a/Version.props +++ b/Version.props @@ -47,7 +47,7 @@ 2.10.0 1.11.0 - 1.2.0 + 1.3.0 8.17.0 10.1.0 1.3.2 @@ -57,5 +57,9 @@ 5.5.1 2.5.0 9.4.0 + + 2.6.0 + 4.11.0 diff --git a/eng/build/BuildTest.cs b/eng/build/BuildTest.cs index a7f209d89..d7fb22988 100644 --- a/eng/build/BuildTest.cs +++ b/eng/build/BuildTest.cs @@ -176,6 +176,12 @@ interface IQylTest : ITest, IHazSourcePaths AssertAtLeastOneTestExecuted("Functional"); }); + Target E2ETests => d => d + .Description("Run end-to-end tests (full Docker topology against freshly built images)") + .DependsOn(static x => x.Compile) + .DependsOn(static x => x.DockerImageBuild) + .Executes(() => RunFilteredE2ETests()); + Target TestSummary => d => d .Unlisted() .Description("Generate Markdown test summary from MTP TRX reports") @@ -198,10 +204,12 @@ interface IQylTest : ITest, IHazSourcePaths .ReportTrx($"{project.Name}.trx") .IgnoreExitCode(8); - // Heavy/opt-in tests (Category=regen — shell out to Weaver, Category=integration etc.) - // are excluded from the default Test run; pass --IQylTest.TestFilter to include them. + // Heavy/opt-in tests (Category=regen — shell out to Weaver; Category=E2E — full + // Docker topology via DockerImageBuild) are excluded from the default Test run; + // pass --IQylTest.TestFilter to include them, or run the dedicated sub-target + // (E2ETests). E2EBootstrap-traited tests (no Docker) intentionally stay in. if (TestFilter is { Length: > 0 } f) mtp.FilterQuery(f); - else mtp.FilterNotTrait("Category", "regen"); + else mtp.FilterNotTrait("Category", "regen").FilterNotTrait("Category", "E2E"); if (StopOnFail == true) mtp.StopOnFail(); if (LiveOutput == true || IsLocalBuild) mtp.ShowLiveOutput(); @@ -236,6 +244,41 @@ sealed void RunFilteredTests(string namespaceFilter, string trxSuffix, bool need }), completeOnFailure: true); } + sealed void RunFilteredE2ETests() + { + EnsureTestcontainersConfigured(); + + var e2eProjects = TestProjects + .Where(static p => p.Name.Equals("qyl.e2e.tests", StringComparison.OrdinalIgnoreCase)) + .ToArray(); + + if (e2eProjects.Length is 0) + { + Log.Warning("E2ETests: no qyl.e2e.tests project found; skipping"); + return; + } + + DotNetTasks.DotNetTest(s => s + .SetNoBuild(true) + .SetNoRestore(true) + .SetResultsDirectory(TestResultsDirectory) + .CombineWith(e2eProjects, (ss, project) => + { + var mtp = MtpExtensions.Mtp() + .ReportTrx($"{project.Name}.E2E.trx") + .IgnoreExitCode(8) + .FilterTrait("Category", "E2E"); + + if (StopOnFail == true) mtp.StopOnFail(); + if (LiveOutput == true || IsLocalBuild) mtp.ShowLiveOutput(); + + var projectPath = project.Path ?? + throw new InvalidOperationException($"Project '{project.Name}' has no path"); + string[] args = ["--project", projectPath.ToString(), .. mtp.BuildArgs().Prepend("--")]; + return ss.SetProcessAdditionalArguments(args); + }), completeOnFailure: true); + } + sealed void EnsureTestcontainersConfigured() { if (IsServerBuild) diff --git a/qyl.slnx b/qyl.slnx index 66a545b87..e51a17d94 100644 --- a/qyl.slnx +++ b/qyl.slnx @@ -45,6 +45,7 @@ + diff --git a/services/qyl.mcp/Formatting/ErrorFormatter.cs b/services/qyl.mcp/Formatting/ErrorFormatter.cs index d67e5b51b..edd4b961e 100644 --- a/services/qyl.mcp/Formatting/ErrorFormatter.cs +++ b/services/qyl.mcp/Formatting/ErrorFormatter.cs @@ -11,10 +11,19 @@ public static string FormatForLlm(Exception error, McpTransportMode transport) = TaskCanceledException { InnerException: TimeoutException or null } => "**Timeout:** The collector did not respond in time. Retry or check if qyl collector is running.", OperationCanceledException opEx => FormatOperationCancelled(opEx), + // ModelContextProtocol 1.3 throws IOException (incl. ClientTransportClosedException) + // for transport connect/closure failures that 1.2 wrapped as InvalidOperationException. + // This arm must precede InvalidOperationException — order matters in C# pattern switches. + IOException ioEx => FormatTransportError(ioEx, transport), InvalidOperationException configEx => FormatConfigError(configEx, transport), _ => FormatUnknown(error, transport) }; + private static string FormatTransportError(IOException ex, McpTransportMode transport) => + transport is McpTransportMode.Stdio + ? $"**Connection Error**\n\n{ex.Message}\n\nCheck if the MCP server process is running." + : $"**Connection Error**\n\n{ex.Message}\n\nCheck the endpoint URL and network reachability."; + private static string FormatHttpError(HttpRequestException ex, McpTransportMode transport) { var (category, hint) = ex.StatusCode switch diff --git a/tests/qyl.e2e.tests/Bootstrap/WireMockLlmSeamTests.cs b/tests/qyl.e2e.tests/Bootstrap/WireMockLlmSeamTests.cs new file mode 100644 index 000000000..9ccacb205 --- /dev/null +++ b/tests/qyl.e2e.tests/Bootstrap/WireMockLlmSeamTests.cs @@ -0,0 +1,48 @@ +using System.Net.Http.Json; +using System.Text.Json; +using WireMock.RequestBuilders; +using WireMock.ResponseBuilders; +using WireMock.Server; + +namespace Qyl.E2E.Tests.Bootstrap; + +[Trait("Category", "E2EBootstrap")] +public sealed class WireMockLlmSeamTests +{ + [Fact] + public async Task ScriptedChatCompletion_RoundtripsAndIsRecordedInLogEntries() + { + using var llm = WireMockServer.Start(); + + llm.Given(Request.Create().WithPath("/v1/chat/completions").UsingPost()) + .RespondWith(Response.Create() + .WithStatusCode(200) + .WithHeader("Content-Type", "application/json") + .WithBodyAsJson(new + { + id = "chatcmpl-bootstrap", + choices = new[] + { + new { message = new { role = "assistant", content = "Bearer secret-token-12345" } }, + }, + })); + + var ct = TestContext.Current.CancellationToken; + using var client = new HttpClient { BaseAddress = new Uri(llm.Url!) }; + var request = new + { + model = "gpt-4o-mini", + messages = new[] { new { role = "user", content = "summarize this" } }, + }; + using var response = await client.PostAsJsonAsync("/v1/chat/completions", request, ct); + + response.IsSuccessStatusCode.Should().BeTrue( + "the WireMock stub must respond 200 for the configured route"); + + using var payload = await JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync(ct), cancellationToken: ct); + payload.RootElement.GetProperty("id").GetString().Should().Be("chatcmpl-bootstrap"); + + llm.LogEntries.Should().ContainSingle( + "WireMock must record the single POST /v1/chat/completions for assertion replay"); + } +} diff --git a/tests/qyl.e2e.tests/E2ECollection.cs b/tests/qyl.e2e.tests/E2ECollection.cs new file mode 100644 index 000000000..7e49609f0 --- /dev/null +++ b/tests/qyl.e2e.tests/E2ECollection.cs @@ -0,0 +1,7 @@ +namespace Qyl.E2E.Tests; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class E2ECollection : ICollectionFixture +{ + public const string Name = "E2E"; +} diff --git a/tests/qyl.e2e.tests/Topology/QylTopologyFixture.cs b/tests/qyl.e2e.tests/Topology/QylTopologyFixture.cs new file mode 100644 index 000000000..0a70f23bb --- /dev/null +++ b/tests/qyl.e2e.tests/Topology/QylTopologyFixture.cs @@ -0,0 +1,149 @@ +using DotNet.Testcontainers.Builders; +using DotNet.Testcontainers.Containers; +using DotNet.Testcontainers.Networks; +using WireMock.RequestBuilders; +using WireMock.ResponseBuilders; +using WireMock.Server; + +namespace Qyl.E2E.Tests.Topology; + +public sealed class QylTopologyFixture : IAsyncLifetime +{ + private const int CollectorInternalPort = 5100; + private const int McpInternalPort = 5200; + + private readonly QylTopologyOptions _options; + + private INetwork? _network; + private IContainer? _collector; + private IContainer? _mcp; + private WireMockServer? _llm; + + public QylTopologyFixture() : this(QylTopologyOptions.Default) + { + } + + public QylTopologyFixture(QylTopologyOptions options) => _options = options; + + public WireMockServer Llm => + _llm ?? throw new InvalidOperationException( + "Topology fixture not initialized — InitializeAsync has not completed."); + + public Uri CollectorBaseUrl => + _collector is null + ? throw new InvalidOperationException("Collector container not started.") + : new Uri($"http://{_collector.Hostname}:{_collector.GetMappedPublicPort(CollectorInternalPort)}/"); + + public Uri McpBaseUrl => + _mcp is null + ? throw new InvalidOperationException("MCP container not started.") + : new Uri($"http://{_mcp.Hostname}:{_mcp.GetMappedPublicPort(McpInternalPort)}/"); + + public async ValueTask InitializeAsync() + { + using var bootstrapCts = new CancellationTokenSource(_options.StartupTimeout); + var ct = bootstrapCts.Token; + + try + { + _llm = WireMockServer.Start(); + ConfigureDefaultLlmResponse(_llm); + + _network = new NetworkBuilder() + .WithName($"qyl-e2e-{Guid.NewGuid():N}") + .Build(); + await _network.CreateAsync(ct).ConfigureAwait(false); + + _collector = new ContainerBuilder() + .WithImage(_options.CollectorImage) + .WithImagePullPolicy(static _ => false) + .WithNetwork(_network) + .WithNetworkAliases("qyl-collector") + .WithPortBinding(CollectorInternalPort, true) + .WithEnvironment("QYL_PORT", CollectorInternalPort.ToString(CultureInfo.InvariantCulture)) + .WithEnvironment("ASPNETCORE_URLS", $"http://+:{CollectorInternalPort}") + .WithWaitStrategy(Wait.ForUnixContainer() + .UntilHttpRequestIsSucceeded(static r => r.ForPath("/health").ForPort(CollectorInternalPort))) + .Build(); + await _collector.StartAsync(ct).ConfigureAwait(false); + + _mcp = new ContainerBuilder() + .WithImage(_options.McpImage) + .WithImagePullPolicy(static _ => false) + .WithNetwork(_network) + .WithNetworkAliases("qyl-mcp") + .WithPortBinding(McpInternalPort, true) + .WithEnvironment("QYL_COLLECTOR_URL", $"http://qyl-collector:{CollectorInternalPort}") + .WithExtraHost("host.docker.internal", "host-gateway") + .WithWaitStrategy(Wait.ForUnixContainer() + .UntilMessageIsLogged("Now listening on:")) + .Build(); + await _mcp.StartAsync(ct).ConfigureAwait(false); + } + catch + { + await DisposeAsync().ConfigureAwait(false); + throw; + } + } + + public async ValueTask DisposeAsync() + { + GC.SuppressFinalize(this); + + if (_mcp is not null) + { + await _mcp.DisposeAsync().ConfigureAwait(false); + _mcp = null; + } + + if (_collector is not null) + { + await _collector.DisposeAsync().ConfigureAwait(false); + _collector = null; + } + + if (_network is not null) + { + await _network.DeleteAsync().ConfigureAwait(false); + _network = null; + } + + if (_llm is not null) + { + _llm.Stop(); + _llm.Dispose(); + _llm = null; + } + } + + public void ResetLlmStub() + { + if (_llm is null) return; + _llm.Reset(); + ConfigureDefaultLlmResponse(_llm); + } + + private static void ConfigureDefaultLlmResponse(WireMockServer llm) => + llm.Given(Request.Create().WithPath("/v1/chat/completions").UsingPost()) + .RespondWith(Response.Create() + .WithStatusCode(200) + .WithHeader("Content-Type", "application/json") + .WithBodyAsJson(new + { + id = "chatcmpl-e2e-default", + @object = "chat.completion", + created = 0, + model = "qyl-e2e-stub", + choices = new[] + { + new + { + index = 0, + message = new { role = "assistant", content = "qyl e2e default response" }, + finish_reason = "stop", + }, + }, + usage = new { prompt_tokens = 1, completion_tokens = 1, total_tokens = 2 }, + })); +} diff --git a/tests/qyl.e2e.tests/Topology/QylTopologyOptions.cs b/tests/qyl.e2e.tests/Topology/QylTopologyOptions.cs new file mode 100644 index 000000000..4be695cf0 --- /dev/null +++ b/tests/qyl.e2e.tests/Topology/QylTopologyOptions.cs @@ -0,0 +1,12 @@ +namespace Qyl.E2E.Tests.Topology; + +public sealed record QylTopologyOptions +{ + public string CollectorImage { get; init; } = "qyl-collector:latest"; + + public string McpImage { get; init; } = "qyl-mcp:latest"; + + public TimeSpan StartupTimeout { get; init; } = TimeSpan.FromSeconds(90); + + public static QylTopologyOptions Default { get; } = new(); +} diff --git a/tests/qyl.e2e.tests/qyl.e2e.tests.csproj b/tests/qyl.e2e.tests/qyl.e2e.tests.csproj new file mode 100644 index 000000000..c790b95e2 --- /dev/null +++ b/tests/qyl.e2e.tests/qyl.e2e.tests.csproj @@ -0,0 +1,20 @@ + + + Qyl.E2E.Tests + Qyl.E2E.Tests + 14 + true + Exe + + + + + + + + + + + + +