From f03a84be819f87a28600ed9fffd509fa60a22f71 Mon Sep 17 00:00:00 2001 From: ancplua Date: Sun, 17 May 2026 03:02:44 +0200 Subject: [PATCH 1/2] chore(tests/e2e): bootstrap E2E test project + topology fixture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First-run output of the qyl-e2e-tests routine — adds the infrastructure needed for the next cycle to land the first end-to-end scenario. - tests/qyl.e2e.tests: new xUnit v3 / MTP project with WireMock.Net + Testcontainers. Programmatic QylTopologyFixture spins up a bridge network, qyl-collector + qyl-mcp containers from local image tags, and a process-local WireMock the containers reach via host.docker.internal. WithImagePullPolicy(_ => false) enforces "test against fresh local builds (DockerImageBuild), never a registry tag". - WireMockLlmSeamTests (Category=E2EBootstrap, no Docker): proves the WireMock stub roundtrips a scripted /v1/chat/completions and surfaces in LogEntries. Ran 3x locally, 0 flakes, ~1s each. - eng/build/BuildTest.cs: new E2ETests Nuke target depending on IDocker.DockerImageBuild + filtering Category=E2E. Default Test target now also excludes Category=E2E (E2EBootstrap stays in). - Version.props + Directory.Packages.props: pin WireMock.Net 2.6.0 and Testcontainers 4.11.0. AspNetCore instrumentation split-pinned to 1.15.2 (forced by WireMock.Net 2.6.0 transitive); Http + Runtime stay at the umbrella 1.15.1 — no 1.15.2 release exists for them. - qyl.slnx: register the project. - .agents/routine-last-run.md: routine ledger update for 2026-05-17. No scenario tests here, per the skill's bootstrap section — scenario work waits until reviewers can verify the seams independently. Co-authored-by: Claude --- .agents/routine-last-run.md | 97 ++++++++---- Directory.Packages.props | 9 +- Version.props | 4 + eng/build/BuildTest.cs | 49 +++++- qyl.slnx | 1 + .../Bootstrap/WireMockLlmSeamTests.cs | 48 ++++++ tests/qyl.e2e.tests/E2ECollection.cs | 7 + .../Topology/QylTopologyFixture.cs | 149 ++++++++++++++++++ .../Topology/QylTopologyOptions.cs | 12 ++ tests/qyl.e2e.tests/qyl.e2e.tests.csproj | 20 +++ 10 files changed, 361 insertions(+), 35 deletions(-) create mode 100644 tests/qyl.e2e.tests/Bootstrap/WireMockLlmSeamTests.cs create mode 100644 tests/qyl.e2e.tests/E2ECollection.cs create mode 100644 tests/qyl.e2e.tests/Topology/QylTopologyFixture.cs create mode 100644 tests/qyl.e2e.tests/Topology/QylTopologyOptions.cs create mode 100644 tests/qyl.e2e.tests/qyl.e2e.tests.csproj diff --git a/.agents/routine-last-run.md b/.agents/routine-last-run.md index 4a7f91bd3..382297c42 100644 --- a/.agents/routine-last-run.md +++ b/.agents/routine-last-run.md @@ -1,41 +1,78 @@ -# Routine Test Run — 2026-05-16 +# Routine Test Run — qyl-e2e-tests 2026-05-17 -## Status: STOPPED (pre-existing environment blocker) +## Status: BOOTSTRAP PR OPENED -## What was broken on arrival +## Outcome -The remote execution container does not have the `dotnet` CLI installed, and the -network policy blocks downloading it: +First-ever run of `qyl-e2e-tests` on a workstation with the full toolchain +present (dotnet 10.0.300, Docker 29.4.0). The previous run (2026-05-16) +exited as a no-op because the remote container lacked `dotnet`. -``` -$ curl -L "https://dot.net/v1/dotnet-install.sh" -Host not in allowlist -``` +Per the skill's bootstrap section, this run produced the **infrastructure-only +PR** (project + topology fixture + Nuke target + central-package pins). **No +scenario tests** were added; the next routine run picks up from here and adds +the first scenario. -`./eng/build.sh Ci` exits with code 22 (curl HTTP error) before any .NET code -is compiled or tested. This is a container/environment configuration issue, not -a code defect. +## Files added / changed -## Actions taken +- `tests/qyl.e2e.tests/qyl.e2e.tests.csproj` — new test project, `ANcpLua.NET.Sdk.Test` +- `tests/qyl.e2e.tests/E2ECollection.cs` — `[CollectionDefinition("E2E", DisableParallelization = true)]` +- `tests/qyl.e2e.tests/Topology/QylTopologyOptions.cs` — image tags + startup timeout +- `tests/qyl.e2e.tests/Topology/QylTopologyFixture.cs` — programmatic Testcontainers + topology: bridge network → `qyl-collector:latest` → `qyl-mcp:latest`; WireMock + process-local; containers reach it via `host.docker.internal`. + `WithImagePullPolicy(_ => false)` enforces local-image-only. +- `tests/qyl.e2e.tests/Bootstrap/WireMockLlmSeamTests.cs` — Category=E2EBootstrap + smoke (no Docker) — proves the WireMock seam roundtrips a scripted + `/v1/chat/completions` and shows up in `LogEntries`. +- `eng/build/BuildTest.cs` — new `E2ETests` target depending on + `IDocker.DockerImageBuild`, filters `Category=E2E`. Default `Test` excludes + `Category=E2E` (bootstrap tests stay in). +- `qyl.slnx` — registered the project. +- `Version.props` + `Directory.Packages.props` — added `WireMock.Net 2.6.0` and + `Testcontainers 4.11.0`. Split-pinned `OpenTelemetry.Instrumentation.AspNetCore` + to 1.15.2 (forced by WireMock.Net 2.6.0 transitive) while keeping Http + + Runtime at the umbrella 1.15.1 (no 1.15.2 release exists for them). -- Read CLAUDE.md / AGENTS.md to understand repo conventions. -- Confirmed `dotnet` is absent from all standard paths (`/usr/bin`, `/usr/local/bin`, - `/opt`, etc.). -- Confirmed the `dot.net` install URL is blocked by the environment's network - allowlist. -- No code was modified. +## Verification -## What needs to happen for the next run +- `dotnet build qyl.slnx` — 0 errors, 1454 warnings (baseline 1393; ~61 new + warnings are all `MultipleGlobalAnalyzerKeys` from the dual-`.globalconfig` + worktree setup — benign, present on every worktree build). +- `dotnet test tests/qyl.e2e.tests --filter-trait Category=E2EBootstrap` — 3 + consecutive runs, all green (~1s each, 0 flakes). +- `nuke E2ETests` target wired up but **not executed** this run (would require + rebuilding all four qyl Docker images — out of scope for the bootstrap PR). -The execution environment must either: -1. Have the .NET 10 SDK pre-installed (matching `global.json`: `10.0.203`, - `rollForward: latestFeature`, `allowPrerelease: true`), or -2. Have `https://dot.net` added to the network allowlist so `eng/build.sh` - can auto-install it on first run. +## Overlap with `Smoke` -## What would be targeted next (test gaps) +`eng/smoke/run.sh` (Nuke target `Smoke`) is the PRD #173 quality gate: real +Ollama + real qyl Compose stack, asserts on cost / activity / conversations / +inventory wiring. It uses a **real LLM** (Ollama). -Once the environment is fixed, priority targets are: -- `services/qyl.collector/Storage/` — DuckDB read/write path integration tests. -- `services/qyl.collector/Errors/` — error ingestion endpoint unit tests. -- `services/qyl.mcp/` — MCP tool registration and telemetry unit tests. +E2E (this routine) uses **WireMock** for deterministic LLM stubbing. The two +don't overlap — Smoke covers "does the stack work against a real model"; E2E +covers "does the stack route data correctly given a known-bad/redacted LLM +response we can assert on". + +## Gaps for the next run + +The next `qyl-e2e-tests` cycle should add the first real scenario. Highest-value +candidates: + +1. **Agent submits chat → trace arrives at downstream sink with credentials + redacted** — exercises qyl.mcp → qyl.collector → OTLP egress with a + scripted LLM response that contains a fake bearer token; asserts the token + is `` in the sink output. +2. **MCP HTTP session reconnect after transient collector failure** — proves + the MCP transport's resumption behavior under collector restart. +3. **Cost rollup updates after a single chat completion** — the smallest + slice of PRD #173 surface that can be tested deterministically. + +Pick exactly one. Add a sink container (e.g. an OTel collector configured to +write to a file volume) to `QylTopologyFixture` for scenario 1. + +## Branch and PR + +- Branch: `tests/auto-e2e-2026-05-17` (off `origin/main`) +- PR: opened against `main` as draft → ready-for-review once green. diff --git a/Directory.Packages.props b/Directory.Packages.props index 13737d9ce..410f10603 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -20,8 +20,10 @@ - + SDK version (1.15.3). AspNetCore is bumped to 1.15.2 because + WireMock.Net 2.6.0 (consumed by tests/qyl.e2e.tests) transitively + requires it; Http and Runtime have no 1.15.2 release, hence the split. --> + @@ -108,6 +110,9 @@ + + + diff --git a/Version.props b/Version.props index 1286b83a9..04c0c96a8 100644 --- a/Version.props +++ b/Version.props @@ -57,5 +57,9 @@ 5.5.1 2.5.0 9.4.0 + + 2.6.0 + 4.11.0 diff --git a/eng/build/BuildTest.cs b/eng/build/BuildTest.cs index 14ffae473..961c9a5cf 100644 --- a/eng/build/BuildTest.cs +++ b/eng/build/BuildTest.cs @@ -161,6 +161,12 @@ interface IQylTest : ITest, IHazSourcePaths .DependsOn(static x => x.Compile) .Executes(() => RunFilteredTests("*.Integration.*", "Integration", true)); + Target E2ETests => d => d + .Description("Run end-to-end tests (full Docker topology against freshly built images)") + .DependsOn(static x => x.Compile) + .DependsOn(static x => x.DockerImageBuild) + .Executes(() => RunFilteredE2ETests()); + Target TestSummary => d => d .Unlisted() .Description("Generate Markdown test summary from MTP TRX reports") @@ -183,10 +189,12 @@ interface IQylTest : ITest, IHazSourcePaths .ReportTrx($"{project.Name}.trx") .IgnoreExitCode(8); - // Heavy/opt-in tests (Category=regen — shell out to Weaver, Category=integration etc.) - // are excluded from the default Test run; pass --IQylTest.TestFilter to include them. + // Heavy/opt-in tests (Category=regen — shell out to Weaver; Category=E2E — full + // Docker topology via DockerImageBuild) are excluded from the default Test run; + // pass --IQylTest.TestFilter to include them, or run the dedicated sub-target + // (E2ETests). E2EBootstrap-traited tests (no Docker) intentionally stay in. if (TestFilter is { Length: > 0 } f) mtp.FilterQuery(f); - else mtp.FilterNotTrait("Category", "regen"); + else mtp.FilterNotTrait("Category", "regen").FilterNotTrait("Category", "E2E"); if (StopOnFail == true) mtp.StopOnFail(); if (LiveOutput == true || IsLocalBuild) mtp.ShowLiveOutput(); @@ -221,6 +229,41 @@ sealed void RunFilteredTests(string namespaceFilter, string trxSuffix, bool need }), completeOnFailure: true); } + sealed void RunFilteredE2ETests() + { + EnsureTestcontainersConfigured(); + + var e2eProjects = TestProjects + .Where(static p => p.Name.Equals("qyl.e2e.tests", StringComparison.OrdinalIgnoreCase)) + .ToArray(); + + if (e2eProjects.Length is 0) + { + Log.Warning("E2ETests: no qyl.e2e.tests project found; skipping"); + return; + } + + DotNetTasks.DotNetTest(s => s + .SetNoBuild(true) + .SetNoRestore(true) + .SetResultsDirectory(TestResultsDirectory) + .CombineWith(e2eProjects, (ss, project) => + { + var mtp = MtpExtensions.Mtp() + .ReportTrx($"{project.Name}.E2E.trx") + .IgnoreExitCode(8) + .FilterTrait("Category", "E2E"); + + if (StopOnFail == true) mtp.StopOnFail(); + if (LiveOutput == true || IsLocalBuild) mtp.ShowLiveOutput(); + + var projectPath = project.Path ?? + throw new InvalidOperationException($"Project '{project.Name}' has no path"); + string[] args = ["--project", projectPath.ToString(), .. mtp.BuildArgs().Prepend("--")]; + return ss.SetProcessAdditionalArguments(args); + }), completeOnFailure: true); + } + sealed void EnsureTestcontainersConfigured() { if (IsServerBuild) diff --git a/qyl.slnx b/qyl.slnx index 95149fd65..cf07e71b5 100644 --- a/qyl.slnx +++ b/qyl.slnx @@ -44,6 +44,7 @@ + diff --git a/tests/qyl.e2e.tests/Bootstrap/WireMockLlmSeamTests.cs b/tests/qyl.e2e.tests/Bootstrap/WireMockLlmSeamTests.cs new file mode 100644 index 000000000..9ccacb205 --- /dev/null +++ b/tests/qyl.e2e.tests/Bootstrap/WireMockLlmSeamTests.cs @@ -0,0 +1,48 @@ +using System.Net.Http.Json; +using System.Text.Json; +using WireMock.RequestBuilders; +using WireMock.ResponseBuilders; +using WireMock.Server; + +namespace Qyl.E2E.Tests.Bootstrap; + +[Trait("Category", "E2EBootstrap")] +public sealed class WireMockLlmSeamTests +{ + [Fact] + public async Task ScriptedChatCompletion_RoundtripsAndIsRecordedInLogEntries() + { + using var llm = WireMockServer.Start(); + + llm.Given(Request.Create().WithPath("/v1/chat/completions").UsingPost()) + .RespondWith(Response.Create() + .WithStatusCode(200) + .WithHeader("Content-Type", "application/json") + .WithBodyAsJson(new + { + id = "chatcmpl-bootstrap", + choices = new[] + { + new { message = new { role = "assistant", content = "Bearer secret-token-12345" } }, + }, + })); + + var ct = TestContext.Current.CancellationToken; + using var client = new HttpClient { BaseAddress = new Uri(llm.Url!) }; + var request = new + { + model = "gpt-4o-mini", + messages = new[] { new { role = "user", content = "summarize this" } }, + }; + using var response = await client.PostAsJsonAsync("/v1/chat/completions", request, ct); + + response.IsSuccessStatusCode.Should().BeTrue( + "the WireMock stub must respond 200 for the configured route"); + + using var payload = await JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync(ct), cancellationToken: ct); + payload.RootElement.GetProperty("id").GetString().Should().Be("chatcmpl-bootstrap"); + + llm.LogEntries.Should().ContainSingle( + "WireMock must record the single POST /v1/chat/completions for assertion replay"); + } +} diff --git a/tests/qyl.e2e.tests/E2ECollection.cs b/tests/qyl.e2e.tests/E2ECollection.cs new file mode 100644 index 000000000..7e49609f0 --- /dev/null +++ b/tests/qyl.e2e.tests/E2ECollection.cs @@ -0,0 +1,7 @@ +namespace Qyl.E2E.Tests; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class E2ECollection : ICollectionFixture +{ + public const string Name = "E2E"; +} diff --git a/tests/qyl.e2e.tests/Topology/QylTopologyFixture.cs b/tests/qyl.e2e.tests/Topology/QylTopologyFixture.cs new file mode 100644 index 000000000..0a70f23bb --- /dev/null +++ b/tests/qyl.e2e.tests/Topology/QylTopologyFixture.cs @@ -0,0 +1,149 @@ +using DotNet.Testcontainers.Builders; +using DotNet.Testcontainers.Containers; +using DotNet.Testcontainers.Networks; +using WireMock.RequestBuilders; +using WireMock.ResponseBuilders; +using WireMock.Server; + +namespace Qyl.E2E.Tests.Topology; + +public sealed class QylTopologyFixture : IAsyncLifetime +{ + private const int CollectorInternalPort = 5100; + private const int McpInternalPort = 5200; + + private readonly QylTopologyOptions _options; + + private INetwork? _network; + private IContainer? _collector; + private IContainer? _mcp; + private WireMockServer? _llm; + + public QylTopologyFixture() : this(QylTopologyOptions.Default) + { + } + + public QylTopologyFixture(QylTopologyOptions options) => _options = options; + + public WireMockServer Llm => + _llm ?? throw new InvalidOperationException( + "Topology fixture not initialized — InitializeAsync has not completed."); + + public Uri CollectorBaseUrl => + _collector is null + ? throw new InvalidOperationException("Collector container not started.") + : new Uri($"http://{_collector.Hostname}:{_collector.GetMappedPublicPort(CollectorInternalPort)}/"); + + public Uri McpBaseUrl => + _mcp is null + ? throw new InvalidOperationException("MCP container not started.") + : new Uri($"http://{_mcp.Hostname}:{_mcp.GetMappedPublicPort(McpInternalPort)}/"); + + public async ValueTask InitializeAsync() + { + using var bootstrapCts = new CancellationTokenSource(_options.StartupTimeout); + var ct = bootstrapCts.Token; + + try + { + _llm = WireMockServer.Start(); + ConfigureDefaultLlmResponse(_llm); + + _network = new NetworkBuilder() + .WithName($"qyl-e2e-{Guid.NewGuid():N}") + .Build(); + await _network.CreateAsync(ct).ConfigureAwait(false); + + _collector = new ContainerBuilder() + .WithImage(_options.CollectorImage) + .WithImagePullPolicy(static _ => false) + .WithNetwork(_network) + .WithNetworkAliases("qyl-collector") + .WithPortBinding(CollectorInternalPort, true) + .WithEnvironment("QYL_PORT", CollectorInternalPort.ToString(CultureInfo.InvariantCulture)) + .WithEnvironment("ASPNETCORE_URLS", $"http://+:{CollectorInternalPort}") + .WithWaitStrategy(Wait.ForUnixContainer() + .UntilHttpRequestIsSucceeded(static r => r.ForPath("/health").ForPort(CollectorInternalPort))) + .Build(); + await _collector.StartAsync(ct).ConfigureAwait(false); + + _mcp = new ContainerBuilder() + .WithImage(_options.McpImage) + .WithImagePullPolicy(static _ => false) + .WithNetwork(_network) + .WithNetworkAliases("qyl-mcp") + .WithPortBinding(McpInternalPort, true) + .WithEnvironment("QYL_COLLECTOR_URL", $"http://qyl-collector:{CollectorInternalPort}") + .WithExtraHost("host.docker.internal", "host-gateway") + .WithWaitStrategy(Wait.ForUnixContainer() + .UntilMessageIsLogged("Now listening on:")) + .Build(); + await _mcp.StartAsync(ct).ConfigureAwait(false); + } + catch + { + await DisposeAsync().ConfigureAwait(false); + throw; + } + } + + public async ValueTask DisposeAsync() + { + GC.SuppressFinalize(this); + + if (_mcp is not null) + { + await _mcp.DisposeAsync().ConfigureAwait(false); + _mcp = null; + } + + if (_collector is not null) + { + await _collector.DisposeAsync().ConfigureAwait(false); + _collector = null; + } + + if (_network is not null) + { + await _network.DeleteAsync().ConfigureAwait(false); + _network = null; + } + + if (_llm is not null) + { + _llm.Stop(); + _llm.Dispose(); + _llm = null; + } + } + + public void ResetLlmStub() + { + if (_llm is null) return; + _llm.Reset(); + ConfigureDefaultLlmResponse(_llm); + } + + private static void ConfigureDefaultLlmResponse(WireMockServer llm) => + llm.Given(Request.Create().WithPath("/v1/chat/completions").UsingPost()) + .RespondWith(Response.Create() + .WithStatusCode(200) + .WithHeader("Content-Type", "application/json") + .WithBodyAsJson(new + { + id = "chatcmpl-e2e-default", + @object = "chat.completion", + created = 0, + model = "qyl-e2e-stub", + choices = new[] + { + new + { + index = 0, + message = new { role = "assistant", content = "qyl e2e default response" }, + finish_reason = "stop", + }, + }, + usage = new { prompt_tokens = 1, completion_tokens = 1, total_tokens = 2 }, + })); +} diff --git a/tests/qyl.e2e.tests/Topology/QylTopologyOptions.cs b/tests/qyl.e2e.tests/Topology/QylTopologyOptions.cs new file mode 100644 index 000000000..4be695cf0 --- /dev/null +++ b/tests/qyl.e2e.tests/Topology/QylTopologyOptions.cs @@ -0,0 +1,12 @@ +namespace Qyl.E2E.Tests.Topology; + +public sealed record QylTopologyOptions +{ + public string CollectorImage { get; init; } = "qyl-collector:latest"; + + public string McpImage { get; init; } = "qyl-mcp:latest"; + + public TimeSpan StartupTimeout { get; init; } = TimeSpan.FromSeconds(90); + + public static QylTopologyOptions Default { get; } = new(); +} diff --git a/tests/qyl.e2e.tests/qyl.e2e.tests.csproj b/tests/qyl.e2e.tests/qyl.e2e.tests.csproj new file mode 100644 index 000000000..c790b95e2 --- /dev/null +++ b/tests/qyl.e2e.tests/qyl.e2e.tests.csproj @@ -0,0 +1,20 @@ + + + Qyl.E2E.Tests + Qyl.E2E.Tests + 14 + true + Exe + + + + + + + + + + + + + From c1ab3823157e05485bb6e5bb6e5e00f65a30524b Mon Sep 17 00:00:00 2001 From: ancplua Date: Sun, 17 May 2026 08:17:05 +0200 Subject: [PATCH 2/2] deps: bump ModelContextProtocol from 1.2.0 to 1.3.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MCP 1.3 changes how transport failures surface from McpClient.CreateAsync: SSE/HTTP connect failures now throw IOException (via the new public ClientTransportClosedException, which derives from IOException) rather than InvalidOperationException. Caller-triggered OperationCanceledException is also no longer wrapped. services/qyl.mcp/Formatting/ErrorFormatter.cs: add an IOException arm before InvalidOperationException so transport-closed failures (including the new ClientTransportClosedException) format as "Connection Error" instead of falling through to FormatUnknown. Arm order matters — C# pattern switches dispatch top-down. Verified: dotnet build qyl.slnx --no-restore -> 0 errors. Ref: https://github.com/modelcontextprotocol/csharp-sdk/releases/tag/v1.3.0 Co-authored-by: Claude --- Version.props | 2 +- services/qyl.mcp/Formatting/ErrorFormatter.cs | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Version.props b/Version.props index 04c0c96a8..5e68298a2 100644 --- a/Version.props +++ b/Version.props @@ -47,7 +47,7 @@ 2.10.0 1.11.0 - 1.2.0 + 1.3.0 8.17.0 10.1.0 1.3.2 diff --git a/services/qyl.mcp/Formatting/ErrorFormatter.cs b/services/qyl.mcp/Formatting/ErrorFormatter.cs index d67e5b51b..edd4b961e 100644 --- a/services/qyl.mcp/Formatting/ErrorFormatter.cs +++ b/services/qyl.mcp/Formatting/ErrorFormatter.cs @@ -11,10 +11,19 @@ public static string FormatForLlm(Exception error, McpTransportMode transport) = TaskCanceledException { InnerException: TimeoutException or null } => "**Timeout:** The collector did not respond in time. Retry or check if qyl collector is running.", OperationCanceledException opEx => FormatOperationCancelled(opEx), + // ModelContextProtocol 1.3 throws IOException (incl. ClientTransportClosedException) + // for transport connect/closure failures that 1.2 wrapped as InvalidOperationException. + // This arm must precede InvalidOperationException — order matters in C# pattern switches. + IOException ioEx => FormatTransportError(ioEx, transport), InvalidOperationException configEx => FormatConfigError(configEx, transport), _ => FormatUnknown(error, transport) }; + private static string FormatTransportError(IOException ex, McpTransportMode transport) => + transport is McpTransportMode.Stdio + ? $"**Connection Error**\n\n{ex.Message}\n\nCheck if the MCP server process is running." + : $"**Connection Error**\n\n{ex.Message}\n\nCheck the endpoint URL and network reachability."; + private static string FormatHttpError(HttpRequestException ex, McpTransportMode transport) { var (category, hint) = ex.StatusCode switch