Skip to content

Restructure tests to Arrange/Act/Assert, and cover the external_links gap #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ivanvyd merged 4 commits into from
Aug 1, 2026
Merged

Restructure tests to Arrange/Act/Assert, and cover the external_links gap #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
34b8070
Restructure tests to Arrange/Act/Assert (5 of 10 files)
ivanvyd Aug 1, 2026
a07b758
Restructure ResultCompletenessTests to AAA, and cover the external_li…
ivanvyd Aug 1, 2026
3adff6a
AAA markers for OutputFidelityTests and LiveGenieTests
ivanvyd Aug 1, 2026
f0c15c6
Complete the AAA restructure: GenieLifecycleTests and QuestionPackLoa…
ivanvyd Aug 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 58 additions & 20 deletions tests/LakeSpeak.Cli.Tests/ExitCodeTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
using LakeSpeak.Cli;
using LakeSpeak.Genie;

namespace LakeSpeak.Cli.Tests;
Expand All @@ -21,10 +20,19 @@ public class ExitCodeTests
[InlineData(GenieFailureKind.RateLimited, ExitCode.GenieFailure)]
[InlineData(GenieFailureKind.PollingTimeout, ExitCode.Timeout)]
[InlineData(GenieFailureKind.MalformedResponse, ExitCode.MalformedResponse)]
[InlineData(GenieFailureKind.UnsupportedResult, ExitCode.MalformedResponse)]
[InlineData(GenieFailureKind.Network, ExitCode.Unexpected)]
[InlineData(GenieFailureKind.Unexpected, ExitCode.Unexpected)]
public void Each_failure_kind_maps_to_its_documented_code(GenieFailureKind kind, int expected) =>
ExitCode.From(kind).ShouldBe(expected);
public void Each_failure_kind_maps_to_its_documented_code(GenieFailureKind kind, int expected)
{
// Arrange — the kind under test arrives as the theory parameter.

// Act
var code = ExitCode.From(kind);

// Assert
code.ShouldBe(expected);
}

/// <summary>
/// Adding a <see cref="GenieFailureKind"/> without extending the mapping would otherwise
Expand All @@ -34,10 +42,14 @@ public void Each_failure_kind_maps_to_its_documented_code(GenieFailureKind kind,
[Fact]
public void Every_failure_kind_is_mapped()
{
foreach (var kind in Enum.GetValues<GenieFailureKind>())
{
Should.NotThrow(() => ExitCode.From(kind), $"{kind} has no exit code mapping.");
}
// Arrange
var kinds = Enum.GetValues<GenieFailureKind>();

// Act
var unmapped = kinds.Where(k => !TryMap(k)).ToList();

// Assert
unmapped.ShouldBeEmpty();
}

/// <summary>
Expand All @@ -47,25 +59,51 @@ public void Every_failure_kind_is_mapped()
[Fact]
public void The_documented_numbers_have_not_moved()
{
ExitCode.Success.ShouldBe(0);
ExitCode.Unexpected.ShouldBe(1);
ExitCode.InvalidUsage.ShouldBe(2);
ExitCode.Authentication.ShouldBe(3);
ExitCode.Authorization.ShouldBe(4);
ExitCode.NotFound.ShouldBe(5);
ExitCode.GenieFailure.ShouldBe(6);
ExitCode.Timeout.ShouldBe(7);
ExitCode.PartialPackFailure.ShouldBe(8);
ExitCode.MalformedResponse.ShouldBe(9);
// Arrange — the documented table from docs/commands.md.
var documented = new[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };

// Act
var actual = new[]
{
ExitCode.Success,
ExitCode.Unexpected,
ExitCode.InvalidUsage,
ExitCode.Authentication,
ExitCode.Authorization,
ExitCode.NotFound,
ExitCode.GenieFailure,
ExitCode.Timeout,
ExitCode.PartialPackFailure,
ExitCode.MalformedResponse,
};

// Assert
actual.ShouldBe(documented);
}

// Success must stay 0 and every failure non-zero, or `set -e` and CI stop working.
[Fact]
public void Only_success_is_zero()
{
foreach (var kind in Enum.GetValues<GenieFailureKind>())
// Arrange — success must stay 0 and every failure non-zero, or `set -e` and CI break.
var kinds = Enum.GetValues<GenieFailureKind>();

// Act
var codes = kinds.Select(ExitCode.From).ToList();

// Assert
codes.ShouldAllBe(c => c != ExitCode.Success);
}

private static bool TryMap(GenieFailureKind kind)
{
try
{
ExitCode.From(kind);
return true;
}
catch (ArgumentOutOfRangeException)
{
ExitCode.From(kind).ShouldNotBe(ExitCode.Success);
return false;
}
}
}
89 changes: 61 additions & 28 deletions tests/LakeSpeak.ContractTests/GenieLifecycleTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ private GenieClient CreateClient(TimeSpan? timeout = null)
var http = new HttpClient { BaseAddress = new Uri(_server.Url!) };
var options = new GenieClientOptions
{
// The server is http, and Validate() rightly refuses a non-https host. Host is only
// The stub is http, and Validate() rightly refuses a non-https host. Host is only
// used to build the base address, which is set directly above.
Host = new Uri("https://example.azuredatabricks.net"),
InitialPollInterval = TimeSpan.FromSeconds(1),
Expand Down Expand Up @@ -74,8 +74,7 @@ private void StubMessageSequence(params string[] statuses)
}

// The last stub must not advance the state, or it stops matching itself after one
// response and WireMock starts 404ing. That turned the polling-timeout test into a
// 404 test without either one failing for the right reason.
// response and WireMock starts 404ing.
if (!isLast)
{
stub = stub.WillSetStateTo($"s{i + 1}");
Expand Down Expand Up @@ -145,15 +144,18 @@ private void StubQueryResult() =>
[Fact]
public async Task Ask_walks_the_full_lifecycle_and_returns_a_normalized_response()
{
// Arrange
StubStartConversation();
StubMessageSequence("SUBMITTED", "FILTERING_CONTEXT", "EXECUTING_QUERY", "COMPLETED");
StubQueryResult();

var client = CreateClient();

// Act
var task = client.AskAsync(Agent, "How did revenue change?", cancellationToken: Ct);
await AdvanceUntilSettledAsync(task);
var response = await task;

// Assert
response.State.ShouldBe(GenieMessageState.Completed);
response.ConversationId.ShouldBe(Conversation);

Expand All @@ -176,19 +178,22 @@ public async Task Ask_walks_the_full_lifecycle_and_returns_a_normalized_response
[Fact]
public async Task Reports_every_state_transition_in_order()
{
// Arrange
StubStartConversation();
StubMessageSequence("SUBMITTED", "PENDING_WAREHOUSE", "EXECUTING_QUERY", "COMPLETED");
StubQueryResult();

var seen = new List<GenieMessageState>();
var client = CreateClient();

// Act
var task = client.AskAsync(Agent, "q", new GenieAskOptions
{
OnStateChanged = s => seen.Add(s),
}, Ct);
await AdvanceUntilSettledAsync(task);
await task;

// Assert
seen.ShouldBe([
GenieMessageState.Submitted,
GenieMessageState.PendingWarehouse,
Expand All @@ -200,6 +205,7 @@ public async Task Reports_every_state_transition_in_order()
[Fact]
public async Task A_failed_message_raises_MessageFailed_carrying_the_platform_reason()
{
// Arrange
StubStartConversation();
_server.Given(Request.Create()
.WithPath($"/api/2.0/genie/spaces/{Agent}/conversations/{Conversation}/messages/{Message}")
Expand All @@ -212,21 +218,24 @@ public async Task A_failed_message_raises_MessageFailed_carrying_the_platform_re
"error": { "error": "Table orders does not exist", "type": "SQL_EXECUTION_EXCEPTION" }
}
"""));

var client = CreateClient();
var ex = await Should.ThrowAsync<GenieException>(() => client.AskAsync(Agent, "q", cancellationToken: Ct));

// Act
var ex = await Should.ThrowAsync<GenieException>(
() => client.AskAsync(Agent, "q", cancellationToken: Ct));

// Assert
ex.Kind.ShouldBe(GenieFailureKind.MessageFailed);
ex.Message.ShouldContain("Table orders does not exist");
ex.ErrorCode.ShouldBe("SQL_EXECUTION_EXCEPTION");
ex.IsRetryable.ShouldBeFalse();
}

// QUERY_RESULT_EXPIRED is terminal but is not a failure: the answer and SQL are still
// valid. Throwing here would discard a good answer over a stale cache entry.
[Fact]
public async Task An_expired_result_returns_the_answer_rather_than_throwing()
{
// Arrange — QUERY_RESULT_EXPIRED is terminal but is not a failure: the answer and SQL
// are still valid, so throwing would discard a good answer over a stale cache entry.
StubStartConversation();
_server.Given(Request.Create()
.WithPath($"/api/2.0/genie/spaces/{Agent}/conversations/{Conversation}/messages/{Message}")
Expand All @@ -241,10 +250,12 @@ public async Task An_expired_result_returns_the_answer_rather_than_throwing()
]
}
"""));

var client = CreateClient();

// Act
var response = await client.AskAsync(Agent, "q", cancellationToken: Ct);

// Assert
response.State.ShouldBe(GenieMessageState.QueryResultExpired);
response.Text.ShouldBe("Revenue rose 14.2%.");
}
Expand All @@ -257,14 +268,18 @@ public async Task An_expired_result_returns_the_answer_rather_than_throwing()
[InlineData(500, GenieFailureKind.Unexpected, false)]
public async Task Maps_http_failures_to_typed_kinds(int status, GenieFailureKind kind, bool retryable)
{
// Arrange
_server.Given(Request.Create()
.WithPath($"/api/2.0/genie/spaces/{Agent}/start-conversation").UsingPost())
.RespondWith(Response.Create().WithStatusCode(status).WithBody(
"""{"error_code":"PERMISSION_DENIED","message":"nope"}"""));

var client = CreateClient();
var ex = await Should.ThrowAsync<GenieException>(() => client.AskAsync(Agent, "q", cancellationToken: Ct));

// Act
var ex = await Should.ThrowAsync<GenieException>(
() => client.AskAsync(Agent, "q", cancellationToken: Ct));

// Assert
ex.Kind.ShouldBe(kind);
ex.StatusCode.ShouldBe(status);
ex.IsRetryable.ShouldBe(retryable);
Expand All @@ -273,30 +288,35 @@ public async Task Maps_http_failures_to_typed_kinds(int status, GenieFailureKind
[Fact]
public async Task A_body_that_is_not_json_fails_as_MalformedResponse_without_echoing_it()
{
// Arrange
_server.Given(Request.Create()
.WithPath($"/api/2.0/genie/spaces/{Agent}/start-conversation").UsingPost())
.RespondWith(Response.Create().WithStatusCode(200).WithBody("<html>gateway</html>"));

var client = CreateClient();
var ex = await Should.ThrowAsync<GenieException>(() => client.AskAsync(Agent, "q", cancellationToken: Ct));

// Act
var ex = await Should.ThrowAsync<GenieException>(
() => client.AskAsync(Agent, "q", cancellationToken: Ct));

// Assert — the body can contain query results, which are governed data, and this message
// may reach a log or a bug report.
ex.Kind.ShouldBe(GenieFailureKind.MalformedResponse);
// The body can contain query results, which are governed data, and this message may
// reach a log or a bug report.
ex.Message.ShouldNotContain("gateway");
}

[Fact]
public async Task Polling_stops_at_the_timeout_and_keeps_the_last_seen_state()
{
// Arrange
StubStartConversation();
StubMessageSequence("EXECUTING_QUERY");

var client = CreateClient(timeout: TimeSpan.FromSeconds(30));
var task = client.AskAsync(Agent, "q", cancellationToken: Ct);

// Act
var task = client.AskAsync(Agent, "q", cancellationToken: Ct);
await AdvanceUntilSettledAsync(task);

// Assert
var ex = await Should.ThrowAsync<GenieException>(() => task);
ex.Kind.ShouldBe(GenieFailureKind.PollingTimeout);
ex.LastKnownResponse!.State.ShouldBe(GenieMessageState.ExecutingQuery);
Expand All @@ -305,31 +325,37 @@ public async Task Polling_stops_at_the_timeout_and_keeps_the_last_seen_state()
[Fact]
public async Task Cancellation_stops_polling_promptly()
{
// Arrange
StubStartConversation();
StubMessageSequence("EXECUTING_QUERY");

using var cts = new CancellationTokenSource();
var client = CreateClient();
var task = client.AskAsync(Agent, "q", cancellationToken: cts.Token);

// Act
var task = client.AskAsync(Agent, "q", cancellationToken: cts.Token);
await Task.Yield();
await cts.CancelAsync();
_clock.Advance(TimeSpan.FromSeconds(5));

// Assert
await Should.ThrowAsync<OperationCanceledException>(() => task);
}

[Fact]
public async Task Feedback_posts_the_rating_and_tolerates_an_empty_body()
{
// Arrange
_server.Given(Request.Create()
.WithPath($"/api/2.0/genie/spaces/{Agent}/conversations/{Conversation}/messages/{Message}/feedback")
.UsingPost())
.RespondWith(Response.Create().WithStatusCode(200).WithBody(string.Empty));

var client = CreateClient();
await client.SendFeedbackAsync(Agent, Conversation, Message, GenieFeedbackRating.Negative, "wrong filter", Ct);

// Act
await client.SendFeedbackAsync(
Agent, Conversation, Message, GenieFeedbackRating.Negative, "wrong filter", Ct);

// Assert
var request = _server.LogEntries.Single().RequestMessage;
Assert.NotNull(request);
Assert.NotNull(request.Body);
Expand All @@ -340,6 +366,7 @@ public async Task Feedback_posts_the_rating_and_tolerates_an_empty_body()
[Fact]
public async Task Agent_listing_follows_pagination()
{
// Arrange
_server.Given(Request.Create().WithPath("/api/2.0/genie/spaces").UsingGet()
.WithParam("page_token", "p2"))
.RespondWith(Response.Create().WithStatusCode(200).WithBody(
Expand All @@ -351,25 +378,30 @@ public async Task Agent_listing_follows_pagination()

var client = CreateClient();
var agents = new List<GenieAgent>();

// Act
await foreach (var agent in client.ListAllAgentsAsync(Ct))
{
agents.Add(agent);
}

// Assert
agents.Select(a => a.Title).ShouldBe(["Sales", "Finance"]);
}

// WireMock hands back the same token forever here. Without the repeated-token guard this
// enumerates until the process is killed.
[Fact]
public async Task Agent_listing_stops_when_the_server_repeats_a_page_token()
{
// Arrange — WireMock hands back the same token forever here. Without the repeated-token
// guard this enumerates until the process is killed.
_server.Given(Request.Create().WithPath("/api/2.0/genie/spaces").UsingGet())
.RespondWith(Response.Create().WithStatusCode(200).WithBody(
"""{"spaces":[{"space_id":"a","title":"Sales"}],"next_page_token":"same"}"""));

var client = CreateClient();
var count = 0;

// Act
await foreach (var _ in client.ListAllAgentsAsync(Ct))
{
if (++count > 50)
Expand All @@ -378,6 +410,7 @@ public async Task Agent_listing_stops_when_the_server_repeats_a_page_token()
}
}

// Assert
count.ShouldBe(2);
}

Expand All @@ -386,10 +419,10 @@ public async Task Agent_listing_stops_when_the_server_repeats_a_page_token()
/// throws.
/// </summary>
/// <remarks>
/// Bounded by real wall time, not by a fixed iteration count. A counted loop can exhaust
/// its iterations while an HTTP round trip is still in flight, after which nothing advances
/// the clock again and the awaiting test hangs forever rather than failing. Falling out of
/// this loop without the task settling is itself a failure, and is reported as one.
/// Bounded by real wall time, not by a fixed iteration count. A counted loop can exhaust its
/// iterations while an HTTP round trip is still in flight, after which nothing advances the
/// clock again and the awaiting test hangs forever rather than failing. Falling out of this
/// loop without the task settling is itself a failure, and is reported as one.
/// </remarks>
private async Task AdvanceUntilSettledAsync(Task task)
{
Expand Down
Loading
Loading