Skip to content

.NET: Fix anthropic integration tests and skip reason #4211

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
westey-m merged 1 commit into from
Feb 24, 2026
Merged

.NET: Fix anthropic integration tests and skip reason #4211

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 4 additions & 21 deletions ...hatCompletion.IntegrationTests/AnthropicChatCompletionChatClientAgentRunStreamingTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,30 +1,13 @@
// Copyright (c) Microsoft. All rights reserved.

using System;
using System.Threading.Tasks;
using AgentConformance.IntegrationTests;

namespace AnthropicChatCompletion.IntegrationTests;

public abstract class SkipAllChatClientRunStreaming(Func<AnthropicChatCompletionFixture> func) : ChatClientAgentRunStreamingTests<AnthropicChatCompletionFixture>(func)
{
public override Task RunWithFunctionsInvokesFunctionsAndReturnsExpectedResultsAsync()
{
Assert.SkipWhen(AnthropicChatCompletionFixture.SkipReason is not null, AnthropicChatCompletionFixture.SkipReason!);
return base.RunWithFunctionsInvokesFunctionsAndReturnsExpectedResultsAsync();
}
public class AnthropicBetaChatCompletionChatClientAgentReasoningRunStreamingTests() : ChatClientAgentRunStreamingTests<AnthropicChatCompletionFixture>(() => new(useReasoningChatModel: true, useBeta: true));

public override Task RunWithInstructionsAndNoMessageReturnsExpectedResultAsync()
{
Assert.SkipWhen(AnthropicChatCompletionFixture.SkipReason is not null, AnthropicChatCompletionFixture.SkipReason!);
return base.RunWithInstructionsAndNoMessageReturnsExpectedResultAsync();
}
}
public class AnthropicBetaChatCompletionChatClientAgentRunStreamingTests() : ChatClientAgentRunStreamingTests<AnthropicChatCompletionFixture>(() => new(useReasoningChatModel: false, useBeta: true));

public class AnthropicBetaChatCompletionChatClientAgentReasoningRunStreamingTests() : SkipAllChatClientRunStreaming(() => new(useReasoningChatModel: true, useBeta: true));
public class AnthropicChatCompletionChatClientAgentRunStreamingTests() : ChatClientAgentRunStreamingTests<AnthropicChatCompletionFixture>(() => new(useReasoningChatModel: false, useBeta: false));

public class AnthropicBetaChatCompletionChatClientAgentRunStreamingTests() : SkipAllChatClientRunStreaming(() => new(useReasoningChatModel: false, useBeta: true));

public class AnthropicChatCompletionChatClientAgentRunStreamingTests() : SkipAllChatClientRunStreaming(() => new(useReasoningChatModel: false, useBeta: false));

public class AnthropicChatCompletionChatClientAgentReasoningRunStreamingTests() : SkipAllChatClientRunStreaming(() => new(useReasoningChatModel: true, useBeta: false));
public class AnthropicChatCompletionChatClientAgentReasoningRunStreamingTests() : ChatClientAgentRunStreamingTests<AnthropicChatCompletionFixture>(() => new(useReasoningChatModel: true, useBeta: false));
25 changes: 4 additions & 21 deletions ...nthropicChatCompletion.IntegrationTests/AnthropicChatCompletionChatClientAgentRunTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,34 +1,17 @@
// Copyright (c) Microsoft. All rights reserved.

using System;
using System.Threading.Tasks;
using AgentConformance.IntegrationTests;

namespace AnthropicChatCompletion.IntegrationTests;

public abstract class SkipAllChatClientAgentRun(Func<AnthropicChatCompletionFixture> func) : ChatClientAgentRunTests<AnthropicChatCompletionFixture>(func)
{
public override Task RunWithFunctionsInvokesFunctionsAndReturnsExpectedResultsAsync()
{
Assert.SkipWhen(AnthropicChatCompletionFixture.SkipReason is not null, AnthropicChatCompletionFixture.SkipReason!);
return base.RunWithFunctionsInvokesFunctionsAndReturnsExpectedResultsAsync();
}

public override Task RunWithInstructionsAndNoMessageReturnsExpectedResultAsync()
{
Assert.SkipWhen(AnthropicChatCompletionFixture.SkipReason is not null, AnthropicChatCompletionFixture.SkipReason!);
return base.RunWithInstructionsAndNoMessageReturnsExpectedResultAsync();
}
}

public class AnthropicBetaChatCompletionChatClientAgentRunTests()
: SkipAllChatClientAgentRun(() => new(useReasoningChatModel: false, useBeta: true));
: ChatClientAgentRunTests<AnthropicChatCompletionFixture>(() => new(useReasoningChatModel: false, useBeta: true));

public class AnthropicBetaChatCompletionChatClientAgentReasoningRunTests()
: SkipAllChatClientAgentRun(() => new(useReasoningChatModel: true, useBeta: true));
: ChatClientAgentRunTests<AnthropicChatCompletionFixture>(() => new(useReasoningChatModel: true, useBeta: true));

public class AnthropicChatCompletionChatClientAgentRunTests()
: SkipAllChatClientAgentRun(() => new(useReasoningChatModel: false, useBeta: false));
: ChatClientAgentRunTests<AnthropicChatCompletionFixture>(() => new(useReasoningChatModel: false, useBeta: false));

public class AnthropicChatCompletionChatClientAgentReasoningRunTests()
: SkipAllChatClientAgentRun(() => new(useReasoningChatModel: true, useBeta: false));
: ChatClientAgentRunTests<AnthropicChatCompletionFixture>(() => new(useReasoningChatModel: true, useBeta: false));
13 changes: 8 additions & 5 deletions dotnet/tests/AnthropicChatCompletion.IntegrationTests/AnthropicChatCompletionFixture.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ public class AnthropicChatCompletionFixture : IChatClientAgentFixture
// All tests for Anthropic are intended to be ran locally as the CI pipeline for Anthropic is not setup.
internal const string SkipReason = "Integrations tests for local execution only";
Copy link

Copilot AI Feb 24, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Typo in the skip reason string: "Integrations tests" should be "Integration tests".

Suggested change
internal const string SkipReason = "Integrations tests for local execution only";
internal const string SkipReason = "Integration tests for local execution only";

Copilot uses AI. Check for mistakes.

private static readonly AnthropicConfiguration s_config = TestConfiguration.LoadSection<AnthropicConfiguration>();
private readonly bool _useReasoningModel;
private readonly bool _useBeta;

Expand Down Expand Up @@ -53,7 +52,8 @@ public Task<ChatClientAgent> CreateChatClientAgentAsync(
string instructions = "You are a helpful assistant.",
IList<AITool>? aiTools = null)
{
var anthropicClient = new AnthropicClient() { ApiKey = s_config.ApiKey };
AnthropicConfiguration config = TestConfiguration.LoadSection<AnthropicConfiguration>();
var anthropicClient = new AnthropicClient() { ApiKey = config.ApiKey };

IChatClient? chatClient = this._useBeta
? anthropicClient
Expand All @@ -64,7 +64,7 @@ public Task<ChatClientAgent> CreateChatClientAgentAsync(
=> options.RawRepresentationFactory = _
=> new Anthropic.Models.Beta.Messages.MessageCreateParams()
{
Model = options.ModelId ?? (this._useReasoningModel ? s_config.ChatReasoningModelId : s_config.ChatModelId),
Model = options.ModelId ?? (this._useReasoningModel ? config.ChatReasoningModelId : config.ChatModelId),
MaxTokens = options.MaxOutputTokens ?? 4096,
Messages = [],
Thinking = this._useReasoningModel
Expand All @@ -79,7 +79,7 @@ public Task<ChatClientAgent> CreateChatClientAgentAsync(
=> options.RawRepresentationFactory = _
=> new Anthropic.Models.Messages.MessageCreateParams()
{
Model = options.ModelId ?? (this._useReasoningModel ? s_config.ChatReasoningModelId : s_config.ChatModelId),
Model = options.ModelId ?? (this._useReasoningModel ? config.ChatReasoningModelId : config.ChatModelId),
MaxTokens = options.MaxOutputTokens ?? 4096,
Messages = [],
Thinking = this._useReasoningModel
Expand All @@ -102,8 +102,11 @@ public Task DeleteSessionAsync(AgentSession session) =>
// Chat Completion does not require/support deleting sessions, so this is a no-op.
Task.CompletedTask;

public async ValueTask InitializeAsync() =>
public async ValueTask InitializeAsync()
{
Assert.SkipWhen(SkipReason is not null, SkipReason ?? string.Empty);
Copy link

Copilot AI Feb 24, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

SkipReason is a non-null const, so the fixture initialization will always skip and the SkipWhen(... is not null, ...) check is redundant. Consider using Assert.Skip(SkipReason) to make the unconditional skip explicit, or make SkipReason/the condition dependent on CI or configuration availability if the fixture should be runnable locally without code changes.

Suggested change
Assert.SkipWhen(SkipReason is not null, SkipReason ?? string.Empty);
Assert.Skip(SkipReason);

Copilot uses AI. Check for mistakes.
this._agent = await this.CreateChatClientAgentAsync();
}

public ValueTask DisposeAsync()
{
Expand Down
43 changes: 4 additions & 39 deletions ...ests/AnthropicChatCompletion.IntegrationTests/AnthropicChatCompletionRunStreamingTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,52 +1,17 @@
// Copyright (c) Microsoft. All rights reserved.

using System;
using System.Threading.Tasks;
using AgentConformance.IntegrationTests;

namespace AnthropicChatCompletion.IntegrationTests;

public abstract class SkipAllRunStreaming(Func<AnthropicChatCompletionFixture> func) : RunStreamingTests<AnthropicChatCompletionFixture>(func)
{
public override Task RunWithChatMessageReturnsExpectedResultAsync()
{
Assert.SkipWhen(AnthropicChatCompletionFixture.SkipReason is not null, AnthropicChatCompletionFixture.SkipReason!);
return base.RunWithChatMessageReturnsExpectedResultAsync();
}

public override Task RunWithNoMessageDoesNotFailAsync()
{
Assert.SkipWhen(AnthropicChatCompletionFixture.SkipReason is not null, AnthropicChatCompletionFixture.SkipReason!);
return base.RunWithNoMessageDoesNotFailAsync();
}

public override Task RunWithChatMessagesReturnsExpectedResultAsync()
{
Assert.SkipWhen(AnthropicChatCompletionFixture.SkipReason is not null, AnthropicChatCompletionFixture.SkipReason!);
return base.RunWithChatMessagesReturnsExpectedResultAsync();
}

public override Task RunWithStringReturnsExpectedResultAsync()
{
Assert.SkipWhen(AnthropicChatCompletionFixture.SkipReason is not null, AnthropicChatCompletionFixture.SkipReason!);
return base.RunWithStringReturnsExpectedResultAsync();
}

public override Task SessionMaintainsHistoryAsync()
{
Assert.SkipWhen(AnthropicChatCompletionFixture.SkipReason is not null, AnthropicChatCompletionFixture.SkipReason!);
return base.SessionMaintainsHistoryAsync();
}
}

public class AnthropicBetaChatCompletionRunStreamingTests()
: SkipAllRunStreaming(() => new(useReasoningChatModel: false, useBeta: true));
: RunStreamingTests<AnthropicChatCompletionFixture>(() => new(useReasoningChatModel: false, useBeta: true));

public class AnthropicBetaChatCompletionReasoningRunStreamingTests()
: SkipAllRunStreaming(() => new(useReasoningChatModel: true, useBeta: true));
: RunStreamingTests<AnthropicChatCompletionFixture>(() => new(useReasoningChatModel: true, useBeta: true));

public class AnthropicChatCompletionRunStreamingTests()
: SkipAllRunStreaming(() => new(useReasoningChatModel: false, useBeta: false));
: RunStreamingTests<AnthropicChatCompletionFixture>(() => new(useReasoningChatModel: false, useBeta: false));

public class AnthropicChatCompletionReasoningRunStreamingTests()
: SkipAllRunStreaming(() => new(useReasoningChatModel: true, useBeta: false));
: RunStreamingTests<AnthropicChatCompletionFixture>(() => new(useReasoningChatModel: true, useBeta: false));
43 changes: 4 additions & 39 deletions dotnet/tests/AnthropicChatCompletion.IntegrationTests/AnthropicChatCompletionRunTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,52 +1,17 @@
// Copyright (c) Microsoft. All rights reserved.

using System;
using System.Threading.Tasks;
using AgentConformance.IntegrationTests;

namespace AnthropicChatCompletion.IntegrationTests;

public abstract class SkipAllRun(Func<AnthropicChatCompletionFixture> func) : RunTests<AnthropicChatCompletionFixture>(func)
{
public override Task RunWithChatMessageReturnsExpectedResultAsync()
{
Assert.SkipWhen(AnthropicChatCompletionFixture.SkipReason is not null, AnthropicChatCompletionFixture.SkipReason!);
return base.RunWithChatMessageReturnsExpectedResultAsync();
}

public override Task RunWithNoMessageDoesNotFailAsync()
{
Assert.SkipWhen(AnthropicChatCompletionFixture.SkipReason is not null, AnthropicChatCompletionFixture.SkipReason!);
return base.RunWithNoMessageDoesNotFailAsync();
}

public override Task RunWithChatMessagesReturnsExpectedResultAsync()
{
Assert.SkipWhen(AnthropicChatCompletionFixture.SkipReason is not null, AnthropicChatCompletionFixture.SkipReason!);
return base.RunWithChatMessagesReturnsExpectedResultAsync();
}

public override Task RunWithStringReturnsExpectedResultAsync()
{
Assert.SkipWhen(AnthropicChatCompletionFixture.SkipReason is not null, AnthropicChatCompletionFixture.SkipReason!);
return base.RunWithStringReturnsExpectedResultAsync();
}

public override Task SessionMaintainsHistoryAsync()
{
Assert.SkipWhen(AnthropicChatCompletionFixture.SkipReason is not null, AnthropicChatCompletionFixture.SkipReason!);
return base.SessionMaintainsHistoryAsync();
}
}

public class AnthropicBetaChatCompletionRunTests()
: SkipAllRun(() => new(useReasoningChatModel: false, useBeta: true));
: RunTests<AnthropicChatCompletionFixture>(() => new(useReasoningChatModel: false, useBeta: true));

public class AnthropicBetaChatCompletionReasoningRunTests()
: SkipAllRun(() => new(useReasoningChatModel: true, useBeta: true));
: RunTests<AnthropicChatCompletionFixture>(() => new(useReasoningChatModel: true, useBeta: true));

public class AnthropicChatCompletionRunTests()
: SkipAllRun(() => new(useReasoningChatModel: false, useBeta: false));
: RunTests<AnthropicChatCompletionFixture>(() => new(useReasoningChatModel: false, useBeta: false));

public class AnthropicChatCompletionReasoningRunTests()
: SkipAllRun(() => new(useReasoningChatModel: true, useBeta: false));
: RunTests<AnthropicChatCompletionFixture>(() => new(useReasoningChatModel: true, useBeta: false));
18 changes: 11 additions & 7 deletions dotnet/tests/AnthropicChatCompletion.IntegrationTests/AnthropicSkillsIntegrationTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,15 @@ public sealed class AnthropicSkillsIntegrationTests
// All tests for Anthropic are intended to be ran locally as the CI pipeline for Anthropic is not setup.
private const string SkipReason = "Integrations tests for local execution only";
Copy link

Copilot AI Feb 24, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Typo in the skip reason string: "Integrations tests" should be "Integration tests".

Suggested change
private const string SkipReason = "Integrations tests for local execution only";
private const string SkipReason = "Integration tests for local execution only";

Copilot uses AI. Check for mistakes.

private static readonly AnthropicConfiguration s_config = TestConfiguration.LoadSection<AnthropicConfiguration>();

[Fact(Skip = SkipReason)]
[Fact]
public async Task CreateAgentWithPptxSkillAsync()
{
Assert.SkipWhen(SkipReason is not null, SkipReason ?? string.Empty);

// Arrange
AnthropicClient anthropicClient = new() { ApiKey = s_config.ApiKey };
string model = s_config.ChatModelId;
AnthropicConfiguration config = TestConfiguration.LoadSection<AnthropicConfiguration>();
AnthropicClient anthropicClient = new() { ApiKey = config.ApiKey };
string model = config.ChatModelId;

BetaSkillParams pptxSkill = new()
{
Expand All @@ -53,11 +54,14 @@ public async Task CreateAgentWithPptxSkillAsync()
Assert.NotEmpty(response.Text);
}

[Fact(Skip = SkipReason)]
[Fact]
public async Task ListAnthropicManagedSkillsAsync()
{
Assert.SkipWhen(SkipReason is not null, SkipReason ?? string.Empty);

// Arrange
AnthropicClient anthropicClient = new() { ApiKey = s_config.ApiKey };
AnthropicConfiguration config = TestConfiguration.LoadSection<AnthropicConfiguration>();
AnthropicClient anthropicClient = new() { ApiKey = config.ApiKey };

// Act
SkillListPage skills = await anthropicClient.Beta.Skills.List(
Expand Down
6 changes: 3 additions & 3 deletions ...ureAIAgentsPersistent.IntegrationTests/AzureAIAgentsPersistentStructuredOutputRunTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,19 +11,19 @@ namespace AzureAIAgentsPersistent.IntegrationTests;

public override Task RunWithResponseFormatReturnsExpectedResultAsync()
{
Assert.SkipWhen(SkipReason is not null, SkipReason!);
Assert.SkipWhen(SkipReason is not null, SkipReason ?? string.Empty);
return base.RunWithResponseFormatReturnsExpectedResultAsync();
}

public override Task RunWithGenericTypeReturnsExpectedResultAsync()
{
Assert.SkipWhen(SkipReason is not null, SkipReason!);
Assert.SkipWhen(SkipReason is not null, SkipReason ?? string.Empty);
return base.RunWithGenericTypeReturnsExpectedResultAsync();
}

public override Task RunWithPrimitiveTypeReturnsExpectedResultAsync()
{
Assert.SkipWhen(SkipReason is not null, SkipReason!);
Assert.SkipWhen(SkipReason is not null, SkipReason ?? string.Empty);
return base.RunWithPrimitiveTypeReturnsExpectedResultAsync();
}
}
8 changes: 4 additions & 4 deletions dotnet/tests/CopilotStudio.IntegrationTests/CopilotStudioRunStreamingTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,25 +18,25 @@ public override Task SessionMaintainsHistoryAsync()

public override Task RunWithChatMessageReturnsExpectedResultAsync()
{
Assert.SkipWhen(ManualVerification is not null, ManualVerification!);
Assert.SkipWhen(ManualVerification is not null, ManualVerification ?? string.Empty);
return base.RunWithChatMessageReturnsExpectedResultAsync();
Comment on lines +21 to 22
Copy link

Copilot AI Feb 24, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same pattern as the non-streaming tests: since ManualVerification is a constant non-null string, this SkipWhen condition is always true and ?? string.Empty is dead code. Consider using Assert.Skip(ManualVerification) or making the skip reason nullable if you want conditional skipping.

Copilot uses AI. Check for mistakes.
}

public override Task RunWithChatMessagesReturnsExpectedResultAsync()
{
Assert.SkipWhen(ManualVerification is not null, ManualVerification!);
Assert.SkipWhen(ManualVerification is not null, ManualVerification ?? string.Empty);
return base.RunWithChatMessagesReturnsExpectedResultAsync();
}

public override Task RunWithNoMessageDoesNotFailAsync()
{
Assert.SkipWhen(ManualVerification is not null, ManualVerification!);
Assert.SkipWhen(ManualVerification is not null, ManualVerification ?? string.Empty);
return base.RunWithNoMessageDoesNotFailAsync();
}

public override Task RunWithStringReturnsExpectedResultAsync()
{
Assert.SkipWhen(ManualVerification is not null, ManualVerification!);
Assert.SkipWhen(ManualVerification is not null, ManualVerification ?? string.Empty);
return base.RunWithStringReturnsExpectedResultAsync();
}
}
8 changes: 4 additions & 4 deletions dotnet/tests/CopilotStudio.IntegrationTests/CopilotStudioRunTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,25 +18,25 @@ public override Task SessionMaintainsHistoryAsync()

public override Task RunWithChatMessageReturnsExpectedResultAsync()
{
Assert.SkipWhen(ManualVerification is not null, ManualVerification!);
Assert.SkipWhen(ManualVerification is not null, ManualVerification ?? string.Empty);
return base.RunWithChatMessageReturnsExpectedResultAsync();
Comment on lines +21 to 22
Copy link

Copilot AI Feb 24, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ManualVerification is a constant non-null string, so Assert.SkipWhen(ManualVerification is not null, ...) will always skip these tests and the ?? string.Empty fallback is never used. Consider simplifying to Assert.Skip(ManualVerification) (or make ManualVerification nullable/non-const if you want to toggle skipping without the redundant condition).

Copilot uses AI. Check for mistakes.
}

public override Task RunWithChatMessagesReturnsExpectedResultAsync()
{
Assert.SkipWhen(ManualVerification is not null, ManualVerification!);
Assert.SkipWhen(ManualVerification is not null, ManualVerification ?? string.Empty);
return base.RunWithChatMessagesReturnsExpectedResultAsync();
}

public override Task RunWithNoMessageDoesNotFailAsync()
{
Assert.SkipWhen(ManualVerification is not null, ManualVerification!);
Assert.SkipWhen(ManualVerification is not null, ManualVerification ?? string.Empty);
return base.RunWithNoMessageDoesNotFailAsync();
}

public override Task RunWithStringReturnsExpectedResultAsync()
{
Assert.SkipWhen(ManualVerification is not null, ManualVerification!);
Assert.SkipWhen(ManualVerification is not null, ManualVerification ?? string.Empty);
return base.RunWithStringReturnsExpectedResultAsync();
}
}
Loading