From a3071b4f1c4a44701d72383fcad4dad68eadeda4 Mon Sep 17 00:00:00 2001 From: "vellum-apollo-bot[bot]" <242025090+vellum-apollo-bot[bot]@users.noreply.github.com> Date: Tue, 12 May 2026 01:02:54 +0000 Subject: [PATCH] fix(macos): emit llm.activeProfile=balanced for managed-inference hatch The macOS onboarding flow has two distinct local-hosting paths that share the same hatch step: 1. Managed inference - user signs in with their Vellum account, picks Local hosting, never enters a provider API key. Chat traffic routes through the managed Anthropic proxy via a platform-injected 'assistant_api_key' credential. 2. BYOK - user provides their own provider API key, hatches without a Vellum account. Both paths were emitting the same hatch-time config overlay ('llm.default.provider: '), which the daemon seeder unconditionally interpreted as 'user has supplied their own API key'. The seeder materialized an 'anthropic-personal' provider connection and - after #30232 (Phase 1.2 PR-D) flipped the activeProfile default - set the active profile to 'custom-balanced'. That profile reads 'credential/anthropic/api_key' from CES, which doesn't exist in the managed flow, so message sends 500'd. Fix: branch on state.skippedAPIKeyEntry (set exclusively when the user is authenticated and skipped the API key entry step) and emit 'llm.activeProfile: balanced' instead of 'llm.default.provider' on the managed path. The daemon seeder already tracks this via 'providedLlmActiveProfile' from mergeDefaultWorkspaceConfig and skips its 'isHatch ? custom-balanced : balanced' branch when preserveActiveProfile is true and the named profile exists - so 'balanced' (managed Anthropic) stays selected through the hatch. The logic moves into a pure top-level helper ('onboardingHatchConfigOverlay') so it can be unit-tested without spinning up OnboardingState, matching the existing convention ('OnboardingHostingModeResolver', 'OnboardingManagedAuthState'). Fixes the 'hello-world-managed-inference' Playwright test that's been red on every scheduled run since 2026-05-10. --- .../Onboarding/HatchingStepView.swift | 16 +-- .../OnboardingHatchConfigOverlay.swift | 40 ++++++++ .../OnboardingHatchConfigOverlayTests.swift | 97 +++++++++++++++++++ 3 files changed, 146 insertions(+), 7 deletions(-) create mode 100644 clients/macos/vellum-assistant/Features/Onboarding/OnboardingHatchConfigOverlay.swift create mode 100644 clients/macos/vellum-assistantTests/OnboardingHatchConfigOverlayTests.swift diff --git a/clients/macos/vellum-assistant/Features/Onboarding/HatchingStepView.swift b/clients/macos/vellum-assistant/Features/Onboarding/HatchingStepView.swift index 5e93e2eb635..9a71f9a2ed4 100644 --- a/clients/macos/vellum-assistant/Features/Onboarding/HatchingStepView.swift +++ b/clients/macos/vellum-assistant/Features/Onboarding/HatchingStepView.swift @@ -584,15 +584,17 @@ struct HatchingStepView: View { private static let dockerReadySentinel = "Docker containers are up and running" /// Build the --config key=value pairs for the onboarding selections. - /// Build config overrides to pass as --config flags during hatch. /// - /// Most config default values are determined by the daemon process and may - /// depend on whether the assistant is hatched on the Vellum Platform or not. + /// Delegates to `onboardingHatchConfigOverlay` — a pure top-level helper + /// that encodes the two distinct shapes (managed inference vs. BYOK) + /// based on whether the user signed in and skipped the provider API key + /// entry step. private func buildOnboardingConfigValues() -> [String: String] { - let provider = state.selectedProvider.isEmpty - ? LLMProviderRegistry.defaultProvider.id - : state.selectedProvider - return ["llm.default.provider": provider] + return onboardingHatchConfigOverlay( + skippedAPIKeyEntry: state.skippedAPIKeyEntry, + selectedProvider: state.selectedProvider, + defaultProvider: LLMProviderRegistry.defaultProvider.id + ) } private func startRemoteHatch() { diff --git a/clients/macos/vellum-assistant/Features/Onboarding/OnboardingHatchConfigOverlay.swift b/clients/macos/vellum-assistant/Features/Onboarding/OnboardingHatchConfigOverlay.swift new file mode 100644 index 00000000000..be91fe9ee8e --- /dev/null +++ b/clients/macos/vellum-assistant/Features/Onboarding/OnboardingHatchConfigOverlay.swift @@ -0,0 +1,40 @@ +import Foundation + +/// Build the `--config key=value` overlay the macOS app sends to the CLI +/// (`vellum hatch`) during the onboarding hatch step. +/// +/// The CLI persists this overlay to a temp JSON file and passes the path to +/// the daemon via `VELLUM_DEFAULT_WORKSPACE_CONFIG_PATH`. At daemon startup, +/// `mergeDefaultWorkspaceConfig` merges it into the workspace config on disk +/// and `seedInferenceProfiles` consumes it via `preserveActiveProfile`. +/// +/// Two distinct shapes: +/// +/// 1. **Managed-inference path** (`skippedAPIKeyEntry == true`). +/// The user signed in with their Vellum account and chose a non-cloud +/// hosting mode (Local / AWS / GCP / SSH). They did *not* supply a +/// provider API key — chat traffic will route through the managed +/// Anthropic proxy via the platform-injected `assistant_api_key`. Emit +/// `llm.activeProfile = "balanced"` so the seeder preserves the managed +/// profile (which points at `anthropic-managed`) and skips +/// user-profile/personal-connection materialization. Without this the +/// seeder would otherwise pick `custom-balanced` and the daemon would +/// fail to send messages on a missing `credential/anthropic/api_key`. +/// +/// 2. **BYOK path** (`skippedAPIKeyEntry == false`). +/// The user entered their own provider API key. Emit +/// `llm.default.provider = ` so the seeder materializes +/// `-personal` and the matching `custom-*` profiles. The +/// seeder picks `custom-balanced` as the active profile in this case. +/// Preserves historical "default to anthropic when empty" behavior. +func onboardingHatchConfigOverlay( + skippedAPIKeyEntry: Bool, + selectedProvider: String, + defaultProvider: String +) -> [String: String] { + if skippedAPIKeyEntry { + return ["llm.activeProfile": "balanced"] + } + let provider = selectedProvider.isEmpty ? defaultProvider : selectedProvider + return ["llm.default.provider": provider] +} diff --git a/clients/macos/vellum-assistantTests/OnboardingHatchConfigOverlayTests.swift b/clients/macos/vellum-assistantTests/OnboardingHatchConfigOverlayTests.swift new file mode 100644 index 00000000000..d67113e691b --- /dev/null +++ b/clients/macos/vellum-assistantTests/OnboardingHatchConfigOverlayTests.swift @@ -0,0 +1,97 @@ +import XCTest +@testable import VellumAssistantLib + +final class OnboardingHatchConfigOverlayTests: XCTestCase { + + // MARK: - Managed-inference path (skippedAPIKeyEntry == true) + + func testManagedInferenceEmitsBalancedActiveProfile() { + let overlay = onboardingHatchConfigOverlay( + skippedAPIKeyEntry: true, + selectedProvider: "anthropic", + defaultProvider: "anthropic" + ) + + XCTAssertEqual(overlay, ["llm.activeProfile": "balanced"]) + } + + func testManagedInferenceIgnoresSelectedProvider() { + // Even if the UI default provider differs, managed inference always + // points at the daemon-seeded `balanced` profile (managed Anthropic). + let overlay = onboardingHatchConfigOverlay( + skippedAPIKeyEntry: true, + selectedProvider: "openai", + defaultProvider: "anthropic" + ) + + XCTAssertEqual(overlay, ["llm.activeProfile": "balanced"]) + } + + func testManagedInferenceDoesNotEmitDefaultProvider() { + // Crucial invariant: no `llm.default.provider` so the seeder does NOT + // materialize an `anthropic-personal` connection during hatch seeding. + let overlay = onboardingHatchConfigOverlay( + skippedAPIKeyEntry: true, + selectedProvider: "anthropic", + defaultProvider: "anthropic" + ) + + XCTAssertNil(overlay["llm.default.provider"]) + } + + // MARK: - BYOK path (skippedAPIKeyEntry == false) + + func testBYOKEmitsSelectedProviderAsDefault() { + let overlay = onboardingHatchConfigOverlay( + skippedAPIKeyEntry: false, + selectedProvider: "openai", + defaultProvider: "anthropic" + ) + + XCTAssertEqual(overlay, ["llm.default.provider": "openai"]) + } + + func testBYOKWithAnthropicProvider() { + let overlay = onboardingHatchConfigOverlay( + skippedAPIKeyEntry: false, + selectedProvider: "anthropic", + defaultProvider: "anthropic" + ) + + XCTAssertEqual(overlay, ["llm.default.provider": "anthropic"]) + } + + func testBYOKFallsBackToDefaultProviderWhenSelectedIsEmpty() { + let overlay = onboardingHatchConfigOverlay( + skippedAPIKeyEntry: false, + selectedProvider: "", + defaultProvider: "anthropic" + ) + + XCTAssertEqual(overlay, ["llm.default.provider": "anthropic"]) + } + + func testBYOKUsesInjectedDefaultProviderWhenSelectedIsEmpty() { + // The fallback isn't hardcoded — it tracks whatever the registry says. + let overlay = onboardingHatchConfigOverlay( + skippedAPIKeyEntry: false, + selectedProvider: "", + defaultProvider: "openai" + ) + + XCTAssertEqual(overlay, ["llm.default.provider": "openai"]) + } + + func testBYOKDoesNotEmitActiveProfile() { + // The BYOK path lets the daemon seeder pick the active profile based + // on whether a user connection is materialized — emitting + // `llm.activeProfile` here would short-circuit that resolution. + let overlay = onboardingHatchConfigOverlay( + skippedAPIKeyEntry: false, + selectedProvider: "anthropic", + defaultProvider: "anthropic" + ) + + XCTAssertNil(overlay["llm.activeProfile"]) + } +}