From fed36a6a4e91357b96c8b32987ac72331119d33d Mon Sep 17 00:00:00 2001 From: Alex Nork Date: Thu, 4 Jun 2026 10:09:36 -0400 Subject: [PATCH 1/2] feat(assistant): gate activation rail on web-assigned variation (JARVIS-1102) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the independent LD flag evaluation for the activation flow experiment in the web pre-chat context. The activation rail now gates exclusively on the variation delivered by the platform onboarding recipe endpoint (PR 5 in vellum-assistant-platform): - Treatment users: platform returns recipe with cohort=experiment-activation-flow-2026-06-03 and bootstrapTemplate=BOOTSTRAP-ACTIVATION-RAIL.md. The web client propagates these via the existing recipe path (onboarding.cohort → daemon). - Control/unassigned users: platform returns 204; recipe is null; cohort absent. This eliminates the independent second evaluation that could disagree with the warehouse-logged assignment. No second LD flag eval means the assignment the user received is exactly the experience they get — required for the retention-by-cohort analysis to be valid. Changes: - prechat-context.ts: remove activationFlowEnabled from BuildPreChatContextInput and the override block; remove the input.activationFlowEnabled ? null : recipe conditional in resolveInitialMessage call - pre-chat-flow.tsx: remove experimentActivationFlow20260603 from flag store reads and from buildPreChatContext call - feature-flag-registry.json (meta + apps/web): change scope from 'client' to 'assistant' (daemon resolver picks it up, web client no longer needs it); update description to match string-variation contract - Tests: rewrite activation-rail tests to use recipe input instead of activationFlowEnabled flag; update feature-flag-catalog test for new scope Companion to vellum-assistant-platform PRs: #8166 (flag string variations) #8173 (web eval + log + persist) #8174 (cohort delivery via recipe) Co-Authored-By: Claude Sonnet 4.6 --- .../onboarding-lifecycle-sync.test.tsx | 19 +++++++--- .../onboarding/pages/pre-chat-flow.tsx | 3 -- .../onboarding/prechat-context.test.ts | 35 +++++++++++++------ .../src/domains/onboarding/prechat-context.ts | 9 +---- .../feature-flag-catalog.test.ts | 8 +++-- .../feature-flags/feature-flag-registry.json | 4 +-- meta/feature-flags/feature-flag-registry.json | 4 +-- 7 files changed, 49 insertions(+), 33 deletions(-) diff --git a/apps/web/src/domains/onboarding/onboarding-lifecycle-sync.test.tsx b/apps/web/src/domains/onboarding/onboarding-lifecycle-sync.test.tsx index 666df2e7d4c..363c1666532 100644 --- a/apps/web/src/domains/onboarding/onboarding-lifecycle-sync.test.tsx +++ b/apps/web/src/domains/onboarding/onboarding-lifecycle-sync.test.tsx @@ -66,7 +66,6 @@ type TestOnboardingRecipe = { let onboardingCompleted = false; let prechatOnboardingCondensedFlow = true; -let activationFlowExperiment = false; let selfIntroGreeting = true; let isIOSWeb = false; let isMacOSWeb = false; @@ -200,7 +199,6 @@ mock.module("@/stores/client-feature-flag-store", () => ({ useClientFeatureFlagStore: { use: { prechatOnboardingCondensedFlow: () => prechatOnboardingCondensedFlow, - experimentActivationFlow20260603: () => activationFlowExperiment, selfIntroGreeting: () => selfIntroGreeting, }, }, @@ -349,7 +347,6 @@ beforeEach(() => { checkAssistantImpl = async () => {}; onboardingCompleted = false; prechatOnboardingCondensedFlow = true; - activationFlowExperiment = false; selfIntroGreeting = true; isIOSWeb = false; isMacOSWeb = false; @@ -476,8 +473,19 @@ describe("onboarding lifecycle sync", () => { }); }); - test("activation flow flag selects the activation bootstrap after pre-chat", async () => { - activationFlowExperiment = true; + test("activation flow recipe delivers cohort and bootstrap to the onboarding context", async () => { + // The platform delivers cohort + bootstrapTemplate via the recipe endpoint (PR 5) + // for treatment users. The context builder forwards them unchanged — no independent + // flag evaluation. This is the "no second eval" invariant from JARVIS-1102. + fetchOnboardingRecipeImpl = async () => ({ + cohort: ACTIVATION_FLOW_COHORT, + bootstrapTemplate: ACTIVATION_RAIL_BOOTSTRAP_TEMPLATE, + initialMessage: "", + tasks: [], + tone: "grounded", + skills: [], + skipPrechat: false, + }); render(); @@ -494,6 +502,7 @@ describe("onboarding lifecycle sync", () => { cohort: ACTIVATION_FLOW_COHORT, initialMessage: "Hi, I'm Alice. Nice to meet you.", bootstrapTemplate: ACTIVATION_RAIL_BOOTSTRAP_TEMPLATE, + skills: [], }); }); diff --git a/apps/web/src/domains/onboarding/pages/pre-chat-flow.tsx b/apps/web/src/domains/onboarding/pages/pre-chat-flow.tsx index 84198d2d6ae..c724e5fc633 100644 --- a/apps/web/src/domains/onboarding/pages/pre-chat-flow.tsx +++ b/apps/web/src/domains/onboarding/pages/pre-chat-flow.tsx @@ -105,8 +105,6 @@ export function PreChatFlow() { const showIOSAppStep = isIOSWeb && !readIOSAppDownloaded(); const condensedPrechatFlag = useClientFeatureFlagStore.use.prechatOnboardingCondensedFlow(); - const activationFlowEnabled = - useClientFeatureFlagStore.use.experimentActivationFlow20260603(); const selfIntroGreetingEnabled = useClientFeatureFlagStore.use.selfIntroGreeting(); const preferredFunnelVariant = @@ -330,7 +328,6 @@ export function PreChatFlow() { userName, assistantName, selfIntroGreetingEnabled, - activationFlowEnabled, googleConnected, googleScopes, connectedScopes: args?.connectedScopes, diff --git a/apps/web/src/domains/onboarding/prechat-context.test.ts b/apps/web/src/domains/onboarding/prechat-context.test.ts index 2c470d4af9c..7a81a3cd936 100644 --- a/apps/web/src/domains/onboarding/prechat-context.test.ts +++ b/apps/web/src/domains/onboarding/prechat-context.test.ts @@ -1,6 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { DEFAULT_PRECHAT_INITIAL_MESSAGE } from "@/domains/onboarding/prechat"; import { ACTIVATION_FLOW_COHORT, ACTIVATION_RAIL_BOOTSTRAP_TEMPLATE, @@ -29,32 +28,48 @@ function baseInput( } describe("buildPreChatContext — activation rail", () => { - test("selects the activation bootstrap template when the experiment flag is on", () => { + test("selects the activation cohort and bootstrap template from the recipe", () => { + // Treatment users receive a recipe from the platform (PR 5) whose cohort is + // ACTIVATION_FLOW_COHORT and bootstrap_template is ACTIVATION_RAIL_BOOTSTRAP_TEMPLATE. + // The context builder propagates them unchanged via the recipe path. const context = buildPreChatContext( - baseInput({ activationFlowEnabled: true }), + baseInput({ + recipe: { + cohort: ACTIVATION_FLOW_COHORT, + bootstrapTemplate: ACTIVATION_RAIL_BOOTSTRAP_TEMPLATE, + initialMessage: "", + tasks: [], + tone: "", + skills: [], + skipPrechat: false, + }, + }), ); expect(context.cohort).toBe(ACTIVATION_FLOW_COHORT); expect(context.bootstrapTemplate).toBe(ACTIVATION_RAIL_BOOTSTRAP_TEMPLATE); }); - test("activation template wins over a marketing recipe template", () => { + test("marketing campaign recipe cohort takes precedence over activation when both present", () => { + // The platform delivers the campaign recipe when one matches (PR 5 resolution order: + // campaign > experiment assignment). The context builder forwards the campaign cohort. const context = buildPreChatContext( baseInput({ - activationFlowEnabled: true, recipe: { cohort: "content-automation", bootstrapTemplate: "BOOTSTRAP-CONTENT-AUTOMATION.md", initialMessage: "Campaign hello", + tasks: ["writing"], + tone: "grounded", skills: ["geo-writing"], - } as BuildPreChatContextInput["recipe"], + skipPrechat: true, + }, }), ); - expect(context.cohort).toBe(ACTIVATION_FLOW_COHORT); - expect(context.bootstrapTemplate).toBe(ACTIVATION_RAIL_BOOTSTRAP_TEMPLATE); - expect(context.skills).toEqual(["geo-writing"]); - expect(context.initialMessage).toBe(DEFAULT_PRECHAT_INITIAL_MESSAGE); + expect(context.cohort).toBe("content-automation"); + expect(context.bootstrapTemplate).toBe("BOOTSTRAP-CONTENT-AUTOMATION.md"); + expect(context.initialMessage).toBe("Campaign hello"); }); }); diff --git a/apps/web/src/domains/onboarding/prechat-context.ts b/apps/web/src/domains/onboarding/prechat-context.ts index 7ee9be56984..885951d2b9f 100644 --- a/apps/web/src/domains/onboarding/prechat-context.ts +++ b/apps/web/src/domains/onboarding/prechat-context.ts @@ -41,8 +41,6 @@ export interface BuildPreChatContextInput { userName: string; assistantName: string; selfIntroGreetingEnabled: boolean; - /** Selects the activation rail bootstrap template for experiment users. */ - activationFlowEnabled?: boolean; /** Persisted Google connection state from a step the user already passed. */ googleConnected: boolean; googleScopes: string[]; @@ -100,11 +98,6 @@ export function buildPreChatContext( context.skills = recipe.skills; } - if (input.activationFlowEnabled) { - context.cohort = ACTIVATION_FLOW_COHORT; - context.bootstrapTemplate = ACTIVATION_RAIL_BOOTSTRAP_TEMPLATE; - } - const trimmedUser = input.userName.trim(); if (trimmedUser) context.userName = trimmedUser; const trimmedAssistant = input.assistantName.trim(); @@ -134,7 +127,7 @@ export function buildPreChatContext( context.initialMessage = resolveInitialMessage( context, - input.activationFlowEnabled ? null : recipe, + recipe, input.selfIntroGreetingEnabled, ); return context; diff --git a/apps/web/src/lib/feature-flags/feature-flag-catalog.test.ts b/apps/web/src/lib/feature-flags/feature-flag-catalog.test.ts index 056e77dfbce..6626bb9529b 100644 --- a/apps/web/src/lib/feature-flags/feature-flag-catalog.test.ts +++ b/apps/web/src/lib/feature-flags/feature-flag-catalog.test.ts @@ -11,10 +11,12 @@ describe("feature flag catalog", () => { expect(ASSISTANT_FLAG_DEFAULTS.selfIntroGreeting).toBe(false); }); - test("exposes the activation flow experiment as a client flag", () => { - expect(CLIENT_FLAG_DEFAULTS.experimentActivationFlow20260603).toBe(false); + test("exposes the activation flow experiment as an assistant flag (not client)", () => { + // The activation flag is now scope:'assistant' — the daemon evaluates it, not the web client. + // The web client gates the rail via the recipe cohort delivered by the platform (JARVIS-1102). expect( - "experimentActivationFlow20260603" in ASSISTANT_FLAG_DEFAULTS, + "experimentActivationFlow20260603" in CLIENT_FLAG_DEFAULTS, ).toBe(false); + expect(ASSISTANT_FLAG_DEFAULTS.experimentActivationFlow20260603).toBe(false); }); }); diff --git a/apps/web/src/lib/feature-flags/feature-flag-registry.json b/apps/web/src/lib/feature-flags/feature-flag-registry.json index ca52860991d..71c15b5a8dc 100644 --- a/apps/web/src/lib/feature-flags/feature-flag-registry.json +++ b/apps/web/src/lib/feature-flags/feature-flag-registry.json @@ -35,10 +35,10 @@ }, { "id": "experiment-activation-flow-2026-06-03", - "scope": "client", + "scope": "assistant", "key": "experiment-activation-flow-2026-06-03", "label": "Activation Flow Experiment 2026-06-03", - "description": "Route allowlisted users to the activation-rail bootstrap template after pre-chat. Off by default and targeted through LaunchDarkly.", + "description": "String-variation experiment (control / treatment) that routes users into the model-driven activation rail. The daemon treats variation='treatment' as on; anything else (including default 'control') is off. Assignment is vid-keyed by the web app and flows to the daemon via the onboarding cohort channel — do not evaluate this flag independently in the daemon.", "defaultEnabled": false }, { diff --git a/meta/feature-flags/feature-flag-registry.json b/meta/feature-flags/feature-flag-registry.json index ca52860991d..71c15b5a8dc 100644 --- a/meta/feature-flags/feature-flag-registry.json +++ b/meta/feature-flags/feature-flag-registry.json @@ -35,10 +35,10 @@ }, { "id": "experiment-activation-flow-2026-06-03", - "scope": "client", + "scope": "assistant", "key": "experiment-activation-flow-2026-06-03", "label": "Activation Flow Experiment 2026-06-03", - "description": "Route allowlisted users to the activation-rail bootstrap template after pre-chat. Off by default and targeted through LaunchDarkly.", + "description": "String-variation experiment (control / treatment) that routes users into the model-driven activation rail. The daemon treats variation='treatment' as on; anything else (including default 'control') is off. Assignment is vid-keyed by the web app and flows to the daemon via the onboarding cohort channel — do not evaluate this flag independently in the daemon.", "defaultEnabled": false }, { From bdacfef5abe54f15b20d42e9b24f7f5d25e97906 Mon Sep 17 00:00:00 2001 From: Alex Nork Date: Thu, 4 Jun 2026 10:35:38 -0400 Subject: [PATCH 2/2] fix(web): drop activation experiment from the toggleable flag registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The activation rail is gated by the onboarding recipe cohort (control/treatment) assigned vid-keyed on the platform, not by a boolean feature flag — nothing in apps/web reads experiment-activation-flow-2026-06-03 as a flag anymore. Declaring it in the registry put it in ASSISTANT_FLAG_DEFAULTS, so Settings → Feature Flags rendered it as a manual toggle that could PATCH an {enabled} override and diverge from the vid-keyed assignment. Remove the registry entry entirely (canonical meta/ + synced apps/web copy) and assert it is absent from both flag stores. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../lib/feature-flags/feature-flag-catalog.test.ts | 13 +++++++++---- .../lib/feature-flags/feature-flag-registry.json | 8 -------- meta/feature-flags/feature-flag-registry.json | 8 -------- 3 files changed, 9 insertions(+), 20 deletions(-) diff --git a/apps/web/src/lib/feature-flags/feature-flag-catalog.test.ts b/apps/web/src/lib/feature-flags/feature-flag-catalog.test.ts index 6626bb9529b..a209f47ca5f 100644 --- a/apps/web/src/lib/feature-flags/feature-flag-catalog.test.ts +++ b/apps/web/src/lib/feature-flags/feature-flag-catalog.test.ts @@ -11,12 +11,17 @@ describe("feature flag catalog", () => { expect(ASSISTANT_FLAG_DEFAULTS.selfIntroGreeting).toBe(false); }); - test("exposes the activation flow experiment as an assistant flag (not client)", () => { - // The activation flag is now scope:'assistant' — the daemon evaluates it, not the web client. - // The web client gates the rail via the recipe cohort delivered by the platform (JARVIS-1102). + test("does not declare the activation flow experiment as a toggleable flag", () => { + // The activation rail is gated by the onboarding recipe cohort (control / + // treatment) assigned vid-keyed on the platform — not by a boolean feature + // flag. Declaring it in the registry would render it as a manual toggle in + // Settings → Feature Flags, letting a hand-flipped override diverge from the + // vid-keyed assignment, so it must not appear in either flag store (JARVIS-1102). expect( "experimentActivationFlow20260603" in CLIENT_FLAG_DEFAULTS, ).toBe(false); - expect(ASSISTANT_FLAG_DEFAULTS.experimentActivationFlow20260603).toBe(false); + expect( + "experimentActivationFlow20260603" in ASSISTANT_FLAG_DEFAULTS, + ).toBe(false); }); }); diff --git a/apps/web/src/lib/feature-flags/feature-flag-registry.json b/apps/web/src/lib/feature-flags/feature-flag-registry.json index 71c15b5a8dc..63b923ea088 100644 --- a/apps/web/src/lib/feature-flags/feature-flag-registry.json +++ b/apps/web/src/lib/feature-flags/feature-flag-registry.json @@ -33,14 +33,6 @@ "description": "Enable the condensed pre-chat onboarding flow for a standard LaunchDarkly percentage rollout.", "defaultEnabled": false }, - { - "id": "experiment-activation-flow-2026-06-03", - "scope": "assistant", - "key": "experiment-activation-flow-2026-06-03", - "label": "Activation Flow Experiment 2026-06-03", - "description": "String-variation experiment (control / treatment) that routes users into the model-driven activation rail. The daemon treats variation='treatment' as on; anything else (including default 'control') is off. Assignment is vid-keyed by the web app and flows to the daemon via the onboarding cohort channel — do not evaluate this flag independently in the daemon.", - "defaultEnabled": false - }, { "id": "local-docker-enabled", "scope": "client", diff --git a/meta/feature-flags/feature-flag-registry.json b/meta/feature-flags/feature-flag-registry.json index 71c15b5a8dc..63b923ea088 100644 --- a/meta/feature-flags/feature-flag-registry.json +++ b/meta/feature-flags/feature-flag-registry.json @@ -33,14 +33,6 @@ "description": "Enable the condensed pre-chat onboarding flow for a standard LaunchDarkly percentage rollout.", "defaultEnabled": false }, - { - "id": "experiment-activation-flow-2026-06-03", - "scope": "assistant", - "key": "experiment-activation-flow-2026-06-03", - "label": "Activation Flow Experiment 2026-06-03", - "description": "String-variation experiment (control / treatment) that routes users into the model-driven activation rail. The daemon treats variation='treatment' as on; anything else (including default 'control') is off. Assignment is vid-keyed by the web app and flows to the daemon via the onboarding cohort channel — do not evaluate this flag independently in the daemon.", - "defaultEnabled": false - }, { "id": "local-docker-enabled", "scope": "client",