From 080bbf423c0036c6763fc0c48713c57c62839d97 Mon Sep 17 00:00:00 2001 From: smakosh Date: Tue, 21 Jul 2026 22:25:30 +0200 Subject: [PATCH 1/9] feat(devpass): add model census survey + reward Yearly DevPass-only "Model Census" asking which coding models are worth their cost, based on verified usage: - model_survey_response table (per user+model+year, value/quality/ speed scores, recommend, use case, usage snapshot) - /model-survey eligibility + submit routes; the org's first entry each year grants a free current-tier Reset Pass ($0 dev_plan_reset_pass_reward transaction, audit-logged) - /public/model-survey/results with a 5-response anonymity threshold - usage-triggered census reminder dialog in the dashboard shell with a 14-day snooze cookie, survey form at /dashboard/survey - public results registry at /data/[year] with Dataset JSON-LD - seeded census data, API spec tests, and Playwright e2e for apps/code (pnpm --filter code test:e2e) Claude-Session: https://claude.ai/code/session_019bW8g9tRS3vXnUjezwoJZN --- apps/api/src/index.ts | 2 + apps/api/src/routes/index.ts | 2 + apps/api/src/routes/model-survey.spec.ts | 413 + apps/api/src/routes/model-survey.ts | 442 + apps/api/src/routes/organization.ts | 1 + apps/api/src/routes/public-model-survey.ts | 152 + apps/api/src/utils/devpass-filter.ts | 2 + apps/code/.gitignore | 2 + apps/code/e2e/census.pw.ts | 91 + apps/code/package.json | 4 +- apps/code/playwright.config.ts | 17 + .../code/src/app/dashboard/DashboardShell.tsx | 13 +- .../components/SurveyReminderDialog.tsx | 162 + .../src/app/dashboard/survey/SurveyClient.tsx | 525 + apps/code/src/app/dashboard/survey/page.tsx | 63 + apps/code/src/app/data/[year]/page.tsx | 340 + apps/code/src/app/data/page.tsx | 8 + apps/code/src/app/sitemap.ts | 6 + apps/code/src/lib/cookies.ts | 21 + apps/code/src/lib/model-survey.ts | 33 + apps/code/tsconfig.json | 1 + .../migrations/1784664094_quick_mercury.sql | 27 + .../migrations/meta/1784664094_snapshot.json | 27796 ++++++++++++++++ packages/db/migrations/meta/_journal.json | 7 + packages/db/src/relations.ts | 8 + packages/db/src/schema.ts | 80 + packages/db/src/seed.ts | 220 + pnpm-lock.yaml | 325 +- 28 files changed, 30633 insertions(+), 130 deletions(-) create mode 100644 apps/api/src/routes/model-survey.spec.ts create mode 100644 apps/api/src/routes/model-survey.ts create mode 100644 apps/api/src/routes/public-model-survey.ts create mode 100644 apps/code/e2e/census.pw.ts create mode 100644 apps/code/playwright.config.ts create mode 100644 apps/code/src/app/dashboard/components/SurveyReminderDialog.tsx create mode 100644 apps/code/src/app/dashboard/survey/SurveyClient.tsx create mode 100644 apps/code/src/app/dashboard/survey/page.tsx create mode 100644 apps/code/src/app/data/[year]/page.tsx create mode 100644 apps/code/src/app/data/page.tsx create mode 100644 apps/code/src/lib/cookies.ts create mode 100644 apps/code/src/lib/model-survey.ts create mode 100644 packages/db/migrations/1784664094_quick_mercury.sql create mode 100644 packages/db/migrations/meta/1784664094_snapshot.json diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index f1d3b16486..ba8be9930a 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -36,6 +36,7 @@ import { publicContact } from "./routes/public-contact.js"; import { publicDiscounts } from "./routes/public-discounts.js"; import { publicLeaderboard } from "./routes/public-leaderboard.js"; import { publicModelRatings } from "./routes/public-model-ratings.js"; +import { publicModelSurvey } from "./routes/public-model-survey.js"; import { publicNewsletter } from "./routes/public-newsletter.js"; import { publicProfile } from "./routes/public-profile.js"; import { publicProvidersStats } from "./routes/public-providers-stats.js"; @@ -313,6 +314,7 @@ app.route("/public/profile", publicProfile); app.route("/public/leaderboard", publicLeaderboard); app.route("/public/providers/stats", publicProvidersStats); app.route("/public/model-ratings", publicModelRatings); +app.route("/public/model-survey", publicModelSurvey); app.doc("/json", config); diff --git a/apps/api/src/routes/index.ts b/apps/api/src/routes/index.ts index a1ac057532..2a5ad507e4 100644 --- a/apps/api/src/routes/index.ts +++ b/apps/api/src/routes/index.ts @@ -19,6 +19,7 @@ import keysProvider from "./keys-provider.js"; import { logs } from "./logs.js"; import masterKeys from "./master-keys.js"; import { modelRatings } from "./model-ratings.js"; +import { modelSurvey } from "./model-survey.js"; import organization from "./organization.js"; import { payments } from "./payments.js"; import playground from "./playground.js"; @@ -78,6 +79,7 @@ routes.route("/dev-plan-cancellation-feedback", devPlanCancellationFeedback); routes.route("/chat-plans", chatPlans); routes.route("/audit-logs", auditLogs); routes.route("/model-ratings", modelRatings); +routes.route("/model-survey", modelSurvey); routes.route("/guardrails", guardrails); routes.route("/routing-config", routingConfig); routes.route("/custom-models", customModels); diff --git a/apps/api/src/routes/model-survey.spec.ts b/apps/api/src/routes/model-survey.spec.ts new file mode 100644 index 0000000000..d100b49fd9 --- /dev/null +++ b/apps/api/src/routes/model-survey.spec.ts @@ -0,0 +1,413 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { app } from "@/index.js"; +import { createTestUser, deleteAll } from "@/testing.js"; + +import { db, desc, eq, tables } from "@llmgateway/db"; + +const ORG_ID = "test-survey-org"; +const PROJECT_ID = "test-survey-project"; +const YEAR = new Date().getUTCFullYear(); +const HOUR_MS = 60 * 60 * 1000; + +async function insertOrg( + overrides: Partial = {}, +) { + await db.insert(tables.organization).values({ + id: ORG_ID, + name: "Personal Org", + billingEmail: "admin@example.com", + kind: "devpass", + devPlan: "pro", + ...overrides, + }); + await db.insert(tables.userOrganization).values({ + userId: "test-user-id", + organizationId: ORG_ID, + role: "owner", + }); + await db.insert(tables.project).values({ + id: PROJECT_ID, + name: "Default Project", + organizationId: ORG_ID, + mode: "credits", + }); +} + +let statsCounter = 0; + +async function seedModelStats( + usedModel: string, + usedProvider: string, + requestCount: number, + { hoursAgo = 2 }: { hoursAgo?: number } = {}, +) { + const offsetMs = hoursAgo * HOUR_MS; + const hourTimestamp = new Date(Date.now() - offsetMs); + hourTimestamp.setMinutes(0, 0, 0); + await db.insert(tables.projectHourlyModelStats).values({ + id: `survey-stats-${statsCounter++}`, + projectId: PROJECT_ID, + hourTimestamp, + usedModel, + usedProvider, + requestCount, + }); +} + +function eligibilityRequest(token?: string) { + return app.request("/model-survey/eligibility", { + headers: token ? { Cookie: token } : {}, + }); +} + +function submitRequest(body: Record, token?: string) { + return app.request("/model-survey", { + method: "POST", + headers: { + ...(token ? { Cookie: token } : {}), + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }); +} + +const validBody = { + modelId: "survey-coder-large", + valueScore: 5, + qualityScore: 4, + speedScore: 3, + wouldRecommend: true, + primaryUseCase: "agentic_coding", +}; + +async function getOrg() { + const org = await db.query.organization.findFirst({ + where: { id: { eq: ORG_ID } }, + }); + if (!org) { + throw new Error("test org disappeared"); + } + return org; +} + +describe("model survey eligibility", () => { + let token: string; + + beforeEach(async () => { + token = await createTestUser(); + }); + + afterEach(async () => { + await deleteAll(); + }); + + it("requires authentication", async () => { + const res = await eligibilityRequest(); + expect(res.status).toBe(401); + }); + + it("is not eligible without a devpass org", async () => { + const res = await eligibilityRequest(token); + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.eligible).toBe(false); + expect(json.devPlan).toBeNull(); + expect(json.topModels).toEqual([]); + }); + + it("is not eligible without an active plan", async () => { + await insertOrg({ devPlan: "none" }); + await seedModelStats("survey-coder-large", "openai", 200); + + const res = await eligibilityRequest(token); + const json = await res.json(); + expect(json.eligible).toBe(false); + expect(json.devPlan).toBe("none"); + expect(json.topModels).toEqual([]); + }); + + it("returns qualifying models most-used first", async () => { + await insertOrg(); + await seedModelStats("survey-coder-large", "openai", 80); + await seedModelStats("anthropic/survey-coder-small:free", "anthropic", 120); + // Below the request threshold: excluded. + await seedModelStats("survey-coder-rare", "openai", 10); + // Old usage outside the window: excluded. + await seedModelStats("survey-coder-stale", "openai", 500, { + hoursAgo: 45 * 24, + }); + + const res = await eligibilityRequest(token); + const json = await res.json(); + expect(json.year).toBe(YEAR); + expect(json.eligible).toBe(true); + expect(json.rewardAvailable).toBe(true); + expect(json.devPlan).toBe("pro"); + expect(json.topModels).toEqual([ + { + modelId: "survey-coder-small", + provider: "anthropic", + requestCount: 120, + alreadySubmitted: false, + }, + { + modelId: "survey-coder-large", + provider: "openai", + requestCount: 80, + alreadySubmitted: false, + }, + ]); + }); +}); + +describe("model survey submit", () => { + let token: string; + + beforeEach(async () => { + token = await createTestUser(); + }); + + afterEach(async () => { + await deleteAll(); + }); + + it("requires authentication", async () => { + const res = await submitRequest(validBody); + expect(res.status).toBe(401); + }); + + it("rejects users without a devpass org", async () => { + const res = await submitRequest(validBody, token); + expect(res.status).toBe(403); + }); + + it("rejects models below the usage threshold", async () => { + await insertOrg(); + await seedModelStats("survey-coder-large", "openai", 49); + + const res = await submitRequest(validBody, token); + expect(res.status).toBe(403); + }); + + it("records the response and grants one reset pass per org per year", async () => { + await insertOrg(); + await seedModelStats("survey-coder-large", "openai", 80); + await seedModelStats("survey-coder-small", "anthropic", 120); + + const res = await submitRequest( + { ...validBody, comment: "Great value for agentic work." }, + token, + ); + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.rewardGranted).toBe(true); + expect(json.rewardTier).toBe("pro"); + expect(json.response.modelId).toBe("survey-coder-large"); + + const org = await getOrg(); + expect(org.devPlanResetPassesPro).toBe(1); + expect(org.devPlanResetPassesLite).toBe(0); + + const stored = await db.query.modelSurveyResponse.findFirst({ + where: { + userId: { eq: "test-user-id" }, + modelId: { eq: "survey-coder-large" }, + }, + }); + expect(stored).toMatchObject({ + year: YEAR, + organizationId: ORG_ID, + valueScore: 5, + qualityScore: 4, + speedScore: 3, + wouldRecommend: true, + primaryUseCase: "agentic_coding", + comment: "Great value for agentic work.", + requestCount: 80, + devPlanTier: "pro", + rewardTier: "pro", + }); + + const [transaction] = await db + .select() + .from(tables.transaction) + .where(eq(tables.transaction.organizationId, ORG_ID)) + .orderBy(desc(tables.transaction.createdAt)); + expect(transaction).toMatchObject({ + type: "dev_plan_reset_pass_reward", + amount: "0", + status: "completed", + }); + + // A second model the same year still records, but never double-rewards. + const second = await submitRequest( + { ...validBody, modelId: "survey-coder-small" }, + token, + ); + expect(second.status).toBe(200); + const secondJson = await second.json(); + expect(secondJson.rewardGranted).toBe(false); + expect(secondJson.rewardTier).toBeNull(); + expect((await getOrg()).devPlanResetPassesPro).toBe(1); + + // Eligibility now reflects both submissions and the spent reward. + const eligibility = await (await eligibilityRequest(token)).json(); + expect(eligibility.eligible).toBe(false); + expect(eligibility.rewardAvailable).toBe(false); + expect( + eligibility.topModels.every( + (m: { alreadySubmitted: boolean }) => m.alreadySubmitted, + ), + ).toBe(true); + }); + + it("rejects a duplicate response for the same model and year", async () => { + await insertOrg(); + await seedModelStats("survey-coder-large", "openai", 80); + + expect((await submitRequest(validBody, token)).status).toBe(200); + const res = await submitRequest(validBody, token); + expect(res.status).toBe(409); + }); + + it("grants the pass on the org's current tier", async () => { + await insertOrg({ devPlan: "lite" }); + await seedModelStats("survey-coder-large", "openai", 80); + + const res = await submitRequest(validBody, token); + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.rewardTier).toBe("lite"); + + const org = await getOrg(); + expect(org.devPlanResetPassesLite).toBe(1); + expect(org.devPlanResetPassesPro).toBe(0); + }); +}); + +describe("public model survey results", () => { + beforeEach(async () => { + await createTestUser(); + await insertOrg(); + }); + + afterEach(async () => { + await deleteAll(); + }); + + async function insertResponses( + modelId: string, + responses: { + valueScore: number; + qualityScore?: number; + speedScore?: number; + wouldRecommend?: boolean; + primaryUseCase?: string; + }[], + ) { + for (const [index, response] of responses.entries()) { + const userId = `survey-user-${modelId}-${index}`; + await db.insert(tables.user).values({ + id: userId, + name: `Survey User ${index}`, + email: `${userId}@example.com`, + }); + await db.insert(tables.modelSurveyResponse).values({ + year: YEAR, + userId, + organizationId: ORG_ID, + modelId, + valueScore: response.valueScore, + qualityScore: response.qualityScore ?? 4, + speedScore: response.speedScore ?? 3, + wouldRecommend: response.wouldRecommend ?? true, + primaryUseCase: (response.primaryUseCase ?? + "agentic_coding") as "agentic_coding", + requestCount: 100, + devPlanTier: "pro", + }); + } + } + + it("returns empty results when nothing was submitted", async () => { + const res = await app.request(`/public/model-survey/results?year=${YEAR}`); + expect(res.status).toBe(200); + const json = await res.json(); + expect(json).toMatchObject({ + year: YEAR, + totalResponses: 0, + totalRespondents: 0, + totalModelsRated: 0, + models: [], + }); + }); + + it("hides models below the anonymity threshold but counts them in totals", async () => { + await insertResponses("survey-coder-small", [ + { valueScore: 5 }, + { valueScore: 4 }, + { valueScore: 5 }, + { valueScore: 3 }, + ]); + + const res = await app.request(`/public/model-survey/results?year=${YEAR}`); + const json = await res.json(); + expect(json.totalResponses).toBe(4); + expect(json.totalModelsRated).toBe(1); + expect(json.models).toEqual([]); + }); + + it("aggregates published models sorted by value score", async () => { + await insertResponses("survey-coder-small", [ + { valueScore: 3, wouldRecommend: false }, + { valueScore: 3 }, + { valueScore: 4 }, + { valueScore: 3, primaryUseCase: "debugging" }, + { valueScore: 3, primaryUseCase: "debugging" }, + ]); + await insertResponses("survey-coder-large", [ + { valueScore: 5, qualityScore: 5 }, + { valueScore: 4 }, + { valueScore: 5 }, + { valueScore: 5 }, + { valueScore: 4, wouldRecommend: false }, + ]); + + const res = await app.request(`/public/model-survey/results?year=${YEAR}`); + const json = await res.json(); + expect(json.totalResponses).toBe(10); + expect(json.totalRespondents).toBe(10); + expect(json.models).toHaveLength(2); + + const [first, second] = json.models; + expect(first.modelId).toBe("survey-coder-large"); + expect(first.responseCount).toBe(5); + expect(first.avgValueScore).toBe(4.6); + expect(first.recommendPercent).toBe(80); + expect(second.modelId).toBe("survey-coder-small"); + expect(second.avgValueScore).toBe(3.2); + expect(second.useCases).toEqual([ + { useCase: "agentic_coding", count: 3 }, + { useCase: "debugging", count: 2 }, + ]); + }); + + it("scopes results to the requested year", async () => { + await insertResponses("survey-coder-large", [ + { valueScore: 5 }, + { valueScore: 5 }, + { valueScore: 5 }, + { valueScore: 5 }, + { valueScore: 5 }, + ]); + + const res = await app.request( + `/public/model-survey/results?year=${YEAR + 1}`, + ); + const json = await res.json(); + expect(json.year).toBe(YEAR + 1); + expect(json.totalResponses).toBe(0); + expect(json.models).toEqual([]); + }); +}); diff --git a/apps/api/src/routes/model-survey.ts b/apps/api/src/routes/model-survey.ts new file mode 100644 index 0000000000..c649ae9511 --- /dev/null +++ b/apps/api/src/routes/model-survey.ts @@ -0,0 +1,442 @@ +import { createRoute, OpenAPIHono } from "@hono/zod-openapi"; +import { HTTPException } from "hono/http-exception"; +import { z } from "zod"; + +import { posthog } from "@/posthog.js"; + +import { logAuditEvent } from "@llmgateway/audit"; +import { + and, + db, + eq, + gte, + inArray, + isNotNull, + sql, + tables, +} from "@llmgateway/db"; + +import type { ServerTypes } from "@/vars.js"; + +export const modelSurvey = new OpenAPIHono(); + +// The yearly DevPass model survey only asks about models a member genuinely +// uses: their DevPass org must have made at least this many requests on a +// model within the qualifying window before it can be rated. +export const MINIMUM_SURVEY_REQUESTS = 50; +export const SURVEY_WINDOW_DAYS = 30; +// How many qualifying models eligibility returns, most-used first. +const TOP_MODELS_LIMIT = 5; + +const useCaseEnum = z.enum([ + "agentic_coding", + "code_completion", + "code_review", + "debugging", + "writing_tests", + "docs_and_explanations", + "other", +]); + +const tierEnum = z.enum(["lite", "pro", "max"]); + +function surveyYear(): number { + return new Date().getUTCFullYear(); +} + +async function findUserDevpassOrg(userId: string) { + const userOrgs = await db.query.userOrganization.findMany({ + where: { userId: { eq: userId } }, + with: { organization: true }, + }); + return userOrgs.find((uo) => uo.organization?.kind === "devpass") + ?.organization; +} + +// Model names in the stats rollups may be stored as "provider/model" and can +// carry a ":variant" suffix; group on the bare model id like model-ratings does. +const normalizedModelExpr = sql`CASE WHEN ${tables.projectHourlyModelStats.usedModel} LIKE '%/%' + THEN SPLIT_PART(SPLIT_PART(${tables.projectHourlyModelStats.usedModel}, '/', 2), ':', 1) + ELSE SPLIT_PART(${tables.projectHourlyModelStats.usedModel}, ':', 1) +END`; + +async function getOrgProjectIds(organizationId: string): Promise { + const projects = await db.query.project.findMany({ + where: { organizationId: { eq: organizationId } }, + columns: { id: true }, + }); + return projects.map((p) => p.id); +} + +const SURVEY_WINDOW_MS = SURVEY_WINDOW_DAYS * 24 * 60 * 60 * 1000; + +function windowStart(): Date { + return new Date(Date.now() - SURVEY_WINDOW_MS); +} + +async function getQualifyingModels( + organizationId: string, +): Promise<{ modelId: string; provider: string; requestCount: number }[]> { + const projectIds = await getOrgProjectIds(organizationId); + if (projectIds.length === 0) { + return []; + } + + const rows = await db + .select({ + modelId: normalizedModelExpr.as("model_id"), + provider: sql`MAX(${tables.projectHourlyModelStats.usedProvider})`, + requestCount: sql`SUM(${tables.projectHourlyModelStats.requestCount})`, + }) + .from(tables.projectHourlyModelStats) + .where( + and( + inArray(tables.projectHourlyModelStats.projectId, projectIds), + gte(tables.projectHourlyModelStats.hourTimestamp, windowStart()), + ), + ) + .groupBy(normalizedModelExpr) + .having( + sql`SUM(${tables.projectHourlyModelStats.requestCount}) >= ${MINIMUM_SURVEY_REQUESTS}`, + ) + .orderBy(sql`SUM(${tables.projectHourlyModelStats.requestCount}) DESC`) + .limit(TOP_MODELS_LIMIT); + + return rows.map((row) => ({ + modelId: row.modelId, + provider: row.provider ?? "", + requestCount: Number(row.requestCount), + })); +} + +async function getModelWindowRequestCount( + organizationId: string, + modelId: string, +): Promise { + const projectIds = await getOrgProjectIds(organizationId); + if (projectIds.length === 0) { + return 0; + } + + const [result] = await db + .select({ + value: sql`COALESCE(SUM(${tables.projectHourlyModelStats.requestCount}), 0)`, + }) + .from(tables.projectHourlyModelStats) + .where( + and( + inArray(tables.projectHourlyModelStats.projectId, projectIds), + gte(tables.projectHourlyModelStats.hourTimestamp, windowStart()), + sql`${normalizedModelExpr} = ${modelId}`, + ), + ); + + return Number(result?.value ?? 0); +} + +async function findRewardedResponse( + dbOrTx: Pick, + organizationId: string, + year: number, +) { + const [row] = await dbOrTx + .select({ id: tables.modelSurveyResponse.id }) + .from(tables.modelSurveyResponse) + .where( + and( + eq(tables.modelSurveyResponse.organizationId, organizationId), + eq(tables.modelSurveyResponse.year, year), + isNotNull(tables.modelSurveyResponse.rewardTier), + ), + ) + .limit(1); + return row; +} + +const topModelSchema = z.object({ + modelId: z.string(), + provider: z.string(), + requestCount: z.number().int(), + alreadySubmitted: z.boolean(), +}); + +const eligibilitySchema = z.object({ + year: z.number().int(), + eligible: z.boolean(), + devPlan: z.enum(["none", "lite", "pro", "max"]).nullable(), + rewardAvailable: z.boolean(), + minimumRequests: z.number().int(), + windowDays: z.number().int(), + topModels: z.array(topModelSchema), +}); + +const getEligibility = createRoute({ + method: "get", + path: "/eligibility", + request: {}, + responses: { + 200: { + content: { + "application/json": { schema: eligibilitySchema }, + }, + description: + "Whether the user can take this year's DevPass model survey, and for which models.", + }, + }, +}); + +modelSurvey.openapi(getEligibility, async (c) => { + const user = c.get("user"); + if (!user) { + throw new HTTPException(401, { message: "Unauthorized" }); + } + + const year = surveyYear(); + const org = await findUserDevpassOrg(user.id); + + if (!org || org.devPlan === "none") { + return c.json({ + year, + eligible: false, + devPlan: org?.devPlan ?? null, + rewardAvailable: false, + minimumRequests: MINIMUM_SURVEY_REQUESTS, + windowDays: SURVEY_WINDOW_DAYS, + topModels: [], + }); + } + + const [models, ownResponses, rewardedRow] = await Promise.all([ + getQualifyingModels(org.id), + db.query.modelSurveyResponse.findMany({ + where: { userId: { eq: user.id }, year: { eq: year } }, + columns: { modelId: true }, + }), + findRewardedResponse(db, org.id, year), + ]); + + const submitted = new Set(ownResponses.map((r) => r.modelId)); + const topModels = models.map((m) => ({ + ...m, + alreadySubmitted: submitted.has(m.modelId), + })); + + return c.json({ + year, + eligible: topModels.some((m) => !m.alreadySubmitted), + devPlan: org.devPlan, + rewardAvailable: !rewardedRow, + minimumRequests: MINIMUM_SURVEY_REQUESTS, + windowDays: SURVEY_WINDOW_DAYS, + topModels, + }); +}); + +const responseSchema = z.object({ + modelId: z.string(), + year: z.number().int(), + valueScore: z.number().int().min(1).max(5), + qualityScore: z.number().int().min(1).max(5), + speedScore: z.number().int().min(1).max(5), + wouldRecommend: z.boolean(), + primaryUseCase: useCaseEnum, + comment: z.string().nullable(), + createdAt: z.string().datetime(), +}); + +const submitSurvey = createRoute({ + method: "post", + path: "/", + request: { + body: { + content: { + "application/json": { + schema: z.object({ + modelId: z.string().min(1), + valueScore: z.number().int().min(1).max(5), + qualityScore: z.number().int().min(1).max(5), + speedScore: z.number().int().min(1).max(5), + wouldRecommend: z.boolean(), + primaryUseCase: useCaseEnum, + comment: z.string().trim().max(2000).optional(), + }), + }, + }, + }, + }, + responses: { + 200: { + content: { + "application/json": { + schema: z.object({ + success: z.boolean(), + rewardGranted: z.boolean(), + rewardTier: tierEnum.nullable(), + response: responseSchema, + }), + }, + }, + description: + "Survey response recorded. The org's first response of the year grants a free Reset Pass.", + }, + }, +}); + +modelSurvey.openapi(submitSurvey, async (c) => { + const user = c.get("user"); + if (!user) { + throw new HTTPException(401, { message: "Unauthorized" }); + } + + const { + modelId, + valueScore, + qualityScore, + speedScore, + wouldRecommend, + primaryUseCase, + comment, + } = c.req.valid("json"); + + const org = await findUserDevpassOrg(user.id); + if (!org || org.devPlan === "none") { + throw new HTTPException(403, { + message: + "The model survey is only open to DevPass members with an active plan.", + }); + } + const tier = org.devPlan; + + const requestCount = await getModelWindowRequestCount(org.id, modelId); + if (requestCount < MINIMUM_SURVEY_REQUESTS) { + throw new HTTPException(403, { + message: `You need at least ${MINIMUM_SURVEY_REQUESTS} requests on this model in the last ${SURVEY_WINDOW_DAYS} days to rate it.`, + }); + } + + const year = surveyYear(); + + let row: typeof tables.modelSurveyResponse.$inferSelect; + let rewardTier: "lite" | "pro" | "max" | null = null; + try { + ({ row, rewardTier } = await db.transaction(async (tx) => { + // Lock the org row so concurrent submissions can't both claim the + // one-per-year reward. + await tx + .select({ id: tables.organization.id }) + .from(tables.organization) + .where(eq(tables.organization.id, org.id)) + .for("update"); + + const rewardedRow = await findRewardedResponse(tx, org.id, year); + const grantTier = rewardedRow ? null : tier; + + const [inserted] = await tx + .insert(tables.modelSurveyResponse) + .values({ + year, + userId: user.id, + organizationId: org.id, + modelId, + valueScore, + qualityScore, + speedScore, + wouldRecommend, + primaryUseCase, + comment: comment || null, + requestCount, + devPlanTier: tier, + rewardTier: grantTier, + }) + .returning(); + + if (grantTier) { + const purchasedIncrement = + grantTier === "lite" + ? { + devPlanResetPassesLite: sql`${tables.organization.devPlanResetPassesLite} + 1`, + } + : grantTier === "pro" + ? { + devPlanResetPassesPro: sql`${tables.organization.devPlanResetPassesPro} + 1`, + } + : { + devPlanResetPassesMax: sql`${tables.organization.devPlanResetPassesMax} + 1`, + }; + + await tx + .update(tables.organization) + .set(purchasedIncrement) + .where(eq(tables.organization.id, org.id)); + + await tx.insert(tables.transaction).values({ + organizationId: org.id, + type: "dev_plan_reset_pass_reward", + amount: "0", + currency: "USD", + status: "completed", + description: `DevPass Reset Pass (${grantTier.toUpperCase()}) — ${year} model survey reward`, + }); + } + + return { row: inserted, rewardTier: grantTier }; + })); + } catch (error) { + if ( + error instanceof Error && + "cause" in error && + (error.cause as { code?: string } | undefined)?.code === "23505" + ) { + throw new HTTPException(409, { + message: `You already rated this model in the ${year} survey.`, + }); + } + throw error; + } + + if (rewardTier) { + await logAuditEvent({ + organizationId: org.id, + userId: user.id, + action: "dev_plan.reset_pass_reward", + resourceType: "dev_plan", + resourceId: row.id, + metadata: { tier: rewardTier, year, modelId }, + }); + } + + posthog.capture({ + distinctId: user.id, + event: "model_survey_submitted", + groups: { organization: org.id }, + properties: { + year, + modelId, + devPlan: tier, + rewardGranted: rewardTier !== null, + valueScore, + qualityScore, + speedScore, + wouldRecommend, + primaryUseCase, + hasComment: Boolean(comment), + }, + }); + + return c.json({ + success: true, + rewardGranted: rewardTier !== null, + rewardTier, + response: { + modelId: row.modelId, + year: row.year, + valueScore: row.valueScore, + qualityScore: row.qualityScore, + speedScore: row.speedScore, + wouldRecommend: row.wouldRecommend, + primaryUseCase: row.primaryUseCase, + comment: row.comment, + createdAt: row.createdAt.toISOString(), + }, + }); +}); diff --git a/apps/api/src/routes/organization.ts b/apps/api/src/routes/organization.ts index 9bec0c488f..3c8dbcf574 100644 --- a/apps/api/src/routes/organization.ts +++ b/apps/api/src/routes/organization.ts @@ -218,6 +218,7 @@ const transactionSchema = z.object({ "dev_plan_end", "dev_plan_renewal", "dev_plan_reset_pass", + "dev_plan_reset_pass_reward", "chat_plan_start", "chat_plan_upgrade", "chat_plan_downgrade", diff --git a/apps/api/src/routes/public-model-survey.ts b/apps/api/src/routes/public-model-survey.ts new file mode 100644 index 0000000000..a7278a57f2 --- /dev/null +++ b/apps/api/src/routes/public-model-survey.ts @@ -0,0 +1,152 @@ +import { createRoute, OpenAPIHono } from "@hono/zod-openapi"; +import { z } from "zod"; + +import { count, countDistinct, db, eq, sql, tables } from "@llmgateway/db"; + +import type { ServerTypes } from "@/vars.js"; + +const publicModelSurvey = new OpenAPIHono(); + +// Models need at least this many responses before their scores are published, +// so no individual respondent's answers can be inferred from the report. +export const MIN_PUBLIC_RESPONSES = 5; + +const useCaseEnum = z.enum([ + "agentic_coding", + "code_completion", + "code_review", + "debugging", + "writing_tests", + "docs_and_explanations", + "other", +]); + +const modelResultSchema = z.object({ + modelId: z.string(), + responseCount: z.number().int(), + avgValueScore: z.number(), + avgQualityScore: z.number(), + avgSpeedScore: z.number(), + recommendPercent: z.number().int(), + useCases: z.array( + z.object({ useCase: useCaseEnum, count: z.number().int() }), + ), +}); + +const resultsSchema = z.object({ + year: z.number().int(), + minResponses: z.number().int(), + totalResponses: z.number().int(), + totalRespondents: z.number().int(), + totalModelsRated: z.number().int(), + models: z.array(modelResultSchema), +}); + +const getResults = createRoute({ + method: "get", + path: "/results", + request: { + query: z.object({ + year: z.coerce.number().int().min(2026).max(2100).optional(), + }), + }, + responses: { + 200: { + content: { + "application/json": { schema: resultsSchema }, + }, + description: + "Aggregated, anonymized results of the yearly DevPass model survey. Only models with enough responses are listed.", + }, + }, +}); + +publicModelSurvey.openapi(getResults, async (c) => { + const { year: queryYear } = c.req.valid("query"); + const year = queryYear ?? new Date().getUTCFullYear(); + + const round2 = (value: string | number | null) => + Math.round(Number(value ?? 0) * 100) / 100; + + const [totals] = await db + .select({ + totalResponses: count(), + totalRespondents: countDistinct(tables.modelSurveyResponse.userId), + totalModelsRated: countDistinct(tables.modelSurveyResponse.modelId), + }) + .from(tables.modelSurveyResponse) + .where(eq(tables.modelSurveyResponse.year, year)); + + const modelRows = await db + .select({ + modelId: tables.modelSurveyResponse.modelId, + responseCount: count(), + avgValueScore: sql`AVG(${tables.modelSurveyResponse.valueScore})`, + avgQualityScore: sql`AVG(${tables.modelSurveyResponse.qualityScore})`, + avgSpeedScore: sql`AVG(${tables.modelSurveyResponse.speedScore})`, + recommendCount: sql`SUM(CASE WHEN ${tables.modelSurveyResponse.wouldRecommend} THEN 1 ELSE 0 END)`, + }) + .from(tables.modelSurveyResponse) + .where(eq(tables.modelSurveyResponse.year, year)) + .groupBy(tables.modelSurveyResponse.modelId) + .having(sql`COUNT(*) >= ${MIN_PUBLIC_RESPONSES}`) + .orderBy( + sql`AVG(${tables.modelSurveyResponse.valueScore}) DESC`, + sql`COUNT(*) DESC`, + ); + + const publishedModelIds = modelRows.map((row) => row.modelId); + const useCaseRows = publishedModelIds.length + ? await db + .select({ + modelId: tables.modelSurveyResponse.modelId, + useCase: tables.modelSurveyResponse.primaryUseCase, + count: count(), + }) + .from(tables.modelSurveyResponse) + .where(eq(tables.modelSurveyResponse.year, year)) + .groupBy( + tables.modelSurveyResponse.modelId, + tables.modelSurveyResponse.primaryUseCase, + ) + : []; + + const useCasesByModel = new Map< + string, + { useCase: (typeof useCaseRows)[number]["useCase"]; count: number }[] + >(); + for (const row of useCaseRows) { + if (!publishedModelIds.includes(row.modelId)) { + continue; + } + const list = useCasesByModel.get(row.modelId) ?? []; + list.push({ useCase: row.useCase, count: Number(row.count) }); + useCasesByModel.set(row.modelId, list); + } + + return c.json({ + year, + minResponses: MIN_PUBLIC_RESPONSES, + totalResponses: Number(totals?.totalResponses ?? 0), + totalRespondents: Number(totals?.totalRespondents ?? 0), + totalModelsRated: Number(totals?.totalModelsRated ?? 0), + models: modelRows.map((row) => { + const responseCount = Number(row.responseCount); + return { + modelId: row.modelId, + responseCount, + avgValueScore: round2(row.avgValueScore), + avgQualityScore: round2(row.avgQualityScore), + avgSpeedScore: round2(row.avgSpeedScore), + recommendPercent: responseCount + ? Math.round((Number(row.recommendCount) / responseCount) * 100) + : 0, + useCases: (useCasesByModel.get(row.modelId) ?? []).sort( + (a, b) => b.count - a.count, + ), + }; + }), + }); +}); + +export { publicModelSurvey }; diff --git a/apps/api/src/utils/devpass-filter.ts b/apps/api/src/utils/devpass-filter.ts index 56426a47cd..52c18c8331 100644 --- a/apps/api/src/utils/devpass-filter.ts +++ b/apps/api/src/utils/devpass-filter.ts @@ -17,6 +17,8 @@ export const planExcludedTypes = [ // creditAmount), reported with the other dev plan rows — not part of the // credits economy. "dev_plan_reset_pass", + // Free survey-reward passes carry amount "0" and are never revenue. + "dev_plan_reset_pass_reward", "subscription_start", "subscription_cancel", "subscription_end", diff --git a/apps/code/.gitignore b/apps/code/.gitignore index d5d8885436..63674b33e5 100644 --- a/apps/code/.gitignore +++ b/apps/code/.gitignore @@ -12,6 +12,8 @@ # testing /coverage +/test-results +/playwright-report # next.js /.next/ diff --git a/apps/code/e2e/census.pw.ts b/apps/code/e2e/census.pw.ts new file mode 100644 index 0000000000..1d33c53ba7 --- /dev/null +++ b/apps/code/e2e/census.pw.ts @@ -0,0 +1,91 @@ +import { expect, test } from "@playwright/test"; + +import type { Page } from "@playwright/test"; + +// Requires a freshly seeded local stack (`pnpm setup` + dev servers): the +// seeded test admin owns a DevPass Pro workspace with 30 days of per-model +// usage, which makes the census reminder eligible. The "files an entry" +// test submits a survey response, so reseed before re-running the suite. + +async function login(page: Page) { + await page.goto("/login"); + await page.fill('input[type="email"]', "admin@example.com"); + await page.fill('input[type="password"]', "admin@example.com"); + await page.click('button[type="submit"]'); + await page.waitForURL("**/dashboard**", { timeout: 45_000 }); +} + +test("census reminder can be snoozed for later", async ({ page }) => { + await login(page); + + const dialog = page.getByTestId("census-dialog"); + await expect(dialog).toBeVisible({ timeout: 20_000 }); + await expect(dialog).toContainText("Model Census"); + + await page.getByTestId("census-dialog-dismiss").click(); + await expect(dialog).not.toBeVisible(); + + const cookies = await page.context().cookies(); + expect( + cookies.some((cookie) => cookie.name.startsWith("devpass_census_snooze_")), + ).toBe(true); + + // Snoozed: the reminder stays quiet on the next visit. + await page.reload(); + await page.waitForTimeout(3000); + await expect(dialog).not.toBeVisible(); +}); + +test("census entry files a response and stamps a reset pass", async ({ + page, +}) => { + await login(page); + + const dialog = page.getByTestId("census-dialog"); + await expect(dialog).toBeVisible({ timeout: 20_000 }); + await page.getByTestId("census-dialog-cta").click(); + + await page.waitForURL("**/dashboard/survey**"); + + await page.getByTestId("census-valueScore-5").click(); + await page.getByTestId("census-qualityScore-4").click(); + await page.getByTestId("census-speedScore-4").click(); + await page.getByTestId("census-recommend-yes").click(); + await page.getByTestId("census-use-case").click(); + await page + .getByRole("option", { name: "Agentic coding — it drives the editor" }) + .click(); + await page + .getByPlaceholder( + "Where it shines, where it face-plants, what you'd tell a teammate.", + ) + .fill("Filed by the census e2e test."); + + await page.getByTestId("census-submit").click(); + + await expect(page.getByTestId("census-success")).toBeVisible({ + timeout: 20_000, + }); + await expect(page.getByTestId("census-reward")).toContainText("Reset Pass"); + + // The reminder never re-prompts after a submission: the top model is + // filed and the year's reward is spent. + await page.goto("/dashboard"); + await page.waitForTimeout(3000); + await expect(page.getByTestId("census-dialog")).not.toBeVisible(); +}); + +test("public census page lists methodology and CTA", async ({ page }) => { + const year = new Date().getUTCFullYear(); + await page.goto(`/data/${year}`); + + await expect( + page.getByRole("heading", { + name: "Which coding models are actually worth the money?", + }), + ).toBeVisible(); + await expect(page.getByText("The rules of the registry")).toBeVisible(); + await expect( + page.getByRole("link", { name: /File your entry/ }), + ).toBeVisible(); +}); diff --git a/apps/code/package.json b/apps/code/package.json index a6a2387f12..d4a63edc89 100644 --- a/apps/code/package.json +++ b/apps/code/package.json @@ -13,7 +13,8 @@ "format": "eslint --fix . && prettier --write .", "generate": "openapi-typescript ../api/openapi.json -o ./src/lib/api/v1.d.ts", "lint": "eslint . && prettier --check .", - "start": "next start" + "start": "next start", + "test:e2e": "playwright test" }, "dependencies": { "@better-auth/passkey": "1.6.23", @@ -66,6 +67,7 @@ }, "devDependencies": { "@eslint/eslintrc": "^3.3.1", + "@playwright/test": "1.61.1", "@tailwindcss/postcss": "^4.1.16", "@tanstack/react-query-devtools": "5.90.2", "@types/node": "^26.1.1", diff --git a/apps/code/playwright.config.ts b/apps/code/playwright.config.ts new file mode 100644 index 0000000000..668b36b94c --- /dev/null +++ b/apps/code/playwright.config.ts @@ -0,0 +1,17 @@ +import { defineConfig } from "@playwright/test"; + +// Browser e2e tests for the DevPass dashboard. These run against a locally +// running stack (API on :4002, code app on :3004) with a freshly seeded +// database (`pnpm setup`), and are not part of the root vitest suites: +// run them with `pnpm --filter code test:e2e`. +export default defineConfig({ + testDir: "./e2e", + testMatch: "**/*.pw.ts", + workers: 1, + timeout: 90_000, + use: { + baseURL: process.env.PW_BASE_URL ?? "http://localhost:3004", + trace: "retain-on-failure", + video: "retain-on-failure", + }, +}); diff --git a/apps/code/src/app/dashboard/DashboardShell.tsx b/apps/code/src/app/dashboard/DashboardShell.tsx index ad92319209..e88d55a29a 100644 --- a/apps/code/src/app/dashboard/DashboardShell.tsx +++ b/apps/code/src/app/dashboard/DashboardShell.tsx @@ -9,6 +9,7 @@ import { Loader2, LogOut, Settings, + Stamp, UserRound, } from "lucide-react"; import dynamic from "next/dynamic"; @@ -18,6 +19,7 @@ import { usePostHog } from "posthog-js/react"; import { useEffect, useRef, useState } from "react"; import { toast } from "sonner"; +import SurveyReminderDialog from "@/app/dashboard/components/SurveyReminderDialog"; import { EmailVerificationBanner } from "@/components/EmailVerificationBanner"; import { AlertDialog, @@ -67,7 +69,14 @@ const resourceNavItems: Array<{ label: string; href: Route; icon: typeof BarChart3; -}> = [{ label: "Coding models", href: "/coding-models" as Route, icon: Code }]; +}> = [ + { label: "Coding models", href: "/coding-models" as Route, icon: Code }, + { + label: "Model census", + href: `/data/${new Date().getUTCFullYear()}` as Route, + icon: Stamp, + }, +]; type SetupActivationStatus = | "loading_stripe" @@ -460,6 +469,8 @@ export default function DashboardShell({ + + {/* Header */}
diff --git a/apps/code/src/app/dashboard/components/SurveyReminderDialog.tsx b/apps/code/src/app/dashboard/components/SurveyReminderDialog.tsx new file mode 100644 index 0000000000..811380e881 --- /dev/null +++ b/apps/code/src/app/dashboard/components/SurveyReminderDialog.tsx @@ -0,0 +1,162 @@ +"use client"; + +import { Stamp } from "lucide-react"; +import { motion } from "motion/react"; +import { useRouter } from "next/navigation"; +import { usePostHog } from "posthog-js/react"; +import { useEffect, useRef, useState } from "react"; + +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { useAppConfig } from "@/lib/config"; +import { getCookie, setCookie } from "@/lib/cookies"; +import { useApi } from "@/lib/fetch-client"; + +// "Maybe later" keeps the census quiet for two weeks; a submission silences it +// for the rest of the year server-side (eligibility flips to false). +const SNOOZE_DAYS = 14; +const snoozeCookie = (year: number) => `devpass_census_snooze_${year}`; + +export default function SurveyReminderDialog({ active }: { active: boolean }) { + const api = useApi(); + const router = useRouter(); + const posthog = usePostHog(); + const { posthogKey } = useAppConfig(); + const [open, setOpen] = useState(false); + const shownTracked = useRef(false); + + const { data } = api.useQuery( + "get", + "/model-survey/eligibility", + {}, + { enabled: active, staleTime: 5 * 60 * 1000 }, + ); + + const topModel = data?.topModels.find((m) => !m.alreadySubmitted); + + useEffect(() => { + if (!data?.eligible || !data.rewardAvailable || !topModel) { + return; + } + if (getCookie(snoozeCookie(data.year))) { + return; + } + const timer = setTimeout(() => { + setOpen(true); + if (!shownTracked.current && posthogKey) { + shownTracked.current = true; + posthog.capture("model_survey_prompt_shown", { + year: data.year, + model: topModel.modelId, + request_count: topModel.requestCount, + }); + } + }, 900); + return () => clearTimeout(timer); + }, [data, topModel, posthog, posthogKey]); + + if (!data || !topModel) { + return null; + } + + const dismiss = () => { + setOpen(false); + setCookie(snoozeCookie(data.year), "1", SNOOZE_DAYS); + if (posthogKey) { + posthog.capture("model_survey_prompt_dismissed", { + year: data.year, + model: topModel.modelId, + }); + } + }; + + const accept = () => { + if (posthogKey) { + posthog.capture("model_survey_prompt_accepted", { + year: data.year, + model: topModel.modelId, + }); + } + setOpen(false); + router.push( + `/dashboard/survey?model=${encodeURIComponent(topModel.modelId)}`, + ); + }; + + return ( + { + if (!next) { + dismiss(); + } + }} + > + + {/* Inked census stamp, slammed into the corner on open */} + + + Census + + + + {data.year} + + + + +
+ DevPass Model Census · {data.year} +
+ + You and {topModel.modelId} have been logging serious miles + + + {topModel.requestCount.toLocaleString()} requests in the last{" "} + {data.windowDays} days. Got a minute to rate it for the {data.year}{" "} + census? A free Reset Pass is stamped into your passport the moment + you do. + + + + + + + + {/* Machine-readable zone, purely decorative */} + + + + ); +} diff --git a/apps/code/src/app/dashboard/survey/SurveyClient.tsx b/apps/code/src/app/dashboard/survey/SurveyClient.tsx new file mode 100644 index 0000000000..c4871e81c8 --- /dev/null +++ b/apps/code/src/app/dashboard/survey/SurveyClient.tsx @@ -0,0 +1,525 @@ +"use client"; + +import { zodResolver } from "@hookform/resolvers/zod"; +import { ArrowUpRight, Stamp } from "lucide-react"; +import { AnimatePresence, motion } from "motion/react"; +import Link from "next/link"; +import { useSearchParams } from "next/navigation"; +import { usePostHog } from "posthog-js/react"; +import { useState } from "react"; +import { useForm } from "react-hook-form"; +import { toast } from "sonner"; +import { z } from "zod/v3"; + +import { Button } from "@/components/ui/button"; +import { + Form, + FormControl, + FormField, + FormItem, + FormLabel, + FormMessage, +} from "@/components/ui/form"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Textarea } from "@/components/ui/textarea"; +import { useAppConfig } from "@/lib/config"; +import { useApi } from "@/lib/fetch-client"; +import { cn } from "@/lib/utils"; + +import type { paths } from "@/lib/api/v1"; + +export type SurveyEligibility = + paths["/model-survey/eligibility"]["get"]["responses"]["200"]["content"]["application/json"]; + +const USE_CASES = [ + { value: "agentic_coding", label: "Agentic coding — it drives the editor" }, + { value: "code_completion", label: "Autocomplete & inline suggestions" }, + { value: "code_review", label: "Code review" }, + { value: "debugging", label: "Debugging & fixing" }, + { value: "writing_tests", label: "Writing tests" }, + { value: "docs_and_explanations", label: "Docs & explanations" }, + { value: "other", label: "Something else" }, +] as const; + +type UseCase = (typeof USE_CASES)[number]["value"]; + +const scoreSchema = z + .number({ required_error: "Score it 1–5." }) + .int() + .min(1, "Score it 1–5.") + .max(5); + +const formSchema = z.object({ + modelId: z.string().min(1, "Pick a model."), + valueScore: scoreSchema, + qualityScore: scoreSchema, + speedScore: scoreSchema, + wouldRecommend: z.boolean({ + required_error: "Approve or deny — customs insists.", + }), + primaryUseCase: z.enum( + USE_CASES.map((u) => u.value) as [UseCase, ...UseCase[]], + { errorMap: () => ({ message: "Pick your main use." }) }, + ), + comment: z.string().max(2000, "Keep it under 2000 characters."), +}); + +type FormValues = z.infer; + +const SCORE_QUESTIONS: { + name: "valueScore" | "qualityScore" | "speedScore"; + label: string; + description: string; + low: string; + high: string; +}[] = [ + { + name: "valueScore", + label: "Worth what it costs?", + description: "Given the credits it burns, the output earns its keep.", + low: "Overpriced", + high: "A steal", + }, + { + name: "qualityScore", + label: "Output quality", + description: "How often its code survives your review.", + low: "Rewrites needed", + high: "Ships as-is", + }, + { + name: "speedScore", + label: "Speed", + description: "From prompt to usable answer.", + low: "Coffee break", + high: "Blink", + }, +]; + +function ScoreRow({ + value, + onChange, + name, + low, + high, +}: { + value: number | undefined; + onChange: (next: number) => void; + name: string; + low: string; + high: string; +}) { + return ( +
+
+ {[1, 2, 3, 4, 5].map((score) => { + const selected = value === score; + return ( + + ); + })} +
+
+ {low} + {high} +
+
+ ); +} + +export default function SurveyClient({ + initialEligibility, +}: { + initialEligibility: SurveyEligibility; +}) { + const api = useApi(); + const posthog = usePostHog(); + const { posthogKey } = useAppConfig(); + const searchParams = useSearchParams(); + const [eligibility, setEligibility] = useState(initialEligibility); + const [result, setResult] = useState<{ + modelId: string; + rewardGranted: boolean; + rewardTier: "lite" | "pro" | "max" | null; + } | null>(null); + + const requestedModel = searchParams.get("model"); + const openModels = eligibility.topModels.filter((m) => !m.alreadySubmitted); + const defaultModel = + openModels.find((m) => m.modelId === requestedModel)?.modelId ?? + openModels[0]?.modelId ?? + ""; + + const submitMutation = api.useMutation("post", "/model-survey"); + + const form = useForm({ + resolver: zodResolver(formSchema), + defaultValues: { + modelId: defaultModel, + comment: "", + }, + }); + + const selectedModel = form.watch("modelId"); + const comment = form.watch("comment"); + const serial = `CS-${eligibility.year}-${(selectedModel || "X").slice(0, 6).toUpperCase()}`; + + async function onSubmit(values: FormValues) { + const trimmedComment = values.comment.trim(); + try { + const response = await submitMutation.mutateAsync({ + body: { + modelId: values.modelId, + valueScore: values.valueScore, + qualityScore: values.qualityScore, + speedScore: values.speedScore, + wouldRecommend: values.wouldRecommend, + primaryUseCase: values.primaryUseCase, + comment: trimmedComment || undefined, + }, + }); + if (posthogKey) { + posthog.capture("model_survey_completed", { + year: eligibility.year, + model: values.modelId, + reward_granted: response.rewardGranted, + }); + } + setEligibility((prev) => ({ + ...prev, + rewardAvailable: prev.rewardAvailable && !response.rewardGranted, + topModels: prev.topModels.map((m) => + m.modelId === values.modelId ? { ...m, alreadySubmitted: true } : m, + ), + })); + setResult({ + modelId: values.modelId, + rewardGranted: response.rewardGranted, + rewardTier: response.rewardTier, + }); + } catch (error) { + const message = + error && typeof error === "object" && "message" in error + ? String((error as { message?: unknown }).message) + : null; + toast.error(message || "Couldn't file your census entry. Try again."); + } + } + + function rateAnother() { + const next = eligibility.topModels.find((m) => !m.alreadySubmitted); + form.reset({ + modelId: next?.modelId ?? "", + comment: "", + }); + setResult(null); + } + + if (result) { + const remaining = eligibility.topModels.filter((m) => !m.alreadySubmitted); + return ( +
+ +
Census filed
+
+ {result.modelId} · {eligibility.year} +
+ + + {result.rewardGranted ? ( + + + A free{" "} + + {result.rewardTier?.toUpperCase()} Reset Pass + {" "} + was stamped into your passport. Redeem it any week you fly too close + to the premium limit. + + ) : ( +

+ Thanks — your verdict is in the {eligibility.year} registry. Your + passport already holds this year's census pass. +

+ )} + +
+ + {remaining.length > 0 && ( + + )} + +
+
+ ); + } + + return ( +
+ +
+
+
+ DevPass Model Census · {eligibility.year} +
+
+ No. {serial} +
+
+

+ One minute of honesty about the models you fly with +

+

+ Which coding models are actually worth their price? Your answers are + anonymous, aggregated into the public census, and your first entry + of the year stamps a free Reset Pass into your passport. + {!eligibility.rewardAvailable && + " (This year's pass is already in your passport.)"} +

+
+ + ( + + + Model under review + + +
+ {eligibility.topModels.map((model) => { + const selected = field.value === model.modelId; + return ( + + ); + })} +
+ + + + )} + /> + + {SCORE_QUESTIONS.map((question) => ( + ( + +
+ + {question.label} + +

+ {question.description} +

+
+ + + + + + )} + /> + ))} + + ( + + + Would you recommend {selectedModel || "it"} to another + developer? + + +
+ + +
+ + + + )} + /> + + ( + + + What do you mainly use it for? + + + + + )} + /> + + ( + + + Field notes (optional) + + +