From cdf407e346e6ba700337bdd71defd8d9fe5c27b5 Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sat, 22 Aug 2026 15:20:01 -0400 Subject: [PATCH 01/29] chore(telemetry-worker): add telemetry ingest worker Cloudflare Worker for bestax.io/api/t writing CLI usage events to the bestax_telemetry Analytics Engine dataset. Strict allowlist validation: only known enum values pass, unknown keys reject the payload, IP and User-Agent are never read. Zero runtime deps; tests via node:test. --- telemetry-worker/README.md | 77 ++++ telemetry-worker/package.json | 16 + .../src/__tests__/handler.test.ts | 262 ++++++++++++++ .../src/__tests__/validate.test.ts | 331 ++++++++++++++++++ telemetry-worker/src/index.ts | 58 +++ telemetry-worker/src/schema.ts | 196 +++++++++++ telemetry-worker/src/types.ts | 22 ++ telemetry-worker/src/validate.ts | 177 ++++++++++ telemetry-worker/tsconfig.json | 17 + telemetry-worker/wrangler.toml | 8 + 10 files changed, 1164 insertions(+) create mode 100644 telemetry-worker/README.md create mode 100644 telemetry-worker/package.json create mode 100644 telemetry-worker/src/__tests__/handler.test.ts create mode 100644 telemetry-worker/src/__tests__/validate.test.ts create mode 100644 telemetry-worker/src/index.ts create mode 100644 telemetry-worker/src/schema.ts create mode 100644 telemetry-worker/src/types.ts create mode 100644 telemetry-worker/src/validate.ts create mode 100644 telemetry-worker/tsconfig.json create mode 100644 telemetry-worker/wrangler.toml diff --git a/telemetry-worker/README.md b/telemetry-worker/README.md new file mode 100644 index 000000000..61cd027c6 --- /dev/null +++ b/telemetry-worker/README.md @@ -0,0 +1,77 @@ +# telemetry-worker + +Cloudflare Worker that ingests anonymous telemetry from the `create-bestax` +and `bestax-migrate` CLIs at `POST bestax.io/api/t` and writes it to the +Cloudflare Analytics Engine dataset `bestax_telemetry`. Fire-and-forget: valid +payloads get an empty `204`, everything else gets a bare error status, and a +missing binding in local dev still answers `204`. + +## Deploy + +```sh +npx wrangler deploy +``` + +Run by Alex — the package is not in the pnpm workspace yet, and Wrangler +bundles `src/index.ts` directly (no build step). The worker is bound to the +zone route `bestax.io/api/t`. If that route ever conflicts with whatever +serves the rest of `bestax.io`, the fallback is a custom domain: attach +`t.bestax.io` to the worker and point the CLIs at `https://t.bestax.io/api/t` +(the handler only checks the `/api/t` pathname, so no code change needed). + +## Analytics Engine mapping + +`blobs[i]` surfaces in SQL as `blob{i+1}`; sampling means counts are +`SUM(_sample_interval)`, not `COUNT(*)`. + +| Column | run event (`scaffold` / `migrate`) | `migrate_todo` point | +| --------- | ----------------------------------- | -------------------- | +| `index1` | tool | `bestax-migrate` | +| `blob1` | event | `migrate_todo` | +| `blob2` | toolVersion | toolVersion | +| `blob3` | platform | platform | +| `blob4` | nodeMajor | nodeMajor | +| `blob5` | template / source | source | +| `blob6` | bulmaFlavor / cssMode | rule | +| `blob7` | iconLibrary / dry (`1`/`0`) | — | +| `blob8` | skills (`1`/`0`) / deps (`1`/`0`) | — | +| `blob9` | packageManager / changedBucket | — | +| `double1` | — / changedCount (capped at 10 000) | count | + +## Example queries + +Scaffolds by template and flavor, last 30 days: + +```sql +SELECT blob5 AS template, blob6 AS flavor, SUM(_sample_interval) AS runs +FROM bestax_telemetry WHERE index1='create-bestax' AND blob1='scaffold' + AND timestamp > NOW() - INTERVAL '30' DAY GROUP BY template, flavor ORDER BY runs DESC; +``` + +Which migration rules leave the most TODOs, last 90 days: + +```sql +SELECT blob6 AS rule, SUM(double1*_sample_interval) AS todos +FROM bestax_telemetry WHERE index1='bestax-migrate' AND blob1='migrate_todo' + AND timestamp > NOW() - INTERVAL '90' DAY GROUP BY rule ORDER BY todos DESC; +``` + +Package manager × skills opt-in, last 30 days: + +```sql +SELECT blob9 AS pm, blob8 AS skills, SUM(_sample_interval) AS runs +FROM bestax_telemetry WHERE index1='create-bestax' AND blob1='scaffold' + AND timestamp > NOW() - INTERVAL '30' DAY GROUP BY pm, skills ORDER BY runs DESC; +``` + +## Privacy + +- Allowlist validation is the backstop: every key and value in a payload must + match a closed allowlist (`src/schema.ts`, `src/validate.ts`) or the whole + payload is rejected with a bare `400`. Unknown values are dropped, never + stored — fail closed. New templates, flavors, icon libraries, sources, or + CSS modes must be added to `src/schema.ts` **first** or their runs are + silently discarded. +- No IP address or User-Agent is ever read or stored; `request.cf` is never + touched; request bodies are never logged. +- No CORS headers: the endpoint is for the CLIs, not browsers. diff --git a/telemetry-worker/package.json b/telemetry-worker/package.json new file mode 100644 index 000000000..632ae3290 --- /dev/null +++ b/telemetry-worker/package.json @@ -0,0 +1,16 @@ +{ + "name": "telemetry-worker", + "version": "0.0.0", + "private": true, + "type": "module", + "scripts": { + "test": "node --test 'src/__tests__/*.test.ts'", + "typecheck": "tsc -p tsconfig.json --noEmit", + "deploy": "wrangler deploy", + "dev": "wrangler dev" + }, + "devDependencies": { + "typescript": "^5", + "wrangler": "^4" + } +} diff --git a/telemetry-worker/src/__tests__/handler.test.ts b/telemetry-worker/src/__tests__/handler.test.ts new file mode 100644 index 000000000..35b0f513b --- /dev/null +++ b/telemetry-worker/src/__tests__/handler.test.ts @@ -0,0 +1,262 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import worker from '../index.ts'; +import type { AnalyticsEngineDataPoint, Env } from '../types.ts'; + +const ENDPOINT = 'https://bestax.io/api/t'; + +// Node's undici Request/Response are structurally compatible with the +// webworker-lib types the handler is written against. +type FetchArgs = Parameters; +const call = (request: unknown, env: Env = {}) => + worker.fetch(request as FetchArgs[0], env as FetchArgs[1]); + +function makeEnv() { + const points: AnalyticsEngineDataPoint[] = []; + const env: Env = { + TELEMETRY: { + writeDataPoint: point => { + points.push(point); + }, + }, + }; + return { env, points }; +} + +function post(body: unknown, contentType = 'application/json') { + return new Request(ENDPOINT, { + method: 'POST', + headers: { 'content-type': contentType }, + body: typeof body === 'string' ? body : JSON.stringify(body), + }); +} + +const createPayload = () => ({ + v: 1, + tool: 'create-bestax', + event: 'scaffold', + toolVersion: '1.4.2', + nodeMajor: 22, + platform: 'darwin', + props: { + template: 'vite-ts', + bulmaFlavor: 'complete', + iconLibrary: 'none', + skills: true, + packageManager: 'pnpm', + }, +}); + +const migratePayload = () => ({ + v: 1, + tool: 'bestax-migrate', + event: 'migrate', + toolVersion: '0.3.0', + nodeMajor: 20, + platform: 'linux', + props: { + source: 'react-bulma-components', + cssMode: 'bestax', + dry: false, + deps: true, + changedBucket: '10-49', + changedCount: 23, + }, + todosByRule: [ + { rule: 'unmapped-prop', count: 3 }, + { rule: 'icon.size', count: 1 }, + ], +}); + +describe('handler: request gate', () => { + it('405 for non-POST', async () => { + const { env, points } = makeEnv(); + const res = await call(new Request(ENDPOINT, { method: 'GET' }), env); + assert.equal(res.status, 405); + assert.equal(points.length, 0); + }); + + it('404 for a different pathname', async () => { + const { env, points } = makeEnv(); + const res = await call( + new Request('https://bestax.io/api/other', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: '{}', + }), + env + ); + assert.equal(res.status, 404); + assert.equal(points.length, 0); + }); + + it('415 for non-JSON content-type', async () => { + const { env, points } = makeEnv(); + const res = await call(post(createPayload(), 'text/plain'), env); + assert.equal(res.status, 415); + assert.equal(points.length, 0); + }); + + it('accepts application/json with a charset parameter', async () => { + const { env } = makeEnv(); + const res = await call( + post(createPayload(), 'application/json; charset=utf-8'), + env + ); + assert.equal(res.status, 204); + }); + + it('413 for an oversized declared content-length, body unread', async () => { + // undici strips content-length from real Requests, so stub the shape. + let read = false; + const request = { + method: 'POST', + url: ENDPOINT, + headers: new Headers({ + 'content-type': 'application/json', + 'content-length': '999999', + }), + text: async () => { + read = true; + return '{}'; + }, + }; + const { env, points } = makeEnv(); + const res = await call(request, env); + assert.equal(res.status, 413); + assert.equal(read, false); + assert.equal(points.length, 0); + }); + + it('413 for a body over 8192 bytes', async () => { + const { env, points } = makeEnv(); + const res = await call(post(`{"pad":"${'a'.repeat(9000)}"}`), env); + assert.equal(res.status, 413); + assert.equal(points.length, 0); + }); + + it('400 for malformed JSON', async () => { + const { env, points } = makeEnv(); + const res = await call(post('{"v":1,'), env); + assert.equal(res.status, 400); + assert.equal(points.length, 0); + }); + + it('400 with empty body for a payload failing validation', async () => { + const { env, points } = makeEnv(); + const res = await call(post({ v: 1, tool: 'create-bestax' }), env); + assert.equal(res.status, 400); + assert.equal(await res.text(), ''); + assert.equal(points.length, 0); + }); +}); + +describe('handler: accepted payloads', () => { + it('create-bestax scaffold writes one exact point', async () => { + const { env, points } = makeEnv(); + const res = await call(post(createPayload()), env); + assert.equal(res.status, 204); + assert.equal(await res.text(), ''); + assert.equal(points.length, 1); + assert.deepEqual(points[0], { + indexes: ['create-bestax'], + blobs: [ + 'scaffold', + '1.4.2', + 'darwin', + '22', + 'vite-ts', + 'complete', + 'none', + '1', + 'pnpm', + ], + doubles: [], + }); + }); + + it('bestax-migrate writes the run point plus one point per todo rule', async () => { + const { env, points } = makeEnv(); + const res = await call(post(migratePayload()), env); + assert.equal(res.status, 204); + assert.equal(points.length, 3); + assert.deepEqual(points[0], { + indexes: ['bestax-migrate'], + blobs: [ + 'migrate', + '0.3.0', + 'linux', + '20', + 'react-bulma-components', + 'bestax', + '0', + '1', + '10-49', + ], + doubles: [23], + }); + assert.deepEqual(points[1], { + indexes: ['bestax-migrate'], + blobs: [ + 'migrate_todo', + '0.3.0', + 'linux', + '20', + 'react-bulma-components', + 'unmapped-prop', + ], + doubles: [3], + }); + assert.deepEqual(points[2], { + indexes: ['bestax-migrate'], + blobs: [ + 'migrate_todo', + '0.3.0', + 'linux', + '20', + 'react-bulma-components', + 'icon.size', + ], + doubles: [1], + }); + }); + + it('caps the changedCount double at 10000', async () => { + const { env, points } = makeEnv(); + const payload = migratePayload(); + payload.props.changedCount = 100000; + const res = await call(post(payload), env); + assert.equal(res.status, 204); + assert.deepEqual(points[0]?.doubles, [10000]); + }); + + it('20 todo rules → exactly 21 writes', async () => { + const { env, points } = makeEnv(); + const payload = migratePayload(); + payload.todosByRule = Array.from({ length: 20 }, (_, i) => ({ + rule: `rule-${i}`, + count: i + 1, + })); + const res = await call(post(payload), env); + assert.equal(res.status, 204); + assert.equal(points.length, 21); + }); + + it('25 todo rules → 400 and zero writes (validation caps at 20)', async () => { + const { env, points } = makeEnv(); + const payload = migratePayload(); + payload.todosByRule = Array.from({ length: 25 }, (_, i) => ({ + rule: `rule-${i}`, + count: 1, + })); + const res = await call(post(payload), env); + assert.equal(res.status, 400); + assert.equal(points.length, 0); + }); + + it('204 even without the TELEMETRY binding (fire-and-forget)', async () => { + const res = await call(post(createPayload()), {}); + assert.equal(res.status, 204); + assert.equal(await res.text(), ''); + }); +}); diff --git a/telemetry-worker/src/__tests__/validate.test.ts b/telemetry-worker/src/__tests__/validate.test.ts new file mode 100644 index 000000000..568464325 --- /dev/null +++ b/telemetry-worker/src/__tests__/validate.test.ts @@ -0,0 +1,331 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { validate } from '../validate.ts'; + +type Payload = Record; + +const createPayload = (): Payload => ({ + v: 1, + tool: 'create-bestax', + event: 'scaffold', + toolVersion: '1.4.2', + nodeMajor: 22, + platform: 'darwin', + props: { + template: 'vite-ts', + bulmaFlavor: 'complete', + iconLibrary: 'none', + skills: true, + packageManager: 'pnpm', + }, +}); + +const migratePayload = (): Payload => ({ + v: 1, + tool: 'bestax-migrate', + event: 'migrate', + toolVersion: '0.3.0', + nodeMajor: 20, + platform: 'linux', + props: { + source: 'react-bulma-components', + cssMode: 'bestax', + dry: false, + deps: true, + changedBucket: '10-49', + changedCount: 23, + }, + todosByRule: [ + { rule: 'unmapped-prop', count: 3 }, + { rule: 'icon.size', count: 1 }, + ], +}); + +const props = (payload: Payload): Payload => payload.props as Payload; + +const assertOk = (payload: unknown) => { + const result = validate(payload); + assert.equal(result.ok, true, JSON.stringify(result)); +}; + +const assertRejected = (payload: unknown, label: string) => { + const result = validate(payload); + assert.equal(result.ok, false, `expected rejection: ${label}`); +}; + +describe('validate: accepts', () => { + it('canonical create-bestax payload', () => { + assertOk(createPayload()); + }); + + it('canonical bestax-migrate payload with todosByRule', () => { + assertOk(migratePayload()); + }); + + it('bestax-migrate payload without todosByRule', () => { + const payload = migratePayload(); + delete payload.todosByRule; + assertOk(payload); + }); + + it('prerelease toolVersion', () => { + const payload = createPayload(); + payload.toolVersion = '2.0.0-beta.1'; + assertOk(payload); + }); + + it('validated payload round-trips unchanged', () => { + const payload = migratePayload(); + const result = validate(payload); + assert.equal(result.ok, true); + if (result.ok) assert.deepEqual(result.payload, payload); + }); +}); + +describe('validate: rejects non-objects', () => { + for (const [label, value] of [ + ['null', null], + ['array', []], + ['string', 'scaffold'], + ['number', 42], + ['boolean', true], + ] as const) { + it(label, () => assertRejected(value, label)); + } +}); + +describe('validate: rejects unknown or missing keys', () => { + it('unknown top-level key', () => { + const payload = createPayload(); + payload.extra = 'nope'; + assertRejected(payload, 'unknown top-level key'); + }); + + it('todosByRule on create-bestax', () => { + const payload = createPayload(); + payload.todosByRule = [{ rule: 'x', count: 1 }]; + assertRejected(payload, 'todosByRule is migrate-only'); + }); + + it('unknown props key', () => { + const payload = createPayload(); + props(payload).ip = '203.0.113.7'; + assertRejected(payload, 'unknown props key'); + }); + + for (const key of [ + 'v', + 'tool', + 'event', + 'toolVersion', + 'nodeMajor', + 'platform', + 'props', + ]) { + it(`missing top-level ${key}`, () => { + const payload = migratePayload(); + delete payload[key]; + assertRejected(payload, `missing ${key}`); + }); + } + + for (const key of [ + 'template', + 'bulmaFlavor', + 'iconLibrary', + 'skills', + 'packageManager', + ]) { + it(`missing create props.${key}`, () => { + const payload = createPayload(); + delete props(payload)[key]; + assertRejected(payload, `missing props.${key}`); + }); + } + + for (const key of [ + 'source', + 'cssMode', + 'dry', + 'deps', + 'changedBucket', + 'changedCount', + ]) { + it(`missing migrate props.${key}`, () => { + const payload = migratePayload(); + delete props(payload)[key]; + assertRejected(payload, `missing props.${key}`); + }); + } +}); + +describe('validate: rejects envelope field values', () => { + for (const v of [2, 0, '1', null]) { + it(`v = ${JSON.stringify(v)}`, () => { + const payload = createPayload(); + payload.v = v; + assertRejected(payload, 'v must be 1'); + }); + } + + it('unknown tool', () => { + const payload = createPayload(); + payload.tool = 'bestax-doctor'; + assertRejected(payload, 'unknown tool'); + }); + + it('create-bestax with migrate event', () => { + const payload = createPayload(); + payload.event = 'migrate'; + assertRejected(payload, 'wrong event for tool'); + }); + + it('bestax-migrate with scaffold event', () => { + const payload = migratePayload(); + payload.event = 'scaffold'; + assertRejected(payload, 'wrong event for tool'); + }); + + for (const version of [ + '1.2', + 'v1.2.3', + '1.2.3_beta', + '1.2.3-', + '1.2.3 ', + `1.0.0-${'a'.repeat(30)}`, // > 32 chars + 123, + ]) { + it(`toolVersion = ${JSON.stringify(version)}`, () => { + const payload = createPayload(); + payload.toolVersion = version; + assertRejected(payload, 'bad toolVersion'); + }); + } + + for (const nodeMajor of [9, 100, 22.5, '22', -22]) { + it(`nodeMajor = ${JSON.stringify(nodeMajor)}`, () => { + const payload = createPayload(); + payload.nodeMajor = nodeMajor; + assertRejected(payload, 'bad nodeMajor'); + }); + } + + it('unknown platform', () => { + const payload = createPayload(); + payload.platform = 'haiku'; + assertRejected(payload, 'unknown platform'); + }); + + it('props as array', () => { + const payload = createPayload(); + payload.props = []; + assertRejected(payload, 'props must be an object'); + }); +}); + +describe('validate: rejects props enum values', () => { + const badEnum: [string, () => Payload, string][] = [ + ['template', createPayload, 'template'], + ['bulmaFlavor', createPayload, 'bulmaFlavor'], + ['iconLibrary', createPayload, 'iconLibrary'], + ['packageManager', createPayload, 'packageManager'], + ['source', migratePayload, 'source'], + ['cssMode', migratePayload, 'cssMode'], + ['changedBucket', migratePayload, 'changedBucket'], + ]; + for (const [label, factory, key] of badEnum) { + it(`unknown ${label}`, () => { + const payload = factory(); + props(payload)[key] = 'not-a-real-value'; + assertRejected(payload, `unknown ${label}`); + }); + } + + it('bucket label as changedBucket', () => { + const payload = migratePayload(); + props(payload).changedBucket = '1-10'; + assertRejected(payload, 'changedBucket not in bucket set'); + }); + + for (const [key, factory] of [ + ['skills', createPayload], + ['dry', migratePayload], + ['deps', migratePayload], + ] as const) { + it(`non-boolean ${key}`, () => { + const payload = factory(); + props(payload)[key] = 'true'; + assertRejected(payload, `${key} must be boolean`); + }); + } + + for (const changedCount of [-1, 100001, 1.5, '23', null]) { + it(`changedCount = ${JSON.stringify(changedCount)}`, () => { + const payload = migratePayload(); + props(payload).changedCount = changedCount; + assertRejected(payload, 'bad changedCount'); + }); + } +}); + +describe('validate: rejects todosByRule shapes', () => { + it('more than 20 entries', () => { + const payload = migratePayload(); + payload.todosByRule = Array.from({ length: 21 }, (_, i) => ({ + rule: `rule-${i}`, + count: 1, + })); + assertRejected(payload, 'oversized todosByRule'); + }); + + it('exactly 20 entries is accepted', () => { + const payload = migratePayload(); + payload.todosByRule = Array.from({ length: 20 }, (_, i) => ({ + rule: `rule-${i}`, + count: 1, + })); + assertOk(payload); + }); + + it('non-array todosByRule', () => { + const payload = migratePayload(); + payload.todosByRule = { rule: 'x', count: 1 }; + assertRejected(payload, 'todosByRule must be an array'); + }); + + it('entry with extra key', () => { + const payload = migratePayload(); + payload.todosByRule = [{ rule: 'x', count: 1, file: 'App.tsx' }]; + assertRejected(payload, 'todo entry with extra key'); + }); + + it('entry missing count', () => { + const payload = migratePayload(); + payload.todosByRule = [{ rule: 'x' }]; + assertRejected(payload, 'todo entry missing count'); + }); + + for (const rule of [ + 'UPPER', + '-leading-dash', + '.leading-dot', + '', + 'has space', + 'a'.repeat(65), + 42, + ]) { + it(`rule = ${JSON.stringify(rule)}`, () => { + const payload = migratePayload(); + payload.todosByRule = [{ rule, count: 1 }]; + assertRejected(payload, 'bad rule slug'); + }); + } + + for (const count of [-1, 100001, 0.5, '3', null]) { + it(`count = ${JSON.stringify(count)}`, () => { + const payload = migratePayload(); + payload.todosByRule = [{ rule: 'ok-rule', count }]; + assertRejected(payload, 'bad todo count'); + }); + } +}); diff --git a/telemetry-worker/src/index.ts b/telemetry-worker/src/index.ts new file mode 100644 index 000000000..98ee51644 --- /dev/null +++ b/telemetry-worker/src/index.ts @@ -0,0 +1,58 @@ +/** + * Anonymous telemetry ingest for the create-bestax and bestax-migrate CLIs. + * + * POST bestax.io/api/t → validate (reject-by-default) → write to Analytics + * Engine → 204. Privacy constraints, deliberately load-bearing: never read + * request.cf, CF-Connecting-IP, or User-Agent; never log the body; no CORS + * headers (the CLIs are not browsers). + */ + +import { runEventPoint, todoPoints } from './schema.ts'; +import type { Env } from './types.ts'; +import { validate } from './validate.ts'; + +const MAX_BODY_BYTES = 8192; + +const status = (code: number): Response => new Response(null, { status: code }); + +export default { + async fetch(request: Request, env: Env): Promise { + if (request.method !== 'POST') return status(405); + if (new URL(request.url).pathname !== '/api/t') return status(404); + + const contentType = request.headers.get('content-type') ?? ''; + if (!contentType.includes('application/json')) return status(415); + + // Declared length first, so an honestly-labeled oversized body is refused + // without buffering it; the byte check after read is the real guard. + const declaredLength = request.headers.get('content-length'); + if (declaredLength !== null && Number(declaredLength) > MAX_BODY_BYTES) { + return status(413); + } + const text = await request.text(); + if (new TextEncoder().encode(text).byteLength > MAX_BODY_BYTES) { + return status(413); + } + + let body: unknown; + try { + body = JSON.parse(text); + } catch { + return status(400); + } + + const result = validate(body); + if (!result.ok) return status(400); + + // Missing binding (wrangler dev without AE) still answers 204 — the CLIs + // fire-and-forget and must never surface an error. Total writes per + // request: 1 run event + ≤20 todo points = ≤21. + if (env.TELEMETRY) { + env.TELEMETRY.writeDataPoint(runEventPoint(result.payload)); + for (const point of todoPoints(result.payload)) { + env.TELEMETRY.writeDataPoint(point); + } + } + return status(204); + }, +}; diff --git a/telemetry-worker/src/schema.ts b/telemetry-worker/src/schema.ts new file mode 100644 index 000000000..334654e1b --- /dev/null +++ b/telemetry-worker/src/schema.ts @@ -0,0 +1,196 @@ +/** + * Telemetry schema: the enum allowlists and the payload → Analytics Engine + * data-point mapping. + * + * Enum sets must stay in sync with create-bestax/src/constants.ts and + * bestax-migrate/src/sources/registry.ts — new values must land HERE FIRST or + * they are silently dropped (fail closed). + */ + +import type { AnalyticsEngineDataPoint } from './types.ts'; + +export const TOOLS = ['create-bestax', 'bestax-migrate'] as const; +export type Tool = (typeof TOOLS)[number]; + +// One run event per tool; `migrate_todo` points are derived, never sent as an +// event by the CLIs. +export const EVENT_FOR_TOOL = { + 'create-bestax': 'scaffold', + 'bestax-migrate': 'migrate', +} as const; + +// create-bestax/src/constants.ts → TEMPLATES +const TEMPLATE_VALUES = ['vite', 'vite-ts'] as const; +// create-bestax/src/constants.ts → BULMA_FLAVORS +const BULMA_FLAVOR_VALUES = [ + 'complete', + 'prefixed', + 'no-helpers', + 'no-helpers-prefixed', + 'no-dark-mode', +] as const; +// create-bestax/src/constants.ts → ICON_LIBRARIES +const ICON_LIBRARY_VALUES = [ + 'none', + 'fontawesome', + 'mdi', + 'ionicons', + 'material-icons', + 'material-symbols', +] as const; +// bestax-migrate/src/sources/registry.ts → SOURCES +const MIGRATE_SOURCE_VALUES = ['react-bulma-components'] as const; +// bestax-migrate/src/cli.ts → CSS_MODES +const CSS_MODE_VALUES = ['bestax', 'bulma', 'keep'] as const; +const PACKAGE_MANAGER_VALUES = ['npm', 'pnpm', 'yarn', 'bun'] as const; +const PLATFORM_VALUES = [ + 'aix', + 'darwin', + 'freebsd', + 'linux', + 'openbsd', + 'sunos', + 'win32', + 'android', +] as const; +const CHANGED_BUCKET_VALUES = ['0', '1-9', '10-49', '50-199', '200+'] as const; + +export type Template = (typeof TEMPLATE_VALUES)[number]; +export type BulmaFlavor = (typeof BULMA_FLAVOR_VALUES)[number]; +export type IconLibrary = (typeof ICON_LIBRARY_VALUES)[number]; +export type MigrateSource = (typeof MIGRATE_SOURCE_VALUES)[number]; +export type CssMode = (typeof CSS_MODE_VALUES)[number]; +export type PackageManager = (typeof PACKAGE_MANAGER_VALUES)[number]; +export type Platform = (typeof PLATFORM_VALUES)[number]; +export type ChangedBucket = (typeof CHANGED_BUCKET_VALUES)[number]; + +export const TEMPLATES: ReadonlySet = new Set(TEMPLATE_VALUES); +export const BULMA_FLAVORS: ReadonlySet = new Set(BULMA_FLAVOR_VALUES); +export const ICON_LIBRARIES: ReadonlySet = new Set(ICON_LIBRARY_VALUES); +export const MIGRATE_SOURCES: ReadonlySet = new Set( + MIGRATE_SOURCE_VALUES +); +export const CSS_MODES: ReadonlySet = new Set(CSS_MODE_VALUES); +export const PACKAGE_MANAGERS: ReadonlySet = new Set( + PACKAGE_MANAGER_VALUES +); +export const PLATFORMS: ReadonlySet = new Set(PLATFORM_VALUES); +export const CHANGED_BUCKETS: ReadonlySet = new Set( + CHANGED_BUCKET_VALUES +); + +export const MAX_TODO_RULES = 20; +export const MAX_CHANGED_COUNT = 100000; +// changedCount above this is a pathological run; cap the stored double so one +// outlier cannot dominate SUM(double1 * _sample_interval) aggregates. +export const CHANGED_COUNT_DOUBLE_CAP = 10000; + +export interface CreateBestaxPayload { + v: 1; + tool: 'create-bestax'; + event: 'scaffold'; + toolVersion: string; + nodeMajor: number; + platform: Platform; + props: { + template: Template; + bulmaFlavor: BulmaFlavor; + iconLibrary: IconLibrary; + skills: boolean; + packageManager: PackageManager; + }; +} + +export interface MigratePayload { + v: 1; + tool: 'bestax-migrate'; + event: 'migrate'; + toolVersion: string; + nodeMajor: number; + platform: Platform; + props: { + source: MigrateSource; + cssMode: CssMode; + dry: boolean; + deps: boolean; + changedBucket: ChangedBucket; + changedCount: number; + }; + todosByRule?: { rule: string; count: number }[]; +} + +export type TelemetryPayload = CreateBestaxPayload | MigratePayload; + +/** + * Analytics Engine layout. blobs[i] surfaces in SQL as blob{i+1}. + * + * | SQL | run event (scaffold / migrate) | migrate_todo point | + * | ------- | ---------------------------------- | ------------------ | + * | index1 | tool | 'bestax-migrate' | + * | blob1 | event | 'migrate_todo' | + * | blob2 | toolVersion | toolVersion | + * | blob3 | platform | platform | + * | blob4 | nodeMajor | nodeMajor | + * | blob5 | template / source | source | + * | blob6 | bulmaFlavor / cssMode | rule | + * | blob7 | iconLibrary / dry '1'|'0' | — | + * | blob8 | skills '1'|'0' / deps '1'|'0' | — | + * | blob9 | packageManager / changedBucket | — | + * | double1 | — / changedCount | count | + */ +export function runEventPoint( + payload: TelemetryPayload +): AnalyticsEngineDataPoint { + const base = [ + payload.event, + payload.toolVersion, + payload.platform, + String(payload.nodeMajor), + ]; + if (payload.tool === 'create-bestax') { + const p = payload.props; + return { + indexes: [payload.tool], + blobs: [ + ...base, + p.template, + p.bulmaFlavor, + p.iconLibrary, + p.skills ? '1' : '0', + p.packageManager, + ], + doubles: [], + }; + } + const p = payload.props; + return { + indexes: [payload.tool], + blobs: [ + ...base, + p.source, + p.cssMode, + p.dry ? '1' : '0', + p.deps ? '1' : '0', + p.changedBucket, + ], + doubles: [Math.min(p.changedCount, CHANGED_COUNT_DOUBLE_CAP)], + }; +} + +export function todoPoints( + payload: TelemetryPayload +): AnalyticsEngineDataPoint[] { + if (payload.tool !== 'bestax-migrate' || !payload.todosByRule) return []; + return payload.todosByRule.slice(0, MAX_TODO_RULES).map(entry => ({ + indexes: [payload.tool], + blobs: [ + 'migrate_todo', + payload.toolVersion, + payload.platform, + String(payload.nodeMajor), + payload.props.source, + entry.rule, + ], + doubles: [entry.count], + })); +} diff --git a/telemetry-worker/src/types.ts b/telemetry-worker/src/types.ts new file mode 100644 index 000000000..909dc13cf --- /dev/null +++ b/telemetry-worker/src/types.ts @@ -0,0 +1,22 @@ +/** + * Minimal structural types for the Workers runtime bindings this worker uses. + * Deliberately hand-rolled: this package sits outside the pnpm workspace (for + * now), so @cloudflare/workers-types cannot be installed. Request/Response/URL + * come from the `webworker` TS lib instead (see tsconfig.json). + */ + +export interface AnalyticsEngineDataPoint { + indexes?: string[]; + blobs?: string[]; + doubles?: number[]; +} + +export interface AnalyticsEngineDataset { + writeDataPoint(point: AnalyticsEngineDataPoint): void; +} + +export interface Env { + // Optional: absent under `wrangler dev` without the AE binding — the handler + // must still 204 (fire-and-forget contract). + TELEMETRY?: AnalyticsEngineDataset; +} diff --git a/telemetry-worker/src/validate.ts b/telemetry-worker/src/validate.ts new file mode 100644 index 000000000..424dac756 --- /dev/null +++ b/telemetry-worker/src/validate.ts @@ -0,0 +1,177 @@ +/** + * Reject-by-default payload validation. Every key, everywhere, must be on an + * allowlist and every value must match its allowlisted shape — anything else + * rejects the whole payload. Rejection reasons are internal (tests/debug + * only); the handler always answers a bare 400. + */ + +import { + BULMA_FLAVORS, + CHANGED_BUCKETS, + CSS_MODES, + EVENT_FOR_TOOL, + ICON_LIBRARIES, + MAX_CHANGED_COUNT, + MAX_TODO_RULES, + MIGRATE_SOURCES, + PACKAGE_MANAGERS, + PLATFORMS, + TEMPLATES, + TOOLS, +} from './schema.ts'; +import type { TelemetryPayload, Tool } from './schema.ts'; + +export type ValidationResult = + { ok: true; payload: TelemetryPayload } | { ok: false; reason: string }; + +const VERSION_RE = /^\d+\.\d+\.\d+(-[\w.]+)?$/; +const MAX_VERSION_LENGTH = 32; +const RULE_RE = /^[a-z0-9][a-z0-9._-]{0,63}$/; + +const TOP_KEYS = [ + 'v', + 'tool', + 'event', + 'toolVersion', + 'nodeMajor', + 'platform', + 'props', +] as const; +const CREATE_PROP_KEYS = [ + 'template', + 'bulmaFlavor', + 'iconLibrary', + 'skills', + 'packageManager', +] as const; +const MIGRATE_PROP_KEYS = [ + 'source', + 'cssMode', + 'dry', + 'deps', + 'changedBucket', + 'changedCount', +] as const; +const TODO_ENTRY_KEYS = ['rule', 'count'] as const; + +const reject = (reason: string): ValidationResult => ({ ok: false, reason }); + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function hasExactKeys( + obj: Record, + required: readonly string[], + optional: readonly string[] = [] +): boolean { + const keys = Object.keys(obj); + return ( + keys.every(key => required.includes(key) || optional.includes(key)) && + required.every(key => keys.includes(key)) + ); +} + +function inSet(set: ReadonlySet, value: unknown): boolean { + return typeof value === 'string' && set.has(value); +} + +export function validate(body: unknown): ValidationResult { + if (!isRecord(body)) return reject('body is not a JSON object'); + + if (typeof body.tool !== 'string' || !TOOLS.includes(body.tool as Tool)) { + return reject('unknown tool'); + } + const tool = body.tool as Tool; + + const optionalTopKeys = tool === 'bestax-migrate' ? ['todosByRule'] : []; + if (!hasExactKeys(body, TOP_KEYS, optionalTopKeys)) { + return reject('unexpected or missing top-level keys'); + } + + if (body.v !== 1) return reject('unsupported schema version'); + if (body.event !== EVENT_FOR_TOOL[tool]) { + return reject('event does not match tool'); + } + if ( + typeof body.toolVersion !== 'string' || + body.toolVersion.length > MAX_VERSION_LENGTH || + !VERSION_RE.test(body.toolVersion) + ) { + return reject('invalid toolVersion'); + } + if ( + typeof body.nodeMajor !== 'number' || + !Number.isInteger(body.nodeMajor) || + body.nodeMajor < 10 || + body.nodeMajor > 99 + ) { + return reject('invalid nodeMajor'); + } + if (!inSet(PLATFORMS, body.platform)) return reject('unknown platform'); + if (!isRecord(body.props)) return reject('props is not an object'); + const props = body.props; + + if (tool === 'create-bestax') { + if (!hasExactKeys(props, CREATE_PROP_KEYS)) { + return reject('unexpected or missing props keys'); + } + if (!inSet(TEMPLATES, props.template)) return reject('unknown template'); + if (!inSet(BULMA_FLAVORS, props.bulmaFlavor)) { + return reject('unknown bulmaFlavor'); + } + if (!inSet(ICON_LIBRARIES, props.iconLibrary)) { + return reject('unknown iconLibrary'); + } + if (typeof props.skills !== 'boolean') return reject('invalid skills'); + if (!inSet(PACKAGE_MANAGERS, props.packageManager)) { + return reject('unknown packageManager'); + } + } else { + if (!hasExactKeys(props, MIGRATE_PROP_KEYS)) { + return reject('unexpected or missing props keys'); + } + if (!inSet(MIGRATE_SOURCES, props.source)) return reject('unknown source'); + if (!inSet(CSS_MODES, props.cssMode)) return reject('unknown cssMode'); + if (typeof props.dry !== 'boolean') return reject('invalid dry'); + if (typeof props.deps !== 'boolean') return reject('invalid deps'); + if (!inSet(CHANGED_BUCKETS, props.changedBucket)) { + return reject('unknown changedBucket'); + } + if ( + typeof props.changedCount !== 'number' || + !Number.isInteger(props.changedCount) || + props.changedCount < 0 || + props.changedCount > MAX_CHANGED_COUNT + ) { + return reject('invalid changedCount'); + } + + if (body.todosByRule !== undefined) { + const todos = body.todosByRule; + if (!Array.isArray(todos) || todos.length > MAX_TODO_RULES) { + return reject('invalid todosByRule'); + } + for (const entry of todos) { + if (!isRecord(entry) || !hasExactKeys(entry, TODO_ENTRY_KEYS)) { + return reject('invalid todosByRule entry'); + } + if (typeof entry.rule !== 'string' || !RULE_RE.test(entry.rule)) { + return reject('invalid todo rule'); + } + if ( + typeof entry.count !== 'number' || + !Number.isInteger(entry.count) || + entry.count < 0 || + entry.count > MAX_CHANGED_COUNT + ) { + return reject('invalid todo count'); + } + } + } + } + + // Every field above has been checked against the allowlists, so the cast is + // the runtime-verified equivalent of the static type. + return { ok: true, payload: body as unknown as TelemetryPayload }; +} diff --git a/telemetry-worker/tsconfig.json b/telemetry-worker/tsconfig.json new file mode 100644 index 000000000..32f4a8597 --- /dev/null +++ b/telemetry-worker/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "es2022", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["es2022", "webworker"], + "types": [], + "strict": true, + "noEmit": true, + "allowImportingTsExtensions": true, + "skipLibCheck": true + }, + "include": ["src/**/*.ts"], + // Tests run under `node --test` (type stripping) and import node:test, whose + // types aren't resolvable here (no @types/node with types: []) — src only. + "exclude": ["src/__tests__"] +} diff --git a/telemetry-worker/wrangler.toml b/telemetry-worker/wrangler.toml new file mode 100644 index 000000000..2441aaccc --- /dev/null +++ b/telemetry-worker/wrangler.toml @@ -0,0 +1,8 @@ +name = "bestax-telemetry" +main = "src/index.ts" +compatibility_date = "2026-08-01" +routes = [{ pattern = "bestax.io/api/t", zone_name = "bestax.io" }] + +[[analytics_engine_datasets]] +binding = "TELEMETRY" +dataset = "bestax_telemetry" From 408e6b47538daa47a3bb673e651bc711a695f55c Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sat, 22 Aug 2026 15:32:00 -0400 Subject: [PATCH 02/29] ci: harden the skills-roster check and derive skill discovery once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes from the #541 deep review: - installFence anchored on the first fence containing '--skill ', so a quick-start example above the real block silently became the validated roster, and its opener regex could not parse info strings. Fenced copies now anchor on explicit markers and are walked with fenceMask. - The Agent Skills table patterns scanned whole files; they now scope to the table under its own '| Skill |' header row, in both directions. - section() was fence-blind: a flush-left '# comment' in a fenced example truncated the scope. It counts headings outside fences now. - The AGENTS.md comma list is split, not pattern-matched: the old regex dropped the last two names without a serial comma. - The 'directory holding a SKILL.md' predicate existed in four copies with three sort comparators (one locale-dependent); all four consumers now import scripts/lib/skills.mjs, and the tests derive the comparison set the same way the check does (rosterSkillNames). - New gates: frontmatter name must equal the directory name (the MCP manifest keys off frontmatter, the rosters off the directory); the docs-site surfaces (per-skill page, sidebars.js entry, intro bullet) are held through the slug transform; install blocks must agree on order (llms/index.md had already drifted — fixed); both sync scripts refuse untracked skill directories, restoring the deleted allowlist's only-vetted-skills guarantee for local builds and manual publishes. --- bestax-mcp/scripts/sync-skills.mjs | 24 +- create-bestax/scripts/sync-skills.mjs | 37 ++- docs/docs/guides/llms/index.md | 4 +- docs/docs/skills/intro.md | 2 + scripts/check-conformance.mjs | 402 +++++++++++++++++++------- scripts/gen-mcp-index.mjs | 10 +- scripts/lib/skills.mjs | 113 ++++++++ scripts/skills-roster.test.mjs | 253 ++++++++++++++++ skills/README.md | 4 + 9 files changed, 716 insertions(+), 133 deletions(-) create mode 100644 scripts/lib/skills.mjs diff --git a/bestax-mcp/scripts/sync-skills.mjs b/bestax-mcp/scripts/sync-skills.mjs index 676a8b272..60a288bbc 100644 --- a/bestax-mcp/scripts/sync-skills.mjs +++ b/bestax-mcp/scripts/sync-skills.mjs @@ -43,6 +43,10 @@ import { createHash } from 'node:crypto'; import { setTimeout as sleep } from 'node:timers/promises'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import { + readSkillNames, + untrackedSkillDirs, +} from '../../scripts/lib/skills.mjs'; const here = path.dirname(fileURLToPath(import.meta.url)); const pkgRoot = path.resolve(here, '..'); @@ -68,18 +72,26 @@ if (!existsSync(skillsSrc)) { process.exit(1); } -const names = (await readdir(skillsSrc, { withFileTypes: true })) - .filter( - e => e.isDirectory() && existsSync(path.join(skillsSrc, e.name, 'SKILL.md')) - ) - .map(e => e.name) - .sort(); +// The shared predicate (scripts/lib/skills.mjs): a directory holding a +// SKILL.md, sorted deterministically — one definition for all four consumers. +const names = await readSkillNames(skillsSrc); if (!names.length) { console.error(`[sync-skills] no skills found in ${skillsSrc}`); process.exit(1); } +// Same vetting gate as create-bestax's sync: data/skills ships in the npm +// tarball, so an untracked scratch skill must not reach a manual publish. +const untrackedHere = untrackedSkillDirs(skillsSrc, names); +if (untrackedHere.length) { + console.error( + `[sync-skills] refusing to bundle untracked skill dir(s): ` + + `${untrackedHere.join(', ')}. \`git add\` them to vet them, or remove them.` + ); + process.exit(1); +} + // Hash of every source path and its bytes. Content rather than mtime on // purpose: a fresh clone or a branch switch rewrites mtimes without changing // what should be copied, and re-copying is the cheap half — it is the diff --git a/create-bestax/scripts/sync-skills.mjs b/create-bestax/scripts/sync-skills.mjs index 2a7177411..0a8b8b2e0 100644 --- a/create-bestax/scripts/sync-skills.mjs +++ b/create-bestax/scripts/sync-skills.mjs @@ -35,6 +35,10 @@ import { fileURLToPath } from 'node:url'; import path from 'node:path'; import fs from 'fs-extra'; +import { + readSkillNames, + untrackedSkillDirs, +} from '../../scripts/lib/skills.mjs'; const here = path.dirname(fileURLToPath(import.meta.url)); const pkgRoot = path.resolve(here, '..'); @@ -46,18 +50,13 @@ if (!fs.existsSync(skillsSrc)) { process.exit(1); } -// A directory with a SKILL.md is a skill. That predicate is what excludes -// `skills/README.md` and `skills/CLAUDE.md` without naming them, so a new -// non-skill file at that level does not become a phantom skill. Sorted so the -// copy order — and the count in the success line — is deterministic. -const skills = (await fs.readdir(skillsSrc, { withFileTypes: true })) - .filter( - entry => - entry.isDirectory() && - fs.existsSync(path.join(skillsSrc, entry.name, 'SKILL.md')) - ) - .map(entry => entry.name) - .sort(); +// A directory with a SKILL.md is a skill — the predicate lives once in +// scripts/lib/skills.mjs, shared with bestax-mcp's sync, gen-mcp-index and +// check-conformance, so the four consumers cannot drift. It is what excludes +// `skills/README.md` and `skills/CLAUDE.md` without naming them, and the list +// comes back sorted so the copy order — and the count in the success line — +// is deterministic. +const skills = await readSkillNames(skillsSrc); // Checked BEFORE emptying the destination, which the hardcoded version did not // have to think about. Reading a roster off disk means a wrong path or a @@ -68,6 +67,20 @@ if (!skills.length) { process.exit(1); } +// The vetting gate the deleted allowlist used to be: discovery bundles +// whatever is on disk, and CI's skills-roster check only sees committed +// state — so an untracked scratch directory would ship in a local build or a +// manual publish with no gate anywhere in the path. `git add` is the act of +// vetting; a tree without git (an exported tarball) skips the gate. +const untracked = untrackedSkillDirs(skillsSrc, skills); +if (untracked.length) { + console.error( + `[sync-skills] refusing to bundle untracked skill dir(s): ` + + `${untracked.join(', ')}. \`git add\` them to vet them, or remove them.` + ); + process.exit(1); +} + await fs.emptyDir(skillsDest); for (const name of skills) { diff --git a/docs/docs/guides/llms/index.md b/docs/docs/guides/llms/index.md index bb1a37da7..8cef87e57 100644 --- a/docs/docs/guides/llms/index.md +++ b/docs/docs/guides/llms/index.md @@ -27,14 +27,16 @@ Beyond the raw docs, bestax ships **Agent Skills** that teach an agent _how_ to with the library (conventions, patterns, and a component catalog). Install one with the [`skills`](https://skills.sh/) CLI: + + ```bash npx skills add https://github.com/allxsmith/bestax --skill bestax-custom-component npx skills add https://github.com/allxsmith/bestax --skill bestax-form npx skills add https://github.com/allxsmith/bestax --skill bestax-theming npx skills add https://github.com/allxsmith/bestax --skill bestax-layout-scaffold npx skills add https://github.com/allxsmith/bestax --skill bestax-icons -npx skills add https://github.com/allxsmith/bestax --skill bestax-optimize npx skills add https://github.com/allxsmith/bestax --skill bestax-migrate +npx skills add https://github.com/allxsmith/bestax --skill bestax-optimize ``` Starting a new app? `pnpm create bestax@latest` offers to **preinstall these skills** diff --git a/docs/docs/skills/intro.md b/docs/docs/skills/intro.md index 84019adc7..0e0c82e6d 100644 --- a/docs/docs/skills/intro.md +++ b/docs/docs/skills/intro.md @@ -11,6 +11,8 @@ other skills.sh-compatible tools — how to build with **@allxsmith/bestax-bulma Install one with the [`skills`](https://skills.sh/) CLI: + + ```bash npx skills add https://github.com/allxsmith/bestax --skill bestax-custom-component npx skills add https://github.com/allxsmith/bestax --skill bestax-form diff --git a/scripts/check-conformance.mjs b/scripts/check-conformance.mjs index b71deeb28..a1ff5c9b4 100644 --- a/scripts/check-conformance.mjs +++ b/scripts/check-conformance.mjs @@ -73,6 +73,12 @@ import { sectionSpans, splitLines, } from './lib/api-page.mjs'; +import { + SKILL_DIR_NAME, + readSkillDirs as libReadSkillDirs, + readSkillNames as libReadSkillNames, + rosterSkillNames, +} from './lib/skills.mjs'; import { renderPage } from './gen-api-docs.mjs'; import { ORDERED_CATEGORIES, @@ -1847,9 +1853,11 @@ async function checkBypassExpiry() { * * The three consumers that CAN derive the roster do: create-bestax's and * bestax-mcp's `sync-skills.mjs` copy each directory holding a `SKILL.md`, and - * `gen-mcp-index.mjs` indexes the same set (#540). What is left is prose, which - * cannot be derived and so drifts silently — the repo-root README was already - * four skills behind seven when this check landed, on the project's front page. + * `gen-mcp-index.mjs` indexes the same set (#540) — all through + * `scripts/lib/skills.mjs`, the single definition of that predicate. What is + * left is prose, which cannot be derived and so drifts silently — the repo-root + * README was already four skills behind seven when this check landed, on the + * project's front page. * * Not to be confused with `skills-sync`, which despite the name is about the * bestax-theming skill's two reference inventories and never reads the roster. @@ -1858,24 +1866,32 @@ async function checkBypassExpiry() { * line — rather than by the bare skill name occurring anywhere in the file. * `bestax-migrate` is why: it is also a package, a CLI, and the marker the * codemod leaves behind, so it appears in prose in most of these files, and a - * bare-name search would pass on a table that had lost its row. The cost is - * that reformatting one of these blocks breaks its pattern, so every message - * prints the exact line the check wanted to find. + * bare-name search would pass on a table that had lost its row. * * Two install lines are deliberately NOT rosters. `bulma-ui/README.md` and * `bulma-ui/AGENTS.md` each show a single `--skill` command as an example; * holding them to all seven would demand a list neither is trying to be. Their - * real rosters (a table and a parenthetical) are covered instead. For the same - * reason the three real install blocks are scoped to their fence, so an example - * elsewhere in those files cannot stand in for a missing entry. + * real rosters (a table and a parenthetical) are covered instead. * - * `fenceMask` is deliberately not used. FOUR of these copies live inside a - * fenced block on purpose — the three install blocks and the layout tree — - * which is the opposite of what masking is for. + * The fenced copies — the three install blocks and the layout tree — anchor on + * a `` marker line and take the fence that follows, + * parsed with `fenceMask`. The first version anchored on "the first fence + * containing `--skill `", so a quick-start example above the real block + * silently became the validated roster, and its `/^```[a-z]*\n/` opener could + * not parse info strings (```bash title=…), frame-shifting every later fence. + * The Agent Skills tables anchor on their own `| Skill |` header row for the + * same reason: `bestax-migrate` in the first cell of some OTHER table must not + * stand in for a deleted row. * * The capture group is the point: reading the names back out checks BOTH * directions, so a roster still advertising a deleted skill fails too. That * half has no other guard — sync-skills.mjs just silently stops copying it. + * + * The docs-site surfaces are held through the slug transform (directory name + * minus `bestax-`, exactly what gen-mcp-index.mjs ships as `promptName`): the + * sidebar entries and the intro bullet roster here, and the per-skill page + * files in `skillsPageViolations`. A new skill fails conformance until its + * docs page, sidebar entry, and intro bullet exist. */ export const SKILL_ROSTERS = [ { @@ -1883,12 +1899,14 @@ export const SKILL_ROSTERS = [ copies: [ { what: 'the Skills table', + scope: skillsTableScope(), list: /^\|[ \t]*\[`([a-z][a-z0-9-]*)`\]/gm, example: n => `| [\`${n}\`](./${n}/SKILL.md) | Use it when… |`, }, skillsAddBlock('the Install block'), { what: 'the Layout tree', + scope: markerFence('skills-roster:tree'), list: /^ {2}([a-z][a-z0-9-]*)\/$/gm, example: n => ` ${n}/`, }, @@ -1910,11 +1928,17 @@ export const SKILL_ROSTERS = [ { file: 'docs/docs/skills/intro.md', copies: [ - // The bullet roster below that block is deliberately not checked: it - // links page slugs (`[Custom Component](./custom-component)`), not skill - // directory names, so matching it would amount to requiring a docs page - // per skill — a separate rule. skillsAddBlock('the skills-add block'), + { + what: 'the per-skill bullet roster', + names: text => + [...text.matchAll(/^- \*\*\[[^\]]+\]\(\.\/([a-z0-9-]+)\)\*\*/gm)].map( + m => m[1] + ), + fromToken: slug => `bestax-${slug}`, + example: n => + `- **[…](./${n.replace(/^bestax-/, '')})** — one line on when to reach for it.`, + }, ], }, { @@ -1926,23 +1950,11 @@ export const SKILL_ROSTERS = [ // already stale when this check landed — four skills against seven — which // is the drift the check exists for, sitting in the most visible place. file: 'README.md', - copies: [ - { - what: 'the Agent Skills table', - list: /^[ \t]*\|[ \t]*`([a-z][a-z0-9-]*)`[ \t]*\|/gm, - example: n => ` | \`${n}\` | Use it when… |`, - }, - ], + copies: [agentSkillsTable()], }, { file: 'bulma-ui/README.md', - copies: [ - { - what: 'the Agent Skills table', - list: /^[ \t]*\|[ \t]*`([a-z][a-z0-9-]*)`[ \t]*\|/gm, - example: n => ` | \`${n}\` | Use it when… |`, - }, - ], + copies: [agentSkillsTable()], }, { file: 'bulma-ui/AGENTS.md', @@ -1952,119 +1964,162 @@ export const SKILL_ROSTERS = [ copies: [ { what: 'the "Agent skills (…)" list', - // Comma-delimited items, not "any lowercase word". A bare - // /([a-z][a-z0-9-]*)/g here would read the "and" out of - // "x, y, and z" and then demand you delete a skill called `and`. - // The optional conjunction is not decoration: serial commas are house - // style, so "x, y, and z" is the likely spelling, and without it the - // final skill reads as missing from a roster that is complete. - list: /(?:^|,)\s*(?:and\s+|or\s+)?([a-z][a-z0-9-]*)\s*(?=,|$)/g, + // A prose comma list, so it is SPLIT rather than pattern-matched: the + // first version's regex only recognized a conjunction after a comma, + // and the non-serial spelling "…, x and y" silently dropped the last + // TWO names. Conjunctions need surrounding whitespace so a hyphenated + // name containing "and" can never be split apart. + names: text => + text + .split(/,|\s+and\s+|\s+or\s+/) + .map(s => s.trim()) + .filter(s => /^[a-z][a-z0-9-]*$/.test(s)), example: n => `${n} (comma-separated, inside the parenthetical)`, }, ], }, + { + file: 'docs/sidebars.js', + copies: [ + { + what: 'the skillsSidebar entries', + names: text => + [...text.matchAll(/'skills\/([a-z0-9-]+)'/g)] + .map(m => m[1]) + .filter(slug => slug !== 'intro'), + fromToken: slug => `bestax-${slug}`, + example: n => `'skills/${n.replace(/^bestax-/, '')}',`, + }, + ], + }, ]; -/** One install block, defined once: three files carry the identical shape. */ +/** + * One install block, defined once: three files carry the identical shape. + * `orderGroup` makes rosterViolations compare their ORDER too — the copies + * are byte-identical on purpose, and a Set comparison alone let two of them + * drift into different orderings without a word. + */ function skillsAddBlock(what) { return { what, - scope: installFence(), + scope: markerFence('skills-roster:install'), + orderGroup: 'the install block', list: /--skill +([a-z][a-z0-9-]*)/g, example: n => `npx skills add https://github.com/allxsmith/bestax --skill ${n}`, }; } +/** The Agent Skills table both READMEs carry, scoped to its own header row. */ +function agentSkillsTable() { + return { + what: 'the Agent Skills table', + scope: skillsTableScope(), + list: /^[ \t]*\|[ \t]*\[?`([a-z][a-z0-9-]*)`/gm, + example: n => ` | \`${n}\` | Use it when… |`, + }; +} + /** - * The fenced block holding the skills-add roster. - * - * Scoped rather than scanning the file, because a single `--skill` line used as - * an EXAMPLE elsewhere would otherwise satisfy a skill that had been dropped - * from the roster itself. + * The body of the table whose header row's first cell is "Skill": every row + * from the separator line to the first non-table line. Anchoring on the header + * is what keeps a kebab-case first cell in some OTHER table (`bestax-migrate` + * is also a package name) from standing in for a roster row — in either + * direction. */ -function installFence() { +function skillsTableScope() { return text => { - for (const [, body] of text.matchAll(/^```[a-z]*\n([\s\S]*?)^```/gm)) { - if (body.includes('--skill ')) return body; - } - return null; + const m = text.match( + /^[ \t]*\|[ \t]*Skill[ \t]*\|.*\n[ \t]*\|[ \t:|-]+\|?[ \t]*\n((?:[ \t]*\|.*(?:\n|$))*)/m + ); + return m ? m[1] : null; }; } /** - * A markdown section by heading, up to the next heading of the same depth or - * shallower. + * The fenced block following a `` line. * - * Line-anchored, not `indexOf`: an unanchored search for `## AI skills` also - * matches the tail of `### AI skills`, and any mention of it in prose or inside - * a fence. The scope would then silently shift to the wrong block and report an - * intact roster as entirely missing — the same failure c8b5d11 fixed in the - * release-docs extractors. The terminator is derived from the heading's own - * depth rather than hardcoded to `##`, so this stays correct if a caller ever - * passes an `###`. + * Anchored on an explicit marker rather than on the first fence whose body + * looks right: content-sniffing is how an example block hijacked the install + * roster. Fences are walked with `fenceMask`, which parses info strings and + * ~~~ fences per CommonMark, so a decorated fence earlier in the file cannot + * frame-shift the pairing. Only blank lines may sit between the marker and + * its fence — anything else is a moved marker, reported as a missing anchor. */ -function section(heading) { - const depth = heading.match(/^#+/)[0].length; - const escaped = heading.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - const start = new RegExp(`^${escaped}[ \\t]*$`, 'm'); - const end = new RegExp(`^#{1,${depth}} `, 'm'); +function markerFence(marker) { + const markerLine = ``; return text => { - const opened = start.exec(text); - if (!opened) return null; - const rest = text.slice(opened.index + opened[0].length); - const closed = end.exec(rest); - return closed ? rest.slice(0, closed.index) : rest; + const { lines } = splitLines(text); + const mask = fenceMask(lines); + const at = lines.findIndex((l, i) => !mask[i] && l.trim() === markerLine); + if (at === -1) return null; + + let open = -1; + for (let i = at + 1; i < lines.length; i++) { + if (mask[i]) { + open = i; + break; + } + if (lines[i].trim() !== '') return null; + } + if (open === -1) return null; + + let close = open + 1; + while (close < lines.length && mask[close]) close++; + // mask marks both delimiters; the last masked line is the closing fence + // unless the fence runs unterminated to EOF. + const body = lines.slice(open + 1, close); + if ( + body.length && + /^ {0,3}(`{3,}|~{3,})[ \t]*$/.test(body[body.length - 1]) + ) { + body.pop(); + } + return body.join('\n'); }; } /** - * Skill directory names have to be expressible in every roster pattern above, - * all of which capture a kebab-case token. A name outside that shape would make - * the check permanently unsatisfiable: it would report the skill missing, and - * the line it tells you to paste still would not match. - */ -const SKILL_DIR_NAME = /^[a-z][a-z0-9]*(-[a-z0-9]+)*$/; - -/** - * Every candidate directory under `skills/`, with whether it actually holds a - * SKILL.md. Returned together so the caller can complain about the ones that - * do not, instead of silently skipping them — a half-landed skill directory is - * exactly the silent omission this check exists to end. + * A markdown section by heading, up to the next heading of the same depth or + * shallower — counting only headings OUTSIDE fenced blocks. The first version + * matched `^#{1,depth} ` anywhere, so a flush-left `# comment` inside a fenced + * bash example truncated the scope: everything below the fence silently went + * unchecked. `fenceMask` is the same guard the release-docs extractors use. */ -export async function readSkillDirs(dir = join(REPO, 'skills')) { - const found = []; - for (const entry of await readdir(dir, { withFileTypes: true })) { - if (!entry.isDirectory()) continue; +function section(heading) { + const depth = heading.match(/^#+/)[0].length; + const endRe = new RegExp(`^#{1,${depth}} `); + return text => { + const { lines } = splitLines(text); + const mask = fenceMask(lines); + const start = lines.findIndex( + (l, i) => !mask[i] && l.trimEnd() === heading + ); + if (start === -1) return null; - let hasSkillFile = true; - try { - await access(join(dir, entry.name, 'SKILL.md')); - } catch { - hasSkillFile = false; + let end = lines.length; + for (let i = start + 1; i < lines.length; i++) { + if (!mask[i] && endRe.test(lines[i])) { + end = i; + break; + } } + return lines.slice(start + 1, end).join('\n'); + }; +} - // A dotted directory is not a skill by convention, but none of the three - // consumers tests for the dot — each takes any directory holding a - // SKILL.md. So `.draft/SKILL.md` really would be bundled and indexed, and - // skipping it here would hide exactly the silent omission this check - // exists to end. Reported, and SKILL_DIR_NAME then rejects the name. - // A dotted directory WITHOUT a SKILL.md is just tooling, so it is ignored. - if (entry.name.startsWith('.') && !hasSkillFile) continue; +// The roster definition itself lives in scripts/lib/skills.mjs, shared with +// both sync scripts and gen-mcp-index.mjs so the four consumers cannot drift. +// Re-exported here because this check and its tests are the historical home. +export { SKILL_DIR_NAME, rosterSkillNames }; - found.push({ name: entry.name, hasSkillFile }); - } - return found.sort((a, b) => a.name.localeCompare(b.name)); +export async function readSkillDirs(dir = join(REPO, 'skills')) { + return libReadSkillDirs(dir); } -/** - * The skill roster, READ from the directory — never a hardcoded list, for the - * same reason gen-mcp-index.mjs says so at its own reader. - */ export async function readSkillNames(dir = join(REPO, 'skills')) { - return (await readSkillDirs(dir)) - .filter(d => d.hasSkillFile) - .map(d => d.name); + return libReadSkillNames(dir); } /** @@ -2104,6 +2159,7 @@ export function skillDirViolations(dirs) { export function rosterViolations(skills, sources) { const violations = []; const known = new Set(skills); + const orderGroups = {}; for (const roster of SKILL_ROSTERS) { const text = sources?.[roster.file]; @@ -2130,7 +2186,21 @@ export function rosterViolations(skills, sources) { continue; } - const listed = new Set([...scoped.matchAll(copy.list)].map(m => m[1])); + // A copy either captures tokens with a regex or parses them with a + // `names` function (the AGENTS.md comma list); `fromToken` maps slugs + // back to directory names for the docs-site copies. + const tokens = copy.names + ? copy.names(scoped) + : [...scoped.matchAll(copy.list)].map(m => m[1]); + const seq = copy.fromToken ? tokens.map(copy.fromToken) : tokens; + const listed = new Set(seq); + + if (copy.orderGroup) { + (orderGroups[copy.orderGroup] ??= []).push({ + file: roster.file, + seq, + }); + } const missing = skills.filter(name => !listed.has(name)); if (missing.length) { @@ -2154,9 +2224,96 @@ export function rosterViolations(skills, sources) { } } + // Copies in an order group are byte-identical by design; when their + // memberships agree but their orders do not, the Set comparison above is + // blind to the drift, so it is reported here. + for (const [group, entries] of Object.entries(orderGroups)) { + const [first, ...rest] = entries; + for (const other of rest) { + const sameMembers = + first.seq.length === other.seq.length && + [...first.seq].sort().join('\n') === [...other.seq].sort().join('\n'); + if (sameMembers && first.seq.join('\n') !== other.seq.join('\n')) { + violations.push( + `${other.file}: ${group} lists the same skills as ${first.file} ` + + `in a different order. The blocks are copies on purpose — match ` + + `the order in ${first.file}.` + ); + } + } + } + return violations; } +/** + * The per-skill docs pages, keyed by slug (directory name minus `bestax-`) — + * the same transform gen-mcp-index.mjs ships as `promptName`. Pure, like + * rosterViolations, so the branches can be driven with fixtures. + */ +export function skillsPageViolations(skills, pageFiles) { + if (!Array.isArray(pageFiles)) { + return [ + 'docs/docs/skills/: could not be read, so the per-skill docs pages ' + + 'went unchecked.', + ]; + } + const violations = []; + const slugs = new Set( + pageFiles + .filter(f => /\.(md|mdx)$/.test(f)) + .map(f => f.replace(/\.(md|mdx)$/, '')) + ); + for (const name of skills) { + const slug = name.replace(/^bestax-/, ''); + if (!slugs.has(slug)) { + violations.push( + `docs/docs/skills/${slug}.mdx: missing — every skill has a docs page ` + + `named by its slug (directory name minus "bestax-"). Add the page, ` + + `its docs/sidebars.js entry, and its intro bullet.` + ); + } + } + const knownSlugs = new Set(skills.map(n => n.replace(/^bestax-/, ''))); + for (const slug of slugs) { + if (slug === 'intro' || knownSlugs.has(slug)) continue; + violations.push( + `docs/docs/skills/${slug}: no skill directory maps to this page ` + + `(directory name minus "bestax-"). Remove the page, or restore the ` + + `skill.` + ); + } + return violations; +} + +/** + * Frontmatter `name:` must equal the directory name. Every prose roster and + * install line is held to the DIRECTORY name, while gen-mcp-index.mjs keys the + * shipped MCP manifest off the FRONTMATTER (`fm.name || name`, and promptName + * derives from it) — with no gate, one edit ships two disagreeing rosters + * while everything stays green. + */ +export function frontmatterNameViolations(entries) { + const violations = []; + for (const { name, fmName } of entries) { + if (fmName && fmName !== name) { + violations.push( + `skills/${name}/SKILL.md: frontmatter says "name: ${fmName}" but the ` + + `directory is ${name}. The rosters follow the directory and the MCP ` + + `manifest follows the frontmatter, so a mismatch ships two ` + + `disagreeing rosters. Rename one to match the other.` + ); + } + } + return violations; +} + +function skillFrontmatterName(text) { + const fm = text.match(/^---\r?\n([\s\S]*?)\r?\n---/); + if (!fm) return null; + return fm[1].match(/^name:\s*['"]?([^'"\r\n]+?)['"]?\s*$/m)?.[1] ?? null; +} + async function checkSkillsRoster() { const skillsDir = join(REPO, 'skills'); @@ -2176,10 +2333,9 @@ async function checkSkillsRoster() { const violations = skillDirViolations(dirs); // Only names a roster could actually express take part in the comparison. // An unexpressible one already has its own violation above, and asking nine - // prose rosters to name something they cannot spell would bury it. - const skills = dirs - .filter(d => d.hasSkillFile && SKILL_DIR_NAME.test(d.name)) - .map(d => d.name); + // prose rosters to name something they cannot spell would bury it. The same + // derivation the tests use — rosterSkillNames — so the two cannot diverge. + const skills = rosterSkillNames(dirs); // Fail rather than pass vacuously: an empty roster would make every // comparison below trivially satisfied. if (!skills.length) { @@ -2190,6 +2346,30 @@ async function checkSkillsRoster() { return violations; } + const fmEntries = []; + for (const name of skills) { + try { + fmEntries.push({ + name, + fmName: skillFrontmatterName( + await readFile(join(skillsDir, name, 'SKILL.md'), 'utf8') + ), + }); + } catch { + // The dir listing said SKILL.md exists; a read race is not this + // check's problem. + } + } + violations.push(...frontmatterNameViolations(fmEntries)); + + let pageFiles = null; + try { + pageFiles = await readdir(join(REPO, 'docs', 'docs', 'skills')); + } catch { + // Reported by skillsPageViolations rather than skipped. + } + violations.push(...skillsPageViolations(skills, pageFiles)); + const sources = {}; for (const { file } of SKILL_ROSTERS) { try { diff --git a/scripts/gen-mcp-index.mjs b/scripts/gen-mcp-index.mjs index a32b428ef..64124d08c 100644 --- a/scripts/gen-mcp-index.mjs +++ b/scripts/gen-mcp-index.mjs @@ -42,6 +42,7 @@ import { fileURLToPath, pathToFileURL } from 'node:url'; import { createRequire } from 'node:module'; import { sectionSpans, sectionBody, firstSentence } from './lib/api-page.mjs'; +import { readSkillNames } from './lib/skills.mjs'; import { extractComponent } from './lib/props-extract.mjs'; import { componentVars } from './lib/scss-vars.mjs'; import { @@ -315,12 +316,15 @@ function propRow(r) { // --------------------------------------------------------------------------- -/** The skills roster, READ from the directory — never a hardcoded list. */ +/** + * The skills roster, READ from the directory — never a hardcoded list. The + * predicate (a directory holding a SKILL.md) lives in scripts/lib/skills.mjs, + * shared with both sync scripts and check-conformance. + */ async function readSkills() { const out = []; - for (const name of await subdirs(SKILLS_DIR)) { + for (const name of await readSkillNames(SKILLS_DIR)) { const skillFile = join(SKILLS_DIR, name, 'SKILL.md'); - if (!existsSync(skillFile)) continue; const src = await readFile(skillFile, 'utf8'); const fm = frontmatter(src); const listing = async sub => { diff --git a/scripts/lib/skills.mjs b/scripts/lib/skills.mjs new file mode 100644 index 000000000..89bc9ebc1 --- /dev/null +++ b/scripts/lib/skills.mjs @@ -0,0 +1,113 @@ +// The ONE definition of "what is a skill": a directory under `skills/` holding +// a SKILL.md. Before this file, four consumers each hand-rolled the predicate +// (create-bestax/scripts/sync-skills.mjs, bestax-mcp/scripts/sync-skills.mjs, +// scripts/gen-mcp-index.mjs, scripts/check-conformance.mjs) and had already +// drifted to three different sort comparators — the same shape of drift +// scripts/lib/shell-words.mjs (#436) exists to prevent. All four import from +// here now; a predicate change lands once or not at all. + +import { readdir, access } from 'node:fs/promises'; +import { join } from 'node:path'; +import { execFileSync } from 'node:child_process'; + +/** + * Skill directory names have to be expressible in every prose roster pattern, + * all of which capture a kebab-case token. A name outside that shape would + * make the skills-roster check permanently unsatisfiable: it would report the + * skill missing, and the line it tells you to paste still would not match. + */ +export const SKILL_DIR_NAME = /^[a-z][a-z0-9]*(-[a-z0-9]+)*$/; + +/** + * Deterministic, locale-independent comparator. Never localeCompare: ICU + * collation varies with the machine's locale (punctuation can be ignorable at + * primary strength), so localeCompare orderings differ across CI runners — + * the flake gen-mcp-index.mjs banned for the same reason. + */ +export function byCodePoint(a, b) { + return a < b ? -1 : a > b ? 1 : 0; +} + +/** + * Every candidate directory under `skills/`, with whether it actually holds a + * SKILL.md. Returned together so callers can complain about the ones that do + * not, instead of silently skipping them — a half-landed skill directory is + * exactly the silent omission the roster check exists to end. + * + * A dotted directory WITHOUT a SKILL.md is tooling and is ignored; one WITH a + * SKILL.md is reported, because every consumer of this predicate really would + * bundle it. + */ +export async function readSkillDirs(dir) { + const found = []; + for (const entry of await readdir(dir, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + + let hasSkillFile = true; + try { + await access(join(dir, entry.name, 'SKILL.md')); + } catch { + hasSkillFile = false; + } + + if (entry.name.startsWith('.') && !hasSkillFile) continue; + + found.push({ name: entry.name, hasSkillFile }); + } + return found.sort((a, b) => byCodePoint(a.name, b.name)); +} + +/** + * The bundling view: what the sync scripts copy and gen-mcp-index indexes. + * READ from the directory — never a hardcoded list (#540). + */ +export async function readSkillNames(dir) { + return (await readSkillDirs(dir)) + .filter(d => d.hasSkillFile) + .map(d => d.name); +} + +/** + * The comparison view the skills-roster check holds prose to: bundled AND + * expressible. An unexpressible name gets its own violation instead — asking + * nine prose rosters to name something they cannot spell would bury it. + * Exported so the check and its tests derive the set the same way. + */ +export function rosterSkillNames(dirs) { + return dirs + .filter(d => d.hasSkillFile && SKILL_DIR_NAME.test(d.name)) + .map(d => d.name); +} + +/** + * The names in `names` whose SKILL.md is not in `trackedPaths` (paths relative + * to the skills dir, as `git ls-files` prints them). Pure so it can be tested + * without a fixture repository. + */ +export function untrackedAgainst(trackedPaths, names) { + const tracked = new Set(trackedPaths); + return names.filter(name => !tracked.has(`${name}/SKILL.md`)); +} + +/** + * The vetting gate the deleted allowlist used to be (#541 review): discovery + * bundles whatever is on disk, so an untracked scratch directory with a + * SKILL.md would ship in a local build or a manual publish with no gate in the + * path — CI's skills-roster check only ever sees committed state. Bundlers + * call this and refuse untracked skills; `git add` is the act of vetting. + * + * Returns [] when git is unavailable or the tree is not a repository (a + * consumer building from an exported tarball has nothing to vet against). + */ +export function untrackedSkillDirs(skillsDir, names) { + let output; + try { + output = execFileSync('git', ['-C', skillsDir, 'ls-files', '--', '.'], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }); + } catch { + return []; + } + return untrackedAgainst(output.split('\n'), names); +} diff --git a/scripts/skills-roster.test.mjs b/scripts/skills-roster.test.mjs index bb7a8a60b..048191091 100644 --- a/scripts/skills-roster.test.mjs +++ b/scripts/skills-roster.test.mjs @@ -20,9 +20,13 @@ import { SKILL_ROSTERS, readSkillDirs, readSkillNames, + rosterSkillNames, rosterViolations, skillDirViolations, + skillsPageViolations, + frontmatterNameViolations, } from './check-conformance.mjs'; +import { untrackedAgainst } from './lib/skills.mjs'; const repoFile = rel => readFileSync(fileURLToPath(new URL(`../${rel}`, import.meta.url)), 'utf8'); @@ -311,3 +315,252 @@ test('an empty roster of skills does not pass vacuously', () => { v.join('\n') ); }); + +// --- regressions from the #541 review ------------------------------------------ + +test('an example fence ABOVE the real install block cannot hijack the anchor', () => { + // The first installFence returned the first fence containing "--skill ", so + // a quick-start example above the real block silently became the validated + // roster — dark in the stale direction, and reporting a complete block as + // six skills short in the other. + const decoyed = REAL['docs/docs/skills/intro.md'] + .replace( + 'Install one with', + 'Quick start:\n\n```bash\nnpx skills add https://github.com/allxsmith/bestax --skill bestax-layout-scaffold\n```\n\nInstall one with' + ) + .replace(/^npx skills add .*--skill bestax-form\n/m, ''); + const v = rosterViolations( + SKILLS, + withFile('docs/docs/skills/intro.md', decoyed) + ); + assert.equal(v.length, 1, v.join('\n')); + assert.match(v[0], /skills-add block/); + assert.match(v[0], /does not name bestax-form/); +}); + +test('a ghost skill in the real block is caught despite a decoy example above', () => { + // The false-green half of the same bug: with the decoy anchored, a deleted + // skill still advertised in the REAL block produced zero violations. + const decoyed = REAL['docs/docs/skills/intro.md'] + .replace( + 'Install one with', + 'Quick start:\n\n```bash\nnpx skills add https://github.com/allxsmith/bestax --skill bestax-layout-scaffold\n```\n\nInstall one with' + ) + .replace( + '--skill bestax-form', + '--skill bestax-form\nnpx skills add https://github.com/allxsmith/bestax --skill bestax-ghost' + ); + const v = rosterViolations( + SKILLS, + withFile('docs/docs/skills/intro.md', decoyed) + ); + assert.equal(v.length, 1, v.join('\n')); + assert.match(v[0], /still names bestax-ghost/); +}); + +test('a metadata info string on an earlier fence does not frame-shift the pairing', () => { + // /^```[a-z]*\n/ could not open ```bash title="…", so every later fence + // paired against the wrong delimiter and the anchor reported itself gone. + // ~1,155 fences across docs/docs use metadata-style openers. + const decorated = REAL['docs/docs/guides/llms/index.md'].replace( + 'Install one with', + '\n```bash title="quick start" showLineNumbers\nnpx skills add x --skill bestax-icons\n```\n\nInstall one with' + ); + assert.notEqual(decorated, REAL['docs/docs/guides/llms/index.md']); + assert.deepEqual( + rosterViolations( + SKILLS, + withFile('docs/docs/guides/llms/index.md', decorated) + ), + [] + ); +}); + +test('a removed marker is a missing anchor, not a silent skip', () => { + const unmarked = REAL['skills/README.md'].replace( + '\n\n', + '' + ); + const v = rosterViolations(SKILLS, withFile('skills/README.md', unmarked)); + assert.equal(v.length, 1, v.join('\n')); + assert.match(v[0], /Install block/); + assert.match(v[0], /anchors on is gone/); +}); + +test('a fenced # comment inside the AI-skills section does not truncate it', () => { + // section()'s end terminator was fence-blind: a flush-left "# comment" in a + // fenced bash example ended the scope, so the roster below it reported as + // entirely missing (or, after the bullets, silently went unchecked). + const fenced = REAL['create-bestax/src/constants.ts'].replace( + '## AI skills', + '## AI skills\n\n```bash\n# install more skills later\nnpx skills add x\n```' + ); + assert.notEqual(fenced, REAL['create-bestax/src/constants.ts']); + assert.deepEqual( + rosterViolations( + SKILLS, + withFile('create-bestax/src/constants.ts', fenced) + ), + [] + ); +}); + +test('dropping the serial comma does not lose the last two names', () => { + // The old comma-list regex only recognized a conjunction after a comma, so + // "…, x and y" un-matched BOTH final names and a complete roster reported + // two skills missing. + const plain = REAL['bulma-ui/AGENTS.md'].replace( + 'bestax-optimize, bestax-theming', + 'bestax-optimize and bestax-theming' + ); + assert.notEqual(plain, REAL['bulma-ui/AGENTS.md']); + assert.deepEqual( + rosterViolations(SKILLS, withFile('bulma-ui/AGENTS.md', plain)), + [] + ); +}); + +test('install blocks with the same members but different orders are caught', () => { + // The copies are byte-identical on purpose; a Set comparison alone let + // docs/docs/guides/llms/index.md drift into a different ordering unseen. + const swapped = REAL['docs/docs/guides/llms/index.md'].replace( + /--skill bestax-migrate(\n.*--skill )bestax-optimize/, + '--skill bestax-optimize$1bestax-migrate' + ); + assert.notEqual(swapped, REAL['docs/docs/guides/llms/index.md']); + const v = rosterViolations( + SKILLS, + withFile('docs/docs/guides/llms/index.md', swapped) + ); + assert.equal(v.length, 1, v.join('\n')); + assert.match(v[0], /different order/); +}); + +test('a kebab-case cell in an unrelated table is not a roster row', () => { + // The table patterns used to scan the whole file: any backticked kebab-case + // first cell anywhere satisfied (or polluted) the Agent Skills table. + const decoyed = + REAL['bulma-ui/README.md'] + + '\n| Hook | Notes |\n| ----------- | ----- |\n| `use-theme` | … |\n'; + assert.deepEqual( + rosterViolations(SKILLS, withFile('bulma-ui/README.md', decoyed)), + [] + ); +}); + +test('a deleted roster row is caught even when the name appears in another table', () => { + // The silent-drift direction of the same bug: bestax-migrate is also a + // package name, so a row for it elsewhere kept the check green after the + // real roster row was deleted. + const gutted = + REAL['bulma-ui/README.md'] + .split('\n') + .filter(l => !/^\s*\|\s*`bestax-migrate`/.test(l)) + .join('\n') + + '\n| Package | Notes |\n| ---------------- | ----- |\n| `bestax-migrate` | … |\n'; + const v = rosterViolations(SKILLS, withFile('bulma-ui/README.md', gutted)); + assert.equal(v.length, 1, v.join('\n')); + assert.match(v[0], /does not name bestax-migrate/); +}); + +test('an indented dir line in an unrelated fence is not a Layout tree entry', () => { + const decoyed = + REAL['skills/README.md'] + + '\nA scaffolded app looks like:\n\n```\nmy-app/\n skills/\n launch/\n```\n'; + assert.deepEqual( + rosterViolations(SKILLS, withFile('skills/README.md', decoyed)), + [] + ); +}); + +test('the docs-site surfaces are held through the slug transform', () => { + const noSidebar = REAL['docs/sidebars.js'].replace( + /^\s*'skills\/optimize',\n/m, + '' + ); + assert.notEqual(noSidebar, REAL['docs/sidebars.js']); + const v = rosterViolations(SKILLS, withFile('docs/sidebars.js', noSidebar)); + assert.equal(v.length, 1, v.join('\n')); + assert.match(v[0], /skillsSidebar/); + assert.match(v[0], /does not name bestax-optimize/); + + const noBullet = REAL['docs/docs/skills/intro.md'].replace( + /^- \*\*\[Migrate\]\(\.\/migrate\)\*\*/m, + '- **Migrate** (link dropped)' + ); + assert.notEqual(noBullet, REAL['docs/docs/skills/intro.md']); + const w = rosterViolations( + SKILLS, + withFile('docs/docs/skills/intro.md', noBullet) + ); + assert.equal(w.length, 1, w.join('\n')); + assert.match(w[0], /bullet roster/); + assert.match(w[0], /does not name bestax-migrate/); +}); + +test('per-skill docs pages: missing and orphaned pages are both caught', () => { + const v = skillsPageViolations( + ['bestax-form', 'bestax-icons'], + ['intro.md', 'form.mdx'] + ); + assert.equal(v.length, 1, v.join('\n')); + assert.match(v[0], /docs\/docs\/skills\/icons\.mdx: missing/); + + const w = skillsPageViolations( + ['bestax-form'], + ['intro.md', 'form.mdx', 'theming.mdx'] + ); + assert.equal(w.length, 1, w.join('\n')); + assert.match(w[0], /docs\/docs\/skills\/theming/); + assert.match(w[0], /no skill directory maps/); + + // intro is the index page, never a skill; an unreadable dir is reported. + assert.deepEqual( + skillsPageViolations(['bestax-form'], ['intro.md', 'form.mdx']), + [] + ); + assert.match( + skillsPageViolations(['bestax-form'], null)[0], + /went unchecked/ + ); +}); + +test('a frontmatter name that disagrees with the directory is caught', () => { + // Every prose roster follows the DIRECTORY name while gen-mcp-index keys the + // shipped MCP manifest off the FRONTMATTER — without this gate a rename in + // one place ships two disagreeing rosters with everything green. + const v = frontmatterNameViolations([ + { name: 'bestax-optimize', fmName: 'bestax-css-optimize' }, + { name: 'bestax-form', fmName: 'bestax-form' }, + { name: 'bestax-icons', fmName: null }, + ]); + assert.equal(v.length, 1, v.join('\n')); + assert.match(v[0], /bestax-optimize/); + assert.match(v[0], /bestax-css-optimize/); +}); + +test('the tests and the check derive the comparison set the same way', () => { + // readSkillNames is the bundling view (what the sync scripts copy); + // rosterSkillNames additionally drops names the prose patterns cannot + // express. Using the former for the roster comparison would demand prose + // name `.draft` — the unsatisfiable noise the check's own comment avoids. + const dirs = [ + { name: '.draft', hasSkillFile: true }, + { name: 'bestax-form', hasSkillFile: true }, + { name: 'half-landed', hasSkillFile: false }, + ]; + assert.deepEqual(rosterSkillNames(dirs), ['bestax-form']); +}); + +test('untracked skill directories are flagged against the tracked set', () => { + const tracked = [ + 'bestax-form/SKILL.md', + 'bestax-form/references/x.md', + 'bestax-icons/SKILL.md', + ]; + assert.deepEqual( + untrackedAgainst(tracked, ['bestax-form', 'bestax-icons', 'bestax-wip']), + ['bestax-wip'] + ); + assert.deepEqual(untrackedAgainst(tracked, ['bestax-form']), []); +}); diff --git a/skills/README.md b/skills/README.md index 7ddb92d4e..d48a71aaf 100644 --- a/skills/README.md +++ b/skills/README.md @@ -27,6 +27,8 @@ examples — useful if your client speaks MCP but not skills. Add a skill to your agent with the [`skills`](https://skills.sh/) CLI: + + ```sh npx skills add https://github.com/allxsmith/bestax --skill bestax-custom-component npx skills add https://github.com/allxsmith/bestax --skill bestax-form @@ -39,6 +41,8 @@ npx skills add https://github.com/allxsmith/bestax --skill bestax-optimize ## Layout + + ``` skills/ bestax-custom-component/ From e59c0cd01c1493694c1615c109b20b0aa633a6d7 Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sat, 22 Aug 2026 15:32:02 -0400 Subject: [PATCH 03/29] docs: repair skills-roster provenance pointers and bundling claims MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - create-bestax/CLAUDE.md: restore the duty to keep docs pages that assert bundling (skills/migrate.mdx, skills/intro.md) in agreement if a per-skill opt-out is ever exercised; trim the #540/#385 provenance retelling to a pointer at its canonical home, the sync script header. - skills/CLAUDE.md: same trim, and replace the incorrect claim that the docs-site surfaces cannot be keyed to directory names — they are held through the slug transform now. - bestax-mcp/CLAUDE.md + server.ts comment: stop pointing at the roster census #541 deleted; cite the skills-roster conformance check instead. --- bestax-mcp/CLAUDE.md | 6 ++++-- bestax-mcp/src/server.ts | 6 ++++-- create-bestax/CLAUDE.md | 10 +++++----- skills/CLAUDE.md | 20 ++++++++++---------- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/bestax-mcp/CLAUDE.md b/bestax-mcp/CLAUDE.md index 365bf0738..3f6b824f4 100644 --- a/bestax-mcp/CLAUDE.md +++ b/bestax-mcp/CLAUDE.md @@ -43,8 +43,10 @@ the API pages and the skill catalog. spends the context the client needs for the actual task. - **No hardcoded roster.** Prompts, resources and skill lookups are all derived from `data/skills.json`, which the generator builds by reading the `skills/` - directory. `skills/CLAUDE.md` already counts five places the roster is - duplicated; do not add a sixth. + directory. The rule repo-wide is derive, don't list (#540); the prose copies + that cannot be derived are held to the directory by the `skills-roster` + conformance check (`pnpm check:conformance --only=skills-roster`). Do not add + a hand-maintained copy here for that check to chase. - **Nothing on stdout but JSON-RPC.** stdout _is_ the transport. Diagnostics go to stderr — a stray `console.log` surfaces to the user as an unexplained disconnect. diff --git a/bestax-mcp/src/server.ts b/bestax-mcp/src/server.ts index f87a162db..128e88f9d 100644 --- a/bestax-mcp/src/server.ts +++ b/bestax-mcp/src/server.ts @@ -778,8 +778,10 @@ export async function createServer( // -- Prompts, one per skill ---------------------------------------------- // - // Generated from the manifest, never listed here. skills/CLAUDE.md counts the - // roster as already duplicated in five places; this would have been a sixth. + // Generated from the manifest, never listed here. The roster is derived, not + // hand-listed (#540); the prose copies that cannot be derived are held to the + // directory by the skills-roster conformance check (`pnpm check:conformance + // --only=skills-roster`). A list here would be one more copy for it to chase. for (const skill of skills) { server.registerPrompt( skill.promptName, diff --git a/create-bestax/CLAUDE.md b/create-bestax/CLAUDE.md index 495db9bbd..441f97876 100644 --- a/create-bestax/CLAUDE.md +++ b/create-bestax/CLAUDE.md @@ -19,11 +19,11 @@ path (`-y` + flags, no TTY) must never hang or regress (#192). - `pnpm build` and `prepack` run `scripts/sync-skills.mjs`, which copies **every directory under the repo-root `skills/` that holds a `SKILL.md`** into the package. The roster is read, - not listed (#540): there is no allowlist, so a new skill bundles without anyone remembering - to add it, and a deleted one stops bundling. That replaced a hardcoded `SKILLS` array which - errored on a listed-but-missing skill and said nothing about the reverse. Every skill now - bundles; #385 settled only `bestax-migrate`'s case and kept the per-skill rule, so dropping - that rule is #540's decision, taken on #385's reasoning. What is _not_ derivable is the + not listed (#540): every skill bundles by construction, with no allowlist to join. Full + provenance (#385 vs #540) and the slot for a future per-skill opt-out live in that script's + header. If such an opt-out is ever exercised, the docs pages that assert bundling — + today `docs/docs/skills/migrate.mdx` and `docs/docs/skills/intro.md` — must be corrected in + the same change: no automated check reads those claims. What is _not_ derivable is the prose rosters, this package's `CLAUDE_MD` roster in `src/constants.ts` among them. The `skills-roster` conformance check holds every one of them to the directory in both directions; `SKILL_ROSTERS` in `scripts/check-conformance.mjs` is the authoritative list, and its failure diff --git a/skills/CLAUDE.md b/skills/CLAUDE.md index 3927743a0..7b9017bf6 100644 --- a/skills/CLAUDE.md +++ b/skills/CLAUDE.md @@ -14,14 +14,13 @@ like library code: they get bug reports (#194, #195, #196, #197) and ship to use ## Adding a skill -The roster is **read, not listed**. `create-bestax/scripts/sync-skills.mjs` and +The roster is **read, not listed** (#540). `create-bestax/scripts/sync-skills.mjs` and `bestax-mcp/scripts/sync-skills.mjs` each copy every directory holding a `SKILL.md` into their package, and `scripts/gen-mcp-index.mjs` discovers the same set to generate the MCP manifest -(it indexes, it does not bundle). A new skill therefore reaches all three by construction -(#540): there is no allowlist to join, and no per-skill bundling call to make. Note the -provenance — #385 settled only `bestax-migrate`'s case and kept the per-skill rule; dropping that -rule is #540's own decision, taken on the reasoning #385 gave for it ("a per-skill carve-out is -exactly the kind of thing that drifts"). +(it indexes, it does not bundle). A new skill therefore reaches all three by construction: +there is no allowlist to join, and no per-skill bundling call to make. Full provenance +(#385 vs #540) and the slot for a future per-skill opt-out live in +`create-bestax/scripts/sync-skills.mjs`'s header. What stays hand-written is prose — rosters spread across the READMEs, the docs, and the scaffolded `CLAUDE_MD`. `pnpm check:conformance --only=skills-roster` holds those to the @@ -30,10 +29,11 @@ skill you deleted. `SKILL_ROSTERS` in `scripts/check-conformance.mjs` is the aut and the failure names every file you missed, so this file deliberately keeps no roster of rosters — that would be the same bug one level up. -Deliberately outside that check, so still yours to remember: a docs page under -`docs/docs/skills/`, its entry in `docs/sidebars.js`, and the intro's bullet roster. Those key -off page slugs rather than skill directory names, and holding them would amount to requiring a -docs page per skill, which is a separate rule nobody has asked for. +The docs-site surfaces — the per-skill page under `docs/docs/skills/`, its entry in +`docs/sidebars.js`, and the intro's bullet roster — are held by the same check through the +slug transform (directory name minus the `bestax-` prefix, exactly what `gen-mcp-index.mjs` +ships as `promptName`). A new skill fails conformance until its docs page, sidebar entry, and +intro bullet exist. ## Rules From 805f90af1498252533b3fc9f505c69c11c4e9a4c Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sat, 22 Aug 2026 15:20:02 -0400 Subject: [PATCH 04/29] docs: add telemetry disclosure guide Canonical disclosure for the CLIs' opt-in anonymous telemetry: exact fields per tool, what is never collected, consent controls (flags, BESTAX_TELEMETRY, DO_NOT_TRACK, config path), retention, and the MCP server's no-telemetry stance. Linked from every consent prompt. --- docs/docs/guides/telemetry.md | 115 ++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 docs/docs/guides/telemetry.md diff --git a/docs/docs/guides/telemetry.md b/docs/docs/guides/telemetry.md new file mode 100644 index 000000000..970b268fa --- /dev/null +++ b/docs/docs/guides/telemetry.md @@ -0,0 +1,115 @@ +--- +title: Telemetry +sidebar_label: Telemetry +sidebar_position: 11 +--- + +# Telemetry + +Two bestax CLIs — `create-bestax` and `bestax-migrate` — can send a single +anonymous usage event per run, and only if you say yes first. Telemetry is +**opt-in**, the event contains nothing that could identify you, and it goes to +a first-party endpoint on the maintainer's own infrastructure — no third-party +analytics service is involved. This page is the complete disclosure: everything +that is sent, everything that never is, and every way to control it. + +## What we collect + +Each CLI sends at most one event, only after a **successful** run, and only if +you opted in. + +### create-bestax + +Sent once after a successful scaffold: + +| Field | Values | +| ------------------ | --------------------------------------------------------------------------------- | +| `template` | `vite` or `vite-ts` | +| `bulmaFlavor` | `complete`, `prefixed`, `no-helpers`, `no-helpers-prefixed`, or `no-dark-mode` | +| `iconLibrary` | `none`, `fontawesome`, `mdi`, `ionicons`, `material-icons`, or `material-symbols` | +| `skills` | `true` or `false` — whether the bestax AI skills were installed | +| `packageManager` | `npm`, `pnpm`, `yarn`, or `bun` | +| CLI version | the `create-bestax` version that ran | +| Node major version | e.g. `22` | +| OS platform | the platform name, e.g. `darwin`, `linux`, `win32` | + +### bestax-migrate + +Sent once after a successful run: + +| Field | Values | +| ------------------ | --------------------------------------------------------------------------- | +| `source` | the source library migrated from, e.g. `react-bulma-components` | +| `cssMode` | `bestax`, `bulma`, or `keep` | +| `dry` | `true` or `false` — whether it was a dry run | +| `deps` | `true` or `false` — whether `package.json` dependencies were updated | +| Changed-file count | a bucket — `0`, `1-9`, `10-49`, `50-199`, or `200+` — plus the capped count | +| TODO counts | per migration rule: the rule name and a count, nothing else | + +Never file paths, never file contents, never code. The TODO report printed in +your terminal lists files and lines; the telemetry event carries only rule +names and counts. + +## What we never collect + +By design, an event can never include: + +- names, emails, or any account information +- IP addresses — the endpoint does not read them and does not store them +- user agents (not stored) +- machine identifiers of any kind +- **random IDs of any kind** — no install ID, session ID, or device ID +- file paths, project names, or file contents +- environment variables + +Because no identifier of any kind exists, there is no way to correlate two +events to the same user — even we cannot tell whether two events came from the +same machine. + +## How consent works + +Telemetry is off until you opt in, and you're asked exactly once: + +- `create-bestax` asks at the end of a successful scaffold. +- `bestax-migrate` asks once after a run, and only on an interactive terminal. + +Your answer — yes or no — is stored in `~/.config/bestax/telemetry.json` (or +`$XDG_CONFIG_HOME/bestax/telemetry.json` if you set `XDG_CONFIG_HOME`). Both +CLIs share this file, so answering either one answers for both and you're only +ever asked once. Delete that file to be asked again. + +### Overrides + +| Control | Effect | +| ------------------------------------------- | --------------------------------------------------------------------------------------- | +| `--telemetry` / `--no-telemetry` | Flags on both CLIs — set the choice for this run **and persist it** to the consent file | +| `BESTAX_TELEMETRY=1` / `BESTAX_TELEMETRY=0` | Enable or disable for this run only — never persisted | +| `DO_NOT_TRACK=1` | Disables everything, including the consent prompt itself | + +## Where the data goes + +Events go to `https://bestax.io/api/t`, a first-party Cloudflare Worker running +on the maintainer's own Cloudflare account. It writes into Workers Analytics +Engine as aggregate counts, retained for roughly 90 days. No third-party +analytics service ever sees the data. + +As a server-side privacy backstop, the endpoint validates every field of every +event against a strict allowlist and rejects anything else — so even a modified +or buggy client cannot get extra data stored. + +## The MCP server + +`bestax-mcp` sends **no telemetry at all** and makes no network requests — it +stays fully offline. The documentation links it prints include a +`utm_source=bestax-mcp` query parameter, so if you visit the docs site through +one of those links, that visit is attributable in the site's own traffic +analytics. That is the only measurement, and it happens on bestax.io like any +other page visit. + +## Why we collect this + +The answers decide where maintenance effort goes: which templates, icon +libraries, Bulma flavors, and migration rules are worth investing in. If +telemetry isn't for you, opting out costs nothing — and the issue tracker is +always the higher-bandwidth channel: +[github.com/allxsmith/bestax/issues](https://github.com/allxsmith/bestax/issues). From 1e5b488f8808ffb09b69fdc7e4f7447f74e0bbf2 Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sat, 22 Aug 2026 15:20:02 -0400 Subject: [PATCH 05/29] feat(create-bestax): add opt-in anonymous telemetry Consent question at the end of a successful scaffold (ask once, saved to ~/.config/bestax/telemetry.json, shared across the CLI family), --telemetry/--no-telemetry flags, DO_NOT_TRACK respected, never prompts without a TTY or under -y (#192). Sends only closed-enum choices: template, bulma flavor, icon library, skills, package manager, versions, OS name. Failures are silent and never affect the scaffold. --- create-bestax/README.md | 12 + create-bestax/src/__tests__/cli.test.ts | 26 ++ .../src/__tests__/project-creator.test.ts | 131 ++++++ create-bestax/src/__tests__/prompts.test.ts | 52 +++ create-bestax/src/__tests__/telemetry.test.ts | 391 ++++++++++++++++++ create-bestax/src/cli.ts | 5 + create-bestax/src/constants.ts | 10 + create-bestax/src/display.ts | 6 + create-bestax/src/project-creator.ts | 38 ++ create-bestax/src/prompts.ts | 20 + create-bestax/src/telemetry.ts | 216 ++++++++++ 11 files changed, 907 insertions(+) create mode 100644 create-bestax/src/__tests__/telemetry.test.ts create mode 100644 create-bestax/src/telemetry.ts diff --git a/create-bestax/README.md b/create-bestax/README.md index ef35761b7..bea5561a7 100644 --- a/create-bestax/README.md +++ b/create-bestax/README.md @@ -151,6 +151,18 @@ A scaffolder runs with write access to your filesystem and picks your starting d Full detail: [`SECURITY.md`](https://github.com/allxsmith/bestax/blob/main/SECURITY.md) · [Security guide](https://bestax.io/docs/guides/security) +## Telemetry + +`create-bestax` can send one **anonymous** usage event after a successful +scaffold — only if you opt in when asked (once, at the end of a scaffold). The +event is just the choices you made — template, Bulma flavor, icon library, +skills, package manager — plus the CLI version, Node major version, and OS +platform; never names, IPs, machine IDs, paths, or file contents, and no +identifier exists that could link two events together. Opt out any time with +`--no-telemetry`, `BESTAX_TELEMETRY=0`, or `DO_NOT_TRACK=1`. + +Full disclosure of every field and control: [Telemetry guide](https://bestax.io/docs/guides/telemetry) + ## Publishing This package uses semantic-release with scope-based rules. Only commits with `feat(create-bestax)` or `fix(create-bestax)` will trigger releases. diff --git a/create-bestax/src/__tests__/cli.test.ts b/create-bestax/src/__tests__/cli.test.ts index f4ff9a014..eefde1a80 100644 --- a/create-bestax/src/__tests__/cli.test.ts +++ b/create-bestax/src/__tests__/cli.test.ts @@ -161,6 +161,32 @@ describe('cli', () => { flags: '-y, --yes', }) ); + expect(options).toContainEqual( + expect.objectContaining({ + flags: '--telemetry', + }) + ); + expect(options).toContainEqual( + expect.objectContaining({ + flags: '--no-telemetry', + }) + ); + }); + + it('should leave telemetry undefined when neither flag is given', () => { + const program = createCLI(); + program.parse(['my-app', '-y'], { from: 'user' }); + expect(program.opts().telemetry).toBeUndefined(); + }); + + it('should parse --telemetry and --no-telemetry into a boolean', () => { + const on = createCLI(); + on.parse(['my-app', '-y', '--telemetry'], { from: 'user' }); + expect(on.opts().telemetry).toBe(true); + + const off = createCLI(); + off.parse(['my-app', '-y', '--no-telemetry'], { from: 'user' }); + expect(off.opts().telemetry).toBe(false); }); it('should have correct template options in help text', () => { diff --git a/create-bestax/src/__tests__/project-creator.test.ts b/create-bestax/src/__tests__/project-creator.test.ts index fde034e20..686945ce9 100644 --- a/create-bestax/src/__tests__/project-creator.test.ts +++ b/create-bestax/src/__tests__/project-creator.test.ts @@ -46,9 +46,15 @@ jest.unstable_mockModule('../file-system.js', () => ({ updatePackageJson: jest.fn(), })); +// Mock telemetry so create() never reads the real config or hits the network +jest.unstable_mockModule('../telemetry.js', () => ({ + reportScaffold: jest.fn(async () => undefined), +})); + const fs = await import('fs-extra'); const prompts = (await import('prompts')).default; const fileSystem = await import('../file-system.js'); +const telemetry = await import('../telemetry.js'); const { ProjectCreator } = await import('../project-creator.js'); const { ICON_LIBRARIES: _ICON_LIBRARIES, BULMA_FLAVORS: _BULMA_FLAVORS } = await import('../constants.js'); @@ -1683,4 +1689,129 @@ describe('ProjectCreator', () => { ); }); }); + + describe('create — telemetry', () => { + const reportScaffold = telemetry.reportScaffold as jest.MockedFunction< + typeof telemetry.reportScaffold + >; + const fullFlags = { + template: 'vite-ts', + bulma: 'prefixed', + icon: 'mdi', + skills: false, + }; + + function mockSuccessfulScaffold(): void { + ( + fileSystem.checkDirectoryExists as jest.MockedFunction< + typeof fileSystem.checkDirectoryExists + > + ).mockResolvedValue(false); + ( + fileSystem.isDirectoryEmpty as jest.MockedFunction< + typeof fileSystem.isDirectoryEmpty + > + ).mockResolvedValue(true); + ( + fs.default.existsSync as jest.MockedFunction + ).mockReturnValue(false); + } + + it('reports the chosen options after a successful scaffold', async () => { + mockSuccessfulScaffold(); + + await projectCreator.create('test-project', { ...fullFlags, yes: true }); + + expect(reportScaffold).toHaveBeenCalledTimes(1); + const [choices, flag, opts] = reportScaffold.mock.calls[0]; + expect(choices).toEqual({ + template: 'vite-ts', + bulmaFlavor: 'prefixed', + iconLibrary: 'mdi', + skills: false, + }); + expect(flag).toBeUndefined(); + // -y means the consent question may not be asked (#192) + expect(opts.interactive).toBe(false); + }); + + it('passes an explicit --telemetry/--no-telemetry flag through', async () => { + mockSuccessfulScaffold(); + + await projectCreator.create('test-project', { + ...fullFlags, + yes: true, + telemetry: false, + }); + + expect(reportScaffold.mock.calls[0][1]).toBe(false); + }); + + it('is interactive only when stdin is a TTY and -y is absent', async () => { + mockSuccessfulScaffold(); + await projectCreator.create('test-project', fullFlags); + expect(reportScaffold.mock.calls[0][2].interactive).toBe(true); + + reportScaffold.mockClear(); + setStdinTTY(false); + mockSuccessfulScaffold(); + await projectCreator.create('test-project', fullFlags); + expect(reportScaffold.mock.calls[0][2].interactive).toBe(false); + }); + + it('the consent callback asks the prompt and acknowledges a yes', async () => { + mockSuccessfulScaffold(); + reportScaffold.mockImplementationOnce(async (_choices, _flag, opts) => { + await expect(opts.promptConsent()).resolves.toBe(true); + }); + (prompts as jest.MockedFunction).mockResolvedValue({ + telemetry: true, + }); + + await projectCreator.create('test-project', fullFlags); + + expect(console.log).toHaveBeenCalledWith( + expect.stringContaining('Opt out anytime') + ); + }); + + it('acknowledges a no without sending', async () => { + mockSuccessfulScaffold(); + reportScaffold.mockImplementationOnce(async (_choices, _flag, opts) => { + await expect(opts.promptConsent()).resolves.toBe(false); + }); + (prompts as jest.MockedFunction).mockResolvedValue({ + telemetry: false, + }); + + await projectCreator.create('test-project', fullFlags); + + expect(console.log).toHaveBeenCalledWith( + expect.stringContaining("we won't ask again") + ); + }); + + it('a cancelled consent prompt prints no ack', async () => { + mockSuccessfulScaffold(); + reportScaffold.mockImplementationOnce(async (_choices, _flag, opts) => { + await expect(opts.promptConsent()).resolves.toBeNull(); + }); + (prompts as jest.MockedFunction).mockResolvedValue({}); + + await projectCreator.create('test-project', fullFlags); + + const logged = (console.log as jest.Mock).mock.calls.flat().join('\n'); + expect(logged).not.toContain('Opt out anytime'); + expect(logged).not.toContain("we won't ask again"); + }); + + it('a telemetry failure never breaks the scaffold', async () => { + mockSuccessfulScaffold(); + reportScaffold.mockRejectedValueOnce(new Error('telemetry boom')); + + await expect( + projectCreator.create('test-project', { ...fullFlags, yes: true }) + ).resolves.toBeUndefined(); + }); + }); }); diff --git a/create-bestax/src/__tests__/prompts.test.ts b/create-bestax/src/__tests__/prompts.test.ts index bcc145d7c..8b95c1ae9 100644 --- a/create-bestax/src/__tests__/prompts.test.ts +++ b/create-bestax/src/__tests__/prompts.test.ts @@ -50,6 +50,7 @@ const { promptBulmaFlavor, promptIconLibrary, promptInstallSkills, + promptTelemetryConsent, } = await import('../prompts.js'); // Import helper functions from cli.js @@ -377,4 +378,55 @@ describe('prompts', () => { } }); }); + + describe('promptTelemetryConsent', () => { + it('returns true when the user opts in', async () => { + (prompts as jest.MockedFunction).mockResolvedValue({ + telemetry: true, + }); + await expect(promptTelemetryConsent()).resolves.toBe(true); + }); + + it('returns false when the user declines', async () => { + (prompts as jest.MockedFunction).mockResolvedValue({ + telemetry: false, + }); + await expect(promptTelemetryConsent()).resolves.toBe(false); + }); + + it('returns null on cancel', async () => { + (prompts as jest.MockedFunction).mockResolvedValue({}); + await expect(promptTelemetryConsent()).resolves.toBeNull(); + }); + + it('is an opt-in confirm defaulting to No, preceded by the notice', async () => { + (prompts as jest.MockedFunction).mockResolvedValue({ + telemetry: false, + }); + await promptTelemetryConsent(); + + const promptCall = (prompts as jest.MockedFunction).mock + .calls[0]?.[0] as { type: string; initial: boolean; message: string }; + expect(promptCall.type).toBe('confirm'); + expect(promptCall.initial).toBe(false); + expect(promptCall.message).toBe('Share anonymous usage stats?'); + expect(console.log).toHaveBeenCalledWith( + expect.stringContaining('https://bestax.io/docs/guides/telemetry') + ); + }); + + it('does not require a TTY (the caller skips it instead)', async () => { + setStdinTTY(false); + const mockExit = jest + .spyOn(process, 'exit') + .mockImplementation((() => undefined) as never); + (prompts as jest.MockedFunction).mockResolvedValue({ + telemetry: true, + }); + + await expect(promptTelemetryConsent()).resolves.toBe(true); + expect(mockExit).not.toHaveBeenCalled(); + mockExit.mockRestore(); + }); + }); }); diff --git a/create-bestax/src/__tests__/telemetry.test.ts b/create-bestax/src/__tests__/telemetry.test.ts new file mode 100644 index 000000000..bd663f4a1 --- /dev/null +++ b/create-bestax/src/__tests__/telemetry.test.ts @@ -0,0 +1,391 @@ +import { + describe, + it, + expect, + jest, + beforeEach, + afterEach, + afterAll, +} from '@jest/globals'; +import { mkdtemp, readFile, writeFile, mkdir } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; + +const { + telemetryConfigPath, + resolveTelemetry, + persistTelemetryDecision, + sendTelemetry, + getToolVersion, + buildScaffoldPayload, + reportScaffold, +} = await import('../telemetry.js'); + +const ENV_KEYS = [ + 'XDG_CONFIG_HOME', + 'DO_NOT_TRACK', + 'BESTAX_TELEMETRY', + 'BESTAX_TELEMETRY_ENDPOINT', +] as const; +const savedEnv: Record = {}; +const originalFetch = globalThis.fetch; + +let configHome: string; +let fetchMock: jest.Mock; + +type FetchArgs = [ + string, + { + method: string; + headers: Record; + body: string; + signal: AbortSignal; + }, +]; + +async function writeConfig(data: unknown): Promise { + await mkdir(join(configHome, 'bestax'), { recursive: true }); + await writeFile( + join(configHome, 'bestax', 'telemetry.json'), + typeof data === 'string' ? data : JSON.stringify(data), + 'utf-8' + ); +} + +async function readConfigFile(): Promise> { + return JSON.parse( + await readFile(join(configHome, 'bestax', 'telemetry.json'), 'utf-8') + ) as Record; +} + +beforeEach(async () => { + for (const key of ENV_KEYS) { + savedEnv[key] = process.env[key]; + delete process.env[key]; + } + configHome = await mkdtemp(join(tmpdir(), 'bestax-telemetry-')); + process.env.XDG_CONFIG_HOME = configHome; + fetchMock = jest.fn(async () => new Response(null, { status: 204 })); + globalThis.fetch = fetchMock as unknown as typeof fetch; +}); + +afterEach(() => { + for (const key of ENV_KEYS) { + if (savedEnv[key] === undefined) delete process.env[key]; + else process.env[key] = savedEnv[key]; + } +}); + +afterAll(() => { + globalThis.fetch = originalFetch; +}); + +describe('telemetryConfigPath', () => { + it('uses XDG_CONFIG_HOME when set', () => { + expect(telemetryConfigPath()).toBe( + join(configHome, 'bestax', 'telemetry.json') + ); + }); + + it('falls back to ~/.config when XDG_CONFIG_HOME is unset', () => { + delete process.env.XDG_CONFIG_HOME; + expect(telemetryConfigPath()).toMatch( + /[/\\]\.config[/\\]bestax[/\\]telemetry\.json$/ + ); + }); +}); + +describe('resolveTelemetry precedence', () => { + it('is undecided (prompt allowed) with no signals at all', async () => { + expect(await resolveTelemetry()).toEqual({ + decision: 'undecided', + source: 'default', + promptAllowed: true, + }); + }); + + it.each([ + [true, 'on'], + [false, 'off'], + ] as const)('flag %s wins over everything', async (flag, decision) => { + process.env.DO_NOT_TRACK = '1'; + process.env.BESTAX_TELEMETRY = flag ? '0' : '1'; + await writeConfig({ version: 1, enabled: !flag }); + expect(await resolveTelemetry(flag)).toEqual({ + decision, + source: 'flag', + promptAllowed: false, + }); + }); + + it('DO_NOT_TRACK beats env var and config', async () => { + process.env.DO_NOT_TRACK = '1'; + process.env.BESTAX_TELEMETRY = '1'; + await writeConfig({ version: 1, enabled: true }); + expect(await resolveTelemetry()).toEqual({ + decision: 'off', + source: 'dnt', + promptAllowed: false, + }); + }); + + it.each(['0', ''])('DO_NOT_TRACK=%j is not a DNT signal', async value => { + process.env.DO_NOT_TRACK = value; + const resolved = await resolveTelemetry(); + expect(resolved.source).toBe('default'); + }); + + it('BESTAX_TELEMETRY=1 forces on without persisting', async () => { + process.env.BESTAX_TELEMETRY = '1'; + await writeConfig({ version: 1, enabled: false }); + expect(await resolveTelemetry()).toEqual({ + decision: 'on', + source: 'env', + promptAllowed: false, + }); + }); + + it('BESTAX_TELEMETRY=0 forces off', async () => { + process.env.BESTAX_TELEMETRY = '0'; + await writeConfig({ version: 1, enabled: true }); + expect((await resolveTelemetry()).decision).toBe('off'); + }); + + it('ignores junk BESTAX_TELEMETRY values', async () => { + process.env.BESTAX_TELEMETRY = 'yes'; + expect((await resolveTelemetry()).source).toBe('default'); + }); + + it.each([ + [true, 'on'], + [false, 'off'], + ] as const)( + 'reads enabled=%s from the config file', + async (enabled, decision) => { + await writeConfig({ version: 1, enabled }); + expect(await resolveTelemetry()).toEqual({ + decision, + source: 'config', + promptAllowed: false, + }); + } + ); + + it('treats corrupt config as undecided', async () => { + await writeConfig('{not json'); + expect((await resolveTelemetry()).decision).toBe('undecided'); + }); + + it('treats a config without a boolean enabled as undecided', async () => { + await writeConfig({ version: 1, enabled: 'yes' }); + expect((await resolveTelemetry()).decision).toBe('undecided'); + }); + + it('treats a non-object config as undecided', async () => { + await writeConfig('null'); + expect((await resolveTelemetry()).decision).toBe('undecided'); + }); +}); + +describe('persistTelemetryDecision', () => { + it('writes the config schema', async () => { + await persistTelemetryDecision(true, 'create-bestax@1.0.0'); + const config = await readConfigFile(); + expect(config.version).toBe(1); + expect(config.enabled).toBe(true); + expect(config.decidedBy).toBe('create-bestax@1.0.0'); + expect(typeof config.decidedAt).toBe('string'); + }); + + it('preserves the noticed map of the sibling CLI', async () => { + await writeConfig({ + version: 1, + noticed: { 'bestax-migrate': '2026-01-01T00:00:00.000Z' }, + }); + await persistTelemetryDecision(false, 'create-bestax@1.0.0'); + const config = await readConfigFile(); + expect(config.enabled).toBe(false); + expect(config.noticed).toEqual({ + 'bestax-migrate': '2026-01-01T00:00:00.000Z', + }); + }); + + it('stays silent when the config dir is unwritable', async () => { + const blocker = join(configHome, 'blocker'); + await writeFile(blocker, 'file, not a dir', 'utf-8'); + process.env.XDG_CONFIG_HOME = blocker; + await expect( + persistTelemetryDecision(true, 'create-bestax@1.0.0') + ).resolves.toBeUndefined(); + }); +}); + +describe('sendTelemetry', () => { + it('POSTs JSON to the default endpoint', async () => { + await sendTelemetry({ v: 1 }); + expect(fetchMock).toHaveBeenCalledTimes(1); + const [url, init] = fetchMock.mock.calls[0] as FetchArgs; + expect(url).toBe('https://bestax.io/api/t'); + expect(init.method).toBe('POST'); + expect(init.headers).toEqual({ 'content-type': 'application/json' }); + expect(init.body).toBe('{"v":1}'); + expect(init.signal).toBeInstanceOf(AbortSignal); + }); + + it('honors BESTAX_TELEMETRY_ENDPOINT', async () => { + process.env.BESTAX_TELEMETRY_ENDPOINT = 'http://127.0.0.1:8787/api/t'; + await sendTelemetry({ v: 1 }); + expect(fetchMock.mock.calls[0]?.[0]).toBe('http://127.0.0.1:8787/api/t'); + }); + + it('resolves silently when fetch rejects', async () => { + fetchMock.mockRejectedValueOnce(new Error('offline')); + await expect(sendTelemetry({ v: 1 })).resolves.toBeUndefined(); + }); + + it('ignores non-2xx responses', async () => { + fetchMock.mockResolvedValueOnce(new Response(null, { status: 400 })); + await expect(sendTelemetry({ v: 1 })).resolves.toBeUndefined(); + }); +}); + +describe('getToolVersion', () => { + it('reads the package version', () => { + expect(getToolVersion()).toMatch(/^\d+\.\d+\.\d+/); + }); +}); + +describe('buildScaffoldPayload', () => { + it('contains exactly the allowlisted fields and nothing else', () => { + const payload = buildScaffoldPayload({ + template: 'vite-ts', + bulmaFlavor: 'complete', + iconLibrary: 'none', + skills: true, + }) as Record; + + expect(Object.keys(payload).sort()).toEqual([ + 'event', + 'nodeMajor', + 'platform', + 'props', + 'tool', + 'toolVersion', + 'v', + ]); + expect(payload.v).toBe(1); + expect(payload.tool).toBe('create-bestax'); + expect(payload.event).toBe('scaffold'); + expect(payload.nodeMajor).toBe(Number(process.versions.node.split('.')[0])); + expect(payload.platform).toBe(process.platform); + expect(Object.keys(payload.props as object).sort()).toEqual([ + 'bulmaFlavor', + 'iconLibrary', + 'packageManager', + 'skills', + 'template', + ]); + const props = payload.props as Record; + expect(props.template).toBe('vite-ts'); + expect(props.skills).toBe(true); + expect(['npm', 'pnpm', 'yarn', 'bun']).toContain(props.packageManager); + }); +}); + +describe('reportScaffold', () => { + const choices = { + template: 'vite', + bulmaFlavor: 'prefixed', + iconLibrary: 'mdi', + skills: false, + }; + + it('flag=true persists and sends without prompting', async () => { + const promptConsent = jest.fn<() => Promise>(); + await reportScaffold(choices, true, { interactive: true, promptConsent }); + expect(promptConsent).not.toHaveBeenCalled(); + expect((await readConfigFile()).enabled).toBe(true); + expect(fetchMock).toHaveBeenCalledTimes(1); + const body = JSON.parse((fetchMock.mock.calls[0] as FetchArgs)[1].body) as { + props: Record; + }; + expect(body.props.template).toBe('vite'); + }); + + it('flag=false persists off and sends nothing', async () => { + const promptConsent = jest.fn<() => Promise>(); + await reportScaffold(choices, false, { interactive: true, promptConsent }); + expect(promptConsent).not.toHaveBeenCalled(); + expect((await readConfigFile()).enabled).toBe(false); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('prompts when undecided and interactive; yes persists and sends', async () => { + const promptConsent = jest + .fn<() => Promise>() + .mockResolvedValue(true); + await reportScaffold(choices, undefined, { + interactive: true, + promptConsent, + }); + expect(promptConsent).toHaveBeenCalledTimes(1); + expect((await readConfigFile()).enabled).toBe(true); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('a "no" answer persists off and sends nothing', async () => { + const promptConsent = jest + .fn<() => Promise>() + .mockResolvedValue(false); + await reportScaffold(choices, undefined, { + interactive: true, + promptConsent, + }); + expect((await readConfigFile()).enabled).toBe(false); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('a cancel persists nothing so the user may be asked again', async () => { + const promptConsent = jest + .fn<() => Promise>() + .mockResolvedValue(null); + await reportScaffold(choices, undefined, { + interactive: true, + promptConsent, + }); + await expect(readConfigFile()).rejects.toThrow(); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('never prompts when not interactive', async () => { + const promptConsent = jest.fn<() => Promise>(); + await reportScaffold(choices, undefined, { + interactive: false, + promptConsent, + }); + expect(promptConsent).not.toHaveBeenCalled(); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('never prompts under DO_NOT_TRACK', async () => { + process.env.DO_NOT_TRACK = '1'; + const promptConsent = jest.fn<() => Promise>(); + await reportScaffold(choices, undefined, { + interactive: true, + promptConsent, + }); + expect(promptConsent).not.toHaveBeenCalled(); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('sends without prompting when the config already says on', async () => { + await writeConfig({ version: 1, enabled: true }); + const promptConsent = jest.fn<() => Promise>(); + await reportScaffold(choices, undefined, { + interactive: true, + promptConsent, + }); + expect(promptConsent).not.toHaveBeenCalled(); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); +}); diff --git a/create-bestax/src/cli.ts b/create-bestax/src/cli.ts index dc909cb63..fc6dcdce5 100644 --- a/create-bestax/src/cli.ts +++ b/create-bestax/src/cli.ts @@ -99,6 +99,11 @@ export function createCLI(): Command { ) .option('--skills', 'install the bestax AI skills into .claude/skills') .option('--no-skills', 'do not install the bestax AI skills') + .option( + '--telemetry', + 'enable anonymous usage telemetry (https://bestax.io/docs/guides/telemetry)' + ) + .option('--no-telemetry', 'disable anonymous usage telemetry') .option('-y, --yes', 'skip prompts and use defaults or provided options') .action((projectDir?: string, options?: unknown) => { projectCreator.create(projectDir, options as CLIOptions); diff --git a/create-bestax/src/constants.ts b/create-bestax/src/constants.ts index 0da648b1f..c6582eff9 100644 --- a/create-bestax/src/constants.ts +++ b/create-bestax/src/constants.ts @@ -39,6 +39,15 @@ export const MESSAGES = { HAPPY_CODING: 'Happy coding! 🎉', SKILLS_ADDED: '✔ Installed bestax AI skills into .claude/skills/ (+ CLAUDE.md, .claude/launch.json)', + TELEMETRY_NOTICE: + 'Help improve bestax — share anonymous usage stats?\n' + + 'Sends only the options you chose (template, Bulma flavor, icon library,\n' + + 'skills, package manager) plus CLI version, Node major, and OS name.\n' + + 'Never your name, paths, project names, or any personal data.\n' + + ' Details & opt-out: https://bestax.io/docs/guides/telemetry\n' + + ' Feedback welcome: https://github.com/allxsmith/bestax/issues', + TELEMETRY_ACK_ON: 'Thanks! Opt out anytime with --no-telemetry.', + TELEMETRY_ACK_OFF: "No problem — we won't ask again.", } as const; // Dev-server manifest written into .claude/ with the skills opt-in so Claude @@ -72,6 +81,7 @@ export const PROMPTS = { SELECT_BULMA_FLAVOR: 'Which Bulma CSS flavor would you like to use?', INSTALL_SKILLS: 'Install the bestax AI skills (.claude/skills) for Claude Code and other agents?', + TELEMETRY_CONSENT: 'Share anonymous usage stats?', } as const; // Icon libraries that require a `ConfigProvider iconLibrary` wrapper, mapped diff --git a/create-bestax/src/display.ts b/create-bestax/src/display.ts index 16de5025e..f714f8b9b 100644 --- a/create-bestax/src/display.ts +++ b/create-bestax/src/display.ts @@ -38,6 +38,12 @@ export function displaySuccess(targetDir: string): void { ); console.log(chalk.dim(' https://github.com/allxsmith/bestax')); console.log(); + console.log( + chalk.dim( + ' Ideas or problems? Open an issue: https://github.com/allxsmith/bestax/issues' + ) + ); + console.log(); } export function displayError(message: string): void { diff --git a/create-bestax/src/project-creator.ts b/create-bestax/src/project-creator.ts index d8ce71532..4de8fa63b 100644 --- a/create-bestax/src/project-creator.ts +++ b/create-bestax/src/project-creator.ts @@ -17,7 +17,9 @@ import { promptIconLibrary, promptBulmaFlavor, promptInstallSkills, + promptTelemetryConsent, } from './prompts.js'; +import { reportScaffold, type ScaffoldChoices } from './telemetry.js'; import { displayHeader, displaySuccess, @@ -57,6 +59,7 @@ export interface CLIOptions { bulma?: string; icon?: string; skills?: boolean; + telemetry?: boolean; yes?: boolean; } @@ -601,9 +604,44 @@ export class ProjectCreator { }); } displaySuccess(targetDir); + await this.handleTelemetry(options, { + template, + bulmaFlavor, + iconLibrary, + skills: installSkills, + }); } catch (error) { displayError((error as Error).message); process.exit(1); } } + + /** + * Consent + beacon after a successful scaffold. Only success events are + * reported, the beacon is awaited (so no process.exit(1) site can kill an + * in-flight request), and nothing in here may throw or alter the exit code. + * The prompt is skipped for -y, missing TTY, an explicit flag, or + * DO_NOT_TRACK (#192: the non-interactive path must never hang). + */ + private async handleTelemetry( + options: CLIOptions | undefined, + choices: ScaffoldChoices + ): Promise { + try { + await reportScaffold(choices, options?.telemetry, { + interactive: process.stdin.isTTY === true && options?.yes !== true, + promptConsent: async () => { + const answer = await promptTelemetryConsent(); + if (answer === true) { + console.log(chalk.dim(MESSAGES.TELEMETRY_ACK_ON)); + } else if (answer === false) { + console.log(chalk.dim(MESSAGES.TELEMETRY_ACK_OFF)); + } + return answer; + }, + }); + } catch { + // Telemetry must never affect the scaffold's outcome. + } + } } diff --git a/create-bestax/src/prompts.ts b/create-bestax/src/prompts.ts index abcbd1b1d..bbb1019a4 100644 --- a/create-bestax/src/prompts.ts +++ b/create-bestax/src/prompts.ts @@ -1,4 +1,5 @@ import prompts from 'prompts'; +import chalk from 'chalk'; import { TEMPLATES, DEFAULT_PROJECT_NAME, @@ -92,6 +93,25 @@ export async function promptIconLibrary(): Promise { return response.iconLibrary || null; } +/** + * Consent for anonymous usage telemetry. Deliberately does NOT call + * ensureInteractive(): consent is optional, so without a TTY the caller skips + * the question instead of failing the run (#192). Returns null when the user + * cancels (Ctrl-C) — a cancel is not an answer and nothing is persisted. + */ +export async function promptTelemetryConsent(): Promise { + console.log(); + console.log(chalk.gray(MESSAGES.TELEMETRY_NOTICE)); + const response = await prompts({ + type: 'confirm', + name: 'telemetry', + message: PROMPT_MESSAGES.TELEMETRY_CONSENT, + initial: false, + }); + + return typeof response.telemetry === 'boolean' ? response.telemetry : null; +} + export async function promptBulmaFlavor(): Promise { ensureInteractive(); const response = await prompts({ diff --git a/create-bestax/src/telemetry.ts b/create-bestax/src/telemetry.ts new file mode 100644 index 000000000..b10d89676 --- /dev/null +++ b/create-bestax/src/telemetry.ts @@ -0,0 +1,216 @@ +import { mkdir, readFile, writeFile } from 'fs/promises'; +import { homedir } from 'os'; +import { dirname, join } from 'path'; +import { fileURLToPath } from 'url'; +import { readFileSync } from 'fs'; +import { detectPackageManager } from './package-manager.js'; + +/** + * Anonymous opt-in telemetry (disclosure: https://bestax.io/docs/guides/telemetry). + * + * A near-identical copy lives in bestax-migrate/src/telemetry.ts — both CLIs + * publish standalone (plain tsc, files:["dist"]), so the ~150 shared lines are + * duplicated rather than published as a package. Keep the copies in sync. + * + * Privacy contract: every transmitted value is a closed-enum member, a version + * string, or a bounded integer — never paths, project names, or free text — + * and no identifier of any kind is generated. Undecided means off. Every + * failure here is silent: telemetry must never affect the CLI's outcome. + */ + +export type TelemetryDecision = 'on' | 'off' | 'undecided'; + +export type TelemetrySource = 'flag' | 'dnt' | 'env' | 'config' | 'default'; + +export interface ResolvedTelemetry { + decision: TelemetryDecision; + source: TelemetrySource; + /** Whether a consent prompt may still be shown (nothing has decided or forbidden it). */ + promptAllowed: boolean; +} + +interface TelemetryConfig { + version: 1; + enabled?: boolean; + decidedAt?: string; + decidedBy?: string; + noticed?: Record; +} + +const DEFAULT_ENDPOINT = 'https://bestax.io/api/t'; +const TIMEOUT_MS = 1500; + +/** Shared across the bestax CLI family so consent is asked at most once. */ +export function telemetryConfigPath(): string { + const base = process.env.XDG_CONFIG_HOME || join(homedir(), '.config'); + return join(base, 'bestax', 'telemetry.json'); +} + +async function readConfig(): Promise { + try { + const parsed: unknown = JSON.parse( + await readFile(telemetryConfigPath(), 'utf-8') + ); + if (parsed !== null && typeof parsed === 'object') { + return parsed as TelemetryConfig; + } + } catch { + // Missing or corrupt config reads as "never asked". + } + return null; +} + +function doNotTrack(): boolean { + const dnt = process.env.DO_NOT_TRACK; + return dnt !== undefined && dnt !== '' && dnt !== '0'; +} + +/** + * Precedence: flag (persisted by the caller) → DO_NOT_TRACK → BESTAX_TELEMETRY + * (per-run, never persisted) → config file → undecided (off). + */ +export async function resolveTelemetry( + flag?: boolean +): Promise { + if (flag !== undefined) { + return { + decision: flag ? 'on' : 'off', + source: 'flag', + promptAllowed: false, + }; + } + if (doNotTrack()) { + return { decision: 'off', source: 'dnt', promptAllowed: false }; + } + const env = process.env.BESTAX_TELEMETRY; + if (env === '0' || env === '1') { + return { + decision: env === '1' ? 'on' : 'off', + source: 'env', + promptAllowed: false, + }; + } + const config = await readConfig(); + if (config && typeof config.enabled === 'boolean') { + return { + decision: config.enabled ? 'on' : 'off', + source: 'config', + promptAllowed: false, + }; + } + return { decision: 'undecided', source: 'default', promptAllowed: true }; +} + +export async function persistTelemetryDecision( + enabled: boolean, + decidedBy: string +): Promise { + try { + const existing = await readConfig(); + const config: TelemetryConfig = { + version: 1, + enabled, + decidedAt: new Date().toISOString(), + decidedBy, + ...(existing?.noticed ? { noticed: existing.noticed } : {}), + }; + const filePath = telemetryConfigPath(); + await mkdir(dirname(filePath), { recursive: true }); + await writeFile(filePath, JSON.stringify(config, null, 2) + '\n', 'utf-8'); + } catch { + // An unwritable config dir must not break the CLI. + } +} + +export async function sendTelemetry(payload: object): Promise { + try { + await fetch(process.env.BESTAX_TELEMETRY_ENDPOINT || DEFAULT_ENDPOINT, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(payload), + signal: AbortSignal.timeout(TIMEOUT_MS), + }); + } catch { + // Fire-and-forget: the response (or its absence) is never surfaced. + } +} + +export function getToolVersion(): string { + try { + const dir = dirname(fileURLToPath(import.meta.url)); + const packageJson = JSON.parse( + readFileSync(join(dir, '..', 'package.json'), 'utf-8') + ) as { version?: string }; + return packageJson.version || '0.0.0'; + } catch { + return '0.0.0'; + } +} + +// --- create-bestax specific (the sibling copy has its own payload builder) --- + +export interface ScaffoldChoices { + template: string; + bulmaFlavor: string; + iconLibrary: string; + skills: boolean; +} + +export function buildScaffoldPayload(choices: ScaffoldChoices): object { + return { + v: 1, + tool: 'create-bestax', + event: 'scaffold', + toolVersion: getToolVersion(), + nodeMajor: Number(process.versions.node.split('.')[0]), + platform: process.platform, + props: { + template: choices.template, + bulmaFlavor: choices.bulmaFlavor, + iconLibrary: choices.iconLibrary, + skills: choices.skills, + packageManager: detectPackageManager(), + }, + }; +} + +export interface ReportScaffoldOptions { + /** True only when a consent question could actually be answered (TTY, not -y). */ + interactive: boolean; + /** Owns all consent UI (notice, question, ack); null means cancelled. */ + promptConsent: () => Promise; +} + +export async function reportScaffold( + choices: ScaffoldChoices, + flag: boolean | undefined, + options: ReportScaffoldOptions +): Promise { + const resolved = await resolveTelemetry(flag); + let decision = resolved.decision; + + if (resolved.source === 'flag') { + // Flags are the scripted equivalent of the prompt: decided once, remembered. + await persistTelemetryDecision( + decision === 'on', + `create-bestax@${getToolVersion()}` + ); + } else if ( + decision === 'undecided' && + resolved.promptAllowed && + options.interactive + ) { + const answer = await options.promptConsent(); + if (answer !== null) { + await persistTelemetryDecision( + answer, + `create-bestax@${getToolVersion()}` + ); + decision = answer ? 'on' : 'off'; + } + } + + if (decision === 'on') { + await sendTelemetry(buildScaffoldPayload(choices)); + } +} From 20c5492b88e5de7212ad47cd687a97a3ecb6a900 Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sat, 22 Aug 2026 15:20:03 -0400 Subject: [PATCH 06/29] feat(bestax-migrate): add opt-in anonymous telemetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Shares create-bestax's consent config so the family asks at most once; TTY-guarded readline consent, --telemetry/--no-telemetry flags, DNT respected, non-TTY runs never prompt. Sends run shape only: source, css mode, dry/deps flags, bucketed changed count, and per-rule TODO counts — never file paths or code. Adds the star-us success message on TTY runs. --- bestax-migrate/README.md | 12 + bestax-migrate/src/__tests__/cli.test.ts | 398 ++++++++++++-- .../src/__tests__/telemetry.test.ts | 497 ++++++++++++++++++ bestax-migrate/src/cli.ts | 111 +++- bestax-migrate/src/index.ts | 2 +- bestax-migrate/src/telemetry.ts | 257 +++++++++ 6 files changed, 1244 insertions(+), 33 deletions(-) create mode 100644 bestax-migrate/src/__tests__/telemetry.test.ts create mode 100644 bestax-migrate/src/telemetry.ts diff --git a/bestax-migrate/README.md b/bestax-migrate/README.md index 7d0cc5a32..ab1069385 100644 --- a/bestax-migrate/README.md +++ b/bestax-migrate/README.md @@ -84,6 +84,18 @@ A codemod rewrites your source in place, so how it is built and published matter Full detail: [`SECURITY.md`](https://github.com/allxsmith/bestax/blob/main/SECURITY.md) · [Security guide](https://bestax.io/docs/guides/security) +## Telemetry + +`bestax-migrate` can send one **anonymous** usage event after a successful +run — only if you opt in when asked (once, on an interactive terminal). The +event is just the run's shape — source library, CSS mode, the dry/deps flags, a +bucketed changed-file count, and per-rule TODO counts — never file paths, file +contents, code, names, IPs, or machine IDs, and no identifier exists that could +link two events together. Opt out any time with `--no-telemetry`, +`BESTAX_TELEMETRY=0`, or `DO_NOT_TRACK=1`. + +Full disclosure of every field and control: [Telemetry guide](https://bestax.io/docs/guides/telemetry) + ## License MIT © Alex Smith diff --git a/bestax-migrate/src/__tests__/cli.test.ts b/bestax-migrate/src/__tests__/cli.test.ts index c57a90a07..523336591 100644 --- a/bestax-migrate/src/__tests__/cli.test.ts +++ b/bestax-migrate/src/__tests__/cli.test.ts @@ -1,17 +1,99 @@ +import { jest } from '@jest/globals'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { collectFiles, createCLI, findPackageJsons } from '../cli.js'; +import { PassThrough } from 'node:stream'; +import { + collectFiles, + createCLI, + findPackageJsons, + handleTelemetry, + promptTelemetryConsent, +} from '../cli.js'; +import type { MigrateRunStats } from '../telemetry.js'; let tempDirs: string[] = []; +// Every successful CLI run now ends in the telemetry handler, so the suite +// must never touch the real ~/.config/bestax, the network, or a TTY prompt: +// XDG_CONFIG_HOME points at a scratch dir, fetch is stubbed, and both stdio +// TTY flags are pinned false (individual tests re-stub them as needed). +const ENV_KEYS = [ + 'XDG_CONFIG_HOME', + 'DO_NOT_TRACK', + 'BESTAX_TELEMETRY', + 'BESTAX_TELEMETRY_ENDPOINT', +] as const; +const savedEnv: Record = {}; +const originalFetch = globalThis.fetch; +const originalStdoutTTY = Object.getOwnPropertyDescriptor( + process.stdout, + 'isTTY' +); +const originalStdinTTY = Object.getOwnPropertyDescriptor( + process.stdin, + 'isTTY' +); + +let fetchMock: jest.Mock; + +type FetchArgs = [ + string, + { + method: string; + headers: Record; + body: string; + signal: AbortSignal; + }, +]; + +function setTTY(stream: object, value: boolean): void { + Object.defineProperty(stream, 'isTTY', { + value, + configurable: true, + writable: true, + }); +} + +beforeEach(() => { + for (const key of ENV_KEYS) { + savedEnv[key] = process.env[key]; + delete process.env[key]; + } + const configHome = fs.mkdtempSync(path.join(os.tmpdir(), 'bestax-cli-cfg-')); + tempDirs.push(configHome); + process.env.XDG_CONFIG_HOME = configHome; + fetchMock = jest.fn(async () => new Response(null, { status: 204 })); + globalThis.fetch = fetchMock as unknown as typeof fetch; + setTTY(process.stdout, false); + setTTY(process.stdin, false); +}); + afterEach(() => { + for (const key of ENV_KEYS) { + if (savedEnv[key] === undefined) delete process.env[key]; + else process.env[key] = savedEnv[key]; + } + if (originalStdoutTTY) { + Object.defineProperty(process.stdout, 'isTTY', originalStdoutTTY); + } else { + delete (process.stdout as { isTTY?: boolean }).isTTY; + } + if (originalStdinTTY) { + Object.defineProperty(process.stdin, 'isTTY', originalStdinTTY); + } else { + delete (process.stdin as { isTTY?: boolean }).isTTY; + } for (const dir of tempDirs) { fs.rmSync(dir, { recursive: true, force: true }); } tempDirs = []; }); +afterAll(() => { + globalThis.fetch = originalFetch; +}); + function makeTempProject(): string { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'bestax-migrate-test-')); tempDirs.push(dir); @@ -55,7 +137,9 @@ function makeTempProject(): string { return dir; } -function runCli(args: string[]): { logs: string[]; errors: string[] } { +async function runCli( + args: string[] +): Promise<{ logs: string[]; errors: string[] }> { const logs: string[] = []; const errors: string[] = []; const program = createCLI({ @@ -63,7 +147,7 @@ function runCli(args: string[]): { logs: string[]; errors: string[] } { error: m => errors.push(m), }); program.exitOverride(); - program.parse(['node', 'bestax-migrate', ...args]); + await program.parseAsync(['node', 'bestax-migrate', ...args]); return { logs, errors }; } @@ -97,9 +181,12 @@ describe('findPackageJsons', () => { }); describe('CLI', () => { - it('transforms files in place and reports', () => { + it('transforms files in place and reports', async () => { const dir = makeTempProject(); - const { logs } = runCli(['react-bulma-components', path.join(dir, 'src')]); + const { logs } = await runCli([ + 'react-bulma-components', + path.join(dir, 'src'), + ]); const transformed = fs.readFileSync( path.join(dir, 'src', 'App.tsx'), 'utf8' @@ -110,10 +197,10 @@ describe('CLI', () => { expect(logs.join('\n')).toContain('3 transformed'); }); - it('leaves files untouched with --dry and prints with --print', () => { + it('leaves files untouched with --dry and prints with --print', async () => { const dir = makeTempProject(); const before = fs.readFileSync(path.join(dir, 'src', 'App.tsx'), 'utf8'); - const { logs } = runCli([ + const { logs } = await runCli([ 'react-bulma-components', path.join(dir, 'src'), '--dry', @@ -126,10 +213,10 @@ describe('CLI', () => { expect(logs.join('\n')).toContain('(dry run)'); }); - it('reports files that fail to parse and continues', () => { + it('reports files that fail to parse and continues', async () => { const dir = makeTempProject(); fs.writeFileSync(path.join(dir, 'src', 'broken.tsx'), 'const = <<>;\n'); - const { logs, errors } = runCli([ + const { logs, errors } = await runCli([ 'react-bulma-components', path.join(dir, 'src'), ]); @@ -137,9 +224,9 @@ describe('CLI', () => { expect(logs.join('\n')).toContain('3 transformed'); }); - it('migrates SCSS files and updates package.json dependencies', () => { + it('migrates SCSS files and updates package.json dependencies', async () => { const dir = makeTempProject(); - runCli(['react-bulma-components', path.join(dir, 'src')]); + await runCli(['react-bulma-components', path.join(dir, 'src')]); const scss = fs.readFileSync(path.join(dir, 'src', 'theme.scss'), 'utf8'); expect(scss).toContain("@use 'bulma/sass' with ("); expect(scss).toContain('$primary: #ff6b35'); @@ -154,45 +241,49 @@ describe('CLI', () => { expect(pkg.devDependencies.sass).toBe('^1.79.0'); }); - it('honors --no-deps and --dry for the package.json step', () => { + it('honors --no-deps and --dry for the package.json step', async () => { const dir = makeTempProject(); const before = fs.readFileSync(path.join(dir, 'package.json'), 'utf8'); - runCli(['react-bulma-components', path.join(dir, 'src'), '--no-deps']); + await runCli([ + 'react-bulma-components', + path.join(dir, 'src'), + '--no-deps', + ]); expect(fs.readFileSync(path.join(dir, 'package.json'), 'utf8')).toBe( before ); - runCli(['react-bulma-components', path.join(dir, 'src'), '--dry']); + await runCli(['react-bulma-components', path.join(dir, 'src'), '--dry']); expect(fs.readFileSync(path.join(dir, 'package.json'), 'utf8')).toBe( before ); }); - it('says "Would update" for the manifest in dry mode', () => { + it('says "Would update" for the manifest in dry mode', async () => { const dir = makeTempProject(); - const dry = runCli([ + const dry = await runCli([ 'react-bulma-components', path.join(dir, 'src'), '--dry', ]); expect(dry.logs.join('\n')).toContain('Would update'); expect(dry.logs.join('\n')).not.toContain('Updated '); - const wet = runCli(['react-bulma-components', path.join(dir, 'src')]); + const wet = await runCli(['react-bulma-components', path.join(dir, 'src')]); expect(wet.logs.join('\n')).toContain('Updated '); }); - it('preserves the manifest indentation style', () => { + it('preserves the manifest indentation style', async () => { const dir = makeTempProject(); fs.writeFileSync( path.join(dir, 'package.json'), '{\n\t"name": "tabbed-app",\n\t"dependencies": {\n\t\t"react-bulma-components": "^4.1.0"\n\t}\n}\n' ); - runCli(['react-bulma-components', path.join(dir, 'src')]); + await runCli(['react-bulma-components', path.join(dir, 'src')]); const raw = fs.readFileSync(path.join(dir, 'package.json'), 'utf8'); expect(raw).toContain('\t"dependencies"'); expect(raw).toContain('\t\t"@allxsmith/bestax-bulma"'); }); - it('flags unparseable component formats that import the source library', () => { + it('flags unparseable component formats that import the source library', async () => { const dir = makeTempProject(); fs.writeFileSync( path.join(dir, 'src', 'Card.astro'), @@ -202,40 +293,285 @@ describe('CLI', () => { path.join(dir, 'src', 'plain.astro'), '---\nconst x = 1;\n---\n

{x}

\n' ); - const { logs } = runCli(['react-bulma-components', path.join(dir, 'src')]); + const { logs } = await runCli([ + 'react-bulma-components', + path.join(dir, 'src'), + ]); const text = logs.join('\n'); expect(text).toContain('unsupported-file'); expect(text).toContain('Card.astro'); expect(text).not.toContain('plain.astro'); }); - it('threads --css keep through to the transforms', () => { + it('threads --css keep through to the transforms', async () => { const dir = makeTempProject(); fs.writeFileSync( path.join(dir, 'src', 'style-entry.ts'), "import 'bulma/css/bulma.min.css';\n" ); - runCli(['react-bulma-components', path.join(dir, 'src'), '--css', 'keep']); + await runCli([ + 'react-bulma-components', + path.join(dir, 'src'), + '--css', + 'keep', + ]); expect( fs.readFileSync(path.join(dir, 'src', 'style-entry.ts'), 'utf8') ).toBe("import 'bulma/css/bulma.min.css';\n"); }); - it('rejects unknown --css modes', () => { + it('rejects unknown --css modes', async () => { const dir = makeTempProject(); - expect(() => + await expect( runCli(['react-bulma-components', path.join(dir, 'src'), '--css', 'nope']) - ).toThrow(); + ).rejects.toThrow(); }); - it('rejects unknown sources', () => { + it('rejects unknown sources', async () => { const dir = makeTempProject(); - expect(() => runCli(['not-a-library', path.join(dir, 'src')])).toThrow(); + await expect( + runCli(['not-a-library', path.join(dir, 'src')]) + ).rejects.toThrow(); }); - it('rejects missing paths', () => { - expect(() => + it('rejects missing paths', async () => { + await expect( runCli(['react-bulma-components', '/definitely/not/here']) - ).toThrow(); + ).rejects.toThrow(); + }); +}); + +describe('CLI telemetry', () => { + it('registers the --telemetry/--no-telemetry option pair', () => { + const flags = createCLI().options.map(option => option.flags); + expect(flags).toContain('--telemetry'); + expect(flags).toContain('--no-telemetry'); + }); + + it('--no-telemetry persists off and never calls fetch', async () => { + const dir = makeTempProject(); + await runCli([ + 'react-bulma-components', + path.join(dir, 'src'), + '--no-telemetry', + ]); + expect(fetchMock).not.toHaveBeenCalled(); + const config = JSON.parse( + fs.readFileSync( + path.join( + process.env.XDG_CONFIG_HOME as string, + 'bestax', + 'telemetry.json' + ), + 'utf8' + ) + ); + expect(config.enabled).toBe(false); + expect(config.decidedBy).toMatch(/^bestax-migrate@/); + }); + + it('--telemetry persists on and sends the run payload', async () => { + const dir = makeTempProject(); + await runCli([ + 'react-bulma-components', + path.join(dir, 'src'), + '--telemetry', + ]); + expect(fetchMock).toHaveBeenCalledTimes(1); + const body = JSON.parse((fetchMock.mock.calls[0] as FetchArgs)[1].body) as { + tool: string; + props: Record; + todosByRule?: Array>; + }; + expect(body.tool).toBe('bestax-migrate'); + expect(body.props.source).toBe('react-bulma-components'); + // Never file paths, lines, or messages — only rule names and counts. + expect(JSON.stringify(body)).not.toContain(dir); + for (const entry of body.todosByRule ?? []) { + expect(Object.keys(entry).sort()).toEqual(['count', 'rule']); + } + }); + + it('an undecided non-TTY run neither prompts, sends, nor persists', async () => { + const dir = makeTempProject(); + await runCli(['react-bulma-components', path.join(dir, 'src')]); + expect(fetchMock).not.toHaveBeenCalled(); + expect( + fs.existsSync( + path.join( + process.env.XDG_CONFIG_HOME as string, + 'bestax', + 'telemetry.json' + ) + ) + ).toBe(false); + }); + + it('never prompts when only stdout is a TTY (stdin piped must not hang)', async () => { + setTTY(process.stdout, true); + const dir = makeTempProject(); + await runCli(['react-bulma-components', path.join(dir, 'src')]); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('never prompts under DO_NOT_TRACK even on a full TTY', async () => { + setTTY(process.stdout, true); + setTTY(process.stdin, true); + process.env.DO_NOT_TRACK = '1'; + const dir = makeTempProject(); + await runCli(['react-bulma-components', path.join(dir, 'src')]); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); + +describe('star nudge', () => { + it('prints the GitHub star message on a TTY', async () => { + setTTY(process.stdout, true); + const dir = makeTempProject(); + const { logs } = await runCli([ + 'react-bulma-components', + path.join(dir, 'src'), + ]); + const text = logs.join('\n'); + expect(text).toContain( + '★ If you enjoy using bestax-bulma, please star us on GitHub!' + ); + expect(text).toContain('https://github.com/allxsmith/bestax'); + }); + + it('stays silent when stdout is not a TTY', async () => { + const dir = makeTempProject(); + const { logs } = await runCli([ + 'react-bulma-components', + path.join(dir, 'src'), + ]); + expect(logs.join('\n')).not.toContain('star us on GitHub'); + }); +}); + +describe('handleTelemetry', () => { + const stats: MigrateRunStats = { + source: 'react-bulma-components', + cssMode: 'bestax', + dry: false, + deps: true, + changedCount: 2, + todosByRule: [], + }; + + function makeIo(): { + logs: string[]; + log: (m: string) => void; + error: (m: string) => void; + } { + const logs: string[] = []; + return { logs, log: m => logs.push(m), error: () => {} }; + } + + it('a "yes" from the prompt persists, acks, and sends', async () => { + setTTY(process.stdout, true); + setTTY(process.stdin, true); + const io = makeIo(); + await handleTelemetry(stats, undefined, io, async () => true); + expect(io.logs.join('\n')).toContain( + 'Thanks! Opt out anytime with --no-telemetry.' + ); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('a "no" from the prompt persists off, acks, and sends nothing', async () => { + setTTY(process.stdout, true); + setTTY(process.stdin, true); + const io = makeIo(); + await handleTelemetry(stats, undefined, io, async () => false); + expect(io.logs.join('\n')).toContain("No problem — we won't ask again."); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('a cancelled prompt acks nothing and persists nothing', async () => { + setTTY(process.stdout, true); + setTTY(process.stdin, true); + const io = makeIo(); + await handleTelemetry(stats, undefined, io, async () => null); + expect(io.logs).toHaveLength(0); + expect(fetchMock).not.toHaveBeenCalled(); + expect( + fs.existsSync( + path.join( + process.env.XDG_CONFIG_HOME as string, + 'bestax', + 'telemetry.json' + ) + ) + ).toBe(false); + }); + + it('never throws, even when the prompt itself does', async () => { + setTTY(process.stdout, true); + setTTY(process.stdin, true); + const io = makeIo(); + await expect( + handleTelemetry(stats, undefined, io, async () => { + throw new Error('boom'); + }) + ).resolves.toBeUndefined(); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); + +describe('promptTelemetryConsent', () => { + function makeIo(): { + logs: string[]; + log: (m: string) => void; + error: (m: string) => void; + } { + const logs: string[] = []; + return { logs, log: m => logs.push(m), error: () => {} }; + } + + it.each([ + ['y\n', true], + ['Y\n', true], + ['yes\n', true], + ['YES\n', true], + ['n\n', false], + ['\n', false], + ['whatever\n', false], + ] as const)('parses %j as %s', async (line, expected) => { + const input = new PassThrough(); + const output = new PassThrough(); + output.resume(); + const io = makeIo(); + const pending = promptTelemetryConsent(io, input, output); + input.write(line); + await expect(pending).resolves.toBe(expected); + }); + + it('prints the privacy notice before asking', async () => { + const input = new PassThrough(); + const output = new PassThrough(); + output.resume(); + const io = makeIo(); + const pending = promptTelemetryConsent(io, input, output); + input.write('n\n'); + await pending; + const text = io.logs.join('\n'); + expect(text).toContain( + 'Help improve bestax-migrate — share anonymous usage stats?' + ); + expect(text).toContain( + 'Sends only rule names and counts — never file paths or code.' + ); + expect(text).toContain('https://bestax.io/docs/guides/telemetry'); + expect(text).toContain('https://github.com/allxsmith/bestax/issues'); + }); + + it('treats a closed input (Ctrl-D) as a cancel, not an answer', async () => { + const input = new PassThrough(); + const output = new PassThrough(); + output.resume(); + const pending = promptTelemetryConsent(makeIo(), input, output); + input.end(); + await expect(pending).resolves.toBeNull(); }); }); diff --git a/bestax-migrate/src/__tests__/telemetry.test.ts b/bestax-migrate/src/__tests__/telemetry.test.ts new file mode 100644 index 000000000..d9748d4f0 --- /dev/null +++ b/bestax-migrate/src/__tests__/telemetry.test.ts @@ -0,0 +1,497 @@ +import { + describe, + it, + expect, + jest, + beforeEach, + afterEach, + afterAll, +} from '@jest/globals'; +import { mkdtemp, readFile, writeFile, mkdir } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; + +const { + telemetryConfigPath, + resolveTelemetry, + persistTelemetryDecision, + markNoticed, + sendTelemetry, + getToolVersion, + buildMigratePayload, + reportMigrateRun, +} = await import('../telemetry.js'); + +const ENV_KEYS = [ + 'XDG_CONFIG_HOME', + 'DO_NOT_TRACK', + 'BESTAX_TELEMETRY', + 'BESTAX_TELEMETRY_ENDPOINT', +] as const; +const savedEnv: Record = {}; +const originalFetch = globalThis.fetch; + +let configHome: string; +let fetchMock: jest.Mock; + +type FetchArgs = [ + string, + { + method: string; + headers: Record; + body: string; + signal: AbortSignal; + }, +]; + +async function writeConfig(data: unknown): Promise { + await mkdir(join(configHome, 'bestax'), { recursive: true }); + await writeFile( + join(configHome, 'bestax', 'telemetry.json'), + typeof data === 'string' ? data : JSON.stringify(data), + 'utf-8' + ); +} + +async function readConfigFile(): Promise> { + return JSON.parse( + await readFile(join(configHome, 'bestax', 'telemetry.json'), 'utf-8') + ) as Record; +} + +beforeEach(async () => { + for (const key of ENV_KEYS) { + savedEnv[key] = process.env[key]; + delete process.env[key]; + } + configHome = await mkdtemp(join(tmpdir(), 'bestax-telemetry-')); + process.env.XDG_CONFIG_HOME = configHome; + fetchMock = jest.fn(async () => new Response(null, { status: 204 })); + globalThis.fetch = fetchMock as unknown as typeof fetch; +}); + +afterEach(() => { + for (const key of ENV_KEYS) { + if (savedEnv[key] === undefined) delete process.env[key]; + else process.env[key] = savedEnv[key]; + } +}); + +afterAll(() => { + globalThis.fetch = originalFetch; +}); + +describe('telemetryConfigPath', () => { + it('uses XDG_CONFIG_HOME when set', () => { + expect(telemetryConfigPath()).toBe( + join(configHome, 'bestax', 'telemetry.json') + ); + }); + + it('falls back to ~/.config when XDG_CONFIG_HOME is unset', () => { + delete process.env.XDG_CONFIG_HOME; + expect(telemetryConfigPath()).toMatch( + /[/\\]\.config[/\\]bestax[/\\]telemetry\.json$/ + ); + }); +}); + +describe('resolveTelemetry precedence', () => { + it('is undecided (prompt allowed) with no signals at all', async () => { + expect(await resolveTelemetry()).toEqual({ + decision: 'undecided', + source: 'default', + promptAllowed: true, + }); + }); + + it.each([ + [true, 'on'], + [false, 'off'], + ] as const)('flag %s wins over everything', async (flag, decision) => { + process.env.DO_NOT_TRACK = '1'; + process.env.BESTAX_TELEMETRY = flag ? '0' : '1'; + await writeConfig({ version: 1, enabled: !flag }); + expect(await resolveTelemetry(flag)).toEqual({ + decision, + source: 'flag', + promptAllowed: false, + }); + }); + + it('DO_NOT_TRACK beats env var and config', async () => { + process.env.DO_NOT_TRACK = '1'; + process.env.BESTAX_TELEMETRY = '1'; + await writeConfig({ version: 1, enabled: true }); + expect(await resolveTelemetry()).toEqual({ + decision: 'off', + source: 'dnt', + promptAllowed: false, + }); + }); + + it.each(['0', ''])('DO_NOT_TRACK=%j is not a DNT signal', async value => { + process.env.DO_NOT_TRACK = value; + const resolved = await resolveTelemetry(); + expect(resolved.source).toBe('default'); + }); + + it('BESTAX_TELEMETRY=1 forces on without persisting', async () => { + process.env.BESTAX_TELEMETRY = '1'; + await writeConfig({ version: 1, enabled: false }); + expect(await resolveTelemetry()).toEqual({ + decision: 'on', + source: 'env', + promptAllowed: false, + }); + }); + + it('BESTAX_TELEMETRY=0 forces off', async () => { + process.env.BESTAX_TELEMETRY = '0'; + await writeConfig({ version: 1, enabled: true }); + expect((await resolveTelemetry()).decision).toBe('off'); + }); + + it('ignores junk BESTAX_TELEMETRY values', async () => { + process.env.BESTAX_TELEMETRY = 'yes'; + expect((await resolveTelemetry()).source).toBe('default'); + }); + + it.each([ + [true, 'on'], + [false, 'off'], + ] as const)( + 'reads enabled=%s from the config file', + async (enabled, decision) => { + await writeConfig({ version: 1, enabled }); + expect(await resolveTelemetry()).toEqual({ + decision, + source: 'config', + promptAllowed: false, + }); + } + ); + + it('treats corrupt config as undecided', async () => { + await writeConfig('{not json'); + expect((await resolveTelemetry()).decision).toBe('undecided'); + }); + + it('treats a config without a boolean enabled as undecided', async () => { + await writeConfig({ version: 1, enabled: 'yes' }); + expect((await resolveTelemetry()).decision).toBe('undecided'); + }); + + it('treats a non-object config as undecided', async () => { + await writeConfig('null'); + expect((await resolveTelemetry()).decision).toBe('undecided'); + }); +}); + +describe('persistTelemetryDecision', () => { + it('writes the config schema', async () => { + await persistTelemetryDecision(true, 'bestax-migrate@1.0.0'); + const config = await readConfigFile(); + expect(config.version).toBe(1); + expect(config.enabled).toBe(true); + expect(config.decidedBy).toBe('bestax-migrate@1.0.0'); + expect(typeof config.decidedAt).toBe('string'); + }); + + it('preserves the noticed map of the sibling CLI', async () => { + await writeConfig({ + version: 1, + noticed: { 'create-bestax': '2026-01-01T00:00:00.000Z' }, + }); + await persistTelemetryDecision(false, 'bestax-migrate@1.0.0'); + const config = await readConfigFile(); + expect(config.enabled).toBe(false); + expect(config.noticed).toEqual({ + 'create-bestax': '2026-01-01T00:00:00.000Z', + }); + }); + + it('stays silent when the config dir is unwritable', async () => { + const blocker = join(configHome, 'blocker'); + await writeFile(blocker, 'file, not a dir', 'utf-8'); + process.env.XDG_CONFIG_HOME = blocker; + await expect( + persistTelemetryDecision(true, 'bestax-migrate@1.0.0') + ).resolves.toBeUndefined(); + }); +}); + +describe('markNoticed', () => { + it('records the first notice and returns true', async () => { + expect(await markNoticed('bestax-migrate')).toBe(true); + const config = await readConfigFile(); + expect(config.version).toBe(1); + const noticed = config.noticed as Record; + expect(new Date(noticed['bestax-migrate']).getTime()).not.toBeNaN(); + }); + + it('returns false once the notice is recorded', async () => { + expect(await markNoticed('bestax-migrate')).toBe(true); + expect(await markNoticed('bestax-migrate')).toBe(false); + }); + + it('preserves every other config field and sibling notices', async () => { + await writeConfig({ + version: 1, + enabled: true, + decidedAt: '2026-01-01T00:00:00.000Z', + decidedBy: 'create-bestax@1.0.0', + noticed: { 'create-bestax': '2026-01-01T00:00:00.000Z' }, + }); + expect(await markNoticed('bestax-migrate')).toBe(true); + const config = await readConfigFile(); + expect(config.enabled).toBe(true); + expect(config.decidedBy).toBe('create-bestax@1.0.0'); + const noticed = config.noticed as Record; + expect(noticed['create-bestax']).toBe('2026-01-01T00:00:00.000Z'); + expect(noticed['bestax-migrate']).toBeDefined(); + }); + + it('returns true without throwing when the config dir is unwritable', async () => { + // The notice is shown this run and may simply repeat later. + const blocker = join(configHome, 'blocker'); + await writeFile(blocker, 'file, not a dir', 'utf-8'); + process.env.XDG_CONFIG_HOME = blocker; + await expect(markNoticed('bestax-migrate')).resolves.toBe(true); + }); +}); + +describe('sendTelemetry', () => { + it('POSTs JSON to the default endpoint', async () => { + await sendTelemetry({ v: 1 }); + expect(fetchMock).toHaveBeenCalledTimes(1); + const [url, init] = fetchMock.mock.calls[0] as FetchArgs; + expect(url).toBe('https://bestax.io/api/t'); + expect(init.method).toBe('POST'); + expect(init.headers).toEqual({ 'content-type': 'application/json' }); + expect(init.body).toBe('{"v":1}'); + expect(init.signal).toBeInstanceOf(AbortSignal); + }); + + it('honors BESTAX_TELEMETRY_ENDPOINT', async () => { + process.env.BESTAX_TELEMETRY_ENDPOINT = 'http://127.0.0.1:8787/api/t'; + await sendTelemetry({ v: 1 }); + expect(fetchMock.mock.calls[0]?.[0]).toBe('http://127.0.0.1:8787/api/t'); + }); + + it('resolves silently when fetch rejects', async () => { + fetchMock.mockRejectedValueOnce(new Error('offline')); + await expect(sendTelemetry({ v: 1 })).resolves.toBeUndefined(); + }); + + it('ignores non-2xx responses', async () => { + fetchMock.mockResolvedValueOnce(new Response(null, { status: 400 })); + await expect(sendTelemetry({ v: 1 })).resolves.toBeUndefined(); + }); +}); + +describe('getToolVersion', () => { + it('reads the package version', () => { + expect(getToolVersion()).toMatch(/^\d+\.\d+\.\d+/); + }); +}); + +describe('buildMigratePayload', () => { + const baseStats = { + source: 'react-bulma-components', + cssMode: 'bestax', + dry: false, + deps: true, + changedCount: 3, + todosByRule: [{ rule: 'unsupported-file', count: 2 }], + }; + + it('contains exactly the allowlisted fields and nothing else', () => { + const payload = buildMigratePayload(baseStats) as Record; + + expect(Object.keys(payload).sort()).toEqual([ + 'event', + 'nodeMajor', + 'platform', + 'props', + 'todosByRule', + 'tool', + 'toolVersion', + 'v', + ]); + expect(payload.v).toBe(1); + expect(payload.tool).toBe('bestax-migrate'); + expect(payload.event).toBe('migrate'); + expect(payload.nodeMajor).toBe(Number(process.versions.node.split('.')[0])); + expect(payload.platform).toBe(process.platform); + expect(Object.keys(payload.props as object).sort()).toEqual([ + 'changedBucket', + 'changedCount', + 'cssMode', + 'deps', + 'dry', + 'source', + ]); + const props = payload.props as Record; + expect(props.source).toBe('react-bulma-components'); + expect(props.cssMode).toBe('bestax'); + expect(props.dry).toBe(false); + expect(props.deps).toBe(true); + expect(props.changedCount).toBe(3); + expect(props.changedBucket).toBe('1-9'); + expect(payload.todosByRule).toEqual([ + { rule: 'unsupported-file', count: 2 }, + ]); + }); + + it('omits todosByRule entirely when there are no TODOs', () => { + const payload = buildMigratePayload({ + ...baseStats, + todosByRule: [], + }) as Record; + expect('todosByRule' in payload).toBe(false); + }); + + it.each([ + [0, '0'], + [1, '1-9'], + [9, '1-9'], + [10, '10-49'], + [49, '10-49'], + [50, '50-199'], + [199, '50-199'], + [200, '200+'], + ] as const)('buckets changedCount=%d as %s', (changedCount, bucket) => { + const payload = buildMigratePayload({ + ...baseStats, + changedCount, + }) as { props: Record }; + expect(payload.props.changedBucket).toBe(bucket); + expect(payload.props.changedCount).toBe(changedCount); + }); + + it('caps changedCount at 10000', () => { + const payload = buildMigratePayload({ + ...baseStats, + changedCount: 123456, + }) as { props: Record }; + expect(payload.props.changedCount).toBe(10000); + expect(payload.props.changedBucket).toBe('200+'); + }); + + it('sends at most 20 rules and caps each count', () => { + const todosByRule = Array.from({ length: 25 }, (_, i) => ({ + rule: `rule-${i}`, + count: i === 0 ? 250000 : i, + })); + const payload = buildMigratePayload({ ...baseStats, todosByRule }) as { + todosByRule: Array<{ rule: string; count: number }>; + }; + expect(payload.todosByRule).toHaveLength(20); + expect(payload.todosByRule[0]).toEqual({ rule: 'rule-0', count: 100000 }); + expect(payload.todosByRule[19]).toEqual({ rule: 'rule-19', count: 19 }); + }); +}); + +describe('reportMigrateRun', () => { + const stats = { + source: 'react-bulma-components', + cssMode: 'keep', + dry: true, + deps: false, + changedCount: 12, + todosByRule: [{ rule: 'unsupported-file', count: 1 }], + }; + + it('flag=true persists and sends without prompting', async () => { + const promptConsent = jest.fn<() => Promise>(); + await reportMigrateRun(stats, true, { interactive: true, promptConsent }); + expect(promptConsent).not.toHaveBeenCalled(); + const config = await readConfigFile(); + expect(config.enabled).toBe(true); + expect(config.decidedBy).toMatch(/^bestax-migrate@/); + expect(fetchMock).toHaveBeenCalledTimes(1); + const body = JSON.parse((fetchMock.mock.calls[0] as FetchArgs)[1].body) as { + props: Record; + }; + expect(body.props.source).toBe('react-bulma-components'); + expect(body.props.changedBucket).toBe('10-49'); + }); + + it('flag=false persists off and sends nothing', async () => { + const promptConsent = jest.fn<() => Promise>(); + await reportMigrateRun(stats, false, { interactive: true, promptConsent }); + expect(promptConsent).not.toHaveBeenCalled(); + expect((await readConfigFile()).enabled).toBe(false); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('prompts when undecided and interactive; yes persists and sends', async () => { + const promptConsent = jest + .fn<() => Promise>() + .mockResolvedValue(true); + await reportMigrateRun(stats, undefined, { + interactive: true, + promptConsent, + }); + expect(promptConsent).toHaveBeenCalledTimes(1); + expect((await readConfigFile()).enabled).toBe(true); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('a "no" answer persists off and sends nothing', async () => { + const promptConsent = jest + .fn<() => Promise>() + .mockResolvedValue(false); + await reportMigrateRun(stats, undefined, { + interactive: true, + promptConsent, + }); + expect((await readConfigFile()).enabled).toBe(false); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('a cancel persists nothing so the user may be asked again', async () => { + const promptConsent = jest + .fn<() => Promise>() + .mockResolvedValue(null); + await reportMigrateRun(stats, undefined, { + interactive: true, + promptConsent, + }); + await expect(readConfigFile()).rejects.toThrow(); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('never prompts when not interactive', async () => { + const promptConsent = jest.fn<() => Promise>(); + await reportMigrateRun(stats, undefined, { + interactive: false, + promptConsent, + }); + expect(promptConsent).not.toHaveBeenCalled(); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('never prompts under DO_NOT_TRACK', async () => { + process.env.DO_NOT_TRACK = '1'; + const promptConsent = jest.fn<() => Promise>(); + await reportMigrateRun(stats, undefined, { + interactive: true, + promptConsent, + }); + expect(promptConsent).not.toHaveBeenCalled(); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('sends without prompting when the config already says on', async () => { + await writeConfig({ version: 1, enabled: true }); + const promptConsent = jest.fn<() => Promise>(); + await reportMigrateRun(stats, undefined, { + interactive: true, + promptConsent, + }); + expect(promptConsent).not.toHaveBeenCalled(); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); +}); diff --git a/bestax-migrate/src/cli.ts b/bestax-migrate/src/cli.ts index 9f3ecd36f..c927f7479 100644 --- a/bestax-migrate/src/cli.ts +++ b/bestax-migrate/src/cli.ts @@ -3,6 +3,7 @@ * * bestax-migrate [--dry] [--print] [--extensions ...] * [--css bestax|bulma|keep] [--no-deps] + * [--telemetry|--no-telemetry] * * Walks the given files/directories, routes each file by type (JS/TSX → * jscodeshift transform, .scss/.sass → the source's stylesheet transform), @@ -13,11 +14,14 @@ import fs from 'node:fs'; import path from 'node:path'; +import { createInterface } from 'node:readline/promises'; +import type { Readable, Writable } from 'node:stream'; import chalk from 'chalk'; import { Command } from 'commander'; import { Reporter } from './report.js'; import { runTransform } from './runner.js'; import { getSource, sourceNames } from './sources/registry.js'; +import { reportMigrateRun, type MigrateRunStats } from './telemetry.js'; import type { CssMode, MigrationSource } from './types.js'; const SKIP_DIRS = new Set([ @@ -212,6 +216,87 @@ function migrateDependencies( } } +const TELEMETRY_NOTICE = + 'Help improve bestax-migrate — share anonymous usage stats?\n' + + 'Sends only rule names and counts — never file paths or code.\n' + + ' Details & opt-out: https://bestax.io/docs/guides/telemetry\n' + + ' Feedback welcome: https://github.com/allxsmith/bestax/issues'; +const TELEMETRY_ACK_ON = 'Thanks! Opt out anytime with --no-telemetry.'; +const TELEMETRY_ACK_OFF = "No problem — we won't ask again."; + +/** + * One-question consent prompt (readline — this package carries no interactive + * prompt dependency). Returns null when the question is cancelled (Ctrl-C / + * Ctrl-D): a cancel is not an answer and nothing is persisted. The streams are + * injectable so tests can drive the question without a TTY. + */ +export async function promptTelemetryConsent( + io: CliIo, + input: Readable = process.stdin, + output: Writable = process.stdout +): Promise { + io.log(''); + io.log(chalk.gray(TELEMETRY_NOTICE)); + const rl = createInterface({ input, output }); + const cancelled = new AbortController(); + // Ctrl-C emits SIGINT on the interface and Ctrl-D closes it; both must + // reject the pending question instead of leaving it hanging forever. + rl.once('SIGINT', () => rl.close()); + rl.once('close', () => cancelled.abort()); + try { + const answer = await rl.question('Share anonymous usage stats? (y/N) ', { + signal: cancelled.signal, + }); + return /^y(es)?$/i.test(answer.trim()); + } catch { + return null; + } finally { + rl.close(); + } +} + +/** + * Consent + beacon after a successful run. Only success events are reported, + * the beacon is awaited (so no error path can kill an in-flight request), and + * nothing in here may throw or alter the exit code. The prompt is skipped + * without a TTY on both ends, under an explicit flag, or with DO_NOT_TRACK + * set (the non-interactive path must never hang). + */ +export async function handleTelemetry( + stats: MigrateRunStats, + flag: boolean | undefined, + io: CliIo, + promptConsent: (io: CliIo) => Promise = promptTelemetryConsent +): Promise { + try { + await reportMigrateRun(stats, flag, { + interactive: + process.stdin.isTTY === true && process.stdout.isTTY === true, + promptConsent: async () => { + const answer = await promptConsent(io); + if (answer === true) { + io.log(chalk.dim(TELEMETRY_ACK_ON)); + } else if (answer === false) { + io.log(chalk.dim(TELEMETRY_ACK_OFF)); + } + return answer; + }, + }); + } catch { + // Telemetry must never affect the migration's outcome. + } +} + +/** Post-run "star us" nudge — TTY only, so piped output stays parseable. */ +function displayStarNudge(io: CliIo): void { + if (process.stdout.isTTY !== true) return; + io.log(''); + io.log( + chalk.yellow('★ If you enjoy using bestax-bulma, please star us on GitHub!') + ); + io.log(chalk.dim(' https://github.com/allxsmith/bestax')); +} + export function createCLI( io: CliIo = { log: console.log, error: console.error } ): Command { @@ -243,7 +328,12 @@ export function createCLI( 'bestax' ) .option('--no-deps', 'skip updating package.json dependencies') - .action((sourceName: string, targets: string[], options) => { + .option( + '--telemetry', + 'enable anonymous usage telemetry (https://bestax.io/docs/guides/telemetry)' + ) + .option('--no-telemetry', 'disable anonymous usage telemetry') + .action(async (sourceName: string, targets: string[], options) => { const source = getSource(sourceName); if (!source) { io.error( @@ -302,6 +392,25 @@ export function createCLI( io.log( reporter.render(source.label + (runOptions.dry ? ' (dry run)' : '')) ); + + displayStarNudge(io); + + // Success path only: rule names and counts, never file/line/message. + const stats: MigrateRunStats = { + source: source.name, + cssMode: runOptions.cssMode, + dry: runOptions.dry, + deps: runOptions.deps, + changedCount: reporter.changedCount, + todosByRule: reporter + .todosByRule() + .map(({ rule, entries }) => ({ rule, count: entries.length })), + }; + await handleTelemetry( + stats, + options.telemetry as boolean | undefined, + io + ); }); return program; } diff --git a/bestax-migrate/src/index.ts b/bestax-migrate/src/index.ts index 293b4fdd3..837288375 100644 --- a/bestax-migrate/src/index.ts +++ b/bestax-migrate/src/index.ts @@ -22,4 +22,4 @@ if (major < 22) { const { createCLI } = await import('./cli.js'); -createCLI().parse(); +await createCLI().parseAsync(process.argv); diff --git a/bestax-migrate/src/telemetry.ts b/bestax-migrate/src/telemetry.ts new file mode 100644 index 000000000..0f68e99df --- /dev/null +++ b/bestax-migrate/src/telemetry.ts @@ -0,0 +1,257 @@ +import { mkdir, readFile, writeFile } from 'fs/promises'; +import { homedir } from 'os'; +import { dirname, join } from 'path'; +import { fileURLToPath } from 'url'; +import { readFileSync } from 'fs'; + +/** + * Anonymous opt-in telemetry (disclosure: https://bestax.io/docs/guides/telemetry). + * + * A near-identical copy lives in create-bestax/src/telemetry.ts — both CLIs + * publish standalone (plain tsc, files:["dist"]), so the ~150 shared lines are + * duplicated rather than published as a package. Keep the copies in sync. + * + * Privacy contract: every transmitted value is a closed-enum member, a version + * string, or a bounded integer — never paths, project names, or free text — + * and no identifier of any kind is generated. Undecided means off. Every + * failure here is silent: telemetry must never affect the CLI's outcome. + */ + +export type TelemetryDecision = 'on' | 'off' | 'undecided'; + +export type TelemetrySource = 'flag' | 'dnt' | 'env' | 'config' | 'default'; + +export interface ResolvedTelemetry { + decision: TelemetryDecision; + source: TelemetrySource; + /** Whether a consent prompt may still be shown (nothing has decided or forbidden it). */ + promptAllowed: boolean; +} + +interface TelemetryConfig { + version: 1; + enabled?: boolean; + decidedAt?: string; + decidedBy?: string; + noticed?: Record; +} + +const DEFAULT_ENDPOINT = 'https://bestax.io/api/t'; +const TIMEOUT_MS = 1500; + +/** Shared across the bestax CLI family so consent is asked at most once. */ +export function telemetryConfigPath(): string { + const base = process.env.XDG_CONFIG_HOME || join(homedir(), '.config'); + return join(base, 'bestax', 'telemetry.json'); +} + +async function readConfig(): Promise { + try { + const parsed: unknown = JSON.parse( + await readFile(telemetryConfigPath(), 'utf-8') + ); + if (parsed !== null && typeof parsed === 'object') { + return parsed as TelemetryConfig; + } + } catch { + // Missing or corrupt config reads as "never asked". + } + return null; +} + +function doNotTrack(): boolean { + const dnt = process.env.DO_NOT_TRACK; + return dnt !== undefined && dnt !== '' && dnt !== '0'; +} + +/** + * Precedence: flag (persisted by the caller) → DO_NOT_TRACK → BESTAX_TELEMETRY + * (per-run, never persisted) → config file → undecided (off). + */ +export async function resolveTelemetry( + flag?: boolean +): Promise { + if (flag !== undefined) { + return { + decision: flag ? 'on' : 'off', + source: 'flag', + promptAllowed: false, + }; + } + if (doNotTrack()) { + return { decision: 'off', source: 'dnt', promptAllowed: false }; + } + const env = process.env.BESTAX_TELEMETRY; + if (env === '0' || env === '1') { + return { + decision: env === '1' ? 'on' : 'off', + source: 'env', + promptAllowed: false, + }; + } + const config = await readConfig(); + if (config && typeof config.enabled === 'boolean') { + return { + decision: config.enabled ? 'on' : 'off', + source: 'config', + promptAllowed: false, + }; + } + return { decision: 'undecided', source: 'default', promptAllowed: true }; +} + +export async function persistTelemetryDecision( + enabled: boolean, + decidedBy: string +): Promise { + try { + const existing = await readConfig(); + const config: TelemetryConfig = { + version: 1, + enabled, + decidedAt: new Date().toISOString(), + decidedBy, + ...(existing?.noticed ? { noticed: existing.noticed } : {}), + }; + const filePath = telemetryConfigPath(); + await mkdir(dirname(filePath), { recursive: true }); + await writeFile(filePath, JSON.stringify(config, null, 2) + '\n', 'utf-8'); + } catch { + // An unwritable config dir must not break the CLI. + } +} + +/** + * Records that `tool` has shown its one-time notice. Returns true when the + * notice should be shown this run (nothing recorded it before), false when it + * was already recorded. Never throws; a failed write still returns true — the + * notice was shown this run and may simply repeat later, which beats crashing + * or silently suppressing it. + */ +export async function markNoticed(tool: string): Promise { + try { + const existing = await readConfig(); + if (existing?.noticed?.[tool]) return false; + const config: TelemetryConfig = { + version: 1, + ...existing, + noticed: { ...existing?.noticed, [tool]: new Date().toISOString() }, + }; + const filePath = telemetryConfigPath(); + await mkdir(dirname(filePath), { recursive: true }); + await writeFile(filePath, JSON.stringify(config, null, 2) + '\n', 'utf-8'); + } catch { + // A failed write only means the notice may show again later. + } + return true; +} + +export async function sendTelemetry(payload: object): Promise { + try { + await fetch(process.env.BESTAX_TELEMETRY_ENDPOINT || DEFAULT_ENDPOINT, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(payload), + signal: AbortSignal.timeout(TIMEOUT_MS), + }); + } catch { + // Fire-and-forget: the response (or its absence) is never surfaced. + } +} + +export function getToolVersion(): string { + try { + const dir = dirname(fileURLToPath(import.meta.url)); + const packageJson = JSON.parse( + readFileSync(join(dir, '..', 'package.json'), 'utf-8') + ) as { version?: string }; + return packageJson.version || '0.0.0'; + } catch { + return '0.0.0'; + } +} + +// --- bestax-migrate specific (the sibling copy has its own payload builder) --- + +export interface MigrateRunStats { + /** Registry source name, e.g. 'react-bulma-components'. */ + source: string; + /** 'bestax' | 'bulma' | 'keep'. */ + cssMode: string; + dry: boolean; + deps: boolean; + changedCount: number; + todosByRule: Array<{ rule: string; count: number }>; +} + +function changedBucket(changedCount: number): string { + if (changedCount <= 0) return '0'; + if (changedCount <= 9) return '1-9'; + if (changedCount <= 49) return '10-49'; + if (changedCount <= 199) return '50-199'; + return '200+'; +} + +export function buildMigratePayload(stats: MigrateRunStats): object { + const todosByRule = stats.todosByRule + .slice(0, 20) + .map(({ rule, count }) => ({ rule, count: Math.min(count, 100000) })); + return { + v: 1, + tool: 'bestax-migrate', + event: 'migrate', + toolVersion: getToolVersion(), + nodeMajor: Number(process.versions.node.split('.')[0]), + platform: process.platform, + props: { + source: stats.source, + cssMode: stats.cssMode, + dry: stats.dry, + deps: stats.deps, + changedBucket: changedBucket(stats.changedCount), + changedCount: Math.min(stats.changedCount, 10000), + }, + ...(todosByRule.length > 0 ? { todosByRule } : {}), + }; +} + +export interface ReportMigrateOptions { + /** True only when a consent question could actually be answered (TTY in and out). */ + interactive: boolean; + /** Owns all consent UI (notice, question, ack); null means cancelled. */ + promptConsent: () => Promise; +} + +export async function reportMigrateRun( + stats: MigrateRunStats, + flag: boolean | undefined, + options: ReportMigrateOptions +): Promise { + const resolved = await resolveTelemetry(flag); + let decision = resolved.decision; + + if (resolved.source === 'flag') { + // Flags are the scripted equivalent of the prompt: decided once, remembered. + await persistTelemetryDecision( + decision === 'on', + `bestax-migrate@${getToolVersion()}` + ); + } else if ( + decision === 'undecided' && + resolved.promptAllowed && + options.interactive + ) { + const answer = await options.promptConsent(); + if (answer !== null) { + await persistTelemetryDecision( + answer, + `bestax-migrate@${getToolVersion()}` + ); + decision = answer ? 'on' : 'off'; + } + } + + if (decision === 'on') { + await sendTelemetry(buildMigratePayload(stats)); + } +} From b6b1a1f42d99ac14ae93c638d1a2ff2559791f40 Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sat, 22 Aug 2026 15:20:04 -0400 Subject: [PATCH 07/29] feat(bestax-mcp): attribute emitted docs links with utm_source Tag the docs/Storybook URLs already printed in tool responses with utm_source=bestax-mcp so follow-up visits are attributable in the site's own traffic analytics. Render-time only via attributed() in format.ts: the server stays fully offline, data/ is untouched, and the read-only tool annotations remain true. --- bestax-mcp/src/__tests__/format.test.ts | 58 ++++++++++++++++++++++++ bestax-mcp/src/__tests__/server.test.ts | 8 ++++ bestax-mcp/src/__tests__/version.test.ts | 5 ++ bestax-mcp/src/format.ts | 18 +++++++- bestax-mcp/src/version.ts | 4 +- 5 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 bestax-mcp/src/__tests__/format.test.ts diff --git a/bestax-mcp/src/__tests__/format.test.ts b/bestax-mcp/src/__tests__/format.test.ts new file mode 100644 index 000000000..050678f5c --- /dev/null +++ b/bestax-mcp/src/__tests__/format.test.ts @@ -0,0 +1,58 @@ +/** + * Unit tests for `attributed`, the link tagger. + * + * The server is offline by design (openWorldHint: false), so the tag on its + * outbound links is the only attribution signal it has — these pin the + * function's contract. The emission paths themselves are asserted where the + * links surface: server.test.ts for the component link line, version.test.ts + * for the drift note. + */ +import { describe, expect, it } from '@jest/globals'; + +import { loadComponent } from '../data.js'; +import { attributed, renderComponent } from '../format.js'; + +describe('attributed', () => { + it('appends with ? when the URL has no query string', () => { + expect(attributed('https://bestax.io/docs/api/elements/button')).toBe( + 'https://bestax.io/docs/api/elements/button?utm_source=bestax-mcp' + ); + }); + + it('appends with & when the URL already has one, as Storybook links do', () => { + expect( + attributed('https://bestax.io/storybook/?path=/story/elements-button') + ).toBe( + 'https://bestax.io/storybook/?path=/story/elements-button&utm_source=bestax-mcp' + ); + }); + + it('is idempotent', () => { + const once = attributed('https://bestax.io/docs'); + expect(attributed(once)).toBe(once); + }); + + it('returns an already-tagged URL unchanged', () => { + const tagged = 'https://bestax.io/docs?utm_source=bestax-mcp'; + expect(attributed(tagged)).toBe(tagged); + }); + + it('passes non-http(s) strings through unchanged', () => { + for (const notAUrl of ['bestax://catalog', 'mailto:a@b.c', 'Button', '']) { + expect(attributed(notAUrl)).toBe(notAUrl); + } + }); +}); + +describe('renderComponent link line', () => { + it('tags both the docs and Storybook links', async () => { + const record = await loadComponent('Button'); + const out = renderComponent(record, ['props']); + // Against the committed index: the docs URL carries no query string, the + // Storybook URL carries `?path=...` — so this exercises both separators. + expect(out).toContain(`Docs: ${record.docsUrl}?utm_source=bestax-mcp`); + expect(out).toContain( + `Storybook: ${record.storybook}&utm_source=bestax-mcp` + ); + }); +}); diff --git a/bestax-mcp/src/__tests__/server.test.ts b/bestax-mcp/src/__tests__/server.test.ts index b5b080dea..27c65f766 100644 --- a/bestax-mcp/src/__tests__/server.test.ts +++ b/bestax-mcp/src/__tests__/server.test.ts @@ -354,6 +354,14 @@ describe('get_component', () => { expect(failed(res)).toBe(true); expect(text(res)).toContain('Button'); }); + + // The server itself never goes online; the utm tag on the links it hands out + // is what makes a follow-through visible in the bestax.io analytics. + it('tags the emitted docs and Storybook links for attribution', async () => { + const out = text(await call('get_component', { name: 'Button' })); + expect(out).toMatch(/Docs: https:\/\/\S+\?utm_source=bestax-mcp/); + expect(out).toMatch(/Storybook: https:\/\/\S+&utm_source=bestax-mcp/); + }); }); describe('get_props', () => { diff --git a/bestax-mcp/src/__tests__/version.test.ts b/bestax-mcp/src/__tests__/version.test.ts index fd4bd017b..c3ebbcdce 100644 --- a/bestax-mcp/src/__tests__/version.test.ts +++ b/bestax-mcp/src/__tests__/version.test.ts @@ -211,6 +211,11 @@ describe('versionNote', () => { expect(note).toContain('4.0.0'); }); + it('tags the docs link it points at, like every emitted link', () => { + const note = versionNote(info({ installed: '4.0.0', drift: 'major' })); + expect(note).toContain('https://bestax.io/docs?utm_source=bestax-mcp'); + }); + it('warns on a minor mismatch, where props can move', () => { expect(versionNote(info({ installed: '5.2.0', drift: 'minor' }))).toContain( '⚠' diff --git a/bestax-mcp/src/format.ts b/bestax-mcp/src/format.ts index e343b1e07..f11b51a76 100644 --- a/bestax-mcp/src/format.ts +++ b/bestax-mcp/src/format.ts @@ -28,6 +28,18 @@ const cell = (text: string) => .replace(/\r?\n/g, ' ') .trim(); +/** + * Tag an outbound link with `utm_source=bestax-mcp` so a docs or Storybook + * visit that started here shows up as such in the site analytics. Render-time + * only — the URLs in `data/` are generated and stay canonical. Idempotent, and + * anything that is not an http(s) URL passes through unchanged. + */ +export function attributed(url: string): string { + if (!/^https?:\/\//.test(url)) return url; + if (url.includes('utm_source=bestax-mcp')) return url; + return `${url}${url.includes('?') ? '&' : '?'}utm_source=bestax-mcp`; +} + export function table(headers: string[], rows: string[][]): string { if (!rows.length) return ''; return [ @@ -145,8 +157,10 @@ export function renderComponent( out.push(`**Related:** ${record.related.map(r => `\`${r}\``).join(', ')}.`); } - const links = [`Docs: ${record.docsUrl}`]; - if (record.storybook) links.push(`Storybook: ${record.storybook}`); + const links = [`Docs: ${attributed(record.docsUrl)}`]; + if (record.storybook) { + links.push(`Storybook: ${attributed(record.storybook)}`); + } out.push(links.join(' · ')); return out.join('\n\n'); } diff --git a/bestax-mcp/src/version.ts b/bestax-mcp/src/version.ts index 9145c1bbb..8788a8a80 100644 --- a/bestax-mcp/src/version.ts +++ b/bestax-mcp/src/version.ts @@ -16,6 +16,8 @@ import { readFile } from 'node:fs/promises'; import { dirname, join, parse } from 'node:path'; +import { attributed } from './format.js'; + const PACKAGE = '@allxsmith/bestax-bulma'; export interface VersionInfo { @@ -171,6 +173,6 @@ export function versionNote(info: VersionInfo): string | null { return ( `⚠ This index documents ${PACKAGE} ${info.indexed}; this project has ` + `${info.installed}. Props, defaults and CSS variables may differ — check ` + - `https://bestax.io/docs before relying on anything above.` + `${attributed('https://bestax.io/docs')} before relying on anything above.` ); } From 453e11b8e2bc496094bdf409d5e3ddfdeb531818 Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sat, 22 Aug 2026 16:42:16 -0400 Subject: [PATCH 08/29] refactor(create-bestax): extract shared telemetry kernel and await parseAsync Move consent/send into telemetry-core.ts (byte-identical copy in bestax-migrate). Drop the dead noticed/promptAllowed state. The CLI action now returns its promise and the bin uses parseAsync so a successful scaffold cannot exit before the beacon finishes. --- create-bestax/src/__tests__/telemetry.test.ts | 26 +-- create-bestax/src/cli.ts | 10 +- create-bestax/src/index.ts | 3 +- create-bestax/src/telemetry-core.ts | 173 ++++++++++++++ create-bestax/src/telemetry.ts | 214 +++--------------- 5 files changed, 215 insertions(+), 211 deletions(-) create mode 100644 create-bestax/src/telemetry-core.ts diff --git a/create-bestax/src/__tests__/telemetry.test.ts b/create-bestax/src/__tests__/telemetry.test.ts index bd663f4a1..d5686fcee 100644 --- a/create-bestax/src/__tests__/telemetry.test.ts +++ b/create-bestax/src/__tests__/telemetry.test.ts @@ -17,9 +17,9 @@ const { persistTelemetryDecision, sendTelemetry, getToolVersion, - buildScaffoldPayload, - reportScaffold, -} = await import('../telemetry.js'); +} = await import('../telemetry-core.js'); +const { buildScaffoldPayload, reportScaffold } = + await import('../telemetry.js'); const ENV_KEYS = [ 'XDG_CONFIG_HOME', @@ -96,11 +96,10 @@ describe('telemetryConfigPath', () => { }); describe('resolveTelemetry precedence', () => { - it('is undecided (prompt allowed) with no signals at all', async () => { + it('is undecided with no signals at all', async () => { expect(await resolveTelemetry()).toEqual({ decision: 'undecided', source: 'default', - promptAllowed: true, }); }); @@ -114,7 +113,6 @@ describe('resolveTelemetry precedence', () => { expect(await resolveTelemetry(flag)).toEqual({ decision, source: 'flag', - promptAllowed: false, }); }); @@ -125,7 +123,6 @@ describe('resolveTelemetry precedence', () => { expect(await resolveTelemetry()).toEqual({ decision: 'off', source: 'dnt', - promptAllowed: false, }); }); @@ -141,7 +138,6 @@ describe('resolveTelemetry precedence', () => { expect(await resolveTelemetry()).toEqual({ decision: 'on', source: 'env', - promptAllowed: false, }); }); @@ -166,7 +162,6 @@ describe('resolveTelemetry precedence', () => { expect(await resolveTelemetry()).toEqual({ decision, source: 'config', - promptAllowed: false, }); } ); @@ -197,19 +192,6 @@ describe('persistTelemetryDecision', () => { expect(typeof config.decidedAt).toBe('string'); }); - it('preserves the noticed map of the sibling CLI', async () => { - await writeConfig({ - version: 1, - noticed: { 'bestax-migrate': '2026-01-01T00:00:00.000Z' }, - }); - await persistTelemetryDecision(false, 'create-bestax@1.0.0'); - const config = await readConfigFile(); - expect(config.enabled).toBe(false); - expect(config.noticed).toEqual({ - 'bestax-migrate': '2026-01-01T00:00:00.000Z', - }); - }); - it('stays silent when the config dir is unwritable', async () => { const blocker = join(configHome, 'blocker'); await writeFile(blocker, 'file, not a dir', 'utf-8'); diff --git a/create-bestax/src/cli.ts b/create-bestax/src/cli.ts index fc6dcdce5..ab0a4ef48 100644 --- a/create-bestax/src/cli.ts +++ b/create-bestax/src/cli.ts @@ -105,8 +105,8 @@ export function createCLI(): Command { ) .option('--no-telemetry', 'disable anonymous usage telemetry') .option('-y, --yes', 'skip prompts and use defaults or provided options') - .action((projectDir?: string, options?: unknown) => { - projectCreator.create(projectDir, options as CLIOptions); + .action(async (projectDir?: string, options?: unknown) => { + await projectCreator.create(projectDir, options as CLIOptions); }); return program; @@ -116,13 +116,13 @@ export function isMainModule(importMetaUrl: string, argv1: string): boolean { return importMetaUrl === `file://${argv1}`; } -export function runCLI(): void { +export async function runCLI(): Promise { const program = createCLI(); - program.parse(); + await program.parseAsync(); } // Only run if this is the main module /* istanbul ignore next */ if (isMainModule(import.meta.url, process.argv[1])) { - runCLI(); + await runCLI(); } diff --git a/create-bestax/src/index.ts b/create-bestax/src/index.ts index 1292b7243..f5cb29dc4 100644 --- a/create-bestax/src/index.ts +++ b/create-bestax/src/index.ts @@ -25,5 +25,4 @@ export {}; const { createCLI } = await import('./cli.js'); -const program = createCLI(); -program.parse(); +await createCLI().parseAsync(process.argv); diff --git a/create-bestax/src/telemetry-core.ts b/create-bestax/src/telemetry-core.ts new file mode 100644 index 000000000..c68cb22b4 --- /dev/null +++ b/create-bestax/src/telemetry-core.ts @@ -0,0 +1,173 @@ +import { mkdir, readFile, writeFile } from 'fs/promises'; +import { homedir } from 'os'; +import { dirname, join } from 'path'; +import { fileURLToPath } from 'url'; +import { readFileSync } from 'fs'; + +/** + * Shared CLI telemetry kernel (disclosure: https://bestax.io/docs/guides/telemetry). + * + * Copied byte-for-byte into create-bestax and bestax-migrate — both publish + * standalone, so this cannot be a workspace package. `check:conformance + * --only=telemetry-core` fails if the copies diverge. Edit this file in + * create-bestax and copy it over the migrate one. + * + * No identifier of any kind is generated. Undecided means off. Every failure + * here is silent: telemetry must never affect the CLI's outcome. + */ + +export type TelemetryDecision = 'on' | 'off' | 'undecided'; + +export type TelemetrySource = 'flag' | 'dnt' | 'env' | 'config' | 'default'; + +export interface ResolvedTelemetry { + decision: TelemetryDecision; + source: TelemetrySource; +} + +interface TelemetryConfig { + version: 1; + enabled?: boolean; + decidedAt?: string; + decidedBy?: string; +} + +const DEFAULT_ENDPOINT = 'https://bestax.io/api/t'; +const TIMEOUT_MS = 1500; + +/** Shared across the bestax CLI family so consent is asked at most once. */ +export function telemetryConfigPath(): string { + const base = process.env.XDG_CONFIG_HOME || join(homedir(), '.config'); + return join(base, 'bestax', 'telemetry.json'); +} + +async function readConfig(): Promise { + try { + const parsed: unknown = JSON.parse( + await readFile(telemetryConfigPath(), 'utf-8') + ); + if (parsed !== null && typeof parsed === 'object') { + return parsed as TelemetryConfig; + } + } catch { + // Missing or corrupt config reads as "never asked". + } + return null; +} + +function doNotTrack(): boolean { + const dnt = process.env.DO_NOT_TRACK; + return dnt !== undefined && dnt !== '' && dnt !== '0'; +} + +/** + * Precedence: flag (persisted by the caller) → DO_NOT_TRACK → BESTAX_TELEMETRY + * (per-run, never persisted) → config file → undecided (off). + */ +export async function resolveTelemetry( + flag?: boolean +): Promise { + if (flag !== undefined) { + return { + decision: flag ? 'on' : 'off', + source: 'flag', + }; + } + if (doNotTrack()) { + return { decision: 'off', source: 'dnt' }; + } + const env = process.env.BESTAX_TELEMETRY; + if (env === '0' || env === '1') { + return { + decision: env === '1' ? 'on' : 'off', + source: 'env', + }; + } + const config = await readConfig(); + if (config && typeof config.enabled === 'boolean') { + return { + decision: config.enabled ? 'on' : 'off', + source: 'config', + }; + } + return { decision: 'undecided', source: 'default' }; +} + +export async function persistTelemetryDecision( + enabled: boolean, + decidedBy: string +): Promise { + try { + const config: TelemetryConfig = { + version: 1, + enabled, + decidedAt: new Date().toISOString(), + decidedBy, + }; + const filePath = telemetryConfigPath(); + await mkdir(dirname(filePath), { recursive: true }); + await writeFile(filePath, JSON.stringify(config, null, 2) + '\n', 'utf-8'); + } catch { + // An unwritable config dir must not break the CLI. + } +} + +export async function sendTelemetry(payload: unknown): Promise { + try { + await fetch(process.env.BESTAX_TELEMETRY_ENDPOINT || DEFAULT_ENDPOINT, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(payload), + signal: AbortSignal.timeout(TIMEOUT_MS), + }); + } catch { + // Fire-and-forget: the response (or its absence) is never surfaced. + } +} + +export function getToolVersion(): string { + try { + const dir = dirname(fileURLToPath(import.meta.url)); + const packageJson = JSON.parse( + readFileSync(join(dir, '..', 'package.json'), 'utf-8') + ) as { version?: string }; + return packageJson.version || '0.0.0'; + } catch { + return '0.0.0'; + } +} + +export interface ReportRunOptions { + /** True only when a consent question could actually be answered. */ + interactive: boolean; + /** Owns all consent UI (notice, question, ack); null means cancelled. */ + promptConsent: () => Promise; +} + +export async function reportRun( + flag: boolean | undefined, + toolName: string, + options: ReportRunOptions, + payload: unknown +): Promise { + const resolved = await resolveTelemetry(flag); + let decision = resolved.decision; + + if (resolved.source === 'flag') { + // Flags are the scripted equivalent of the prompt: decided once, remembered. + await persistTelemetryDecision( + decision === 'on', + `${toolName}@${getToolVersion()}` + ); + } else if (decision === 'undecided' && options.interactive) { + const answer = await options.promptConsent(); + if (answer !== null) { + await persistTelemetryDecision(answer, `${toolName}@${getToolVersion()}`); + decision = answer ? 'on' : 'off'; + } + } + + if (decision === 'on') { + await sendTelemetry(payload); + } +} diff --git a/create-bestax/src/telemetry.ts b/create-bestax/src/telemetry.ts index b10d89676..6ff65ebc7 100644 --- a/create-bestax/src/telemetry.ts +++ b/create-bestax/src/telemetry.ts @@ -1,154 +1,14 @@ -import { mkdir, readFile, writeFile } from 'fs/promises'; -import { homedir } from 'os'; -import { dirname, join } from 'path'; -import { fileURLToPath } from 'url'; -import { readFileSync } from 'fs'; import { detectPackageManager } from './package-manager.js'; +import { getToolVersion, reportRun } from './telemetry-core.js'; +import type { ReportRunOptions } from './telemetry-core.js'; /** - * Anonymous opt-in telemetry (disclosure: https://bestax.io/docs/guides/telemetry). - * - * A near-identical copy lives in bestax-migrate/src/telemetry.ts — both CLIs - * publish standalone (plain tsc, files:["dist"]), so the ~150 shared lines are - * duplicated rather than published as a package. Keep the copies in sync. - * - * Privacy contract: every transmitted value is a closed-enum member, a version - * string, or a bounded integer — never paths, project names, or free text — - * and no identifier of any kind is generated. Undecided means off. Every - * failure here is silent: telemetry must never affect the CLI's outcome. + * create-bestax payload + reporter. The kernel lives in telemetry-core.ts + * (copied into bestax-migrate; see that file). Envelope and props are closed + * enums, a version string, or a bounded integer — never paths, project names, + * or free text. */ -export type TelemetryDecision = 'on' | 'off' | 'undecided'; - -export type TelemetrySource = 'flag' | 'dnt' | 'env' | 'config' | 'default'; - -export interface ResolvedTelemetry { - decision: TelemetryDecision; - source: TelemetrySource; - /** Whether a consent prompt may still be shown (nothing has decided or forbidden it). */ - promptAllowed: boolean; -} - -interface TelemetryConfig { - version: 1; - enabled?: boolean; - decidedAt?: string; - decidedBy?: string; - noticed?: Record; -} - -const DEFAULT_ENDPOINT = 'https://bestax.io/api/t'; -const TIMEOUT_MS = 1500; - -/** Shared across the bestax CLI family so consent is asked at most once. */ -export function telemetryConfigPath(): string { - const base = process.env.XDG_CONFIG_HOME || join(homedir(), '.config'); - return join(base, 'bestax', 'telemetry.json'); -} - -async function readConfig(): Promise { - try { - const parsed: unknown = JSON.parse( - await readFile(telemetryConfigPath(), 'utf-8') - ); - if (parsed !== null && typeof parsed === 'object') { - return parsed as TelemetryConfig; - } - } catch { - // Missing or corrupt config reads as "never asked". - } - return null; -} - -function doNotTrack(): boolean { - const dnt = process.env.DO_NOT_TRACK; - return dnt !== undefined && dnt !== '' && dnt !== '0'; -} - -/** - * Precedence: flag (persisted by the caller) → DO_NOT_TRACK → BESTAX_TELEMETRY - * (per-run, never persisted) → config file → undecided (off). - */ -export async function resolveTelemetry( - flag?: boolean -): Promise { - if (flag !== undefined) { - return { - decision: flag ? 'on' : 'off', - source: 'flag', - promptAllowed: false, - }; - } - if (doNotTrack()) { - return { decision: 'off', source: 'dnt', promptAllowed: false }; - } - const env = process.env.BESTAX_TELEMETRY; - if (env === '0' || env === '1') { - return { - decision: env === '1' ? 'on' : 'off', - source: 'env', - promptAllowed: false, - }; - } - const config = await readConfig(); - if (config && typeof config.enabled === 'boolean') { - return { - decision: config.enabled ? 'on' : 'off', - source: 'config', - promptAllowed: false, - }; - } - return { decision: 'undecided', source: 'default', promptAllowed: true }; -} - -export async function persistTelemetryDecision( - enabled: boolean, - decidedBy: string -): Promise { - try { - const existing = await readConfig(); - const config: TelemetryConfig = { - version: 1, - enabled, - decidedAt: new Date().toISOString(), - decidedBy, - ...(existing?.noticed ? { noticed: existing.noticed } : {}), - }; - const filePath = telemetryConfigPath(); - await mkdir(dirname(filePath), { recursive: true }); - await writeFile(filePath, JSON.stringify(config, null, 2) + '\n', 'utf-8'); - } catch { - // An unwritable config dir must not break the CLI. - } -} - -export async function sendTelemetry(payload: object): Promise { - try { - await fetch(process.env.BESTAX_TELEMETRY_ENDPOINT || DEFAULT_ENDPOINT, { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify(payload), - signal: AbortSignal.timeout(TIMEOUT_MS), - }); - } catch { - // Fire-and-forget: the response (or its absence) is never surfaced. - } -} - -export function getToolVersion(): string { - try { - const dir = dirname(fileURLToPath(import.meta.url)); - const packageJson = JSON.parse( - readFileSync(join(dir, '..', 'package.json'), 'utf-8') - ) as { version?: string }; - return packageJson.version || '0.0.0'; - } catch { - return '0.0.0'; - } -} - -// --- create-bestax specific (the sibling copy has its own payload builder) --- - export interface ScaffoldChoices { template: string; bulmaFlavor: string; @@ -156,7 +16,25 @@ export interface ScaffoldChoices { skills: boolean; } -export function buildScaffoldPayload(choices: ScaffoldChoices): object { +export interface ScaffoldPayload { + v: 1; + tool: 'create-bestax'; + event: 'scaffold'; + toolVersion: string; + nodeMajor: number; + platform: string; + props: { + template: string; + bulmaFlavor: string; + iconLibrary: string; + skills: boolean; + packageManager: string; + }; +} + +export function buildScaffoldPayload( + choices: ScaffoldChoices +): ScaffoldPayload { return { v: 1, tool: 'create-bestax', @@ -174,43 +52,15 @@ export function buildScaffoldPayload(choices: ScaffoldChoices): object { }; } -export interface ReportScaffoldOptions { - /** True only when a consent question could actually be answered (TTY, not -y). */ - interactive: boolean; - /** Owns all consent UI (notice, question, ack); null means cancelled. */ - promptConsent: () => Promise; -} - export async function reportScaffold( choices: ScaffoldChoices, flag: boolean | undefined, - options: ReportScaffoldOptions + options: ReportRunOptions ): Promise { - const resolved = await resolveTelemetry(flag); - let decision = resolved.decision; - - if (resolved.source === 'flag') { - // Flags are the scripted equivalent of the prompt: decided once, remembered. - await persistTelemetryDecision( - decision === 'on', - `create-bestax@${getToolVersion()}` - ); - } else if ( - decision === 'undecided' && - resolved.promptAllowed && - options.interactive - ) { - const answer = await options.promptConsent(); - if (answer !== null) { - await persistTelemetryDecision( - answer, - `create-bestax@${getToolVersion()}` - ); - decision = answer ? 'on' : 'off'; - } - } - - if (decision === 'on') { - await sendTelemetry(buildScaffoldPayload(choices)); - } + await reportRun( + flag, + 'create-bestax', + options, + buildScaffoldPayload(choices) + ); } From c4c74269dc301b6248af1f880ee97dac8df3406a Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sat, 22 Aug 2026 16:42:22 -0400 Subject: [PATCH 09/29] refactor(bestax-migrate): share telemetry kernel with create-bestax Same telemetry-core.ts as create-bestax (conformance fails if they diverge). Payload builder stays here; markNoticed is gone. Core unit tests live in create-bestax; this package covers payload shape and reportMigrateRun. --- bestax-migrate/jest.config.mjs | 3 + .../src/__tests__/telemetry.test.ts | 276 +++--------------- bestax-migrate/src/telemetry-core.ts | 173 +++++++++++ bestax-migrate/src/telemetry.ts | 234 ++------------- 4 files changed, 244 insertions(+), 442 deletions(-) create mode 100644 bestax-migrate/src/telemetry-core.ts diff --git a/bestax-migrate/jest.config.mjs b/bestax-migrate/jest.config.mjs index 5245d50df..8dd46478c 100644 --- a/bestax-migrate/jest.config.mjs +++ b/bestax-migrate/jest.config.mjs @@ -14,6 +14,9 @@ export default { '!src/**/__tests__/**', '!src/**/__testfixtures__/**', '!src/index.ts', // bin entry — exercised via the e2e/CLI tests, not unit-covered + // Byte-identical to create-bestax; covered there. Drift fails + // `check:conformance --only=telemetry-core`. + '!src/telemetry-core.ts', ], coverageThreshold: { global: { diff --git a/bestax-migrate/src/__tests__/telemetry.test.ts b/bestax-migrate/src/__tests__/telemetry.test.ts index d9748d4f0..37308b5bf 100644 --- a/bestax-migrate/src/__tests__/telemetry.test.ts +++ b/bestax-migrate/src/__tests__/telemetry.test.ts @@ -11,16 +11,9 @@ import { mkdtemp, readFile, writeFile, mkdir } from 'fs/promises'; import { tmpdir } from 'os'; import { join } from 'path'; -const { - telemetryConfigPath, - resolveTelemetry, - persistTelemetryDecision, - markNoticed, - sendTelemetry, - getToolVersion, - buildMigratePayload, - reportMigrateRun, -} = await import('../telemetry.js'); +const { getToolVersion } = await import('../telemetry-core.js'); +const { buildMigratePayload, reportMigrateRun } = + await import('../telemetry.js'); const ENV_KEYS = [ 'XDG_CONFIG_HOME', @@ -81,221 +74,25 @@ afterAll(() => { globalThis.fetch = originalFetch; }); -describe('telemetryConfigPath', () => { - it('uses XDG_CONFIG_HOME when set', () => { - expect(telemetryConfigPath()).toBe( - join(configHome, 'bestax', 'telemetry.json') - ); - }); - - it('falls back to ~/.config when XDG_CONFIG_HOME is unset', () => { - delete process.env.XDG_CONFIG_HOME; - expect(telemetryConfigPath()).toMatch( - /[/\\]\.config[/\\]bestax[/\\]telemetry\.json$/ - ); - }); -}); - -describe('resolveTelemetry precedence', () => { - it('is undecided (prompt allowed) with no signals at all', async () => { - expect(await resolveTelemetry()).toEqual({ - decision: 'undecided', - source: 'default', - promptAllowed: true, - }); - }); - - it.each([ - [true, 'on'], - [false, 'off'], - ] as const)('flag %s wins over everything', async (flag, decision) => { - process.env.DO_NOT_TRACK = '1'; - process.env.BESTAX_TELEMETRY = flag ? '0' : '1'; - await writeConfig({ version: 1, enabled: !flag }); - expect(await resolveTelemetry(flag)).toEqual({ - decision, - source: 'flag', - promptAllowed: false, - }); - }); - - it('DO_NOT_TRACK beats env var and config', async () => { - process.env.DO_NOT_TRACK = '1'; - process.env.BESTAX_TELEMETRY = '1'; - await writeConfig({ version: 1, enabled: true }); - expect(await resolveTelemetry()).toEqual({ - decision: 'off', - source: 'dnt', - promptAllowed: false, - }); - }); - - it.each(['0', ''])('DO_NOT_TRACK=%j is not a DNT signal', async value => { - process.env.DO_NOT_TRACK = value; - const resolved = await resolveTelemetry(); - expect(resolved.source).toBe('default'); - }); - - it('BESTAX_TELEMETRY=1 forces on without persisting', async () => { - process.env.BESTAX_TELEMETRY = '1'; - await writeConfig({ version: 1, enabled: false }); - expect(await resolveTelemetry()).toEqual({ - decision: 'on', - source: 'env', - promptAllowed: false, - }); - }); - - it('BESTAX_TELEMETRY=0 forces off', async () => { - process.env.BESTAX_TELEMETRY = '0'; - await writeConfig({ version: 1, enabled: true }); - expect((await resolveTelemetry()).decision).toBe('off'); - }); - - it('ignores junk BESTAX_TELEMETRY values', async () => { - process.env.BESTAX_TELEMETRY = 'yes'; - expect((await resolveTelemetry()).source).toBe('default'); - }); - - it.each([ - [true, 'on'], - [false, 'off'], - ] as const)( - 'reads enabled=%s from the config file', - async (enabled, decision) => { - await writeConfig({ version: 1, enabled }); - expect(await resolveTelemetry()).toEqual({ - decision, - source: 'config', - promptAllowed: false, - }); - } - ); - - it('treats corrupt config as undecided', async () => { - await writeConfig('{not json'); - expect((await resolveTelemetry()).decision).toBe('undecided'); - }); - - it('treats a config without a boolean enabled as undecided', async () => { - await writeConfig({ version: 1, enabled: 'yes' }); - expect((await resolveTelemetry()).decision).toBe('undecided'); - }); - - it('treats a non-object config as undecided', async () => { - await writeConfig('null'); - expect((await resolveTelemetry()).decision).toBe('undecided'); - }); -}); - -describe('persistTelemetryDecision', () => { - it('writes the config schema', async () => { - await persistTelemetryDecision(true, 'bestax-migrate@1.0.0'); - const config = await readConfigFile(); - expect(config.version).toBe(1); - expect(config.enabled).toBe(true); - expect(config.decidedBy).toBe('bestax-migrate@1.0.0'); - expect(typeof config.decidedAt).toBe('string'); - }); - - it('preserves the noticed map of the sibling CLI', async () => { - await writeConfig({ - version: 1, - noticed: { 'create-bestax': '2026-01-01T00:00:00.000Z' }, - }); - await persistTelemetryDecision(false, 'bestax-migrate@1.0.0'); - const config = await readConfigFile(); - expect(config.enabled).toBe(false); - expect(config.noticed).toEqual({ - 'create-bestax': '2026-01-01T00:00:00.000Z', - }); - }); - - it('stays silent when the config dir is unwritable', async () => { - const blocker = join(configHome, 'blocker'); - await writeFile(blocker, 'file, not a dir', 'utf-8'); - process.env.XDG_CONFIG_HOME = blocker; - await expect( - persistTelemetryDecision(true, 'bestax-migrate@1.0.0') - ).resolves.toBeUndefined(); - }); -}); - -describe('markNoticed', () => { - it('records the first notice and returns true', async () => { - expect(await markNoticed('bestax-migrate')).toBe(true); - const config = await readConfigFile(); - expect(config.version).toBe(1); - const noticed = config.noticed as Record; - expect(new Date(noticed['bestax-migrate']).getTime()).not.toBeNaN(); - }); - - it('returns false once the notice is recorded', async () => { - expect(await markNoticed('bestax-migrate')).toBe(true); - expect(await markNoticed('bestax-migrate')).toBe(false); - }); - - it('preserves every other config field and sibling notices', async () => { - await writeConfig({ - version: 1, - enabled: true, - decidedAt: '2026-01-01T00:00:00.000Z', - decidedBy: 'create-bestax@1.0.0', - noticed: { 'create-bestax': '2026-01-01T00:00:00.000Z' }, - }); - expect(await markNoticed('bestax-migrate')).toBe(true); - const config = await readConfigFile(); - expect(config.enabled).toBe(true); - expect(config.decidedBy).toBe('create-bestax@1.0.0'); - const noticed = config.noticed as Record; - expect(noticed['create-bestax']).toBe('2026-01-01T00:00:00.000Z'); - expect(noticed['bestax-migrate']).toBeDefined(); - }); - - it('returns true without throwing when the config dir is unwritable', async () => { - // The notice is shown this run and may simply repeat later. - const blocker = join(configHome, 'blocker'); - await writeFile(blocker, 'file, not a dir', 'utf-8'); - process.env.XDG_CONFIG_HOME = blocker; - await expect(markNoticed('bestax-migrate')).resolves.toBe(true); - }); -}); - -describe('sendTelemetry', () => { - it('POSTs JSON to the default endpoint', async () => { - await sendTelemetry({ v: 1 }); - expect(fetchMock).toHaveBeenCalledTimes(1); - const [url, init] = fetchMock.mock.calls[0] as FetchArgs; - expect(url).toBe('https://bestax.io/api/t'); - expect(init.method).toBe('POST'); - expect(init.headers).toEqual({ 'content-type': 'application/json' }); - expect(init.body).toBe('{"v":1}'); - expect(init.signal).toBeInstanceOf(AbortSignal); - }); - - it('honors BESTAX_TELEMETRY_ENDPOINT', async () => { - process.env.BESTAX_TELEMETRY_ENDPOINT = 'http://127.0.0.1:8787/api/t'; - await sendTelemetry({ v: 1 }); - expect(fetchMock.mock.calls[0]?.[0]).toBe('http://127.0.0.1:8787/api/t'); - }); - - it('resolves silently when fetch rejects', async () => { - fetchMock.mockRejectedValueOnce(new Error('offline')); - await expect(sendTelemetry({ v: 1 })).resolves.toBeUndefined(); - }); - - it('ignores non-2xx responses', async () => { - fetchMock.mockResolvedValueOnce(new Response(null, { status: 400 })); - await expect(sendTelemetry({ v: 1 })).resolves.toBeUndefined(); - }); -}); - describe('getToolVersion', () => { it('reads the package version', () => { expect(getToolVersion()).toMatch(/^\d+\.\d+\.\d+/); }); }); +/** Production slugs the worker must accept — keep in lockstep with telemetry-worker tests. */ +const PRODUCTION_RULES = [ + 'prop:className', + 'prop:renderAs', + 'prop:size', + 'unsupported-file', + 'peer-deps', + 'plain-element', + 'imports', + 'responsive', + 'deps', +] as const; + describe('buildMigratePayload', () => { const baseStats = { source: 'react-bulma-components', @@ -307,7 +104,7 @@ describe('buildMigratePayload', () => { }; it('contains exactly the allowlisted fields and nothing else', () => { - const payload = buildMigratePayload(baseStats) as Record; + const payload = buildMigratePayload(baseStats); expect(Object.keys(payload).sort()).toEqual([ 'event', @@ -324,7 +121,7 @@ describe('buildMigratePayload', () => { expect(payload.event).toBe('migrate'); expect(payload.nodeMajor).toBe(Number(process.versions.node.split('.')[0])); expect(payload.platform).toBe(process.platform); - expect(Object.keys(payload.props as object).sort()).toEqual([ + expect(Object.keys(payload.props).sort()).toEqual([ 'changedBucket', 'changedCount', 'cssMode', @@ -332,13 +129,12 @@ describe('buildMigratePayload', () => { 'dry', 'source', ]); - const props = payload.props as Record; - expect(props.source).toBe('react-bulma-components'); - expect(props.cssMode).toBe('bestax'); - expect(props.dry).toBe(false); - expect(props.deps).toBe(true); - expect(props.changedCount).toBe(3); - expect(props.changedBucket).toBe('1-9'); + expect(payload.props.source).toBe('react-bulma-components'); + expect(payload.props.cssMode).toBe('bestax'); + expect(payload.props.dry).toBe(false); + expect(payload.props.deps).toBe(true); + expect(payload.props.changedCount).toBe(3); + expect(payload.props.changedBucket).toBe('1-9'); expect(payload.todosByRule).toEqual([ { rule: 'unsupported-file', count: 2 }, ]); @@ -348,10 +144,20 @@ describe('buildMigratePayload', () => { const payload = buildMigratePayload({ ...baseStats, todosByRule: [], - }) as Record; + }); expect('todosByRule' in payload).toBe(false); }); + it('passes through production rule slugs including prop:className', () => { + const payload = buildMigratePayload({ + ...baseStats, + todosByRule: PRODUCTION_RULES.map(rule => ({ rule, count: 1 })), + }); + expect(payload.todosByRule?.map(entry => entry.rule)).toEqual([ + ...PRODUCTION_RULES, + ]); + }); + it.each([ [0, '0'], [1, '1-9'], @@ -365,7 +171,7 @@ describe('buildMigratePayload', () => { const payload = buildMigratePayload({ ...baseStats, changedCount, - }) as { props: Record }; + }); expect(payload.props.changedBucket).toBe(bucket); expect(payload.props.changedCount).toBe(changedCount); }); @@ -374,7 +180,7 @@ describe('buildMigratePayload', () => { const payload = buildMigratePayload({ ...baseStats, changedCount: 123456, - }) as { props: Record }; + }); expect(payload.props.changedCount).toBe(10000); expect(payload.props.changedBucket).toBe('200+'); }); @@ -384,12 +190,10 @@ describe('buildMigratePayload', () => { rule: `rule-${i}`, count: i === 0 ? 250000 : i, })); - const payload = buildMigratePayload({ ...baseStats, todosByRule }) as { - todosByRule: Array<{ rule: string; count: number }>; - }; + const payload = buildMigratePayload({ ...baseStats, todosByRule }); expect(payload.todosByRule).toHaveLength(20); - expect(payload.todosByRule[0]).toEqual({ rule: 'rule-0', count: 100000 }); - expect(payload.todosByRule[19]).toEqual({ rule: 'rule-19', count: 19 }); + expect(payload.todosByRule?.[0]).toEqual({ rule: 'rule-0', count: 100000 }); + expect(payload.todosByRule?.[19]).toEqual({ rule: 'rule-19', count: 19 }); }); }); diff --git a/bestax-migrate/src/telemetry-core.ts b/bestax-migrate/src/telemetry-core.ts new file mode 100644 index 000000000..c68cb22b4 --- /dev/null +++ b/bestax-migrate/src/telemetry-core.ts @@ -0,0 +1,173 @@ +import { mkdir, readFile, writeFile } from 'fs/promises'; +import { homedir } from 'os'; +import { dirname, join } from 'path'; +import { fileURLToPath } from 'url'; +import { readFileSync } from 'fs'; + +/** + * Shared CLI telemetry kernel (disclosure: https://bestax.io/docs/guides/telemetry). + * + * Copied byte-for-byte into create-bestax and bestax-migrate — both publish + * standalone, so this cannot be a workspace package. `check:conformance + * --only=telemetry-core` fails if the copies diverge. Edit this file in + * create-bestax and copy it over the migrate one. + * + * No identifier of any kind is generated. Undecided means off. Every failure + * here is silent: telemetry must never affect the CLI's outcome. + */ + +export type TelemetryDecision = 'on' | 'off' | 'undecided'; + +export type TelemetrySource = 'flag' | 'dnt' | 'env' | 'config' | 'default'; + +export interface ResolvedTelemetry { + decision: TelemetryDecision; + source: TelemetrySource; +} + +interface TelemetryConfig { + version: 1; + enabled?: boolean; + decidedAt?: string; + decidedBy?: string; +} + +const DEFAULT_ENDPOINT = 'https://bestax.io/api/t'; +const TIMEOUT_MS = 1500; + +/** Shared across the bestax CLI family so consent is asked at most once. */ +export function telemetryConfigPath(): string { + const base = process.env.XDG_CONFIG_HOME || join(homedir(), '.config'); + return join(base, 'bestax', 'telemetry.json'); +} + +async function readConfig(): Promise { + try { + const parsed: unknown = JSON.parse( + await readFile(telemetryConfigPath(), 'utf-8') + ); + if (parsed !== null && typeof parsed === 'object') { + return parsed as TelemetryConfig; + } + } catch { + // Missing or corrupt config reads as "never asked". + } + return null; +} + +function doNotTrack(): boolean { + const dnt = process.env.DO_NOT_TRACK; + return dnt !== undefined && dnt !== '' && dnt !== '0'; +} + +/** + * Precedence: flag (persisted by the caller) → DO_NOT_TRACK → BESTAX_TELEMETRY + * (per-run, never persisted) → config file → undecided (off). + */ +export async function resolveTelemetry( + flag?: boolean +): Promise { + if (flag !== undefined) { + return { + decision: flag ? 'on' : 'off', + source: 'flag', + }; + } + if (doNotTrack()) { + return { decision: 'off', source: 'dnt' }; + } + const env = process.env.BESTAX_TELEMETRY; + if (env === '0' || env === '1') { + return { + decision: env === '1' ? 'on' : 'off', + source: 'env', + }; + } + const config = await readConfig(); + if (config && typeof config.enabled === 'boolean') { + return { + decision: config.enabled ? 'on' : 'off', + source: 'config', + }; + } + return { decision: 'undecided', source: 'default' }; +} + +export async function persistTelemetryDecision( + enabled: boolean, + decidedBy: string +): Promise { + try { + const config: TelemetryConfig = { + version: 1, + enabled, + decidedAt: new Date().toISOString(), + decidedBy, + }; + const filePath = telemetryConfigPath(); + await mkdir(dirname(filePath), { recursive: true }); + await writeFile(filePath, JSON.stringify(config, null, 2) + '\n', 'utf-8'); + } catch { + // An unwritable config dir must not break the CLI. + } +} + +export async function sendTelemetry(payload: unknown): Promise { + try { + await fetch(process.env.BESTAX_TELEMETRY_ENDPOINT || DEFAULT_ENDPOINT, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(payload), + signal: AbortSignal.timeout(TIMEOUT_MS), + }); + } catch { + // Fire-and-forget: the response (or its absence) is never surfaced. + } +} + +export function getToolVersion(): string { + try { + const dir = dirname(fileURLToPath(import.meta.url)); + const packageJson = JSON.parse( + readFileSync(join(dir, '..', 'package.json'), 'utf-8') + ) as { version?: string }; + return packageJson.version || '0.0.0'; + } catch { + return '0.0.0'; + } +} + +export interface ReportRunOptions { + /** True only when a consent question could actually be answered. */ + interactive: boolean; + /** Owns all consent UI (notice, question, ack); null means cancelled. */ + promptConsent: () => Promise; +} + +export async function reportRun( + flag: boolean | undefined, + toolName: string, + options: ReportRunOptions, + payload: unknown +): Promise { + const resolved = await resolveTelemetry(flag); + let decision = resolved.decision; + + if (resolved.source === 'flag') { + // Flags are the scripted equivalent of the prompt: decided once, remembered. + await persistTelemetryDecision( + decision === 'on', + `${toolName}@${getToolVersion()}` + ); + } else if (decision === 'undecided' && options.interactive) { + const answer = await options.promptConsent(); + if (answer !== null) { + await persistTelemetryDecision(answer, `${toolName}@${getToolVersion()}`); + decision = answer ? 'on' : 'off'; + } + } + + if (decision === 'on') { + await sendTelemetry(payload); + } +} diff --git a/bestax-migrate/src/telemetry.ts b/bestax-migrate/src/telemetry.ts index 0f68e99df..36e6f88cf 100644 --- a/bestax-migrate/src/telemetry.ts +++ b/bestax-migrate/src/telemetry.ts @@ -1,178 +1,15 @@ -import { mkdir, readFile, writeFile } from 'fs/promises'; -import { homedir } from 'os'; -import { dirname, join } from 'path'; -import { fileURLToPath } from 'url'; -import { readFileSync } from 'fs'; +import { getToolVersion, reportRun } from './telemetry-core.js'; +import type { ReportRunOptions } from './telemetry-core.js'; /** - * Anonymous opt-in telemetry (disclosure: https://bestax.io/docs/guides/telemetry). + * bestax-migrate payload + reporter. The kernel lives in telemetry-core.ts + * (copied from create-bestax; `check:conformance --only=telemetry-core`). * - * A near-identical copy lives in create-bestax/src/telemetry.ts — both CLIs - * publish standalone (plain tsc, files:["dist"]), so the ~150 shared lines are - * duplicated rather than published as a package. Keep the copies in sync. - * - * Privacy contract: every transmitted value is a closed-enum member, a version - * string, or a bounded integer — never paths, project names, or free text — - * and no identifier of any kind is generated. Undecided means off. Every - * failure here is silent: telemetry must never affect the CLI's outcome. + * Envelope and props are closed enums, versions, or bounded integers. + * `todosByRule` slugs are migrate's own rule names (including `prop:`), + * never file paths, messages, or code. */ -export type TelemetryDecision = 'on' | 'off' | 'undecided'; - -export type TelemetrySource = 'flag' | 'dnt' | 'env' | 'config' | 'default'; - -export interface ResolvedTelemetry { - decision: TelemetryDecision; - source: TelemetrySource; - /** Whether a consent prompt may still be shown (nothing has decided or forbidden it). */ - promptAllowed: boolean; -} - -interface TelemetryConfig { - version: 1; - enabled?: boolean; - decidedAt?: string; - decidedBy?: string; - noticed?: Record; -} - -const DEFAULT_ENDPOINT = 'https://bestax.io/api/t'; -const TIMEOUT_MS = 1500; - -/** Shared across the bestax CLI family so consent is asked at most once. */ -export function telemetryConfigPath(): string { - const base = process.env.XDG_CONFIG_HOME || join(homedir(), '.config'); - return join(base, 'bestax', 'telemetry.json'); -} - -async function readConfig(): Promise { - try { - const parsed: unknown = JSON.parse( - await readFile(telemetryConfigPath(), 'utf-8') - ); - if (parsed !== null && typeof parsed === 'object') { - return parsed as TelemetryConfig; - } - } catch { - // Missing or corrupt config reads as "never asked". - } - return null; -} - -function doNotTrack(): boolean { - const dnt = process.env.DO_NOT_TRACK; - return dnt !== undefined && dnt !== '' && dnt !== '0'; -} - -/** - * Precedence: flag (persisted by the caller) → DO_NOT_TRACK → BESTAX_TELEMETRY - * (per-run, never persisted) → config file → undecided (off). - */ -export async function resolveTelemetry( - flag?: boolean -): Promise { - if (flag !== undefined) { - return { - decision: flag ? 'on' : 'off', - source: 'flag', - promptAllowed: false, - }; - } - if (doNotTrack()) { - return { decision: 'off', source: 'dnt', promptAllowed: false }; - } - const env = process.env.BESTAX_TELEMETRY; - if (env === '0' || env === '1') { - return { - decision: env === '1' ? 'on' : 'off', - source: 'env', - promptAllowed: false, - }; - } - const config = await readConfig(); - if (config && typeof config.enabled === 'boolean') { - return { - decision: config.enabled ? 'on' : 'off', - source: 'config', - promptAllowed: false, - }; - } - return { decision: 'undecided', source: 'default', promptAllowed: true }; -} - -export async function persistTelemetryDecision( - enabled: boolean, - decidedBy: string -): Promise { - try { - const existing = await readConfig(); - const config: TelemetryConfig = { - version: 1, - enabled, - decidedAt: new Date().toISOString(), - decidedBy, - ...(existing?.noticed ? { noticed: existing.noticed } : {}), - }; - const filePath = telemetryConfigPath(); - await mkdir(dirname(filePath), { recursive: true }); - await writeFile(filePath, JSON.stringify(config, null, 2) + '\n', 'utf-8'); - } catch { - // An unwritable config dir must not break the CLI. - } -} - -/** - * Records that `tool` has shown its one-time notice. Returns true when the - * notice should be shown this run (nothing recorded it before), false when it - * was already recorded. Never throws; a failed write still returns true — the - * notice was shown this run and may simply repeat later, which beats crashing - * or silently suppressing it. - */ -export async function markNoticed(tool: string): Promise { - try { - const existing = await readConfig(); - if (existing?.noticed?.[tool]) return false; - const config: TelemetryConfig = { - version: 1, - ...existing, - noticed: { ...existing?.noticed, [tool]: new Date().toISOString() }, - }; - const filePath = telemetryConfigPath(); - await mkdir(dirname(filePath), { recursive: true }); - await writeFile(filePath, JSON.stringify(config, null, 2) + '\n', 'utf-8'); - } catch { - // A failed write only means the notice may show again later. - } - return true; -} - -export async function sendTelemetry(payload: object): Promise { - try { - await fetch(process.env.BESTAX_TELEMETRY_ENDPOINT || DEFAULT_ENDPOINT, { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify(payload), - signal: AbortSignal.timeout(TIMEOUT_MS), - }); - } catch { - // Fire-and-forget: the response (or its absence) is never surfaced. - } -} - -export function getToolVersion(): string { - try { - const dir = dirname(fileURLToPath(import.meta.url)); - const packageJson = JSON.parse( - readFileSync(join(dir, '..', 'package.json'), 'utf-8') - ) as { version?: string }; - return packageJson.version || '0.0.0'; - } catch { - return '0.0.0'; - } -} - -// --- bestax-migrate specific (the sibling copy has its own payload builder) --- - export interface MigrateRunStats { /** Registry source name, e.g. 'react-bulma-components'. */ source: string; @@ -184,6 +21,24 @@ export interface MigrateRunStats { todosByRule: Array<{ rule: string; count: number }>; } +export interface MigratePayload { + v: 1; + tool: 'bestax-migrate'; + event: 'migrate'; + toolVersion: string; + nodeMajor: number; + platform: string; + props: { + source: string; + cssMode: string; + dry: boolean; + deps: boolean; + changedBucket: string; + changedCount: number; + }; + todosByRule?: Array<{ rule: string; count: number }>; +} + function changedBucket(changedCount: number): string { if (changedCount <= 0) return '0'; if (changedCount <= 9) return '1-9'; @@ -192,7 +47,7 @@ function changedBucket(changedCount: number): string { return '200+'; } -export function buildMigratePayload(stats: MigrateRunStats): object { +export function buildMigratePayload(stats: MigrateRunStats): MigratePayload { const todosByRule = stats.todosByRule .slice(0, 20) .map(({ rule, count }) => ({ rule, count: Math.min(count, 100000) })); @@ -215,43 +70,10 @@ export function buildMigratePayload(stats: MigrateRunStats): object { }; } -export interface ReportMigrateOptions { - /** True only when a consent question could actually be answered (TTY in and out). */ - interactive: boolean; - /** Owns all consent UI (notice, question, ack); null means cancelled. */ - promptConsent: () => Promise; -} - export async function reportMigrateRun( stats: MigrateRunStats, flag: boolean | undefined, - options: ReportMigrateOptions + options: ReportRunOptions ): Promise { - const resolved = await resolveTelemetry(flag); - let decision = resolved.decision; - - if (resolved.source === 'flag') { - // Flags are the scripted equivalent of the prompt: decided once, remembered. - await persistTelemetryDecision( - decision === 'on', - `bestax-migrate@${getToolVersion()}` - ); - } else if ( - decision === 'undecided' && - resolved.promptAllowed && - options.interactive - ) { - const answer = await options.promptConsent(); - if (answer !== null) { - await persistTelemetryDecision( - answer, - `bestax-migrate@${getToolVersion()}` - ); - decision = answer ? 'on' : 'off'; - } - } - - if (decision === 'on') { - await sendTelemetry(buildMigratePayload(stats)); - } + await reportRun(flag, 'bestax-migrate', options, buildMigratePayload(stats)); } From 1ef43ce5c11e30eee4eb6e8dab65058baa4f8c59 Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sat, 22 Aug 2026 16:42:25 -0400 Subject: [PATCH 10/29] fix(bestax-mcp): keep URL fragments when attributing docs links Insert utm_source on the pre-hash substring so heading links stay fragments. Do not use URL.searchParams: it re-encodes Storybook path=/story/... slashes. --- bestax-mcp/src/__tests__/format.test.ts | 16 ++++++++++++++++ bestax-mcp/src/format.ts | 8 +++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/bestax-mcp/src/__tests__/format.test.ts b/bestax-mcp/src/__tests__/format.test.ts index 050678f5c..3b0ad4338 100644 --- a/bestax-mcp/src/__tests__/format.test.ts +++ b/bestax-mcp/src/__tests__/format.test.ts @@ -42,6 +42,22 @@ describe('attributed', () => { expect(attributed(notAUrl)).toBe(notAUrl); } }); + + it('keeps a fragment after the query string', () => { + expect(attributed('https://bestax.io/docs/guides/foo#bar')).toBe( + 'https://bestax.io/docs/guides/foo?utm_source=bestax-mcp#bar' + ); + }); + + it('inserts ahead of the fragment when a query string is already present', () => { + expect( + attributed( + 'https://bestax.io/storybook/?path=/story/elements-button#anchor' + ) + ).toBe( + 'https://bestax.io/storybook/?path=/story/elements-button&utm_source=bestax-mcp#anchor' + ); + }); }); describe('renderComponent link line', () => { diff --git a/bestax-mcp/src/format.ts b/bestax-mcp/src/format.ts index f11b51a76..f70108d58 100644 --- a/bestax-mcp/src/format.ts +++ b/bestax-mcp/src/format.ts @@ -37,7 +37,13 @@ const cell = (text: string) => export function attributed(url: string): string { if (!/^https?:\/\//.test(url)) return url; if (url.includes('utm_source=bestax-mcp')) return url; - return `${url}${url.includes('?') ? '&' : '?'}utm_source=bestax-mcp`; + // Insert before the fragment so `#heading` stays a fragment. Do not use + // URL.searchParams: it re-encodes Storybook `path=/story/...` as `%2F`. + const hashIndex = url.indexOf('#'); + const hash = hashIndex === -1 ? '' : url.slice(hashIndex); + const withoutHash = hashIndex === -1 ? url : url.slice(0, hashIndex); + const sep = withoutHash.includes('?') ? '&' : '?'; + return `${withoutHash}${sep}utm_source=bestax-mcp${hash}`; } export function table(headers: string[], rows: string[][]): string { From 7f350fc9aae45e51baf68f5cb98ff447a8c78835 Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sat, 22 Aug 2026 16:42:28 -0400 Subject: [PATCH 11/29] docs: note prop: slugs in the telemetry disclosure TODO counts include migrate's prop: rule names, not a closed enum of family labels. --- docs/docs/guides/telemetry.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/docs/guides/telemetry.md b/docs/docs/guides/telemetry.md index 970b268fa..52e03f0dc 100644 --- a/docs/docs/guides/telemetry.md +++ b/docs/docs/guides/telemetry.md @@ -37,14 +37,14 @@ Sent once after a successful scaffold: Sent once after a successful run: -| Field | Values | -| ------------------ | --------------------------------------------------------------------------- | -| `source` | the source library migrated from, e.g. `react-bulma-components` | -| `cssMode` | `bestax`, `bulma`, or `keep` | -| `dry` | `true` or `false` — whether it was a dry run | -| `deps` | `true` or `false` — whether `package.json` dependencies were updated | -| Changed-file count | a bucket — `0`, `1-9`, `10-49`, `50-199`, or `200+` — plus the capped count | -| TODO counts | per migration rule: the rule name and a count, nothing else | +| Field | Values | +| ------------------ | ---------------------------------------------------------------------------------------------- | +| `source` | the source library migrated from, e.g. `react-bulma-components` | +| `cssMode` | `bestax`, `bulma`, or `keep` | +| `dry` | `true` or `false` — whether it was a dry run | +| `deps` | `true` or `false` — whether `package.json` dependencies were updated | +| Changed-file count | a bucket — `0`, `1-9`, `10-49`, `50-199`, or `200+` — plus the capped count | +| TODO counts | per migration rule: the rule name (including `prop:` slugs) and a count, nothing else | Never file paths, never file contents, never code. The TODO report printed in your terminal lists files and lines; the telemetry event carries only rule From 5029bd2a6c74ce0b6931567a737d7f991f1e2134 Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sat, 22 Aug 2026 16:42:33 -0400 Subject: [PATCH 12/29] chore: align telemetry ingest with CLI payloads and gate it in CI Accept production migrate slugs (prop:className); drop a bad todosByRule entry instead of 400ing the run. Conformance now byte-compares the CLI kernels and requires the worker allowlists to be a superset of CLI enums (including package managers). Root test/typecheck/all run the worker suite, and a contract test round-trips build*Payload() through validate(). --- package.json | 6 +- scripts/check-conformance.mjs | 163 ++++++++++++++++++ scripts/lib/resolve-ts-from-js.mjs | 25 +++ scripts/telemetry-contract.test.mjs | 82 +++++++++ telemetry-worker/README.md | 16 +- .../src/__tests__/handler.test.ts | 26 +++ .../src/__tests__/validate.test.ts | 70 ++++++-- telemetry-worker/src/validate.ts | 27 ++- 8 files changed, 384 insertions(+), 31 deletions(-) create mode 100644 scripts/lib/resolve-ts-from-js.mjs create mode 100644 scripts/telemetry-contract.test.mjs diff --git a/package.json b/package.json index 8f8a0dcdd..82b6d02b2 100644 --- a/package.json +++ b/package.json @@ -6,10 +6,10 @@ "scripts": { "build": "turbo run build", "dev": "turbo run dev", - "test": "turbo run test && node --test \"scripts/*.test.mjs\"", + "test": "turbo run test && node --test \"scripts/*.test.mjs\" && node --test \"telemetry-worker/src/__tests__/*.test.ts\"", "test:coverage": "turbo run test:coverage", "bundle:stats": "turbo run bundle:stats", - "typecheck": "turbo run typecheck", + "typecheck": "turbo run typecheck && tsc -p telemetry-worker/tsconfig.json --noEmit", "format": "prettier --write \"**/*.{ts,tsx,js,jsx,mjs,md,mdx}\"", "format:check": "prettier --check \"**/*.{ts,tsx,js,jsx,mjs,md,mdx}\"", "lint": "turbo run lint && eslint scripts docs/scripts", @@ -23,7 +23,7 @@ "gen:mcp": "node scripts/gen-mcp-index.mjs", "gen:mcp:check": "node scripts/gen-mcp-index.mjs && git add --intent-to-add --all bestax-mcp/data && git diff --exit-code -- bestax-mcp/data", "gen": "pnpm gen:api-docs && pnpm gen:catalog && pnpm gen:mcp", - "all": "turbo run build typecheck test test:coverage bundle:stats && node --test \"scripts/*.test.mjs\" && pnpm run lint && pnpm run format:check && turbo run build-storybook --filter=@allxsmith/bestax-bulma", + "all": "turbo run build typecheck test test:coverage bundle:stats && node --test \"scripts/*.test.mjs\" && node --test \"telemetry-worker/src/__tests__/*.test.ts\" && tsc -p telemetry-worker/tsconfig.json --noEmit && pnpm run lint && pnpm run format:check && turbo run build-storybook --filter=@allxsmith/bestax-bulma", "storybook": "turbo run storybook --filter=@allxsmith/bestax-bulma", "docs": "turbo run docs --filter=@allxsmith/bestax-docs", "release": "turbo run release", diff --git a/scripts/check-conformance.mjs b/scripts/check-conformance.mjs index b71deeb28..983dda181 100644 --- a/scripts/check-conformance.mjs +++ b/scripts/check-conformance.mjs @@ -48,6 +48,10 @@ * carries a `# bestax:review ` or * `# bestax:permanent` marker, and no review date has * passed (#391) + * telemetry-core create-bestax and bestax-migrate telemetry-core.ts + * copies are byte-identical + * telemetry-allowlists worker schema enums are a superset of the CLI values + * (templates, flavors, icons, sources, css modes, PMs) */ import { readFile, readdir, writeFile, access } from 'node:fs/promises'; import { join, relative, dirname, isAbsolute } from 'node:path'; @@ -2203,6 +2207,163 @@ async function checkSkillsRoster() { return [...violations, ...rosterViolations(skills, sources)]; } +// --------------------------------------------------------------------------- +// Telemetry: two standalone CLIs cannot share a package, so the kernel is +// copied. The worker allowlists are the privacy backstop — a new template +// that isn't listed there is silently dropped. +// --------------------------------------------------------------------------- + +const TELEMETRY_CORE_CANONICAL = 'create-bestax/src/telemetry-core.ts'; +const TELEMETRY_CORE_COPY = 'bestax-migrate/src/telemetry-core.ts'; + +async function checkTelemetryCore() { + const violations = []; + let canonical; + let copy; + try { + canonical = await readFile(join(REPO, TELEMETRY_CORE_CANONICAL), 'utf8'); + } catch { + return [ + `${TELEMETRY_CORE_CANONICAL} is missing (canonical telemetry kernel).`, + ]; + } + try { + copy = await readFile(join(REPO, TELEMETRY_CORE_COPY), 'utf8'); + } catch { + return [ + `${TELEMETRY_CORE_COPY} is missing. Copy ${TELEMETRY_CORE_CANONICAL} ` + + `over it so the two CLI kernels stay in lockstep.`, + ]; + } + if (canonical !== copy) { + violations.push( + `${TELEMETRY_CORE_COPY} differs from ${TELEMETRY_CORE_CANONICAL} ` + + `(canonical). Copy ${TELEMETRY_CORE_CANONICAL} over ${TELEMETRY_CORE_COPY} ` + + `so the two CLI kernels stay in lockstep — they cannot share a package ` + + `(published standalone).` + ); + } + return violations; +} + +function quotedStringsIn(block) { + return [...block.matchAll(/'([^']+)'/g)].map(m => m[1]); +} + +function constStringArray(src, name) { + const m = src.match( + new RegExp(`const ${name} = \\[([\\s\\S]*?)\\] as const`) + ); + return m ? quotedStringsIn(m[1]) : null; +} + +function exportedNamedArray(src, name) { + const m = src.match( + new RegExp(`export const ${name}[^=]*= \\[([\\s\\S]*?)\\];`) + ); + if (!m) return null; + return [...m[1].matchAll(/name: '([^']+)'/g)].map(x => x[1]); +} + +function cssModes(src) { + const m = src.match(/const CSS_MODES: CssMode\[\] = \[([\s\S]*?)\];/); + return m ? quotedStringsIn(m[1]) : null; +} + +async function migrateSourceNames() { + const dir = join(REPO, 'bestax-migrate/src/sources'); + const names = []; + for (const entry of await readdir(dir, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + try { + const src = await readFile(join(dir, entry.name, 'index.ts'), 'utf8'); + const m = src.match(/: MigrationSource = \{[\s\S]*?name: '([^']+)'/); + if (m) names.push(m[1]); + } catch { + // Directory without an index.ts is not a source. + } + } + return names; +} + +function missingFromWorker(label, producer, worker, workerFile) { + if (producer === null) { + return [`could not parse producer values for ${label}`]; + } + if (worker === null) { + return [ + `could not parse ${label} from ${workerFile} — the worker schema ` + + `array is missing or malformed.`, + ]; + } + const missing = producer.filter(value => !worker.includes(value)); + return missing.map( + value => + `${label} value '${value}' is used by a CLI but missing from ` + + `${workerFile} — add it there first or its events are silently dropped.` + ); +} + +async function checkTelemetryAllowlists() { + const workerFile = 'telemetry-worker/src/schema.ts'; + const constantsFile = 'create-bestax/src/constants.ts'; + const pmFile = 'create-bestax/src/package-manager.ts'; + const cliFile = 'bestax-migrate/src/cli.ts'; + const [schema, constants, pm, cli] = await Promise.all([ + readFile(join(REPO, workerFile), 'utf8'), + readFile(join(REPO, constantsFile), 'utf8'), + readFile(join(REPO, pmFile), 'utf8'), + readFile(join(REPO, cliFile), 'utf8'), + ]); + const sources = await migrateSourceNames(); + const violations = []; + if (!sources.length) { + violations.push( + 'bestax-migrate/src/sources/*/index.ts: no MigrationSource `name` was ' + + 'found, so the worker allowlist went unchecked.' + ); + } + violations.push( + ...missingFromWorker( + 'template', + exportedNamedArray(constants, 'TEMPLATES'), + constStringArray(schema, 'TEMPLATE_VALUES'), + workerFile + ), + ...missingFromWorker( + 'bulmaFlavor', + exportedNamedArray(constants, 'BULMA_FLAVORS'), + constStringArray(schema, 'BULMA_FLAVOR_VALUES'), + workerFile + ), + ...missingFromWorker( + 'iconLibrary', + exportedNamedArray(constants, 'ICON_LIBRARIES'), + constStringArray(schema, 'ICON_LIBRARY_VALUES'), + workerFile + ), + ...missingFromWorker( + 'packageManager', + constStringArray(pm, 'KNOWN'), + constStringArray(schema, 'PACKAGE_MANAGER_VALUES'), + workerFile + ), + ...missingFromWorker( + 'migrate source', + sources, + constStringArray(schema, 'MIGRATE_SOURCE_VALUES'), + workerFile + ), + ...missingFromWorker( + 'cssMode', + cssModes(cli), + constStringArray(schema, 'CSS_MODE_VALUES'), + workerFile + ) + ); + return violations; +} + // --------------------------------------------------------------------------- const CHECKS = { @@ -2221,6 +2382,8 @@ const CHECKS = { 'autodocs-tag': checkAutodocsTag, 'publishable-manifests': checkPublishableManifests, 'bypass-expiry': checkBypassExpiry, + 'telemetry-core': checkTelemetryCore, + 'telemetry-allowlists': checkTelemetryAllowlists, 'inline-style': null, // handled below (takes the flag) }; diff --git a/scripts/lib/resolve-ts-from-js.mjs b/scripts/lib/resolve-ts-from-js.mjs new file mode 100644 index 000000000..b9ea302ec --- /dev/null +++ b/scripts/lib/resolve-ts-from-js.mjs @@ -0,0 +1,25 @@ +/** + * Node ESM resolver hook: when a .ts parent imports `./foo.js` and `foo.ts` + * exists, load the TypeScript. Lets root `node --test` import the CLIs' source + * (TypeScript ESM `.js` specifiers) without a build step. + */ +import { existsSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +export async function resolve(specifier, context, nextResolve) { + if ( + context.parentURL && + specifier.startsWith('.') && + specifier.endsWith('.js') + ) { + const asTs = join( + dirname(fileURLToPath(context.parentURL)), + specifier.replace(/\.js$/, '.ts') + ); + if (existsSync(asTs)) { + return nextResolve(specifier.replace(/\.js$/, '.ts'), context); + } + } + return nextResolve(specifier, context); +} diff --git a/scripts/telemetry-contract.test.mjs b/scripts/telemetry-contract.test.mjs new file mode 100644 index 000000000..aac20ff9a --- /dev/null +++ b/scripts/telemetry-contract.test.mjs @@ -0,0 +1,82 @@ +/** + * Round-trip: CLI payload builders → JSON → worker validate(). + * + * The worker is outside the pnpm workspace and the CLIs use `.js` specifiers + * for `.ts` files, so this test registers a small resolver hook and imports + * both sides' source. A payload the CLIs actually emit that the worker + * rejects is the class of bug that silently drops every event. + */ +import { register } from 'node:module'; +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { validate } from '../telemetry-worker/src/validate.ts'; + +register('./lib/resolve-ts-from-js.mjs', import.meta.url); + +const { buildScaffoldPayload } = + await import('../create-bestax/src/telemetry.ts'); +const { buildMigratePayload } = + await import('../bestax-migrate/src/telemetry.ts'); + +const PRODUCTION_RULES = [ + 'prop:className', + 'prop:renderAs', + 'prop:size', + 'unsupported-file', + 'peer-deps', + 'plain-element', + 'imports', + 'responsive', + 'deps', +]; + +function roundTrip(payload) { + return validate(JSON.parse(JSON.stringify(payload))); +} + +test('create-bestax scaffold payload validates', () => { + const result = roundTrip( + buildScaffoldPayload({ + template: 'vite-ts', + bulmaFlavor: 'complete', + iconLibrary: 'none', + skills: true, + }) + ); + assert.equal(result.ok, true, JSON.stringify(result)); +}); + +test('bestax-migrate payload with production rule slugs validates', () => { + const result = roundTrip( + buildMigratePayload({ + source: 'react-bulma-components', + cssMode: 'bestax', + dry: false, + deps: true, + changedCount: 23, + todosByRule: PRODUCTION_RULES.map(rule => ({ rule, count: 1 })), + }) + ); + assert.equal(result.ok, true, JSON.stringify(result)); + if (result.ok && result.payload.tool === 'bestax-migrate') { + assert.deepEqual( + result.payload.todosByRule?.map(entry => entry.rule), + PRODUCTION_RULES + ); + } +}); + +test('bestax-migrate payload with no TODOs validates', () => { + const result = roundTrip( + buildMigratePayload({ + source: 'react-bulma-components', + cssMode: 'keep', + dry: true, + deps: false, + changedCount: 0, + todosByRule: [], + }) + ); + assert.equal(result.ok, true, JSON.stringify(result)); + if (result.ok) assert.equal('todosByRule' in result.payload, false); +}); diff --git a/telemetry-worker/README.md b/telemetry-worker/README.md index 61cd027c6..b89a6c941 100644 --- a/telemetry-worker/README.md +++ b/telemetry-worker/README.md @@ -12,12 +12,16 @@ missing binding in local dev still answers `204`. npx wrangler deploy ``` -Run by Alex — the package is not in the pnpm workspace yet, and Wrangler -bundles `src/index.ts` directly (no build step). The worker is bound to the -zone route `bestax.io/api/t`. If that route ever conflicts with whatever -serves the rest of `bestax.io`, the fallback is a custom domain: attach -`t.bestax.io` to the worker and point the CLIs at `https://t.bestax.io/api/t` -(the handler only checks the `/api/t` pathname, so no code change needed). +Run by Alex — the package is not in the pnpm workspace (Wrangler's native +`workerd` build is blocked repo-wide). Wrangler bundles `src/index.ts` +directly (no build step). Tests and typecheck still run from the repo root: +`pnpm test` includes `node --test telemetry-worker/src/__tests__/*.test.ts`, +and `pnpm typecheck` includes `tsc -p telemetry-worker`. The worker is bound +to the zone route `bestax.io/api/t`. If that route ever conflicts with +whatever serves the rest of `bestax.io`, the fallback is a custom domain: +attach `t.bestax.io` to the worker and point the CLIs at +`https://t.bestax.io/api/t` (the handler only checks the `/api/t` pathname, +so no code change needed). ## Analytics Engine mapping diff --git a/telemetry-worker/src/__tests__/handler.test.ts b/telemetry-worker/src/__tests__/handler.test.ts index 35b0f513b..cb9c2ffbd 100644 --- a/telemetry-worker/src/__tests__/handler.test.ts +++ b/telemetry-worker/src/__tests__/handler.test.ts @@ -254,6 +254,32 @@ describe('handler: accepted payloads', () => { assert.equal(points.length, 0); }); + it('a bad todo entry is dropped; the run and good slugs still write', async () => { + const { env, points } = makeEnv(); + const payload = migratePayload(); + payload.todosByRule = [ + { rule: 'prop:className', count: 4 }, + { rule: 'has space', count: 1 }, + { rule: 'x', count: 1, file: 'App.tsx' }, + ]; + const res = await call(post(payload), env); + assert.equal(res.status, 204); + assert.equal(points.length, 2); + assert.equal(points[0]?.blobs?.[0], 'migrate'); + assert.deepEqual(points[1], { + indexes: ['bestax-migrate'], + blobs: [ + 'migrate_todo', + '0.3.0', + 'linux', + '20', + 'react-bulma-components', + 'prop:className', + ], + doubles: [4], + }); + }); + it('204 even without the TELEMETRY binding (fire-and-forget)', async () => { const res = await call(post(createPayload()), {}); assert.equal(res.status, 204); diff --git a/telemetry-worker/src/__tests__/validate.test.ts b/telemetry-worker/src/__tests__/validate.test.ts index 568464325..96196313c 100644 --- a/telemetry-worker/src/__tests__/validate.test.ts +++ b/telemetry-worker/src/__tests__/validate.test.ts @@ -268,8 +268,20 @@ describe('validate: rejects props enum values', () => { } }); -describe('validate: rejects todosByRule shapes', () => { - it('more than 20 entries', () => { +describe('validate: todosByRule', () => { + const PRODUCTION_RULES = [ + 'prop:className', + 'prop:renderAs', + 'prop:size', + 'unsupported-file', + 'peer-deps', + 'plain-element', + 'imports', + 'responsive', + 'deps', + ] as const; + + it('more than 20 entries rejects the whole payload', () => { const payload = migratePayload(); payload.todosByRule = Array.from({ length: 21 }, (_, i) => ({ rule: `rule-${i}`, @@ -287,26 +299,52 @@ describe('validate: rejects todosByRule shapes', () => { assertOk(payload); }); - it('non-array todosByRule', () => { + it('non-array todosByRule rejects the whole payload', () => { const payload = migratePayload(); payload.todosByRule = { rule: 'x', count: 1 }; assertRejected(payload, 'todosByRule must be an array'); }); - it('entry with extra key', () => { + it('accepts production migrate rule slugs', () => { const payload = migratePayload(); - payload.todosByRule = [{ rule: 'x', count: 1, file: 'App.tsx' }]; - assertRejected(payload, 'todo entry with extra key'); + payload.todosByRule = PRODUCTION_RULES.map(rule => ({ rule, count: 1 })); + const result = validate(payload); + assert.equal(result.ok, true, JSON.stringify(result)); + if (result.ok && result.payload.tool === 'bestax-migrate') { + assert.deepEqual( + result.payload.todosByRule?.map(entry => entry.rule), + [...PRODUCTION_RULES] + ); + } + }); + + it('drops a bad entry and keeps the run plus good slugs', () => { + const payload = migratePayload(); + payload.todosByRule = [ + { rule: 'prop:className', count: 2 }, + { rule: 'has space', count: 1 }, + { rule: 'x', count: 1, file: 'App.tsx' }, + { rule: 'ok-rule' }, + { rule: 'ok-rule', count: -1 }, + ]; + const result = validate(payload); + assert.equal(result.ok, true, JSON.stringify(result)); + if (result.ok && result.payload.tool === 'bestax-migrate') { + assert.deepEqual(result.payload.todosByRule, [ + { rule: 'prop:className', count: 2 }, + ]); + } }); - it('entry missing count', () => { + it('omits todosByRule when every entry is dropped', () => { const payload = migratePayload(); - payload.todosByRule = [{ rule: 'x' }]; - assertRejected(payload, 'todo entry missing count'); + payload.todosByRule = [{ rule: 'x', count: 1, file: 'App.tsx' }]; + const result = validate(payload); + assert.equal(result.ok, true, JSON.stringify(result)); + if (result.ok) assert.equal('todosByRule' in result.payload, false); }); for (const rule of [ - 'UPPER', '-leading-dash', '.leading-dot', '', @@ -314,18 +352,22 @@ describe('validate: rejects todosByRule shapes', () => { 'a'.repeat(65), 42, ]) { - it(`rule = ${JSON.stringify(rule)}`, () => { + it(`drops rule = ${JSON.stringify(rule)} and keeps the run`, () => { const payload = migratePayload(); payload.todosByRule = [{ rule, count: 1 }]; - assertRejected(payload, 'bad rule slug'); + const result = validate(payload); + assert.equal(result.ok, true, JSON.stringify(result)); + if (result.ok) assert.equal('todosByRule' in result.payload, false); }); } for (const count of [-1, 100001, 0.5, '3', null]) { - it(`count = ${JSON.stringify(count)}`, () => { + it(`drops count = ${JSON.stringify(count)} and keeps the run`, () => { const payload = migratePayload(); payload.todosByRule = [{ rule: 'ok-rule', count }]; - assertRejected(payload, 'bad todo count'); + const result = validate(payload); + assert.equal(result.ok, true, JSON.stringify(result)); + if (result.ok) assert.equal('todosByRule' in result.payload, false); }); } }); diff --git a/telemetry-worker/src/validate.ts b/telemetry-worker/src/validate.ts index 424dac756..f753c2f87 100644 --- a/telemetry-worker/src/validate.ts +++ b/telemetry-worker/src/validate.ts @@ -1,8 +1,10 @@ /** - * Reject-by-default payload validation. Every key, everywhere, must be on an - * allowlist and every value must match its allowlisted shape — anything else - * rejects the whole payload. Rejection reasons are internal (tests/debug - * only); the handler always answers a bare 400. + * Reject-by-default payload validation. Unknown top-level or props keys, and + * a malformed envelope, reject the whole payload (privacy: extra fields never + * land). A single bad `todosByRule` entry is dropped and the run is kept — + * migrate's production slugs include `prop:className`, and one ugly slug must + * not zero out the event. Rejection reasons are internal (tests/debug only); + * the handler always answers a bare 400. */ import { @@ -26,7 +28,9 @@ export type ValidationResult = const VERSION_RE = /^\d+\.\d+\.\d+(-[\w.]+)?$/; const MAX_VERSION_LENGTH = 32; -const RULE_RE = /^[a-z0-9][a-z0-9._-]{0,63}$/; +// Production slugs are `prop:` (colon + camelCase) plus lowercase +// families like `unsupported-file`. Length cap is the whole slug. +const RULE_RE = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/; const TOP_KEYS = [ 'v', @@ -152,12 +156,13 @@ export function validate(body: unknown): ValidationResult { if (!Array.isArray(todos) || todos.length > MAX_TODO_RULES) { return reject('invalid todosByRule'); } + const kept: { rule: string; count: number }[] = []; for (const entry of todos) { if (!isRecord(entry) || !hasExactKeys(entry, TODO_ENTRY_KEYS)) { - return reject('invalid todosByRule entry'); + continue; } if (typeof entry.rule !== 'string' || !RULE_RE.test(entry.rule)) { - return reject('invalid todo rule'); + continue; } if ( typeof entry.count !== 'number' || @@ -165,8 +170,14 @@ export function validate(body: unknown): ValidationResult { entry.count < 0 || entry.count > MAX_CHANGED_COUNT ) { - return reject('invalid todo count'); + continue; } + kept.push({ rule: entry.rule, count: entry.count }); + } + if (kept.length > 0) { + body.todosByRule = kept; + } else { + delete body.todosByRule; } } } From 8def8da80ce70160d50efba1ae2869827c5814b3 Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sat, 22 Aug 2026 20:23:12 -0400 Subject: [PATCH 13/29] fix(create-bestax): gate consent on both TTYs, honor DNT and Ctrl-C Review fixes: the consent question now also requires stdout to be a TTY (with output redirected it rendered into the log while the CLI blocked on an invisible prompt); a ^C at the question sets exit code 130 instead of being swallowed into a clean exit; --telemetry under DO_NOT_TRACK applies to that single run but is never persisted, so a copied command cannot enable telemetry beyond the run it was typed for; and the we-won't-ask-again acknowledgement is only printed when the decision was actually written. --- .../src/__tests__/project-creator.test.ts | 80 ++++++++++++++++--- create-bestax/src/__tests__/prompts.test.ts | 17 +++- create-bestax/src/__tests__/telemetry.test.ts | 72 +++++++++++++++-- create-bestax/src/constants.ts | 2 + create-bestax/src/project-creator.ts | 26 +++--- create-bestax/src/prompts.ts | 31 +++++-- create-bestax/src/telemetry-core.ts | 38 +++++++-- 7 files changed, 223 insertions(+), 43 deletions(-) diff --git a/create-bestax/src/__tests__/project-creator.test.ts b/create-bestax/src/__tests__/project-creator.test.ts index 686945ce9..af01c8eb3 100644 --- a/create-bestax/src/__tests__/project-creator.test.ts +++ b/create-bestax/src/__tests__/project-creator.test.ts @@ -82,6 +82,27 @@ function restoreStdinTTY(): void { } } +const originalStdoutTTYDescriptor = Object.getOwnPropertyDescriptor( + process.stdout, + 'isTTY' +); + +function setStdoutTTY(value: boolean | undefined): void { + Object.defineProperty(process.stdout, 'isTTY', { + value, + configurable: true, + writable: true, + }); +} + +function restoreStdoutTTY(): void { + if (originalStdoutTTYDescriptor) { + Object.defineProperty(process.stdout, 'isTTY', originalStdoutTTYDescriptor); + } else { + delete (process.stdout as { isTTY?: boolean }).isTTY; + } +} + beforeEach(() => { console.log = jest.fn(); console.error = jest.fn(); @@ -1747,22 +1768,37 @@ describe('ProjectCreator', () => { expect(reportScaffold.mock.calls[0][1]).toBe(false); }); - it('is interactive only when stdin is a TTY and -y is absent', async () => { - mockSuccessfulScaffold(); - await projectCreator.create('test-project', fullFlags); - expect(reportScaffold.mock.calls[0][2].interactive).toBe(true); - - reportScaffold.mockClear(); - setStdinTTY(false); - mockSuccessfulScaffold(); - await projectCreator.create('test-project', fullFlags); - expect(reportScaffold.mock.calls[0][2].interactive).toBe(false); + it('is interactive only with a TTY on both ends and -y absent', async () => { + try { + setStdoutTTY(true); + mockSuccessfulScaffold(); + await projectCreator.create('test-project', fullFlags); + expect(reportScaffold.mock.calls[0][2].interactive).toBe(true); + + // stdout redirected: the question would render into the log while + // the CLI blocked on an invisible prompt. + reportScaffold.mockClear(); + setStdoutTTY(false); + mockSuccessfulScaffold(); + await projectCreator.create('test-project', fullFlags); + expect(reportScaffold.mock.calls[0][2].interactive).toBe(false); + + reportScaffold.mockClear(); + setStdoutTTY(true); + setStdinTTY(false); + mockSuccessfulScaffold(); + await projectCreator.create('test-project', fullFlags); + expect(reportScaffold.mock.calls[0][2].interactive).toBe(false); + } finally { + restoreStdoutTTY(); + } }); - it('the consent callback asks the prompt and acknowledges a yes', async () => { + it('the consent callback asks the prompt, and a saved yes is acknowledged', async () => { mockSuccessfulScaffold(); reportScaffold.mockImplementationOnce(async (_choices, _flag, opts) => { await expect(opts.promptConsent()).resolves.toBe(true); + opts.onDecided?.(true, true); }); (prompts as jest.MockedFunction).mockResolvedValue({ telemetry: true, @@ -1775,10 +1811,11 @@ describe('ProjectCreator', () => { ); }); - it('acknowledges a no without sending', async () => { + it('a saved no is acknowledged without sending', async () => { mockSuccessfulScaffold(); reportScaffold.mockImplementationOnce(async (_choices, _flag, opts) => { await expect(opts.promptConsent()).resolves.toBe(false); + opts.onDecided?.(false, true); }); (prompts as jest.MockedFunction).mockResolvedValue({ telemetry: false, @@ -1791,6 +1828,25 @@ describe('ProjectCreator', () => { ); }); + it('an unsaved decision is acknowledged truthfully, not with a promise', async () => { + // "We won't ask again" after a swallowed write failure would be false + // on every future run; the ack is worded on whether the write stuck. + mockSuccessfulScaffold(); + reportScaffold.mockImplementationOnce(async (_choices, _flag, opts) => { + await expect(opts.promptConsent()).resolves.toBe(false); + opts.onDecided?.(false, false); + }); + (prompts as jest.MockedFunction).mockResolvedValue({ + telemetry: false, + }); + + await projectCreator.create('test-project', fullFlags); + + const logged = (console.log as jest.Mock).mock.calls.flat().join('\n'); + expect(logged).toContain("Couldn't save your choice"); + expect(logged).not.toContain("we won't ask again"); + }); + it('a cancelled consent prompt prints no ack', async () => { mockSuccessfulScaffold(); reportScaffold.mockImplementationOnce(async (_choices, _flag, opts) => { diff --git a/create-bestax/src/__tests__/prompts.test.ts b/create-bestax/src/__tests__/prompts.test.ts index 8b95c1ae9..d667e8c74 100644 --- a/create-bestax/src/__tests__/prompts.test.ts +++ b/create-bestax/src/__tests__/prompts.test.ts @@ -394,9 +394,24 @@ describe('prompts', () => { await expect(promptTelemetryConsent()).resolves.toBe(false); }); - it('returns null on cancel', async () => { + it('returns null on cancel and honors the interrupt via exit code 130', async () => { + const originalExitCode = process.exitCode; + (prompts as jest.MockedFunction).mockImplementationOnce( + (async (_questions: unknown, opts?: { onCancel?: () => void }) => { + opts?.onCancel?.(); + return {}; + }) as unknown as typeof prompts + ); + await expect(promptTelemetryConsent()).resolves.toBeNull(); + expect(process.exitCode).toBe(130); + process.exitCode = originalExitCode; + }); + + it('a missing answer without a cancel leaves the exit code alone', async () => { + const originalExitCode = process.exitCode; (prompts as jest.MockedFunction).mockResolvedValue({}); await expect(promptTelemetryConsent()).resolves.toBeNull(); + expect(process.exitCode).toBe(originalExitCode); }); it('is an opt-in confirm defaulting to No, preceded by the notice', async () => { diff --git a/create-bestax/src/__tests__/telemetry.test.ts b/create-bestax/src/__tests__/telemetry.test.ts index d5686fcee..2cb2fe4d2 100644 --- a/create-bestax/src/__tests__/telemetry.test.ts +++ b/create-bestax/src/__tests__/telemetry.test.ts @@ -7,7 +7,7 @@ import { afterEach, afterAll, } from '@jest/globals'; -import { mkdtemp, readFile, writeFile, mkdir } from 'fs/promises'; +import { mkdtemp, readFile, writeFile, mkdir, rm } from 'fs/promises'; import { tmpdir } from 'os'; import { join } from 'path'; @@ -69,7 +69,9 @@ beforeEach(async () => { globalThis.fetch = fetchMock as unknown as typeof fetch; }); -afterEach(() => { +afterEach(async () => { + // ~80 scratch dirs per run otherwise outlive the suite in the OS tmpdir. + await rm(configHome, { recursive: true, force: true }); for (const key of ENV_KEYS) { if (savedEnv[key] === undefined) delete process.env[key]; else process.env[key] = savedEnv[key]; @@ -183,8 +185,10 @@ describe('resolveTelemetry precedence', () => { }); describe('persistTelemetryDecision', () => { - it('writes the config schema', async () => { - await persistTelemetryDecision(true, 'create-bestax@1.0.0'); + it('writes the config schema and reports success', async () => { + await expect( + persistTelemetryDecision(true, 'create-bestax@1.0.0') + ).resolves.toBe(true); const config = await readConfigFile(); expect(config.version).toBe(1); expect(config.enabled).toBe(true); @@ -192,13 +196,13 @@ describe('persistTelemetryDecision', () => { expect(typeof config.decidedAt).toBe('string'); }); - it('stays silent when the config dir is unwritable', async () => { + it('stays silent when the config dir is unwritable, and says so', async () => { const blocker = join(configHome, 'blocker'); await writeFile(blocker, 'file, not a dir', 'utf-8'); process.env.XDG_CONFIG_HOME = blocker; await expect( persistTelemetryDecision(true, 'create-bestax@1.0.0') - ).resolves.toBeUndefined(); + ).resolves.toBe(false); }); }); @@ -282,6 +286,62 @@ describe('reportScaffold', () => { skills: false, }; + it('a flag under DO_NOT_TRACK applies to this run only, never persisted', async () => { + // A copied command containing --telemetry must not enable telemetry + // beyond the run it was typed for when the user has opted out globally. + process.env.DO_NOT_TRACK = '1'; + const promptConsent = jest.fn<() => Promise>(); + await reportScaffold(choices, true, { interactive: true, promptConsent }); + expect(promptConsent).not.toHaveBeenCalled(); + expect(fetchMock).toHaveBeenCalledTimes(1); + await expect(readConfigFile()).rejects.toThrow(); + + fetchMock.mockClear(); + await reportScaffold(choices, false, { interactive: true, promptConsent }); + expect(fetchMock).not.toHaveBeenCalled(); + await expect(readConfigFile()).rejects.toThrow(); + }); + + it('tells onDecided whether the prompted decision was saved', async () => { + const onDecided = jest.fn<(enabled: boolean, persisted: boolean) => void>(); + const promptYes = jest + .fn<() => Promise>() + .mockResolvedValue(true); + await reportScaffold(choices, undefined, { + interactive: true, + promptConsent: promptYes, + onDecided, + }); + expect(onDecided).toHaveBeenCalledWith(true, true); + + onDecided.mockClear(); + const blocker = join(configHome, 'blocker'); + await writeFile(blocker, 'file, not a dir', 'utf-8'); + process.env.XDG_CONFIG_HOME = blocker; + const promptNo = jest + .fn<() => Promise>() + .mockResolvedValue(false); + await reportScaffold(choices, undefined, { + interactive: true, + promptConsent: promptNo, + onDecided, + }); + expect(onDecided).toHaveBeenCalledWith(false, false); + }); + + it('does not call onDecided on a cancel', async () => { + const onDecided = jest.fn<(enabled: boolean, persisted: boolean) => void>(); + const promptConsent = jest + .fn<() => Promise>() + .mockResolvedValue(null); + await reportScaffold(choices, undefined, { + interactive: true, + promptConsent, + onDecided, + }); + expect(onDecided).not.toHaveBeenCalled(); + }); + it('flag=true persists and sends without prompting', async () => { const promptConsent = jest.fn<() => Promise>(); await reportScaffold(choices, true, { interactive: true, promptConsent }); diff --git a/create-bestax/src/constants.ts b/create-bestax/src/constants.ts index c6582eff9..1b6929aef 100644 --- a/create-bestax/src/constants.ts +++ b/create-bestax/src/constants.ts @@ -48,6 +48,8 @@ export const MESSAGES = { ' Feedback welcome: https://github.com/allxsmith/bestax/issues', TELEMETRY_ACK_ON: 'Thanks! Opt out anytime with --no-telemetry.', TELEMETRY_ACK_OFF: "No problem — we won't ask again.", + TELEMETRY_ACK_UNSAVED: + "Couldn't save your choice (config dir not writable) — you may be asked again.", } as const; // Dev-server manifest written into .claude/ with the skills opt-in so Claude diff --git a/create-bestax/src/project-creator.ts b/create-bestax/src/project-creator.ts index 4de8fa63b..ea1e57330 100644 --- a/create-bestax/src/project-creator.ts +++ b/create-bestax/src/project-creator.ts @@ -619,9 +619,11 @@ export class ProjectCreator { /** * Consent + beacon after a successful scaffold. Only success events are * reported, the beacon is awaited (so no process.exit(1) site can kill an - * in-flight request), and nothing in here may throw or alter the exit code. - * The prompt is skipped for -y, missing TTY, an explicit flag, or - * DO_NOT_TRACK (#192: the non-interactive path must never hang). + * in-flight request), and nothing in here may throw or fail the scaffold. + * The prompt is skipped for -y, an explicit flag, DO_NOT_TRACK, or a + * missing TTY on EITHER end — with stdout redirected the question would + * render into the log while the CLI blocked on an invisible prompt + * (#192: the non-interactive path must never hang). */ private async handleTelemetry( options: CLIOptions | undefined, @@ -629,15 +631,21 @@ export class ProjectCreator { ): Promise { try { await reportScaffold(choices, options?.telemetry, { - interactive: process.stdin.isTTY === true && options?.yes !== true, - promptConsent: async () => { - const answer = await promptTelemetryConsent(); - if (answer === true) { + interactive: + process.stdin.isTTY === true && + process.stdout.isTTY === true && + options?.yes !== true, + promptConsent: promptTelemetryConsent, + onDecided: (enabled, persisted) => { + // Worded on whether the write actually stuck: promising "we won't + // ask again" after a swallowed write failure would be false. + if (!persisted) { + console.log(chalk.dim(MESSAGES.TELEMETRY_ACK_UNSAVED)); + } else if (enabled) { console.log(chalk.dim(MESSAGES.TELEMETRY_ACK_ON)); - } else if (answer === false) { + } else { console.log(chalk.dim(MESSAGES.TELEMETRY_ACK_OFF)); } - return answer; }, }); } catch { diff --git a/create-bestax/src/prompts.ts b/create-bestax/src/prompts.ts index bbb1019a4..ba2739fa9 100644 --- a/create-bestax/src/prompts.ts +++ b/create-bestax/src/prompts.ts @@ -97,18 +97,35 @@ export async function promptIconLibrary(): Promise { * Consent for anonymous usage telemetry. Deliberately does NOT call * ensureInteractive(): consent is optional, so without a TTY the caller skips * the question instead of failing the run (#192). Returns null when the user - * cancels (Ctrl-C) — a cancel is not an answer and nothing is persisted. + * cancels (Ctrl-C) — a cancel is not an answer and nothing is persisted, but + * the interrupt is honored via exit code 130 rather than swallowed. */ export async function promptTelemetryConsent(): Promise { console.log(); console.log(chalk.gray(MESSAGES.TELEMETRY_NOTICE)); - const response = await prompts({ - type: 'confirm', - name: 'telemetry', - message: PROMPT_MESSAGES.TELEMETRY_CONSENT, - initial: false, - }); + let cancelled = false; + const response = await prompts( + { + type: 'confirm', + name: 'telemetry', + message: PROMPT_MESSAGES.TELEMETRY_CONSENT, + initial: false, + }, + { + onCancel: () => { + cancelled = true; + }, + } + ); + if (cancelled) { + // A cancel is not an answer, but it IS an interrupt: without this, a ^C + // at the question would be swallowed and the run would exit 0 as if it + // had never been interrupted. 130 = 128 + SIGINT, the shell convention. + // The scaffold itself already succeeded, so no abrupt process.exit here. + process.exitCode = 130; + return null; + } return typeof response.telemetry === 'boolean' ? response.telemetry : null; } diff --git a/create-bestax/src/telemetry-core.ts b/create-bestax/src/telemetry-core.ts index c68cb22b4..5175cc0ca 100644 --- a/create-bestax/src/telemetry-core.ts +++ b/create-bestax/src/telemetry-core.ts @@ -93,10 +93,15 @@ export async function resolveTelemetry( return { decision: 'undecided', source: 'default' }; } +/** + * Returns whether the decision was actually written: callers word their + * acknowledgement on it, because promising "we won't ask again" after a + * swallowed write failure would be false on every future run. + */ export async function persistTelemetryDecision( enabled: boolean, decidedBy: string -): Promise { +): Promise { try { const config: TelemetryConfig = { version: 1, @@ -107,8 +112,10 @@ export async function persistTelemetryDecision( const filePath = telemetryConfigPath(); await mkdir(dirname(filePath), { recursive: true }); await writeFile(filePath, JSON.stringify(config, null, 2) + '\n', 'utf-8'); + return true; } catch { // An unwritable config dir must not break the CLI. + return false; } } @@ -140,8 +147,13 @@ export function getToolVersion(): string { export interface ReportRunOptions { /** True only when a consent question could actually be answered. */ interactive: boolean; - /** Owns all consent UI (notice, question, ack); null means cancelled. */ + /** Owns the consent UI up to the answer (notice, question); null = cancelled. */ promptConsent: () => Promise; + /** + * Called after a prompted answer has been (attempted to be) saved, so the + * acknowledgement can tell the truth about whether it will stick. + */ + onDecided?: (enabled: boolean, persisted: boolean) => void; } export async function reportRun( @@ -154,16 +166,26 @@ export async function reportRun( let decision = resolved.decision; if (resolved.source === 'flag') { - // Flags are the scripted equivalent of the prompt: decided once, remembered. - await persistTelemetryDecision( - decision === 'on', - `${toolName}@${getToolVersion()}` - ); + // Flags are the scripted equivalent of the prompt: decided once, + // remembered — EXCEPT under DO_NOT_TRACK, where the flag still applies to + // this one run (an explicit ask on the command line wins) but is never + // written down, so a copied command containing --telemetry cannot enable + // telemetry beyond the run it was typed for. + if (!doNotTrack()) { + await persistTelemetryDecision( + decision === 'on', + `${toolName}@${getToolVersion()}` + ); + } } else if (decision === 'undecided' && options.interactive) { const answer = await options.promptConsent(); if (answer !== null) { - await persistTelemetryDecision(answer, `${toolName}@${getToolVersion()}`); + const persisted = await persistTelemetryDecision( + answer, + `${toolName}@${getToolVersion()}` + ); decision = answer ? 'on' : 'off'; + options.onDecided?.(answer, persisted); } } From 271ba4c3f19ef2846cad80be696c8b3af1991030 Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sat, 22 Aug 2026 20:23:13 -0400 Subject: [PATCH 14/29] fix(bestax-migrate): stop sending changedBucket, honor Ctrl-C at consent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 0/1-9/10-49/50-199/200+ bucket is now derived by the ingest worker from the capped changed count — sending it too gave the boundaries three sources of truth with no consistency check. A ^C at the consent question sets exit code 130 instead of reading as a clean exit, the saved acknowledgement is only printed when the write actually stuck, and the kernel picks up the same DNT-flag rules as create-bestax. README now lists the CLI version, Node major, and platform fields the payload always carried. --- bestax-migrate/README.md | 7 ++-- bestax-migrate/src/__tests__/cli.test.ts | 15 ++++++++ .../src/__tests__/telemetry.test.ts | 34 +++++------------ bestax-migrate/src/cli.ts | 23 ++++++++--- bestax-migrate/src/telemetry-core.ts | 38 +++++++++++++++---- bestax-migrate/src/telemetry.ts | 12 +----- 6 files changed, 77 insertions(+), 52 deletions(-) diff --git a/bestax-migrate/README.md b/bestax-migrate/README.md index ab1069385..cb7448f0f 100644 --- a/bestax-migrate/README.md +++ b/bestax-migrate/README.md @@ -89,9 +89,10 @@ Full detail: [`SECURITY.md`](https://github.com/allxsmith/bestax/blob/main/SECUR `bestax-migrate` can send one **anonymous** usage event after a successful run — only if you opt in when asked (once, on an interactive terminal). The event is just the run's shape — source library, CSS mode, the dry/deps flags, a -bucketed changed-file count, and per-rule TODO counts — never file paths, file -contents, code, names, IPs, or machine IDs, and no identifier exists that could -link two events together. Opt out any time with `--no-telemetry`, +capped changed-file count, and per-rule TODO counts — plus the CLI version, +Node major version, and OS platform name. Never file paths, file contents, +code, names, IPs, or machine IDs, and no identifier exists that could link two +events together. Opt out any time with `--no-telemetry`, `BESTAX_TELEMETRY=0`, or `DO_NOT_TRACK=1`. Full disclosure of every field and control: [Telemetry guide](https://bestax.io/docs/guides/telemetry) diff --git a/bestax-migrate/src/__tests__/cli.test.ts b/bestax-migrate/src/__tests__/cli.test.ts index 523336591..98f2ac3ff 100644 --- a/bestax-migrate/src/__tests__/cli.test.ts +++ b/bestax-migrate/src/__tests__/cli.test.ts @@ -488,6 +488,21 @@ describe('handleTelemetry', () => { expect(fetchMock).not.toHaveBeenCalled(); }); + it('acknowledges truthfully when the choice could not be saved', async () => { + // "We won't ask again" after a swallowed write failure would be false on + // every future run; the ack is worded on whether the write stuck. + setTTY(process.stdout, true); + setTTY(process.stdin, true); + const blocker = path.join(process.env.XDG_CONFIG_HOME as string, 'blocker'); + fs.writeFileSync(blocker, 'file, not a dir'); + process.env.XDG_CONFIG_HOME = blocker; + const io = makeIo(); + await handleTelemetry(stats, undefined, io, async () => false); + expect(io.logs.join('\n')).toContain("Couldn't save your choice"); + expect(io.logs.join('\n')).not.toContain("we won't ask again"); + expect(fetchMock).not.toHaveBeenCalled(); + }); + it('a cancelled prompt acks nothing and persists nothing', async () => { setTTY(process.stdout, true); setTTY(process.stdin, true); diff --git a/bestax-migrate/src/__tests__/telemetry.test.ts b/bestax-migrate/src/__tests__/telemetry.test.ts index 37308b5bf..40098bd08 100644 --- a/bestax-migrate/src/__tests__/telemetry.test.ts +++ b/bestax-migrate/src/__tests__/telemetry.test.ts @@ -7,7 +7,7 @@ import { afterEach, afterAll, } from '@jest/globals'; -import { mkdtemp, readFile, writeFile, mkdir } from 'fs/promises'; +import { mkdtemp, readFile, writeFile, mkdir, rm } from 'fs/promises'; import { tmpdir } from 'os'; import { join } from 'path'; @@ -63,7 +63,9 @@ beforeEach(async () => { globalThis.fetch = fetchMock as unknown as typeof fetch; }); -afterEach(() => { +afterEach(async () => { + // ~80 scratch dirs per run otherwise outlive the suite in the OS tmpdir. + await rm(configHome, { recursive: true, force: true }); for (const key of ENV_KEYS) { if (savedEnv[key] === undefined) delete process.env[key]; else process.env[key] = savedEnv[key]; @@ -122,7 +124,6 @@ describe('buildMigratePayload', () => { expect(payload.nodeMajor).toBe(Number(process.versions.node.split('.')[0])); expect(payload.platform).toBe(process.platform); expect(Object.keys(payload.props).sort()).toEqual([ - 'changedBucket', 'changedCount', 'cssMode', 'deps', @@ -134,7 +135,6 @@ describe('buildMigratePayload', () => { expect(payload.props.dry).toBe(false); expect(payload.props.deps).toBe(true); expect(payload.props.changedCount).toBe(3); - expect(payload.props.changedBucket).toBe('1-9'); expect(payload.todosByRule).toEqual([ { rule: 'unsupported-file', count: 2 }, ]); @@ -158,31 +158,15 @@ describe('buildMigratePayload', () => { ]); }); - it.each([ - [0, '0'], - [1, '1-9'], - [9, '1-9'], - [10, '10-49'], - [49, '10-49'], - [50, '50-199'], - [199, '50-199'], - [200, '200+'], - ] as const)('buckets changedCount=%d as %s', (changedCount, bucket) => { - const payload = buildMigratePayload({ - ...baseStats, - changedCount, - }); - expect(payload.props.changedBucket).toBe(bucket); - expect(payload.props.changedCount).toBe(changedCount); - }); - - it('caps changedCount at 10000', () => { + // The 0/1-9/10-49/50-199/200+ bucket is derived by the ingest worker from + // changedCount — sending it too gave the boundaries three sources of truth. + it('caps changedCount at 10000 and sends no bucket', () => { const payload = buildMigratePayload({ ...baseStats, changedCount: 123456, }); expect(payload.props.changedCount).toBe(10000); - expect(payload.props.changedBucket).toBe('200+'); + expect('changedBucket' in payload.props).toBe(false); }); it('sends at most 20 rules and caps each count', () => { @@ -219,7 +203,7 @@ describe('reportMigrateRun', () => { props: Record; }; expect(body.props.source).toBe('react-bulma-components'); - expect(body.props.changedBucket).toBe('10-49'); + expect(body.props.changedCount).toBe(12); }); it('flag=false persists off and sends nothing', async () => { diff --git a/bestax-migrate/src/cli.ts b/bestax-migrate/src/cli.ts index c927f7479..0e127bf79 100644 --- a/bestax-migrate/src/cli.ts +++ b/bestax-migrate/src/cli.ts @@ -223,6 +223,8 @@ const TELEMETRY_NOTICE = ' Feedback welcome: https://github.com/allxsmith/bestax/issues'; const TELEMETRY_ACK_ON = 'Thanks! Opt out anytime with --no-telemetry.'; const TELEMETRY_ACK_OFF = "No problem — we won't ask again."; +const TELEMETRY_ACK_UNSAVED = + "Couldn't save your choice (config dir not writable) — you may be asked again."; /** * One-question consent prompt (readline — this package carries no interactive @@ -241,7 +243,13 @@ export async function promptTelemetryConsent( const cancelled = new AbortController(); // Ctrl-C emits SIGINT on the interface and Ctrl-D closes it; both must // reject the pending question instead of leaving it hanging forever. - rl.once('SIGINT', () => rl.close()); + rl.once('SIGINT', () => { + // Honor the interrupt instead of swallowing it into a zero exit: the run + // itself already succeeded, so no abrupt kill — but the ^C must be + // visible to callers. 130 = 128 + SIGINT, the shell convention. + process.exitCode = 130; + rl.close(); + }); rl.once('close', () => cancelled.abort()); try { const answer = await rl.question('Share anonymous usage stats? (y/N) ', { @@ -272,14 +280,17 @@ export async function handleTelemetry( await reportMigrateRun(stats, flag, { interactive: process.stdin.isTTY === true && process.stdout.isTTY === true, - promptConsent: async () => { - const answer = await promptConsent(io); - if (answer === true) { + promptConsent: () => promptConsent(io), + onDecided: (enabled, persisted) => { + // Worded on whether the write actually stuck: promising "we won't + // ask again" after a swallowed write failure would be false. + if (!persisted) { + io.log(chalk.dim(TELEMETRY_ACK_UNSAVED)); + } else if (enabled) { io.log(chalk.dim(TELEMETRY_ACK_ON)); - } else if (answer === false) { + } else { io.log(chalk.dim(TELEMETRY_ACK_OFF)); } - return answer; }, }); } catch { diff --git a/bestax-migrate/src/telemetry-core.ts b/bestax-migrate/src/telemetry-core.ts index c68cb22b4..5175cc0ca 100644 --- a/bestax-migrate/src/telemetry-core.ts +++ b/bestax-migrate/src/telemetry-core.ts @@ -93,10 +93,15 @@ export async function resolveTelemetry( return { decision: 'undecided', source: 'default' }; } +/** + * Returns whether the decision was actually written: callers word their + * acknowledgement on it, because promising "we won't ask again" after a + * swallowed write failure would be false on every future run. + */ export async function persistTelemetryDecision( enabled: boolean, decidedBy: string -): Promise { +): Promise { try { const config: TelemetryConfig = { version: 1, @@ -107,8 +112,10 @@ export async function persistTelemetryDecision( const filePath = telemetryConfigPath(); await mkdir(dirname(filePath), { recursive: true }); await writeFile(filePath, JSON.stringify(config, null, 2) + '\n', 'utf-8'); + return true; } catch { // An unwritable config dir must not break the CLI. + return false; } } @@ -140,8 +147,13 @@ export function getToolVersion(): string { export interface ReportRunOptions { /** True only when a consent question could actually be answered. */ interactive: boolean; - /** Owns all consent UI (notice, question, ack); null means cancelled. */ + /** Owns the consent UI up to the answer (notice, question); null = cancelled. */ promptConsent: () => Promise; + /** + * Called after a prompted answer has been (attempted to be) saved, so the + * acknowledgement can tell the truth about whether it will stick. + */ + onDecided?: (enabled: boolean, persisted: boolean) => void; } export async function reportRun( @@ -154,16 +166,26 @@ export async function reportRun( let decision = resolved.decision; if (resolved.source === 'flag') { - // Flags are the scripted equivalent of the prompt: decided once, remembered. - await persistTelemetryDecision( - decision === 'on', - `${toolName}@${getToolVersion()}` - ); + // Flags are the scripted equivalent of the prompt: decided once, + // remembered — EXCEPT under DO_NOT_TRACK, where the flag still applies to + // this one run (an explicit ask on the command line wins) but is never + // written down, so a copied command containing --telemetry cannot enable + // telemetry beyond the run it was typed for. + if (!doNotTrack()) { + await persistTelemetryDecision( + decision === 'on', + `${toolName}@${getToolVersion()}` + ); + } } else if (decision === 'undecided' && options.interactive) { const answer = await options.promptConsent(); if (answer !== null) { - await persistTelemetryDecision(answer, `${toolName}@${getToolVersion()}`); + const persisted = await persistTelemetryDecision( + answer, + `${toolName}@${getToolVersion()}` + ); decision = answer ? 'on' : 'off'; + options.onDecided?.(answer, persisted); } } diff --git a/bestax-migrate/src/telemetry.ts b/bestax-migrate/src/telemetry.ts index 36e6f88cf..6d3b7f571 100644 --- a/bestax-migrate/src/telemetry.ts +++ b/bestax-migrate/src/telemetry.ts @@ -33,20 +33,13 @@ export interface MigratePayload { cssMode: string; dry: boolean; deps: boolean; - changedBucket: string; + // The 0/1-9/10-49/50-199/200+ bucket is derived by the ingest worker from + // this count — sending it too gave the boundaries three sources of truth. changedCount: number; }; todosByRule?: Array<{ rule: string; count: number }>; } -function changedBucket(changedCount: number): string { - if (changedCount <= 0) return '0'; - if (changedCount <= 9) return '1-9'; - if (changedCount <= 49) return '10-49'; - if (changedCount <= 199) return '50-199'; - return '200+'; -} - export function buildMigratePayload(stats: MigrateRunStats): MigratePayload { const todosByRule = stats.todosByRule .slice(0, 20) @@ -63,7 +56,6 @@ export function buildMigratePayload(stats: MigrateRunStats): MigratePayload { cssMode: stats.cssMode, dry: stats.dry, deps: stats.deps, - changedBucket: changedBucket(stats.changedCount), changedCount: Math.min(stats.changedCount, 10000), }, ...(todosByRule.length > 0 ? { todosByRule } : {}), From 493aad25594053bfd6dfed5bd37050eba2c3d3cd Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sat, 22 Aug 2026 20:23:14 -0400 Subject: [PATCH 15/29] chore(telemetry-worker): cap todo counts, derive buckets, stream the body cap Review fixes: migrate_todo doubles get the same 10000 cap as the run event (one outlier could dominate SUM(double1 * _sample_interval)); changedBucket is derived server-side from the capped count and rejected on the wire; the body is read as a stream and cut off at 8KB so a chunked POST cannot buffer to the plan cap; netbsd and haiku join the platform allowlist; devDependency versions are pinned exact while the package sits outside the pnpm workspace; README documents the unauthenticated-ingest tradeoff. --- telemetry-worker/README.md | 17 ++- telemetry-worker/package.json | 4 +- .../src/__tests__/handler.test.ts | 106 +++++++++++++++++- .../src/__tests__/validate.test.ts | 33 +++--- telemetry-worker/src/index.ts | 49 +++++++- telemetry-worker/src/schema.ts | 45 ++++++-- telemetry-worker/src/validate.ts | 7 +- 7 files changed, 211 insertions(+), 50 deletions(-) diff --git a/telemetry-worker/README.md b/telemetry-worker/README.md index b89a6c941..0440bd46e 100644 --- a/telemetry-worker/README.md +++ b/telemetry-worker/README.md @@ -13,8 +13,9 @@ npx wrangler deploy ``` Run by Alex — the package is not in the pnpm workspace (Wrangler's native -`workerd` build is blocked repo-wide). Wrangler bundles `src/index.ts` -directly (no build step). Tests and typecheck still run from the repo root: +`workerd` build is blocked repo-wide), so the exact-pinned devDependencies in +`package.json` are the only version control these tools get until it joins. +Wrangler bundles `src/index.ts` directly (no build step). Tests and typecheck still run from the repo root: `pnpm test` includes `node --test telemetry-worker/src/__tests__/*.test.ts`, and `pnpm typecheck` includes `tsc -p telemetry-worker`. The worker is bound to the zone route `bestax.io/api/t`. If that route ever conflicts with @@ -39,8 +40,12 @@ so no code change needed). | `blob6` | bulmaFlavor / cssMode | rule | | `blob7` | iconLibrary / dry (`1`/`0`) | — | | `blob8` | skills (`1`/`0`) / deps (`1`/`0`) | — | -| `blob9` | packageManager / changedBucket | — | -| `double1` | — / changedCount (capped at 10 000) | count | +| `blob9` | packageManager / changedBucket¹ | — | +| `double1` | — / changedCount (capped at 10 000) | count (same cap) | + +¹ `changedBucket` is not a wire field: the worker derives it from the capped +`changedCount` (`0`, `1-9`, `10-49`, `50-199`, `200+`), so the bucket and the +double can never disagree. ## Example queries @@ -78,4 +83,8 @@ FROM bestax_telemetry WHERE index1='create-bestax' AND blob1='scaffold' silently discarded. - No IP address or User-Agent is ever read or stored; `request.cf` is never touched; request bodies are never logged. +- The ingest is unauthenticated by design — any client can POST synthetic + events, because per-sender identification would contradict the no-identifier + privacy contract — so aggregate queries should treat absolute counts as + indicative, not audited. - No CORS headers: the endpoint is for the CLIs, not browsers. diff --git a/telemetry-worker/package.json b/telemetry-worker/package.json index 632ae3290..e0c82cdb0 100644 --- a/telemetry-worker/package.json +++ b/telemetry-worker/package.json @@ -10,7 +10,7 @@ "dev": "wrangler dev" }, "devDependencies": { - "typescript": "^5", - "wrangler": "^4" + "typescript": "5.9.2", + "wrangler": "4.33.1" } } diff --git a/telemetry-worker/src/__tests__/handler.test.ts b/telemetry-worker/src/__tests__/handler.test.ts index cb9c2ffbd..366149ad4 100644 --- a/telemetry-worker/src/__tests__/handler.test.ts +++ b/telemetry-worker/src/__tests__/handler.test.ts @@ -59,7 +59,6 @@ const migratePayload = () => ({ cssMode: 'bestax', dry: false, deps: true, - changedBucket: '10-49', changedCount: 23, }, todosByRule: [ @@ -116,9 +115,9 @@ describe('handler: request gate', () => { 'content-type': 'application/json', 'content-length': '999999', }), - text: async () => { + get body() { read = true; - return '{}'; + return null; }, }; const { env, points } = makeEnv(); @@ -135,6 +134,67 @@ describe('handler: request gate', () => { assert.equal(points.length, 0); }); + it('413 for an oversized chunked body, without buffering it all', async () => { + // A streamed body declares no content-length, so only the running byte + // count can refuse it. 1 KiB chunks: the cap must trip on chunk 9 and + // stop pulling — chunk 20 must never be requested. + let pulls = 0; + const chunk = new TextEncoder().encode('a'.repeat(1024)); + const stream = new ReadableStream({ + pull(controller) { + pulls += 1; + if (pulls > 20) controller.close(); + else controller.enqueue(chunk); + }, + }); + const request = new Request(ENDPOINT, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: stream, + duplex: 'half', + } as RequestInit); + const { env, points } = makeEnv(); + const res = await call(request, env); + assert.equal(res.status, 413); + assert.ok(pulls < 20, `stopped pulling early (pulled ${pulls} chunks)`); + assert.equal(points.length, 0); + }); + + it('accepts a valid payload sent as a chunked stream', async () => { + const bytes = new TextEncoder().encode(JSON.stringify(createPayload())); + const stream = new ReadableStream({ + start(controller) { + // Split mid-payload so decoding must happen after reassembly. + controller.enqueue(bytes.slice(0, 40)); + controller.enqueue(bytes.slice(40)); + controller.close(); + }, + }); + const request = new Request(ENDPOINT, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: stream, + duplex: 'half', + } as RequestInit); + const { env, points } = makeEnv(); + const res = await call(request, env); + assert.equal(res.status, 204); + assert.equal(points.length, 1); + }); + + it('400 for a POST with no body', async () => { + const { env, points } = makeEnv(); + const res = await call( + new Request(ENDPOINT, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + }), + env + ); + assert.equal(res.status, 400); + assert.equal(points.length, 0); + }); + it('400 for malformed JSON', async () => { const { env, points } = makeEnv(); const res = await call(post('{"v":1,'), env); @@ -221,13 +281,51 @@ describe('handler: accepted payloads', () => { }); }); - it('caps the changedCount double at 10000', async () => { + it('caps the changedCount double at 10000 and buckets the capped value', async () => { const { env, points } = makeEnv(); const payload = migratePayload(); payload.props.changedCount = 100000; const res = await call(post(payload), env); assert.equal(res.status, 204); assert.deepEqual(points[0]?.doubles, [10000]); + assert.equal(points[0]?.blobs?.[8], '200+'); + }); + + it('derives changedBucket (blob9) from changedCount server-side', async () => { + const boundaries: [number, string][] = [ + [0, '0'], + [1, '1-9'], + [9, '1-9'], + [10, '10-49'], + [49, '10-49'], + [50, '50-199'], + [199, '50-199'], + [200, '200+'], + [10000, '200+'], + [100000, '200+'], // capped to 10000 first, still 200+ + ]; + for (const [changedCount, bucket] of boundaries) { + const { env, points } = makeEnv(); + const payload = migratePayload(); + payload.props.changedCount = changedCount; + const res = await call(post(payload), env); + assert.equal(res.status, 204); + assert.equal( + points[0]?.blobs?.[8], + bucket, + `changedCount ${changedCount} → bucket ${bucket}` + ); + } + }); + + it('caps each todo count double at 10000', async () => { + const { env, points } = makeEnv(); + const payload = migratePayload(); + payload.todosByRule = [{ rule: 'prop:className', count: 100000 }]; + const res = await call(post(payload), env); + assert.equal(res.status, 204); + assert.equal(points.length, 2); + assert.deepEqual(points[1]?.doubles, [10000]); }); it('20 todo rules → exactly 21 writes', async () => { diff --git a/telemetry-worker/src/__tests__/validate.test.ts b/telemetry-worker/src/__tests__/validate.test.ts index 96196313c..2c2a440e2 100644 --- a/telemetry-worker/src/__tests__/validate.test.ts +++ b/telemetry-worker/src/__tests__/validate.test.ts @@ -32,7 +32,6 @@ const migratePayload = (): Payload => ({ cssMode: 'bestax', dry: false, deps: true, - changedBucket: '10-49', changedCount: 23, }, todosByRule: [ @@ -74,6 +73,14 @@ describe('validate: accepts', () => { assertOk(payload); }); + for (const platform of ['netbsd', 'haiku']) { + it(`${platform} platform`, () => { + const payload = createPayload(); + payload.platform = platform; + assertOk(payload); + }); + } + it('validated payload round-trips unchanged', () => { const payload = migratePayload(); const result = validate(payload); @@ -113,6 +120,12 @@ describe('validate: rejects unknown or missing keys', () => { assertRejected(payload, 'unknown props key'); }); + it('changedBucket in migrate props (derived server-side, not a wire field)', () => { + const payload = migratePayload(); + props(payload).changedBucket = '10-49'; + assertRejected(payload, 'changedBucket is no longer accepted'); + }); + for (const key of [ 'v', 'tool', @@ -143,14 +156,7 @@ describe('validate: rejects unknown or missing keys', () => { }); } - for (const key of [ - 'source', - 'cssMode', - 'dry', - 'deps', - 'changedBucket', - 'changedCount', - ]) { + for (const key of ['source', 'cssMode', 'dry', 'deps', 'changedCount']) { it(`missing migrate props.${key}`, () => { const payload = migratePayload(); delete props(payload)[key]; @@ -212,7 +218,7 @@ describe('validate: rejects envelope field values', () => { it('unknown platform', () => { const payload = createPayload(); - payload.platform = 'haiku'; + payload.platform = 'beos'; assertRejected(payload, 'unknown platform'); }); @@ -231,7 +237,6 @@ describe('validate: rejects props enum values', () => { ['packageManager', createPayload, 'packageManager'], ['source', migratePayload, 'source'], ['cssMode', migratePayload, 'cssMode'], - ['changedBucket', migratePayload, 'changedBucket'], ]; for (const [label, factory, key] of badEnum) { it(`unknown ${label}`, () => { @@ -241,12 +246,6 @@ describe('validate: rejects props enum values', () => { }); } - it('bucket label as changedBucket', () => { - const payload = migratePayload(); - props(payload).changedBucket = '1-10'; - assertRejected(payload, 'changedBucket not in bucket set'); - }); - for (const [key, factory] of [ ['skills', createPayload], ['dry', migratePayload], diff --git a/telemetry-worker/src/index.ts b/telemetry-worker/src/index.ts index 98ee51644..5ed34f3f2 100644 --- a/telemetry-worker/src/index.ts +++ b/telemetry-worker/src/index.ts @@ -15,6 +15,43 @@ const MAX_BODY_BYTES = 8192; const status = (code: number): Response => new Response(null, { status: code }); +/** + * Read the body as a stream, refusing the moment the running byte total + * exceeds MAX_BODY_BYTES — a chunked POST carries no content-length, and + * buffering it whole before checking would let one request hold the plan cap + * in memory. Returns the raw bytes, or the error status to answer with. + */ +async function readBody( + body: ReadableStream +): Promise<{ ok: true; bytes: Uint8Array } | { ok: false; status: number }> { + const reader = body.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + try { + for (;;) { + const result = await reader.read(); + if (result.done) break; + total += result.value.byteLength; + if (total > MAX_BODY_BYTES) { + await reader.cancel(); + return { ok: false, status: 413 }; + } + chunks.push(result.value); + } + } catch { + // A stream error mid-read (client abort, malformed chunking): treat like + // any other unusable body. + return { ok: false, status: 400 }; + } + const bytes = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return { ok: true, bytes }; +} + export default { async fetch(request: Request, env: Env): Promise { if (request.method !== 'POST') return status(405); @@ -24,19 +61,19 @@ export default { if (!contentType.includes('application/json')) return status(415); // Declared length first, so an honestly-labeled oversized body is refused - // without buffering it; the byte check after read is the real guard. + // without reading a byte; the streamed count in readBody is the real + // guard (a chunked POST declares no length). const declaredLength = request.headers.get('content-length'); if (declaredLength !== null && Number(declaredLength) > MAX_BODY_BYTES) { return status(413); } - const text = await request.text(); - if (new TextEncoder().encode(text).byteLength > MAX_BODY_BYTES) { - return status(413); - } + if (request.body === null) return status(400); // no body: never valid JSON + const read = await readBody(request.body); + if (!read.ok) return status(read.status); let body: unknown; try { - body = JSON.parse(text); + body = JSON.parse(new TextDecoder().decode(read.bytes)); } catch { return status(400); } diff --git a/telemetry-worker/src/schema.ts b/telemetry-worker/src/schema.ts index 334654e1b..e9bd7bc13 100644 --- a/telemetry-worker/src/schema.ts +++ b/telemetry-worker/src/schema.ts @@ -48,12 +48,13 @@ const PLATFORM_VALUES = [ 'darwin', 'freebsd', 'linux', + 'netbsd', 'openbsd', 'sunos', 'win32', 'android', + 'haiku', ] as const; -const CHANGED_BUCKET_VALUES = ['0', '1-9', '10-49', '50-199', '200+'] as const; export type Template = (typeof TEMPLATE_VALUES)[number]; export type BulmaFlavor = (typeof BULMA_FLAVOR_VALUES)[number]; @@ -62,7 +63,6 @@ export type MigrateSource = (typeof MIGRATE_SOURCE_VALUES)[number]; export type CssMode = (typeof CSS_MODE_VALUES)[number]; export type PackageManager = (typeof PACKAGE_MANAGER_VALUES)[number]; export type Platform = (typeof PLATFORM_VALUES)[number]; -export type ChangedBucket = (typeof CHANGED_BUCKET_VALUES)[number]; export const TEMPLATES: ReadonlySet = new Set(TEMPLATE_VALUES); export const BULMA_FLAVORS: ReadonlySet = new Set(BULMA_FLAVOR_VALUES); @@ -75,16 +75,34 @@ export const PACKAGE_MANAGERS: ReadonlySet = new Set( PACKAGE_MANAGER_VALUES ); export const PLATFORMS: ReadonlySet = new Set(PLATFORM_VALUES); -export const CHANGED_BUCKETS: ReadonlySet = new Set( - CHANGED_BUCKET_VALUES -); export const MAX_TODO_RULES = 20; export const MAX_CHANGED_COUNT = 100000; -// changedCount above this is a pathological run; cap the stored double so one -// outlier cannot dominate SUM(double1 * _sample_interval) aggregates. +// A count above this is a pathological run; cap every stored double — the run +// event's changedCount and each migrate_todo count alike — so one outlier +// cannot dominate SUM(double1 * _sample_interval) aggregates. export const CHANGED_COUNT_DOUBLE_CAP = 10000; +// changedBucket (blob9 on the migrate run event) is derived HERE, not sent on +// the wire. Single boundary definition: ascending lower bounds; each label is +// derived ('0', '1-9', '10-49', '50-199', '200+'), so the buckets cannot +// drift from the boundaries. +const CHANGED_BUCKET_BOUNDS = [0, 1, 10, 50, 200] as const; + +export function changedBucket(count: number): string { + let i = 0; + while ( + i + 1 < CHANGED_BUCKET_BOUNDS.length && + count >= CHANGED_BUCKET_BOUNDS[i + 1] + ) { + i += 1; + } + const lo = CHANGED_BUCKET_BOUNDS[i]; + if (i === CHANGED_BUCKET_BOUNDS.length - 1) return `${lo}+`; + const next = CHANGED_BUCKET_BOUNDS[i + 1]; + return next - lo === 1 ? `${lo}` : `${lo}-${next - 1}`; +} + export interface CreateBestaxPayload { v: 1; tool: 'create-bestax'; @@ -113,7 +131,6 @@ export interface MigratePayload { cssMode: CssMode; dry: boolean; deps: boolean; - changedBucket: ChangedBucket; changedCount: number; }; todosByRule?: { rule: string; count: number }[]; @@ -135,8 +152,11 @@ export type TelemetryPayload = CreateBestaxPayload | MigratePayload; * | blob6 | bulmaFlavor / cssMode | rule | * | blob7 | iconLibrary / dry '1'|'0' | — | * | blob8 | skills '1'|'0' / deps '1'|'0' | — | - * | blob9 | packageManager / changedBucket | — | + * | blob9 | packageManager / changedBucket* | — | * | double1 | — / changedCount | count | + * + * *changedBucket is derived server-side from the capped changedCount — it is + * not a wire field. */ export function runEventPoint( payload: TelemetryPayload @@ -163,6 +183,7 @@ export function runEventPoint( }; } const p = payload.props; + const cappedCount = Math.min(p.changedCount, CHANGED_COUNT_DOUBLE_CAP); return { indexes: [payload.tool], blobs: [ @@ -171,9 +192,9 @@ export function runEventPoint( p.cssMode, p.dry ? '1' : '0', p.deps ? '1' : '0', - p.changedBucket, + changedBucket(cappedCount), ], - doubles: [Math.min(p.changedCount, CHANGED_COUNT_DOUBLE_CAP)], + doubles: [cappedCount], }; } @@ -191,6 +212,6 @@ export function todoPoints( payload.props.source, entry.rule, ], - doubles: [entry.count], + doubles: [Math.min(entry.count, CHANGED_COUNT_DOUBLE_CAP)], })); } diff --git a/telemetry-worker/src/validate.ts b/telemetry-worker/src/validate.ts index f753c2f87..406e7774a 100644 --- a/telemetry-worker/src/validate.ts +++ b/telemetry-worker/src/validate.ts @@ -9,7 +9,6 @@ import { BULMA_FLAVORS, - CHANGED_BUCKETS, CSS_MODES, EVENT_FOR_TOOL, ICON_LIBRARIES, @@ -48,12 +47,13 @@ const CREATE_PROP_KEYS = [ 'skills', 'packageManager', ] as const; +// changedBucket is NOT a wire field: the worker derives it from changedCount +// (schema.ts), so a payload sending it is rejected as an unknown key. const MIGRATE_PROP_KEYS = [ 'source', 'cssMode', 'dry', 'deps', - 'changedBucket', 'changedCount', ] as const; const TODO_ENTRY_KEYS = ['rule', 'count'] as const; @@ -139,9 +139,6 @@ export function validate(body: unknown): ValidationResult { if (!inSet(CSS_MODES, props.cssMode)) return reject('unknown cssMode'); if (typeof props.dry !== 'boolean') return reject('invalid dry'); if (typeof props.deps !== 'boolean') return reject('invalid deps'); - if (!inSet(CHANGED_BUCKETS, props.changedBucket)) { - return reject('unknown changedBucket'); - } if ( typeof props.changedCount !== 'number' || !Number.isInteger(props.changedCount) || From b3b2a31ef715b2b6b4fae7586c417827add5164d Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sat, 22 Aug 2026 20:23:15 -0400 Subject: [PATCH 16/29] docs: true up the telemetry disclosure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the CLI version / Node major / platform fields the migrate table omitted, describe the changed-count bucket as derived at the ingest endpoint, spell out the DO_NOT_TRACK interaction with an explicit --telemetry flag (applies to that run, never saved), and stop claiming 'exactly once' — a cancelled question may be asked again. --- docs/docs/guides/telemetry.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/docs/guides/telemetry.md b/docs/docs/guides/telemetry.md index 52e03f0dc..187676415 100644 --- a/docs/docs/guides/telemetry.md +++ b/docs/docs/guides/telemetry.md @@ -43,8 +43,11 @@ Sent once after a successful run: | `cssMode` | `bestax`, `bulma`, or `keep` | | `dry` | `true` or `false` — whether it was a dry run | | `deps` | `true` or `false` — whether `package.json` dependencies were updated | -| Changed-file count | a bucket — `0`, `1-9`, `10-49`, `50-199`, or `200+` — plus the capped count | +| Changed-file count | capped at 10,000 (the ingest endpoint buckets it as `0`/`1-9`/`10-49`/`50-199`/`200+`) | | TODO counts | per migration rule: the rule name (including `prop:` slugs) and a count, nothing else | +| CLI version | the `bestax-migrate` version that ran | +| Node major version | e.g. `22` | +| OS platform | the platform name, e.g. `darwin`, `linux`, `win32` | Never file paths, never file contents, never code. The TODO report printed in your terminal lists files and lines; the telemetry event carries only rule @@ -68,7 +71,9 @@ same machine. ## How consent works -Telemetry is off until you opt in, and you're asked exactly once: +Telemetry is off until you opt in, and once you answer you're never asked +again (cancelling the question with Ctrl-C is not an answer — you may be asked +on a later run): - `create-bestax` asks at the end of a successful scaffold. - `bestax-migrate` asks once after a run, and only on an interactive terminal. @@ -84,7 +89,13 @@ ever asked once. Delete that file to be asked again. | ------------------------------------------- | --------------------------------------------------------------------------------------- | | `--telemetry` / `--no-telemetry` | Flags on both CLIs — set the choice for this run **and persist it** to the consent file | | `BESTAX_TELEMETRY=1` / `BESTAX_TELEMETRY=0` | Enable or disable for this run only — never persisted | -| `DO_NOT_TRACK=1` | Disables everything, including the consent prompt itself | +| `DO_NOT_TRACK=1` | Disables telemetry and the consent prompt itself | + +One interaction worth spelling out: with `DO_NOT_TRACK` set, an explicit +`--telemetry` flag still applies **to that single run** (typing the flag is a +direct ask, which the DNT convention lets win) — but under `DO_NOT_TRACK` the +flag is **never saved**, so a copied command containing `--telemetry` cannot +enable telemetry beyond the run it was typed for. ## Where the data goes From 9cf0c237e26598b6ce28acc7d0b69105fd6d67e4 Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sat, 22 Aug 2026 20:23:15 -0400 Subject: [PATCH 17/29] chore: normalize utm_source out of eval docs metrics; list telemetry-worker The MCP server tags its printed docs links with utm_source=bestax-mcp; collect-metrics now strips it so tagged and bare forms of one page count once across guidance channels. Root CLAUDE.md gains the telemetry-worker/ line its package roster was missing. --- CLAUDE.md | 2 ++ eval/agent-loop/bin/collect-metrics.mjs | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 645c44cd1..8ce142e00 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,6 +9,8 @@ React component library for **Bulma v1** in TypeScript. pnpm monorepo orchestrat - `bestax-mcp/` — the `bestax-mcp` MCP server; its `data/` index is **generated** (has its own CLAUDE.md) - `skills/` — Agent Skills, a **shipped product** bundled into create-bestax (has its own CLAUDE.md) +- `telemetry-worker/` — Cloudflare Worker ingesting the CLIs' opt-in telemetry + (deployed manually with wrangler; not yet in the pnpm workspace) - `eval/agent-loop/` — cold-start eval harness: does an unassisted agent build correctly when given one of our guidance channels (the skills, the MCP server, both)? Frozen rubric, its own README diff --git a/eval/agent-loop/bin/collect-metrics.mjs b/eval/agent-loop/bin/collect-metrics.mjs index 818d1e2c3..18308a73d 100644 --- a/eval/agent-loop/bin/collect-metrics.mjs +++ b/eval/agent-loop/bin/collect-metrics.mjs @@ -254,7 +254,15 @@ if (transcriptPath && existsSync(transcriptPath)) { for (const m of inputStr.matchAll( /https?:\/\/[^"\\\s]*bestax\.io[^"\\\s]*/g )) - docsUrls.add(m[0]); + // The MCP server tags the links it prints with utm_source=bestax-mcp + // (passive attribution). Strip it here so the tagged and bare forms + // of one page count once, keeping docs_fetches comparable across + // guidance channels instead of double-counting the MCP one. + docsUrls.add( + m[0] + .replace(/([?&])utm_source=bestax-mcp&?/, '$1') + .replace(/[?&]$/, '') + ); } } } From da27d67217c1f54977f947957b4645821e1a9895 Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sun, 23 Aug 2026 10:22:01 -0400 Subject: [PATCH 18/29] chore: add telemetry-worker to the pnpm workspace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Its devDependencies now sit under the lockfile, audit gate, and 3-day cooldown like every other package. Wrangler bumps from the 4.33.1 pin (inside the GHSA range the audit flagged, along with two ws advisories via miniflare) to 4.124.0 — the audit is back to main's baseline with no telemetry-worker paths. workerd's postinstall stays blocked; deploy never needs it and local dev can use --remote. --- CLAUDE.md | 2 +- pnpm-lock.yaml | 122 ++++++++++++++++++++++++++++++++++ pnpm-workspace.yaml | 1 + telemetry-worker/README.md | 12 ++-- telemetry-worker/package.json | 2 +- 5 files changed, 133 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 8ce142e00..67d5e4c65 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,7 +10,7 @@ React component library for **Bulma v1** in TypeScript. pnpm monorepo orchestrat (has its own CLAUDE.md) - `skills/` — Agent Skills, a **shipped product** bundled into create-bestax (has its own CLAUDE.md) - `telemetry-worker/` — Cloudflare Worker ingesting the CLIs' opt-in telemetry - (deployed manually with wrangler; not yet in the pnpm workspace) + (deployed manually with wrangler) - `eval/agent-loop/` — cold-start eval harness: does an unassisted agent build correctly when given one of our guidance channels (the skills, the MCP server, both)? Frozen rubric, its own README diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5311363c4..930fe566b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -442,6 +442,15 @@ importers: specifier: ^4.0.2 version: 4.0.2(webpack@5.108.1(postcss@8.5.26)) + telemetry-worker: + devDependencies: + typescript: + specifier: 5.9.2 + version: 5.9.2 + wrangler: + specifier: 4.124.0 + version: 4.124.0 + packages: '@11ty/gray-matter@1.0.0': @@ -1244,30 +1253,60 @@ packages: cpu: [x64] os: [darwin] + '@cloudflare/workerd-darwin-64@1.20260815.1': + resolution: {integrity: sha512-7PsLdcz6pT9EMd1EJGZEgMyYRfs0CHxGs62PS2L1w3s6+xGmQcRXKm/zoMftmqZF45JBa4MzFeownRKbRt/x5g==} + engines: {node: '>=16'} + cpu: [x64] + os: [darwin] + '@cloudflare/workerd-darwin-arm64@1.20260722.1': resolution: {integrity: sha512-EmIQymihDq6WNdER4+LF8Qn80yqayBUpJ+tkOO7wmY8pmgfyXjIUFNXotl21AHovTeu2seR7HdVUgeN/BilCWw==} engines: {node: '>=16'} cpu: [arm64] os: [darwin] + '@cloudflare/workerd-darwin-arm64@1.20260815.1': + resolution: {integrity: sha512-60wtg8ng7FVWeOg/UMbZ9Ye0sslpRRAKoftPbdtuH2volq676quxVr6Zm2EjVULH/JFZeCn72dbLlrnbh0Mpcw==} + engines: {node: '>=16'} + cpu: [arm64] + os: [darwin] + '@cloudflare/workerd-linux-64@1.20260722.1': resolution: {integrity: sha512-jvZ3k9fxcnEn04s80CgIYxQfpOyAiz/8qC42DP8EBa9tR27qWyg9wmm31zIobVlrgBZn/+8NfdP73avRGcQOjQ==} engines: {node: '>=16'} cpu: [x64] os: [linux] + '@cloudflare/workerd-linux-64@1.20260815.1': + resolution: {integrity: sha512-MuqKIHPo0Qyo8MZMmy0lP2B5PeAL7f4T9Fu4Usk3QdbV4JIrKG/OoybN3Ign7m/Dff+L1Oo/ZHydB+hEg1ueFw==} + engines: {node: '>=16'} + cpu: [x64] + os: [linux] + '@cloudflare/workerd-linux-arm64@1.20260722.1': resolution: {integrity: sha512-BOSB55SMNdy+DA5uj2WirgiNanpHGis5PVvXH1wSfvjRKr4JGgWK+EZzxz0RFUo6QjjQQC/NimEzNZ7va7jmKg==} engines: {node: '>=16'} cpu: [arm64] os: [linux] + '@cloudflare/workerd-linux-arm64@1.20260815.1': + resolution: {integrity: sha512-XNFtJ5rIqJxnY6ISjkfbhT/ODiWJ6LcBvNbntuPD6I/F2k7aZeKgPaXrvWvKde66LXyzFKzc8Hn+Ydx4shevQg==} + engines: {node: '>=16'} + cpu: [arm64] + os: [linux] + '@cloudflare/workerd-windows-64@1.20260722.1': resolution: {integrity: sha512-sYM8YgUpKnRz2xjvdJLX1Ojzoi4MlA4gk8WTTExhGydjYB2UTs5NIbv0ZmpKgMoK9io3ixgmiW56ZnTbcWOdiA==} engines: {node: '>=16'} cpu: [x64] os: [win32] + '@cloudflare/workerd-windows-64@1.20260815.1': + resolution: {integrity: sha512-PiIUWrhbMg3quolwjgMvPOd75vKESjT4aDm7nL6mSjL5IOgmpO/zKstXnYfnEH3pq7sC0UCvKlF8ZPcfsh8NMw==} + engines: {node: '>=16'} + cpu: [x64] + os: [win32] + '@colors/colors@1.5.0': resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==} engines: {node: '>=0.1.90'} @@ -7630,6 +7669,10 @@ packages: engines: {node: '>=22.0.0'} hasBin: true + miniflare@5.20260815.0-alpha: + resolution: {integrity: sha512-YAaGj4Sh5f4fqHKiMQ8zRHDOOM5IGUVtMhnLIeyjuQfU+9P6hcOTrHUVtbfj/ZPay9Kzik4pWELB39pGgefjiQ==} + engines: {node: '>=22.0.0'} + minimalistic-assert@1.0.1: resolution: {integrity: sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==} @@ -9846,6 +9889,11 @@ packages: typedarray-to-buffer@3.1.5: resolution: {integrity: sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==} + typescript@5.9.2: + resolution: {integrity: sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==} + engines: {node: '>=14.17'} + hasBin: true + typescript@6.0.3: resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} engines: {node: '>=14.17'} @@ -10227,6 +10275,11 @@ packages: engines: {node: '>=16'} hasBin: true + workerd@1.20260815.1: + resolution: {integrity: sha512-8bArFkHmlp7qFEKVPyNzDzHzS35gc2fg0PYBcDtaNLF7UCDryCX2BQnpkUkTHYIy824IRrHOTwOEoTj0sUO2Fg==} + engines: {node: '>=16'} + hasBin: true + wrangler@4.115.0: resolution: {integrity: sha512-+upG2VW66M1sjb43yzgUZ6Ss8iYpJ6+7F3U4GF8TY5EYd+08sYdS54d24AEwCnhuef3J9KlSPusqiqR9WYy1UA==} engines: {node: '>=22.0.0'} @@ -10237,6 +10290,16 @@ packages: '@cloudflare/workers-types': optional: true + wrangler@4.124.0: + resolution: {integrity: sha512-75euoZKjVTJYFy+Xhctt/5JlZL4M6A4xmovZsUlep+6GHcCm14n9VtdGzNIybOW2t8wuNxRj5iMUwjT5E7Ctog==} + engines: {node: '>=22.0.0'} + hasBin: true + peerDependencies: + '@cloudflare/workers-types': ^5.20260815.1 + peerDependenciesMeta: + '@cloudflare/workers-types': + optional: true + wrap-ansi@6.2.0: resolution: {integrity: sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==} engines: {node: '>=8'} @@ -11441,21 +11504,42 @@ snapshots: optionalDependencies: workerd: 1.20260722.1 + '@cloudflare/unenv-preset@2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260815.1)': + dependencies: + unenv: 2.0.0-rc.24 + optionalDependencies: + workerd: 1.20260815.1 + '@cloudflare/workerd-darwin-64@1.20260722.1': optional: true + '@cloudflare/workerd-darwin-64@1.20260815.1': + optional: true + '@cloudflare/workerd-darwin-arm64@1.20260722.1': optional: true + '@cloudflare/workerd-darwin-arm64@1.20260815.1': + optional: true + '@cloudflare/workerd-linux-64@1.20260722.1': optional: true + '@cloudflare/workerd-linux-64@1.20260815.1': + optional: true + '@cloudflare/workerd-linux-arm64@1.20260722.1': optional: true + '@cloudflare/workerd-linux-arm64@1.20260815.1': + optional: true + '@cloudflare/workerd-windows-64@1.20260722.1': optional: true + '@cloudflare/workerd-windows-64@1.20260815.1': + optional: true + '@colors/colors@1.5.0': optional: true @@ -19577,6 +19661,18 @@ snapshots: - bufferutil - utf-8-validate + miniflare@5.20260815.0-alpha: + dependencies: + '@cspotcode/source-map-support': 0.8.1 + sharp: 0.35.2 + undici: 7.29.0 + workerd: 1.20260815.1 + ws: 8.21.0 + youch: 4.1.0-beta.10 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + minimalistic-assert@1.0.1: {} minimatch@10.2.5: @@ -21989,6 +22085,8 @@ snapshots: dependencies: is-typedarray: 1.0.0 + typescript@5.9.2: {} + typescript@6.0.3: {} uglify-js@3.19.3: @@ -22517,6 +22615,14 @@ snapshots: '@cloudflare/workerd-linux-arm64': 1.20260722.1 '@cloudflare/workerd-windows-64': 1.20260722.1 + workerd@1.20260815.1: + optionalDependencies: + '@cloudflare/workerd-darwin-64': 1.20260815.1 + '@cloudflare/workerd-darwin-arm64': 1.20260815.1 + '@cloudflare/workerd-linux-64': 1.20260815.1 + '@cloudflare/workerd-linux-arm64': 1.20260815.1 + '@cloudflare/workerd-windows-64': 1.20260815.1 + wrangler@4.115.0: dependencies: '@cloudflare/kv-asset-handler': 0.5.0 @@ -22533,6 +22639,22 @@ snapshots: - bufferutil - utf-8-validate + wrangler@4.124.0: + dependencies: + '@cloudflare/kv-asset-handler': 0.5.0 + '@cloudflare/unenv-preset': 2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260815.1) + blake3-wasm: 2.1.5 + esbuild: 0.28.1 + miniflare: 5.20260815.0-alpha + path-to-regexp: 6.3.0 + unenv: 2.0.0-rc.24 + workerd: 1.20260815.1 + optionalDependencies: + fsevents: 2.3.3 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + wrap-ansi@6.2.0: dependencies: ansi-styles: 4.3.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 839d95a5a..dfe21bf4c 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -4,6 +4,7 @@ packages: - create-bestax - bestax-migrate - bestax-mcp + - telemetry-worker # --- Supply-chain hardening --- diff --git a/telemetry-worker/README.md b/telemetry-worker/README.md index 0440bd46e..1e2a18390 100644 --- a/telemetry-worker/README.md +++ b/telemetry-worker/README.md @@ -12,10 +12,14 @@ missing binding in local dev still answers `204`. npx wrangler deploy ``` -Run by Alex — the package is not in the pnpm workspace (Wrangler's native -`workerd` build is blocked repo-wide), so the exact-pinned devDependencies in -`package.json` are the only version control these tools get until it joins. -Wrangler bundles `src/index.ts` directly (no build step). Tests and typecheck still run from the repo root: +Run by Alex — the deploy is manual (nothing in `.github/workflows` touches +this worker). The package IS in the pnpm workspace, so the lockfile, audit +gate, and 3-day cooldown govern its devDependencies; they stay exact-pinned +anyway so a deploy is reproducible from the manifest alone. Wrangler's native +`workerd` postinstall is blocked repo-wide (`allowBuilds`), which `wrangler +deploy` never needs — for local `wrangler dev`, use `--remote` or flip the +`workerd` entry. Wrangler bundles `src/index.ts` directly (no build step). +Tests and typecheck still run from the repo root: `pnpm test` includes `node --test telemetry-worker/src/__tests__/*.test.ts`, and `pnpm typecheck` includes `tsc -p telemetry-worker`. The worker is bound to the zone route `bestax.io/api/t`. If that route ever conflicts with diff --git a/telemetry-worker/package.json b/telemetry-worker/package.json index e0c82cdb0..d92198e57 100644 --- a/telemetry-worker/package.json +++ b/telemetry-worker/package.json @@ -11,6 +11,6 @@ }, "devDependencies": { "typescript": "5.9.2", - "wrangler": "4.33.1" + "wrangler": "4.124.0" } } From 1ecb5ee09dcc8ef751115fb979bd20e57221706d Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sun, 23 Aug 2026 14:29:44 -0400 Subject: [PATCH 19/29] ci: deploy the telemetry worker from main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors the Pages deploy: GitHub builds (wrangler bundles src/index.ts), Cloudflare receives the artifact. Triggers on main-branch changes under telemetry-worker/** plus manual dispatch. Credential is the dedicated CLOUDFLARE_WORKERS_DEPLOY_TOKEN, scoped to Workers Scripts:Edit and Workers Routes:Edit on bestax.io only — separate from the Pages token so each pipeline holds only what it deploys. Read-only GITHUB_TOKEN; action SHAs on the repo-wide pins. --- .github/workflows/deploy-worker.yml | 41 +++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/deploy-worker.yml diff --git a/.github/workflows/deploy-worker.yml b/.github/workflows/deploy-worker.yml new file mode 100644 index 000000000..3df790be7 --- /dev/null +++ b/.github/workflows/deploy-worker.yml @@ -0,0 +1,41 @@ +# Deploys telemetry-worker to Cloudflare Workers on every main-branch change +# under telemetry-worker/**, mirroring the Pages deploy in deploy.yml: build +# happens here (wrangler bundles src/index.ts), Cloudflare only receives the +# artifact. workflow_dispatch allows a manual redeploy from the Actions tab. +# +# The only credential in this job is CLOUDFLARE_WORKERS_DEPLOY_TOKEN, an API +# token scoped to Workers Scripts:Edit + Workers Routes:Edit on bestax.io — +# deliberately separate from CLOUDFLARE_API_TOKEN (Pages) so each pipeline +# holds only what it deploys. +name: Deploy telemetry worker + +on: + push: + branches: + - main + paths: + - 'telemetry-worker/**' + workflow_dispatch: + +permissions: + contents: read + +jobs: + deploy: + name: Deploy to Cloudflare Workers + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + # No pnpm/Node setup: the worker is stdlib-only TypeScript with no build + # step of its own — wrangler-action installs the pinned wrangler and its + # bundler does the rest. + - name: Deploy with wrangler + uses: cloudflare/wrangler-action@ebbaa1584979971c8614a24965b4405ff95890e0 # v4 + with: + apiToken: ${{ secrets.CLOUDFLARE_WORKERS_DEPLOY_TOKEN }} + accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + workingDirectory: telemetry-worker + wranglerVersion: '4.124.0' + command: deploy From a6db10299c77368622988c1cfc1687b433f5d602 Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sun, 23 Aug 2026 15:36:05 -0400 Subject: [PATCH 20/29] fix(create-bestax): never send telemetry from the e2e scaffold harness The exec inherited the developer's real environment, so on an opted-in machine every e2e scaffold would POST a synthetic event to production (-y only skips the consent prompt, not the send). BESTAX_TELEMETRY=0 pins it off per-run regardless of the machine's consent config. --- create-bestax/e2e/utils/scaffold.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/create-bestax/e2e/utils/scaffold.ts b/create-bestax/e2e/utils/scaffold.ts index f30e10c5d..d2fd69260 100644 --- a/create-bestax/e2e/utils/scaffold.ts +++ b/create-bestax/e2e/utils/scaffold.ts @@ -62,6 +62,11 @@ export async function scaffoldApp(config: ScaffoldConfig): Promise { execSync(scaffoldCmd, { cwd: path.dirname(outputDir), stdio: 'inherit', + // Never send telemetry from tests: -y only skips the consent prompt, and + // this exec inherits the developer's real environment — on an opted-in + // machine every e2e scaffold would POST a synthetic event to production. + // BESTAX_TELEMETRY=0 is the per-run kill switch and beats the config. + env: { ...process.env, BESTAX_TELEMETRY: '0' }, }); // -y implies the AI-skills opt-in, so every real scaffold must contain the From 3a430bc9cf856ea791d44dd646b46b0e49203e62 Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sun, 23 Aug 2026 15:47:06 -0400 Subject: [PATCH 21/29] ci: make the skills vetting gate repo-aware and file-granular MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two #550 review findings: an exported (git-less) tree nested inside any outer repository resolved that repo's index and reported every skill untracked, hard-failing the build — the gate now verifies git's toplevel actually contains the skills directory and stands down otherwise. And the gate only checked SKILL.md while sync copies whole directories, so an untracked scratch file inside a tracked skill still shipped — it now lists untracked files via --others --exclude-standard (gitignored noise stays exempt). Fixture-repo tests cover both scenarios. --- bestax-mcp/scripts/sync-skills.mjs | 6 +- create-bestax/scripts/sync-skills.mjs | 6 +- scripts/lib/skills.mjs | 62 +++++++++++++------ scripts/skills-roster.test.mjs | 89 ++++++++++++++++++++++++--- 4 files changed, 128 insertions(+), 35 deletions(-) diff --git a/bestax-mcp/scripts/sync-skills.mjs b/bestax-mcp/scripts/sync-skills.mjs index 60a288bbc..175116e11 100644 --- a/bestax-mcp/scripts/sync-skills.mjs +++ b/bestax-mcp/scripts/sync-skills.mjs @@ -45,7 +45,7 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { readSkillNames, - untrackedSkillDirs, + untrackedSkillPaths, } from '../../scripts/lib/skills.mjs'; const here = path.dirname(fileURLToPath(import.meta.url)); @@ -83,10 +83,10 @@ if (!names.length) { // Same vetting gate as create-bestax's sync: data/skills ships in the npm // tarball, so an untracked scratch skill must not reach a manual publish. -const untrackedHere = untrackedSkillDirs(skillsSrc, names); +const untrackedHere = untrackedSkillPaths(skillsSrc, names); if (untrackedHere.length) { console.error( - `[sync-skills] refusing to bundle untracked skill dir(s): ` + + `[sync-skills] refusing to bundle untracked file(s) under skills/: ` + `${untrackedHere.join(', ')}. \`git add\` them to vet them, or remove them.` ); process.exit(1); diff --git a/create-bestax/scripts/sync-skills.mjs b/create-bestax/scripts/sync-skills.mjs index 0a8b8b2e0..450599822 100644 --- a/create-bestax/scripts/sync-skills.mjs +++ b/create-bestax/scripts/sync-skills.mjs @@ -37,7 +37,7 @@ import path from 'node:path'; import fs from 'fs-extra'; import { readSkillNames, - untrackedSkillDirs, + untrackedSkillPaths, } from '../../scripts/lib/skills.mjs'; const here = path.dirname(fileURLToPath(import.meta.url)); @@ -72,10 +72,10 @@ if (!skills.length) { // state — so an untracked scratch directory would ship in a local build or a // manual publish with no gate anywhere in the path. `git add` is the act of // vetting; a tree without git (an exported tarball) skips the gate. -const untracked = untrackedSkillDirs(skillsSrc, skills); +const untracked = untrackedSkillPaths(skillsSrc, skills); if (untracked.length) { console.error( - `[sync-skills] refusing to bundle untracked skill dir(s): ` + + `[sync-skills] refusing to bundle untracked file(s) under skills/: ` + `${untracked.join(', ')}. \`git add\` them to vet them, or remove them.` ); process.exit(1); diff --git a/scripts/lib/skills.mjs b/scripts/lib/skills.mjs index 89bc9ebc1..bff99d3c8 100644 --- a/scripts/lib/skills.mjs +++ b/scripts/lib/skills.mjs @@ -7,7 +7,8 @@ // here now; a predicate change lands once or not at all. import { readdir, access } from 'node:fs/promises'; -import { join } from 'node:path'; +import { realpathSync } from 'node:fs'; +import { join, dirname, resolve } from 'node:path'; import { execFileSync } from 'node:child_process'; /** @@ -80,34 +81,55 @@ export function rosterSkillNames(dirs) { } /** - * The names in `names` whose SKILL.md is not in `trackedPaths` (paths relative - * to the skills dir, as `git ls-files` prints them). Pure so it can be tested - * without a fixture repository. + * The paths in `candidatePaths` (relative to the skills dir, as git prints + * them) that sit inside one of the bundled skill directories in `names`. Pure + * so it can be tested without a fixture repository. */ -export function untrackedAgainst(trackedPaths, names) { - const tracked = new Set(trackedPaths); - return names.filter(name => !tracked.has(`${name}/SKILL.md`)); +export function pathsInsideSkills(candidatePaths, names) { + const bundled = new Set(names); + return candidatePaths.filter(p => p && bundled.has(p.split('/')[0])); +} + +function git(cwd, args) { + return execFileSync('git', ['-C', cwd, ...args], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }); } /** * The vetting gate the deleted allowlist used to be (#541 review): discovery - * bundles whatever is on disk, so an untracked scratch directory with a - * SKILL.md would ship in a local build or a manual publish with no gate in the - * path — CI's skills-roster check only ever sees committed state. Bundlers - * call this and refuse untracked skills; `git add` is the act of vetting. + * bundles whatever is on disk, so untracked content would ship in a local + * build or a manual publish with no gate in the path — CI's skills-roster + * check only ever sees committed state. Bundlers call this and refuse + * untracked FILES, not just directories: sync copies whole skill directories + * off disk, so a scratch note dropped into a tracked skill ships exactly like + * a scratch skill would. `git add` is the act of vetting; `.gitignore`d noise + * (`.DS_Store`) stays exempt via --exclude-standard. * - * Returns [] when git is unavailable or the tree is not a repository (a - * consumer building from an exported tarball has nothing to vet against). + * Returns [] when there is nothing to vet against: + * - git is unavailable, or the tree is not inside any repository, or + * - git resolves some OTHER repository — an exported (git-less) bestax tree + * nested under a git-managed directory would otherwise report every skill + * untracked and hard-fail the build (#550 review). The gate only speaks for + * the repository whose root actually contains this skills directory. */ -export function untrackedSkillDirs(skillsDir, names) { - let output; +export function untrackedSkillPaths(skillsDir, names) { try { - output = execFileSync('git', ['-C', skillsDir, 'ls-files', '--', '.'], { - encoding: 'utf8', - stdio: ['ignore', 'pipe', 'ignore'], - }); + const toplevel = git(skillsDir, ['rev-parse', '--show-toplevel']).trim(); + // realpath both sides: git prints physical paths (macOS /tmp → /private/tmp). + if (realpathSync(toplevel) !== realpathSync(resolve(dirname(skillsDir)))) { + return []; + } + const others = git(skillsDir, [ + 'ls-files', + '--others', + '--exclude-standard', + '--', + '.', + ]); + return pathsInsideSkills(others.split('\n'), names); } catch { return []; } - return untrackedAgainst(output.split('\n'), names); } diff --git a/scripts/skills-roster.test.mjs b/scripts/skills-roster.test.mjs index 048191091..ebf56aac9 100644 --- a/scripts/skills-roster.test.mjs +++ b/scripts/skills-roster.test.mjs @@ -26,7 +26,7 @@ import { skillsPageViolations, frontmatterNameViolations, } from './check-conformance.mjs'; -import { untrackedAgainst } from './lib/skills.mjs'; +import { pathsInsideSkills, untrackedSkillPaths } from './lib/skills.mjs'; const repoFile = rel => readFileSync(fileURLToPath(new URL(`../${rel}`, import.meta.url)), 'utf8'); @@ -552,15 +552,86 @@ test('the tests and the check derive the comparison set the same way', () => { assert.deepEqual(rosterSkillNames(dirs), ['bestax-form']); }); -test('untracked skill directories are flagged against the tracked set', () => { - const tracked = [ - 'bestax-form/SKILL.md', - 'bestax-form/references/x.md', - 'bestax-icons/SKILL.md', +test('only paths inside bundled skill directories count against the gate', () => { + const others = [ + 'bestax-form/notes.md', + 'bestax-wip/SKILL.md', + 'scratch.txt', + '', ]; + assert.deepEqual(pathsInsideSkills(others, ['bestax-form', 'bestax-wip']), [ + 'bestax-form/notes.md', + 'bestax-wip/SKILL.md', + ]); + assert.deepEqual(pathsInsideSkills(others, ['bestax-icons']), []); +}); + +test('the vetting gate flags untracked files only in its OWN repository', async t => { + const { mkdtempSync, mkdirSync, writeFileSync, rmSync } = + await import('node:fs'); + const { execFileSync } = await import('node:child_process'); + const os = await import('node:os'); + const path = await import('node:path'); + + const root = mkdtempSync(path.join(os.tmpdir(), 'bestax-gate-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const git = (cwd, ...args) => + execFileSync('git', ['-C', cwd, ...args], { stdio: 'ignore' }); + + // Scenario 1: skills/ inside its own repository — a tracked skill with an + // untracked scratch file inside it is flagged FILE-granularly (sync copies + // whole directories, so the scratch file would ship), and a fully-tracked + // skill passes. + const own = path.join(root, 'own'); + mkdirSync(path.join(own, 'skills', 'bestax-form'), { recursive: true }); + writeFileSync(path.join(own, 'skills', 'bestax-form', 'SKILL.md'), '# s\n'); + git(own, 'init', '-q'); + git(own, 'add', '-A'); + const skillsDir = path.join(own, 'skills'); + assert.deepEqual(untrackedSkillPaths(skillsDir, ['bestax-form']), []); + writeFileSync(path.join(skillsDir, 'bestax-form', 'notes.md'), 'draft\n'); + assert.deepEqual(untrackedSkillPaths(skillsDir, ['bestax-form']), [ + 'bestax-form/notes.md', + ]); + // A file outside the bundled skill dirs never blocks the bundle. + writeFileSync(path.join(skillsDir, 'scratch.txt'), 'x\n'); + assert.deepEqual(untrackedSkillPaths(skillsDir, ['bestax-form']), [ + 'bestax-form/notes.md', + ]); + + // Scenario 2: an exported (git-less) tree nested inside some OUTER + // repository. git resolves the outer repo, which knows none of these files — + // the gate must return [] instead of refusing every skill (#550 review + // reproduced exactly this against a git-managed $HOME). + const outer = path.join(root, 'outer'); + mkdirSync(path.join(outer, 'exported', 'skills', 'bestax-form'), { + recursive: true, + }); + writeFileSync( + path.join(outer, 'exported', 'skills', 'bestax-form', 'SKILL.md'), + '# s\n' + ); + git(outer, 'init', '-q'); assert.deepEqual( - untrackedAgainst(tracked, ['bestax-form', 'bestax-icons', 'bestax-wip']), - ['bestax-wip'] + untrackedSkillPaths(path.join(outer, 'exported', 'skills'), [ + 'bestax-form', + ]), + [] + ); +}); + +test('an adjacent fence with no blank line cannot merge into the scope', () => { + // fenceMask marks delimiters and interiors alike, so back-to-back fences + // form one continuous masked run; the close-scan must stop at the block's + // own closer, not the run's end, or the second block's tokens join the + // roster comparison. + const merged = REAL['docs/docs/skills/intro.md'].replace( + /^```$/m, + '```\n```bash\nnpx skills add https://github.com/allxsmith/bestax --skill bestax-ghost\n```' + ); + assert.ok(merged.includes('bestax-ghost'), 'fixture must inject the decoy'); + assert.deepEqual( + rosterViolations(SKILLS, withFile('docs/docs/skills/intro.md', merged)), + [] ); - assert.deepEqual(untrackedAgainst(tracked, ['bestax-form']), []); }); From d2e568bb53555a285eb370c3c11af4f5fb562cbf Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sun, 23 Aug 2026 15:47:07 -0400 Subject: [PATCH 22/29] ci: import telemetry producer enums instead of scraping, close scope gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The allowlist check regex-scraped CLI source; a reshaped declaration silently narrowed the comparison and a source directory that stopped matching dropped out entirely — exactly the silent 400-at-ingest drop the check exists to catch. constants.ts and package-manager.ts are leaf modules, so their enums are now imported (loudly null on failure); unparseable source dirs are violations; cli.ts stays scraped with the existing loud-null guard to keep jscodeshift out of conformance runs. markerFence now matches a block's own closing delimiter instead of the fenceMask run, so an adjacent fence with no blank line between them can no longer merge into the scope. --- create-bestax/src/package-manager.ts | 5 +- scripts/check-conformance.mjs | 107 +++++++++++++++++++-------- 2 files changed, 79 insertions(+), 33 deletions(-) diff --git a/create-bestax/src/package-manager.ts b/create-bestax/src/package-manager.ts index d26df4169..df97054b8 100644 --- a/create-bestax/src/package-manager.ts +++ b/create-bestax/src/package-manager.ts @@ -1,6 +1,9 @@ export type PackageManager = 'npm' | 'pnpm' | 'yarn' | 'bun'; -const KNOWN = ['npm', 'pnpm', 'yarn', 'bun'] as const; +// Exported so check:conformance can compare it against the telemetry +// worker's allowlist by importing it, rather than regex-scraping the source. +export const KNOWN_PACKAGE_MANAGERS = ['npm', 'pnpm', 'yarn', 'bun'] as const; +const KNOWN = KNOWN_PACKAGE_MANAGERS; /** Detect the package manager that invoked the CLI from npm_config_user_agent * (e.g. "pnpm/9.12.0 npm/? node/v22 darwin arm64"). Falls back to npm. */ diff --git a/scripts/check-conformance.mjs b/scripts/check-conformance.mjs index 480f1ff89..8270c2c47 100644 --- a/scripts/check-conformance.mjs +++ b/scripts/check-conformance.mjs @@ -2069,18 +2069,24 @@ function markerFence(marker) { } if (open === -1) return null; - let close = open + 1; - while (close < lines.length && mask[close]) close++; - // mask marks both delimiters; the last masked line is the closing fence - // unless the fence runs unterminated to EOF. - const body = lines.slice(open + 1, close); - if ( - body.length && - /^ {0,3}(`{3,}|~{3,})[ \t]*$/.test(body[body.length - 1]) - ) { - body.pop(); + // The closer is matched against the OPENER (same char, at least as long, + // no info string — CommonMark), not against the fenceMask run: mask marks + // delimiters and interiors alike, so two fences with no blank line between + // them form one continuous run and the first version merged them into a + // single scope, letting tokens from an unrelated adjacent block satisfy or + // pollute the roster (#550 review). + const opener = lines[open].match(/^ {0,3}(`{3,}|~{3,})/); + const char = opener[1][0]; + const len = opener[1].length; + let close = lines.length; + for (let i = open + 1; i < lines.length; i++) { + const m = lines[i].match(/^ {0,3}(`{3,}|~{3,})[ \t]*$/); + if (m && m[1][0] === char && m[1].length >= len) { + close = i; + break; + } } - return body.join('\n'); + return lines.slice(open + 1, close).join('\n'); }; } @@ -2437,33 +2443,56 @@ function constStringArray(src, name) { return m ? quotedStringsIn(m[1]) : null; } -function exportedNamedArray(src, name) { - const m = src.match( - new RegExp(`export const ${name}[^=]*= \\[([\\s\\S]*?)\\];`) - ); - if (!m) return null; - return [...m[1].matchAll(/name: '([^']+)'/g)].map(x => x[1]); +/** + * Import a CLI module and pick the producer values out of it. Importing beats + * regex-scraping wherever the module is a leaf (#550 review: a scrape that + * stops matching a reshaped declaration silently narrows the comparison — an + * import either yields the real array or fails loudly as null here). + * constants.ts and package-manager.ts import nothing but chalk, so pulling + * them into a conformance run is cheap; node's type stripping loads the .ts + * directly. + */ +async function importProducer(relPath, pick) { + try { + const mod = await import(pathToFileURL(join(REPO, relPath)).href); + return pick(mod); + } catch { + return null; + } } +// bestax-migrate/src/cli.ts stays regex-scraped: importing it would drag the +// whole transform chain (jscodeshift included) into every conformance run. +// The scrape fails loudly as null when the declaration stops matching. function cssModes(src) { const m = src.match(/const CSS_MODES: CssMode\[\] = \[([\s\S]*?)\];/); return m ? quotedStringsIn(m[1]) : null; } +/** + * Every source's registry name, plus the directories whose index.ts exists + * but did not yield one — a non-matching declaration must be REPORTED, not + * skipped (#550 review): a silently-dropped source would leave its enum + * unchecked against the worker, and its production events would 400 at + * ingest with every gate green. + */ async function migrateSourceNames() { const dir = join(REPO, 'bestax-migrate/src/sources'); const names = []; + const unparsed = []; for (const entry of await readdir(dir, { withFileTypes: true })) { if (!entry.isDirectory()) continue; + let src; try { - const src = await readFile(join(dir, entry.name, 'index.ts'), 'utf8'); - const m = src.match(/: MigrationSource = \{[\s\S]*?name: '([^']+)'/); - if (m) names.push(m[1]); + src = await readFile(join(dir, entry.name, 'index.ts'), 'utf8'); } catch { - // Directory without an index.ts is not a source. + continue; // a directory without an index.ts is not a source } + const m = src.match(/: MigrationSource = \{[\s\S]*?name: '([^']+)'/); + if (m) names.push(m[1]); + else unparsed.push(entry.name); } - return names; + return { names, unparsed }; } function missingFromWorker(label, producer, worker, workerFile) { @@ -2487,50 +2516,64 @@ function missingFromWorker(label, producer, worker, workerFile) { async function checkTelemetryAllowlists() { const workerFile = 'telemetry-worker/src/schema.ts'; const constantsFile = 'create-bestax/src/constants.ts'; - const pmFile = 'create-bestax/src/package-manager.ts'; const cliFile = 'bestax-migrate/src/cli.ts'; - const [schema, constants, pm, cli] = await Promise.all([ + const [schema, cli] = await Promise.all([ readFile(join(REPO, workerFile), 'utf8'), - readFile(join(REPO, constantsFile), 'utf8'), - readFile(join(REPO, pmFile), 'utf8'), readFile(join(REPO, cliFile), 'utf8'), ]); + const [templates, flavors, icons, pms] = await Promise.all([ + importProducer(constantsFile, m => m.TEMPLATES.map(t => t.name)), + importProducer(constantsFile, m => m.BULMA_FLAVORS.map(f => f.name)), + importProducer(constantsFile, m => m.ICON_LIBRARIES.map(i => i.name)), + importProducer('create-bestax/src/package-manager.ts', m => [ + ...m.KNOWN_PACKAGE_MANAGERS, + ]), + ]); const sources = await migrateSourceNames(); const violations = []; - if (!sources.length) { + if (!sources.names.length) { violations.push( 'bestax-migrate/src/sources/*/index.ts: no MigrationSource `name` was ' + 'found, so the worker allowlist went unchecked.' ); } + for (const dir of sources.unparsed) { + violations.push( + `bestax-migrate/src/sources/${dir}/index.ts: has an index.ts but no ` + + `parseable \`: MigrationSource = { name: '…' }\` declaration, so its ` + + `enum would silently skip the worker comparison and its production ` + + `events would be dropped at ingest. Match the shape, or update ` + + `migrateSourceNames in scripts/check-conformance.mjs.` + ); + } violations.push( ...missingFromWorker( 'template', - exportedNamedArray(constants, 'TEMPLATES'), + templates, constStringArray(schema, 'TEMPLATE_VALUES'), workerFile ), ...missingFromWorker( 'bulmaFlavor', - exportedNamedArray(constants, 'BULMA_FLAVORS'), + flavors, constStringArray(schema, 'BULMA_FLAVOR_VALUES'), workerFile ), ...missingFromWorker( 'iconLibrary', - exportedNamedArray(constants, 'ICON_LIBRARIES'), + icons, constStringArray(schema, 'ICON_LIBRARY_VALUES'), workerFile ), ...missingFromWorker( 'packageManager', - constStringArray(pm, 'KNOWN'), + pms, constStringArray(schema, 'PACKAGE_MANAGER_VALUES'), workerFile ), ...missingFromWorker( 'migrate source', - sources, + sources.names, constStringArray(schema, 'MIGRATE_SOURCE_VALUES'), workerFile ), From 7bf859f075e94ed75ee4a146432e6f6424ae6cc7 Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sun, 23 Aug 2026 15:47:08 -0400 Subject: [PATCH 23/29] ci: harden-runner on the worker deploy job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rule 10: new jobs ship with egress-policy: block — this is the job holding CLOUDFLARE_WORKERS_DEPLOY_TOKEN. Allowed endpoints are checkout, the npm registry for the wrangler install, and api.cloudflare.com for the upload; wrangler's metrics beacon is disabled instead of allow-listing its host. --- .github/workflows/deploy-worker.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/deploy-worker.yml b/.github/workflows/deploy-worker.yml index 3df790be7..c5aeb7565 100644 --- a/.github/workflows/deploy-worker.yml +++ b/.github/workflows/deploy-worker.yml @@ -25,6 +25,22 @@ jobs: name: Deploy to Cloudflare Workers runs-on: ubuntu-latest steps: + # Rule 10 (.github/CLAUDE.md): new jobs ship with egress-policy: block — + # this is the job holding the Cloudflare deploy token. Endpoints: + # checkout (github.com trio), wrangler install (npm registry), and the + # upload itself (api.cloudflare.com). Wrangler's own metrics beacon is + # disabled below instead of allow-listing its host. + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: block + allowed-endpoints: > + api.cloudflare.com:443 + api.github.com:443 + github.com:443 + objects.githubusercontent.com:443 + registry.npmjs.org:443 + - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -33,6 +49,8 @@ jobs: # bundler does the rest. - name: Deploy with wrangler uses: cloudflare/wrangler-action@ebbaa1584979971c8614a24965b4405ff95890e0 # v4 + env: + WRANGLER_SEND_METRICS: 'false' with: apiToken: ${{ secrets.CLOUDFLARE_WORKERS_DEPLOY_TOKEN }} accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} From 71b18ba6c90351a22f4f60d8c8297e9a15285957 Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sun, 23 Aug 2026 15:47:08 -0400 Subject: [PATCH 24/29] chore: guard the ts resolver hook and the eval utm strip The resolver hook now falls through for non-file: parents instead of throwing on fileURLToPath, and the eval metrics utm strip no longer leaves a dangling '?' when a fragment follows the stripped parameter. --- eval/agent-loop/bin/collect-metrics.mjs | 10 +++++++--- scripts/lib/resolve-ts-from-js.mjs | 5 ++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/eval/agent-loop/bin/collect-metrics.mjs b/eval/agent-loop/bin/collect-metrics.mjs index 18308a73d..094507714 100644 --- a/eval/agent-loop/bin/collect-metrics.mjs +++ b/eval/agent-loop/bin/collect-metrics.mjs @@ -257,11 +257,15 @@ if (transcriptPath && existsSync(transcriptPath)) { // The MCP server tags the links it prints with utm_source=bestax-mcp // (passive attribution). Strip it here so the tagged and bare forms // of one page count once, keeping docs_fetches comparable across - // guidance channels instead of double-counting the MCP one. + // guidance channels instead of double-counting the MCP one. Two + // cases, because attributed() supports fragments: mid-query the + // param yields its delimiter to the next param; as the last param + // its own leading ?/& goes too, whether the URL ends there or a + // #fragment follows (the old end-anchored cleanup left '?#'). docsUrls.add( m[0] - .replace(/([?&])utm_source=bestax-mcp&?/, '$1') - .replace(/[?&]$/, '') + .replace(/([?&])utm_source=bestax-mcp&/, '$1') + .replace(/[?&]utm_source=bestax-mcp(?=#|$)/, '') ); } } diff --git a/scripts/lib/resolve-ts-from-js.mjs b/scripts/lib/resolve-ts-from-js.mjs index b9ea302ec..578d777be 100644 --- a/scripts/lib/resolve-ts-from-js.mjs +++ b/scripts/lib/resolve-ts-from-js.mjs @@ -8,8 +8,11 @@ import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; export async function resolve(specifier, context, nextResolve) { + // file: parents only — fileURLToPath throws on any other scheme, and this + // hook must fall through (never fail) for modules it was not written for + // (a data: URL, another loader's in-memory parent). if ( - context.parentURL && + context.parentURL?.startsWith('file:') && specifier.startsWith('.') && specifier.endsWith('.js') ) { From 06b3653c5489e5f35bdc082c8dac476eef173de2 Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sun, 23 Aug 2026 15:47:09 -0400 Subject: [PATCH 25/29] fix(bestax-migrate): name the wire caps and pin them to the worker TODO_RULES_CAP/TODO_COUNT_CAP/CHANGED_COUNT_CAP replace bare literals and the contract test pins each equal to the worker's constant, plus a boundary payload at every cap. Worker side: PLATFORM_VALUES gains cygwin and os400 (all 12 documented process.platform values, cited so the next omission is checkable), the round-trip test clones its input so it can actually fail, and stale out-of-workspace comments are gone. --- .../src/__tests__/telemetry.test.ts | 33 ++++++++----- bestax-migrate/src/telemetry.ts | 19 +++++-- scripts/telemetry-contract.test.mjs | 49 ++++++++++++++++--- telemetry-worker/README.md | 29 ++++++++--- .../src/__tests__/validate.test.ts | 18 ++++++- telemetry-worker/src/schema.ts | 11 ++++- telemetry-worker/src/types.ts | 6 +-- 7 files changed, 132 insertions(+), 33 deletions(-) diff --git a/bestax-migrate/src/__tests__/telemetry.test.ts b/bestax-migrate/src/__tests__/telemetry.test.ts index 40098bd08..b308984d1 100644 --- a/bestax-migrate/src/__tests__/telemetry.test.ts +++ b/bestax-migrate/src/__tests__/telemetry.test.ts @@ -12,8 +12,13 @@ import { tmpdir } from 'os'; import { join } from 'path'; const { getToolVersion } = await import('../telemetry-core.js'); -const { buildMigratePayload, reportMigrateRun } = - await import('../telemetry.js'); +const { + buildMigratePayload, + reportMigrateRun, + TODO_RULES_CAP, + TODO_COUNT_CAP, + CHANGED_COUNT_CAP, +} = await import('../telemetry.js'); const ENV_KEYS = [ 'XDG_CONFIG_HOME', @@ -160,24 +165,30 @@ describe('buildMigratePayload', () => { // The 0/1-9/10-49/50-199/200+ bucket is derived by the ingest worker from // changedCount — sending it too gave the boundaries three sources of truth. - it('caps changedCount at 10000 and sends no bucket', () => { + it('caps changedCount at CHANGED_COUNT_CAP and sends no bucket', () => { const payload = buildMigratePayload({ ...baseStats, - changedCount: 123456, + changedCount: CHANGED_COUNT_CAP + 113456, }); - expect(payload.props.changedCount).toBe(10000); + expect(payload.props.changedCount).toBe(CHANGED_COUNT_CAP); expect('changedBucket' in payload.props).toBe(false); }); - it('sends at most 20 rules and caps each count', () => { - const todosByRule = Array.from({ length: 25 }, (_, i) => ({ + it('sends at most TODO_RULES_CAP rules and caps each count', () => { + const todosByRule = Array.from({ length: TODO_RULES_CAP + 5 }, (_, i) => ({ rule: `rule-${i}`, - count: i === 0 ? 250000 : i, + count: i === 0 ? TODO_COUNT_CAP + 150000 : i, })); const payload = buildMigratePayload({ ...baseStats, todosByRule }); - expect(payload.todosByRule).toHaveLength(20); - expect(payload.todosByRule?.[0]).toEqual({ rule: 'rule-0', count: 100000 }); - expect(payload.todosByRule?.[19]).toEqual({ rule: 'rule-19', count: 19 }); + expect(payload.todosByRule).toHaveLength(TODO_RULES_CAP); + expect(payload.todosByRule?.[0]).toEqual({ + rule: 'rule-0', + count: TODO_COUNT_CAP, + }); + expect(payload.todosByRule?.[TODO_RULES_CAP - 1]).toEqual({ + rule: `rule-${TODO_RULES_CAP - 1}`, + count: TODO_RULES_CAP - 1, + }); }); }); diff --git a/bestax-migrate/src/telemetry.ts b/bestax-migrate/src/telemetry.ts index 6d3b7f571..4475580e1 100644 --- a/bestax-migrate/src/telemetry.ts +++ b/bestax-migrate/src/telemetry.ts @@ -10,6 +10,16 @@ import type { ReportRunOptions } from './telemetry-core.js'; * never file paths, messages, or code. */ +/** + * Client half of the wire-contract caps. The worker's schema.ts / validate.ts + * (telemetry-worker) hold the other half; scripts/telemetry-contract.test.mjs + * pins each pair equal (TODO_RULES_CAP === MAX_TODO_RULES, TODO_COUNT_CAP === + * MAX_CHANGED_COUNT, CHANGED_COUNT_CAP === CHANGED_COUNT_DOUBLE_CAP). + */ +export const TODO_RULES_CAP = 20; +export const TODO_COUNT_CAP = 100000; +export const CHANGED_COUNT_CAP = 10000; + export interface MigrateRunStats { /** Registry source name, e.g. 'react-bulma-components'. */ source: string; @@ -42,8 +52,11 @@ export interface MigratePayload { export function buildMigratePayload(stats: MigrateRunStats): MigratePayload { const todosByRule = stats.todosByRule - .slice(0, 20) - .map(({ rule, count }) => ({ rule, count: Math.min(count, 100000) })); + .slice(0, TODO_RULES_CAP) + .map(({ rule, count }) => ({ + rule, + count: Math.min(count, TODO_COUNT_CAP), + })); return { v: 1, tool: 'bestax-migrate', @@ -56,7 +69,7 @@ export function buildMigratePayload(stats: MigrateRunStats): MigratePayload { cssMode: stats.cssMode, dry: stats.dry, deps: stats.deps, - changedCount: Math.min(stats.changedCount, 10000), + changedCount: Math.min(stats.changedCount, CHANGED_COUNT_CAP), }, ...(todosByRule.length > 0 ? { todosByRule } : {}), }; diff --git a/scripts/telemetry-contract.test.mjs b/scripts/telemetry-contract.test.mjs index aac20ff9a..3415c1294 100644 --- a/scripts/telemetry-contract.test.mjs +++ b/scripts/telemetry-contract.test.mjs @@ -1,22 +1,32 @@ /** * Round-trip: CLI payload builders → JSON → worker validate(). * - * The worker is outside the pnpm workspace and the CLIs use `.js` specifiers - * for `.ts` files, so this test registers a small resolver hook and imports - * both sides' source. A payload the CLIs actually emit that the worker - * rejects is the class of bug that silently drops every event. + * The worker imports directly (its dependency-free source uses explicit `.ts` + * specifiers), but the CLIs use `.js` specifiers for `.ts` files, so this + * test registers a small resolver hook and imports both sides' source. A + * payload the CLIs actually emit that the worker rejects is the class of bug + * that silently drops every event. */ import { register } from 'node:module'; import { test } from 'node:test'; import assert from 'node:assert/strict'; import { validate } from '../telemetry-worker/src/validate.ts'; +import { + CHANGED_COUNT_DOUBLE_CAP, + MAX_CHANGED_COUNT, + MAX_TODO_RULES, +} from '../telemetry-worker/src/schema.ts'; register('./lib/resolve-ts-from-js.mjs', import.meta.url); const { buildScaffoldPayload } = await import('../create-bestax/src/telemetry.ts'); -const { buildMigratePayload } = - await import('../bestax-migrate/src/telemetry.ts'); +const { + buildMigratePayload, + TODO_RULES_CAP, + TODO_COUNT_CAP, + CHANGED_COUNT_CAP, +} = await import('../bestax-migrate/src/telemetry.ts'); const PRODUCTION_RULES = [ 'prop:className', @@ -66,6 +76,33 @@ test('bestax-migrate payload with production rule slugs validates', () => { } }); +test('client caps equal the worker bounds', () => { + assert.equal(TODO_RULES_CAP, MAX_TODO_RULES); + assert.equal(TODO_COUNT_CAP, MAX_CHANGED_COUNT); + assert.equal(CHANGED_COUNT_CAP, CHANGED_COUNT_DOUBLE_CAP); +}); + +test('boundary bestax-migrate payload at every cap validates', () => { + const result = roundTrip( + buildMigratePayload({ + source: 'react-bulma-components', + cssMode: 'bulma', + dry: false, + deps: true, + changedCount: CHANGED_COUNT_CAP, + todosByRule: Array.from({ length: TODO_RULES_CAP }, (_, i) => ({ + rule: `rule-${i}`, + count: TODO_COUNT_CAP, + })), + }) + ); + assert.equal(result.ok, true, JSON.stringify(result)); + if (result.ok && result.payload.tool === 'bestax-migrate') { + assert.equal(result.payload.todosByRule?.length, TODO_RULES_CAP); + assert.equal(result.payload.props.changedCount, CHANGED_COUNT_CAP); + } +}); + test('bestax-migrate payload with no TODOs validates', () => { const result = roundTrip( buildMigratePayload({ diff --git a/telemetry-worker/README.md b/telemetry-worker/README.md index 1e2a18390..220c68bfc 100644 --- a/telemetry-worker/README.md +++ b/telemetry-worker/README.md @@ -8,12 +8,23 @@ missing binding in local dev still answers `204`. ## Deploy +CI deploys this worker: `.github/workflows/deploy-worker.yml` runs `wrangler +deploy` on every `main` push that touches `telemetry-worker/**`, plus a manual +re-run via `workflow_dispatch` from the Actions tab. Its only credential is +the dedicated `CLOUDFLARE_WORKERS_DEPLOY_TOKEN`, an API token scoped to +Workers Scripts:Edit + Workers Routes:Edit on bestax.io — separate from the +Pages token, so each pipeline holds only what it deploys. Mind the merge-time +consequence: a merged change under `telemetry-worker/**` ships to production +immediately. + +Deploying from a laptop remains the fallback: + ```sh npx wrangler deploy ``` -Run by Alex — the deploy is manual (nothing in `.github/workflows` touches -this worker). The package IS in the pnpm workspace, so the lockfile, audit +(authenticate with `wrangler login` OAuth and keep the granted scopes +least-privilege). The package IS in the pnpm workspace, so the lockfile, audit gate, and 3-day cooldown govern its devDependencies; they stay exact-pinned anyway so a deploy is reproducible from the manifest alone. Wrangler's native `workerd` postinstall is blocked repo-wide (`allowBuilds`), which `wrangler @@ -79,10 +90,16 @@ FROM bestax_telemetry WHERE index1='create-bestax' AND blob1='scaffold' ## Privacy -- Allowlist validation is the backstop: every key and value in a payload must - match a closed allowlist (`src/schema.ts`, `src/validate.ts`) or the whole - payload is rejected with a bare `400`. Unknown values are dropped, never - stored — fail closed. New templates, flavors, icon libraries, sources, or +- Allowlist validation is the backstop, with one deliberate exception: every + key must match the schema, and every value must be a closed enum, a version + string, or a bounded integer (`src/schema.ts`, `src/validate.ts`) or the + whole payload is rejected with a bare `400` — fail closed. The exception is + `todosByRule[].rule`: migrate rule names like `prop:className` are + open-ended, so an enum is impossible by design. A rule is pattern-validated + free text instead (1–64 chars of `[A-Za-z0-9._:-]`, at most 20 entries) and + stored verbatim, which means a modified client can store short arbitrary + strings in that one column — aggregate queries should treat unrecognized + rule names as noise. New templates, flavors, icon libraries, sources, or CSS modes must be added to `src/schema.ts` **first** or their runs are silently discarded. - No IP address or User-Agent is ever read or stored; `request.cf` is never diff --git a/telemetry-worker/src/__tests__/validate.test.ts b/telemetry-worker/src/__tests__/validate.test.ts index 2c2a440e2..cebee4eed 100644 --- a/telemetry-worker/src/__tests__/validate.test.ts +++ b/telemetry-worker/src/__tests__/validate.test.ts @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; import { describe, it } from 'node:test'; +import { PLATFORM_VALUES } from '../schema.ts'; import { validate } from '../validate.ts'; type Payload = Record; @@ -73,7 +74,9 @@ describe('validate: accepts', () => { assertOk(payload); }); - for (const platform of ['netbsd', 'haiku']) { + // Parameterized over the schema's own allowlist so a platform added there + // is covered automatically; the reject side is 'beos' below. + for (const platform of PLATFORM_VALUES) { it(`${platform} platform`, () => { const payload = createPayload(); payload.platform = platform; @@ -83,9 +86,20 @@ describe('validate: accepts', () => { it('validated payload round-trips unchanged', () => { const payload = migratePayload(); + // Clone BEFORE validate(): it returns (and may mutate) the very object it + // was given, so comparing result.payload against `payload` is vacuous. + const expected = structuredClone(payload); const result = validate(payload); assert.equal(result.ok, true); - if (result.ok) assert.deepEqual(result.payload, payload); + if (result.ok) { + // Content contract: a fully-valid payload passes through with no field + // altered or dropped. + assert.deepEqual(result.payload, expected); + // Identity contract: validate() intentionally returns the same object + // it was passed (no defensive copy) — the handler forwards this + // reference straight to the Analytics Engine point mappers. + assert.equal(result.payload, payload); + } }); }); diff --git a/telemetry-worker/src/schema.ts b/telemetry-worker/src/schema.ts index e9bd7bc13..bdf80e781 100644 --- a/telemetry-worker/src/schema.ts +++ b/telemetry-worker/src/schema.ts @@ -43,17 +43,24 @@ const MIGRATE_SOURCE_VALUES = ['react-bulma-components'] as const; // bestax-migrate/src/cli.ts → CSS_MODES const CSS_MODE_VALUES = ['bestax', 'bulma', 'keep'] as const; const PACKAGE_MANAGER_VALUES = ['npm', 'pnpm', 'yarn', 'bun'] as const; -const PLATFORM_VALUES = [ +// Node's documented process.platform values (nodejs.org/api/process.html): +// aix, darwin, freebsd, linux, openbsd, sunos, win32, plus android, cygwin, +// haiku, netbsd, os400 on some builds. All 12, listed in that order so an +// omission is checkable against the sentence above at a glance. Exported so +// the worker tests parameterize acceptance over the full list. +export const PLATFORM_VALUES = [ 'aix', 'darwin', 'freebsd', 'linux', - 'netbsd', 'openbsd', 'sunos', 'win32', 'android', + 'cygwin', 'haiku', + 'netbsd', + 'os400', ] as const; export type Template = (typeof TEMPLATE_VALUES)[number]; diff --git a/telemetry-worker/src/types.ts b/telemetry-worker/src/types.ts index 909dc13cf..b63c16659 100644 --- a/telemetry-worker/src/types.ts +++ b/telemetry-worker/src/types.ts @@ -1,8 +1,8 @@ /** * Minimal structural types for the Workers runtime bindings this worker uses. - * Deliberately hand-rolled: this package sits outside the pnpm workspace (for - * now), so @cloudflare/workers-types cannot be installed. Request/Response/URL - * come from the `webworker` TS lib instead (see tsconfig.json). + * Deliberately hand-rolled to keep the worker dependency-free — no + * @cloudflare/workers-types. Request/Response/URL come from the `webworker` + * TS lib instead (see tsconfig.json). */ export interface AnalyticsEngineDataPoint { From 00545b469942b0607b435e9dbd3d345f79604595 Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sun, 23 Aug 2026 15:47:10 -0400 Subject: [PATCH 26/29] docs: true up deploy, backstop, and utm-coverage claims MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The worker deploys from CI (deploy-worker.yml), not manually; the validation backstop has one deliberate exception (rule names are bounded pattern text, not an enum — stated plainly with its abuse implication); utm tagging covers the links the server composes itself, while skill and reference bodies are served verbatim and say so, with a format.ts comment marking the boundary as deliberate. --- CLAUDE.md | 3 ++- bestax-mcp/src/format.ts | 7 +++++++ docs/docs/guides/telemetry.md | 25 ++++++++++++++++++------- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 67d5e4c65..9ca0fb668 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,7 +10,8 @@ React component library for **Bulma v1** in TypeScript. pnpm monorepo orchestrat (has its own CLAUDE.md) - `skills/` — Agent Skills, a **shipped product** bundled into create-bestax (has its own CLAUDE.md) - `telemetry-worker/` — Cloudflare Worker ingesting the CLIs' opt-in telemetry - (deployed manually with wrangler) + (deployed from CI by `deploy-worker.yml` — a merged change under it ships to + production immediately) - `eval/agent-loop/` — cold-start eval harness: does an unassisted agent build correctly when given one of our guidance channels (the skills, the MCP server, both)? Frozen rubric, its own README diff --git a/bestax-mcp/src/format.ts b/bestax-mcp/src/format.ts index f70108d58..9a0e57a0f 100644 --- a/bestax-mcp/src/format.ts +++ b/bestax-mcp/src/format.ts @@ -33,6 +33,13 @@ const cell = (text: string) => * visit that started here shows up as such in the site analytics. Render-time * only — the URLs in `data/` are generated and stay canonical. Idempotent, and * anything that is not an http(s) URL passes through unchanged. + * + * Coverage is deliberately partial: only links this server composes itself — + * the Docs/Storybook footer on component responses and the version-drift + * notice — go through here. Skill bodies and reference docs are served + * verbatim from the bundled markdown, untagged links included, because + * rewriting URLs inside arbitrary markdown/code examples risks corrupting + * them. Do not "fix" that by running this over served markdown. */ export function attributed(url: string): string { if (!/^https?:\/\//.test(url)) return url; diff --git a/docs/docs/guides/telemetry.md b/docs/docs/guides/telemetry.md index 187676415..157e118fc 100644 --- a/docs/docs/guides/telemetry.md +++ b/docs/docs/guides/telemetry.md @@ -105,17 +105,28 @@ Engine as aggregate counts, retained for roughly 90 days. No third-party analytics service ever sees the data. As a server-side privacy backstop, the endpoint validates every field of every -event against a strict allowlist and rejects anything else — so even a modified -or buggy client cannot get extra data stored. +event and rejects anything else. With one exception, every field is a closed +enum, a version string, or a bounded integer, so a modified or buggy client +cannot get extra data stored in those fields. The exception is migration rule +names, which are open-ended by design (the `prop:` slugs mean no fixed +list exists): the endpoint bounds them — at most 20 per event, each 1–64 +characters of `A-Za-z0-9._:-` — but stores them verbatim, so a modified client +could record short arbitrary strings there. Aggregate queries treat +unrecognized rule names as noise. ## The MCP server `bestax-mcp` sends **no telemetry at all** and makes no network requests — it -stays fully offline. The documentation links it prints include a -`utm_source=bestax-mcp` query parameter, so if you visit the docs site through -one of those links, that visit is attributable in the site's own traffic -analytics. That is the only measurement, and it happens on bestax.io like any -other page visit. +stays fully offline. Some of the links it prints carry a +`utm_source=bestax-mcp` query parameter: the Docs and Storybook links on +component responses and the link in the version-drift notice are tagged at +render time. Skill bodies and reference docs (`get_skill`, the MCP prompts and +resources) are served verbatim from the bundled markdown — rewriting URLs +inside arbitrary markdown and code examples risks corrupting them — so the +bestax.io links in those are untagged. If you visit the docs site through a +tagged link, that visit is attributable in the site's own traffic analytics. +That is the only measurement, and it happens on bestax.io like any other page +visit. ## Why we collect this From 750b53e052ccb1acf803e01c5ee7bdafbba21b1a Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sun, 23 Aug 2026 16:08:49 -0400 Subject: [PATCH 27/29] fix(create-bestax): review-thread fixes across the telemetry surface CodeRabbit/Copilot triage on PR #550, the items that were real: - attributed() now checks the QUERY for prior attribution, so the tag text inside a fragment or another parameter's value no longer skips tagging (regression cases added). - A flag whose persistence fails now surfaces through onDecided, so --no-telemetry against a read-only config warns instead of silently re-enabling on the next family run (kernel copied to both CLIs). - bestax-migrate's consent notice lists every field the payload sends, matching the disclosure page instead of underclaiming. - SKILL.md must be a regular file before a directory counts as a skill. - telemetry-worker gains a lint script and tsconfig.eslint.json on the house per-package pattern, so pnpm lint now covers its source. - deploy-worker.yml: serialized concurrency group (an older run must not finish after a newer one and ship stale code) and persist-credentials: false on checkout. - Both CLI CLAUDE.mds document the kernel copy rule and cap pinning; the disclosure's overrides table lists BESTAX_TELEMETRY_ENDPOINT. --- .github/workflows/deploy-worker.yml | 11 +++++++ bestax-mcp/src/__tests__/format.test.ts | 13 ++++++++ bestax-mcp/src/format.ts | 14 ++++++--- bestax-migrate/CLAUDE.md | 6 ++++ bestax-migrate/src/__tests__/cli.test.ts | 4 +-- bestax-migrate/src/cli.ts | 4 ++- bestax-migrate/src/telemetry-core.ts | 8 ++++- create-bestax/CLAUDE.md | 8 +++++ create-bestax/src/__tests__/telemetry.test.ts | 30 +++++++++++++++++++ create-bestax/src/telemetry-core.ts | 8 ++++- docs/docs/guides/telemetry.md | 1 + scripts/lib/skills.mjs | 10 ++++--- telemetry-worker/package.json | 3 +- telemetry-worker/tsconfig.eslint.json | 5 ++++ 14 files changed, 110 insertions(+), 15 deletions(-) create mode 100644 telemetry-worker/tsconfig.eslint.json diff --git a/.github/workflows/deploy-worker.yml b/.github/workflows/deploy-worker.yml index c5aeb7565..f715c60c0 100644 --- a/.github/workflows/deploy-worker.yml +++ b/.github/workflows/deploy-worker.yml @@ -20,6 +20,13 @@ on: permissions: contents: read +# Serialize deploys: two quick main pushes must not race — an older run +# finishing last would ship stale worker code. No cancel-in-progress: a +# deploy already uploading should finish, then the newer one runs. +concurrency: + group: deploy-worker + cancel-in-progress: false + jobs: deploy: name: Deploy to Cloudflare Workers @@ -43,6 +50,10 @@ jobs: - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + # No later step needs authenticated git; don't leave the token in + # the local git config for the rest of the job. + persist-credentials: false # No pnpm/Node setup: the worker is stdlib-only TypeScript with no build # step of its own — wrangler-action installs the pinned wrangler and its diff --git a/bestax-mcp/src/__tests__/format.test.ts b/bestax-mcp/src/__tests__/format.test.ts index 3b0ad4338..4b9f08557 100644 --- a/bestax-mcp/src/__tests__/format.test.ts +++ b/bestax-mcp/src/__tests__/format.test.ts @@ -37,6 +37,19 @@ describe('attributed', () => { expect(attributed(tagged)).toBe(tagged); }); + it('is not fooled by the tag text inside a fragment', () => { + // The analytics request never sees the fragment, so this URL is NOT + // attributed yet — the already-tagged check must look at the query only. + expect(attributed('https://bestax.io/docs#utm_source=bestax-mcp')).toBe( + 'https://bestax.io/docs?utm_source=bestax-mcp#utm_source=bestax-mcp' + ); + }); + + it('is not fooled by the tag text inside another parameter value', () => { + const url = 'https://bestax.io/docs?ref=utm_source=bestax-mcp'; + expect(attributed(url)).toBe(`${url}&utm_source=bestax-mcp`); + }); + it('passes non-http(s) strings through unchanged', () => { for (const notAUrl of ['bestax://catalog', 'mailto:a@b.c', 'Button', '']) { expect(attributed(notAUrl)).toBe(notAUrl); diff --git a/bestax-mcp/src/format.ts b/bestax-mcp/src/format.ts index 9a0e57a0f..1effdcf1c 100644 --- a/bestax-mcp/src/format.ts +++ b/bestax-mcp/src/format.ts @@ -43,13 +43,19 @@ const cell = (text: string) => */ export function attributed(url: string): string { if (!/^https?:\/\//.test(url)) return url; - if (url.includes('utm_source=bestax-mcp')) return url; - // Insert before the fragment so `#heading` stays a fragment. Do not use - // URL.searchParams: it re-encodes Storybook `path=/story/...` as `%2F`. + // Split the fragment FIRST: the already-tagged check below must inspect + // only the query. A `#utm_source=bestax-mcp` inside a fragment (or the + // same text inside another parameter's value) is not attribution — the + // analytics request never sees it — so matching it would skip tagging. + // Do not use URL.searchParams: it re-encodes Storybook `path=/story/...` + // as `%2F`. const hashIndex = url.indexOf('#'); const hash = hashIndex === -1 ? '' : url.slice(hashIndex); const withoutHash = hashIndex === -1 ? url : url.slice(0, hashIndex); - const sep = withoutHash.includes('?') ? '&' : '?'; + const queryIndex = withoutHash.indexOf('?'); + const query = queryIndex === -1 ? '' : withoutHash.slice(queryIndex + 1); + if (query.split('&').includes('utm_source=bestax-mcp')) return url; + const sep = queryIndex === -1 ? '?' : '&'; return `${withoutHash}${sep}utm_source=bestax-mcp${hash}`; } diff --git a/bestax-migrate/CLAUDE.md b/bestax-migrate/CLAUDE.md index a1e5ea98c..b5465e6d1 100644 --- a/bestax-migrate/CLAUDE.md +++ b/bestax-migrate/CLAUDE.md @@ -34,6 +34,12 @@ each source library registers in `src/sources/registry.ts`; the first is (package.json updater; never runs an install), `jsx-utils.ts` (AST helpers). - Components with no bestax equivalent (Element, Tile) keep a trimmed, TODO-annotated RBC import so the code still runs during gradual migration. +- `src/telemetry-core.ts` is a byte-for-byte copy of + `create-bestax/src/telemetry-core.ts` — never edit it here; edit the + create-bestax original and copy it over (`check:conformance +--only=telemetry-core` enforces identity). This package's payload builder and + caps live in `src/telemetry.ts`; the caps are pinned to the ingest worker's + bounds by `scripts/telemetry-contract.test.mjs`. ## Testing diff --git a/bestax-migrate/src/__tests__/cli.test.ts b/bestax-migrate/src/__tests__/cli.test.ts index 98f2ac3ff..e528725c0 100644 --- a/bestax-migrate/src/__tests__/cli.test.ts +++ b/bestax-migrate/src/__tests__/cli.test.ts @@ -574,9 +574,7 @@ describe('promptTelemetryConsent', () => { expect(text).toContain( 'Help improve bestax-migrate — share anonymous usage stats?' ); - expect(text).toContain( - 'Sends only rule names and counts — never file paths or code.' - ); + expect(text).toContain('Never file paths, file contents, or code.'); expect(text).toContain('https://bestax.io/docs/guides/telemetry'); expect(text).toContain('https://github.com/allxsmith/bestax/issues'); }); diff --git a/bestax-migrate/src/cli.ts b/bestax-migrate/src/cli.ts index 0e127bf79..8832ee6df 100644 --- a/bestax-migrate/src/cli.ts +++ b/bestax-migrate/src/cli.ts @@ -218,7 +218,9 @@ function migrateDependencies( const TELEMETRY_NOTICE = 'Help improve bestax-migrate — share anonymous usage stats?\n' + - 'Sends only rule names and counts — never file paths or code.\n' + + "Sends only the run's shape (source library, CSS mode, dry/deps flags,\n" + + 'changed-file count, rule names and counts) plus CLI version, Node major,\n' + + 'and OS name. Never file paths, file contents, or code.\n' + ' Details & opt-out: https://bestax.io/docs/guides/telemetry\n' + ' Feedback welcome: https://github.com/allxsmith/bestax/issues'; const TELEMETRY_ACK_ON = 'Thanks! Opt out anytime with --no-telemetry.'; diff --git a/bestax-migrate/src/telemetry-core.ts b/bestax-migrate/src/telemetry-core.ts index 5175cc0ca..71a5ba77b 100644 --- a/bestax-migrate/src/telemetry-core.ts +++ b/bestax-migrate/src/telemetry-core.ts @@ -172,10 +172,16 @@ export async function reportRun( // written down, so a copied command containing --telemetry cannot enable // telemetry beyond the run it was typed for. if (!doNotTrack()) { - await persistTelemetryDecision( + const persisted = await persistTelemetryDecision( decision === 'on', `${toolName}@${getToolVersion()}` ); + // A silently unsaved flag is worse than an unsaved answer: with an + // opposite value already in the shared config, --no-telemetry would + // apply to this run only and the next family run would re-enable. + if (!persisted) { + options.onDecided?.(decision === 'on', false); + } } } else if (decision === 'undecided' && options.interactive) { const answer = await options.promptConsent(); diff --git a/create-bestax/CLAUDE.md b/create-bestax/CLAUDE.md index 441f97876..7a2c61921 100644 --- a/create-bestax/CLAUDE.md +++ b/create-bestax/CLAUDE.md @@ -32,6 +32,14 @@ path (`-y` + flags, no TTY) must never hang or regress (#192). - The `CLAUDE_MD` template in `constants.ts` is what every generated app tells its AI agents. When library conventions, skills, or the canonical docs entrypoint change (#203), check whether this template must change too. +- `src/telemetry-core.ts` is the shared consent/beacon kernel, duplicated + byte-for-byte into `bestax-migrate/src/telemetry-core.ts` (standalone + publishes, no bundler — so no workspace package). **Edit the copy here, then + copy it over migrate's**; `check:conformance --only=telemetry-core` fails on + any divergence. Tool-specific payload builders stay in each package's + `src/telemetry.ts`, and the worker's allowlists must gain new enum values + FIRST (`check:conformance --only=telemetry-allowlists`) or events are + silently dropped at ingest. - Templates pin the library's CSS import and icon setup — a change to bulma-ui's published exports or flavors (`bestax.css`, `versions/*.css`) may require a template update. diff --git a/create-bestax/src/__tests__/telemetry.test.ts b/create-bestax/src/__tests__/telemetry.test.ts index 2cb2fe4d2..9d198f3e8 100644 --- a/create-bestax/src/__tests__/telemetry.test.ts +++ b/create-bestax/src/__tests__/telemetry.test.ts @@ -329,6 +329,36 @@ describe('reportScaffold', () => { expect(onDecided).toHaveBeenCalledWith(false, false); }); + it('surfaces a failed flag persistence through onDecided', async () => { + // With an opposite value already saved and the config unwritable, + // --no-telemetry would apply to this run only and the next family run + // would silently re-enable — the caller must get to warn about that. + const blocker = join(configHome, 'blocker'); + await writeFile(blocker, 'file, not a dir', 'utf-8'); + process.env.XDG_CONFIG_HOME = blocker; + const onDecided = jest.fn<(enabled: boolean, persisted: boolean) => void>(); + const promptConsent = jest.fn<() => Promise>(); + await reportScaffold(choices, false, { + interactive: false, + promptConsent, + onDecided, + }); + expect(onDecided).toHaveBeenCalledWith(false, false); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('does not call onDecided when a flag persists cleanly', async () => { + const onDecided = jest.fn<(enabled: boolean, persisted: boolean) => void>(); + const promptConsent = jest.fn<() => Promise>(); + await reportScaffold(choices, true, { + interactive: false, + promptConsent, + onDecided, + }); + expect(onDecided).not.toHaveBeenCalled(); + expect((await readConfigFile()).enabled).toBe(true); + }); + it('does not call onDecided on a cancel', async () => { const onDecided = jest.fn<(enabled: boolean, persisted: boolean) => void>(); const promptConsent = jest diff --git a/create-bestax/src/telemetry-core.ts b/create-bestax/src/telemetry-core.ts index 5175cc0ca..71a5ba77b 100644 --- a/create-bestax/src/telemetry-core.ts +++ b/create-bestax/src/telemetry-core.ts @@ -172,10 +172,16 @@ export async function reportRun( // written down, so a copied command containing --telemetry cannot enable // telemetry beyond the run it was typed for. if (!doNotTrack()) { - await persistTelemetryDecision( + const persisted = await persistTelemetryDecision( decision === 'on', `${toolName}@${getToolVersion()}` ); + // A silently unsaved flag is worse than an unsaved answer: with an + // opposite value already in the shared config, --no-telemetry would + // apply to this run only and the next family run would re-enable. + if (!persisted) { + options.onDecided?.(decision === 'on', false); + } } } else if (decision === 'undecided' && options.interactive) { const answer = await options.promptConsent(); diff --git a/docs/docs/guides/telemetry.md b/docs/docs/guides/telemetry.md index 157e118fc..1041ebe59 100644 --- a/docs/docs/guides/telemetry.md +++ b/docs/docs/guides/telemetry.md @@ -90,6 +90,7 @@ ever asked once. Delete that file to be asked again. | `--telemetry` / `--no-telemetry` | Flags on both CLIs — set the choice for this run **and persist it** to the consent file | | `BESTAX_TELEMETRY=1` / `BESTAX_TELEMETRY=0` | Enable or disable for this run only — never persisted | | `DO_NOT_TRACK=1` | Disables telemetry and the consent prompt itself | +| `BESTAX_TELEMETRY_ENDPOINT=` | Redirects the event for this run — a test hook; only meaningful if you set it yourself | One interaction worth spelling out: with `DO_NOT_TRACK` set, an explicit `--telemetry` flag still applies **to that single run** (typing the flag is a diff --git a/scripts/lib/skills.mjs b/scripts/lib/skills.mjs index bff99d3c8..d3492af00 100644 --- a/scripts/lib/skills.mjs +++ b/scripts/lib/skills.mjs @@ -6,7 +6,7 @@ // scripts/lib/shell-words.mjs (#436) exists to prevent. All four import from // here now; a predicate change lands once or not at all. -import { readdir, access } from 'node:fs/promises'; +import { readdir, stat } from 'node:fs/promises'; import { realpathSync } from 'node:fs'; import { join, dirname, resolve } from 'node:path'; import { execFileSync } from 'node:child_process'; @@ -44,11 +44,13 @@ export async function readSkillDirs(dir) { for (const entry of await readdir(dir, { withFileTypes: true })) { if (!entry.isDirectory()) continue; - let hasSkillFile = true; + // A regular file specifically: a directory named SKILL.md would satisfy + // a bare existence probe and then break every consumer that reads it. + let hasSkillFile = false; try { - await access(join(dir, entry.name, 'SKILL.md')); + hasSkillFile = (await stat(join(dir, entry.name, 'SKILL.md'))).isFile(); } catch { - hasSkillFile = false; + // missing → not a skill } if (entry.name.startsWith('.') && !hasSkillFile) continue; diff --git a/telemetry-worker/package.json b/telemetry-worker/package.json index d92198e57..1de039ee6 100644 --- a/telemetry-worker/package.json +++ b/telemetry-worker/package.json @@ -7,7 +7,8 @@ "test": "node --test 'src/__tests__/*.test.ts'", "typecheck": "tsc -p tsconfig.json --noEmit", "deploy": "wrangler deploy", - "dev": "wrangler dev" + "dev": "wrangler dev", + "lint": "eslint src" }, "devDependencies": { "typescript": "5.9.2", diff --git a/telemetry-worker/tsconfig.eslint.json b/telemetry-worker/tsconfig.eslint.json new file mode 100644 index 000000000..eb55636ff --- /dev/null +++ b/telemetry-worker/tsconfig.eslint.json @@ -0,0 +1,5 @@ +{ + "extends": "./tsconfig.json", + "include": ["src/**/*"], + "exclude": ["node_modules"] +} From 81df91954cd421c57df7df0495698178be083f5a Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sun, 23 Aug 2026 16:13:40 -0400 Subject: [PATCH 28/29] fix(create-bestax): fail closed on foreign consent records, gate ignored files Second review-round fixes: - readConfig trusts only a well-formed v1 record: a bare {"enabled":true}, an array, or an unknown future schema version reads as never-asked, not as an opt-in (kernel, both copies). - The vetting gate drops --exclude-standard: gitignored content inside a skill (.env, logs, node_modules) would ship via the recursive copy while an exclude-standard gate passed. Only .DS_Store stays exempt, and both sync scripts now filter it out of the copy so the exemption never becomes shipped content. - bestax-migrate's SIGINT consent path is genuinely exercised: the readline interface is injectable, and a test emits SIGINT and asserts the null answer, exit code 130, and no persisted config. - The telemetry-allowlists helpers are exported with focused failure tests: null producer, malformed worker array, missing value named, unparseable source directory reported, and the real tree green. --- bestax-mcp/scripts/sync-skills.mjs | 3 + bestax-migrate/src/__tests__/cli.test.ts | 27 ++++++ bestax-migrate/src/cli.ts | 7 +- bestax-migrate/src/telemetry-core.ts | 10 ++- create-bestax/scripts/sync-skills.mjs | 6 +- create-bestax/src/__tests__/telemetry.test.ts | 12 +++ create-bestax/src/telemetry-core.ts | 10 ++- scripts/check-conformance.mjs | 13 +-- scripts/lib/skills.mjs | 18 ++-- scripts/telemetry-allowlists.test.mjs | 82 +++++++++++++++++++ 10 files changed, 169 insertions(+), 19 deletions(-) create mode 100644 scripts/telemetry-allowlists.test.mjs diff --git a/bestax-mcp/scripts/sync-skills.mjs b/bestax-mcp/scripts/sync-skills.mjs index 175116e11..105c50bb0 100644 --- a/bestax-mcp/scripts/sync-skills.mjs +++ b/bestax-mcp/scripts/sync-skills.mjs @@ -217,6 +217,9 @@ try { for (const name of names) { await cp(path.join(skillsSrc, name), path.join(skillsDest, name), { recursive: true, + // .DS_Store is the one path the vetting gate exempts; keep it out of + // the bundle too so the exemption never becomes shipped content. + filter: src => !src.endsWith('.DS_Store'), }); } // Stamped last: a run killed mid-copy leaves no stamp, so the next caller diff --git a/bestax-migrate/src/__tests__/cli.test.ts b/bestax-migrate/src/__tests__/cli.test.ts index e528725c0..72d82f9a8 100644 --- a/bestax-migrate/src/__tests__/cli.test.ts +++ b/bestax-migrate/src/__tests__/cli.test.ts @@ -579,6 +579,33 @@ describe('promptTelemetryConsent', () => { expect(text).toContain('https://github.com/allxsmith/bestax/issues'); }); + it('a SIGINT at the question cancels, exits 130, and persists nothing', async () => { + // PassThrough streams never emit SIGINT (only real TTY input does), so + // capture the interface via the injectable factory and emit it directly — + // exercising the real handler: exitCode 130, question aborted to null. + const { createInterface } = await import('node:readline/promises'); + const originalExitCode = process.exitCode; + const input = new PassThrough(); + const output = new PassThrough(); + output.resume(); + let captured: ReturnType | undefined; + const capture: typeof createInterface = ((opts: unknown) => { + captured = createInterface(opts as Parameters[0]); + return captured; + }) as typeof createInterface; + + const pending = promptTelemetryConsent(makeIo(), input, output, capture); + await new Promise(resolve => setImmediate(resolve)); + captured?.emit('SIGINT'); + + await expect(pending).resolves.toBeNull(); + expect(process.exitCode).toBe(130); + process.exitCode = originalExitCode; + expect( + fs.existsSync(path.join(process.env.XDG_CONFIG_HOME as string, 'bestax')) + ).toBe(false); + }); + it('treats a closed input (Ctrl-D) as a cancel, not an answer', async () => { const input = new PassThrough(); const output = new PassThrough(); diff --git a/bestax-migrate/src/cli.ts b/bestax-migrate/src/cli.ts index 8832ee6df..351adf4a5 100644 --- a/bestax-migrate/src/cli.ts +++ b/bestax-migrate/src/cli.ts @@ -237,11 +237,14 @@ const TELEMETRY_ACK_UNSAVED = export async function promptTelemetryConsent( io: CliIo, input: Readable = process.stdin, - output: Writable = process.stdout + output: Writable = process.stdout, + // Injectable for tests: SIGINT is only emitted by real TTY input, so a test + // captures the interface through this factory and emits it directly. + makeInterface: typeof createInterface = createInterface ): Promise { io.log(''); io.log(chalk.gray(TELEMETRY_NOTICE)); - const rl = createInterface({ input, output }); + const rl = makeInterface({ input, output }); const cancelled = new AbortController(); // Ctrl-C emits SIGINT on the interface and Ctrl-D closes it; both must // reject the pending question instead of leaving it hanging forever. diff --git a/bestax-migrate/src/telemetry-core.ts b/bestax-migrate/src/telemetry-core.ts index 71a5ba77b..fde77a90a 100644 --- a/bestax-migrate/src/telemetry-core.ts +++ b/bestax-migrate/src/telemetry-core.ts @@ -46,7 +46,15 @@ async function readConfig(): Promise { const parsed: unknown = JSON.parse( await readFile(telemetryConfigPath(), 'utf-8') ); - if (parsed !== null && typeof parsed === 'object') { + // Consent fails closed: only a well-formed v1 record is trusted. A bare + // {"enabled":true}, an array, or an unknown future schema version reads + // as "never asked" (prompt again) — never as an opt-in. + if ( + parsed !== null && + typeof parsed === 'object' && + !Array.isArray(parsed) && + (parsed as { version?: unknown }).version === 1 + ) { return parsed as TelemetryConfig; } } catch { diff --git a/create-bestax/scripts/sync-skills.mjs b/create-bestax/scripts/sync-skills.mjs index 450599822..771860e02 100644 --- a/create-bestax/scripts/sync-skills.mjs +++ b/create-bestax/scripts/sync-skills.mjs @@ -84,7 +84,11 @@ if (untracked.length) { await fs.emptyDir(skillsDest); for (const name of skills) { - await fs.copy(path.join(skillsSrc, name), path.join(skillsDest, name)); + await fs.copy(path.join(skillsSrc, name), path.join(skillsDest, name), { + // .DS_Store is the one path the vetting gate exempts; keep it out of the + // bundle too so the exemption never becomes shipped content. + filter: src => !src.endsWith('.DS_Store'), + }); } console.log(`[sync-skills] copied ${skills.length} skills -> templates/skills`); diff --git a/create-bestax/src/__tests__/telemetry.test.ts b/create-bestax/src/__tests__/telemetry.test.ts index 9d198f3e8..a0d7d7680 100644 --- a/create-bestax/src/__tests__/telemetry.test.ts +++ b/create-bestax/src/__tests__/telemetry.test.ts @@ -178,6 +178,18 @@ describe('resolveTelemetry precedence', () => { expect((await resolveTelemetry()).decision).toBe('undecided'); }); + it('a bare {"enabled":true} never enables — consent fails closed', async () => { + // No version field: a hand-written or foreign record is not a valid v1 + // opt-in and must read as "never asked", not as consent. + await writeConfig({ enabled: true }); + expect((await resolveTelemetry()).decision).toBe('undecided'); + }); + + it('an unknown future schema version is undecided, not trusted', async () => { + await writeConfig({ version: 2, enabled: true }); + expect((await resolveTelemetry()).decision).toBe('undecided'); + }); + it('treats a non-object config as undecided', async () => { await writeConfig('null'); expect((await resolveTelemetry()).decision).toBe('undecided'); diff --git a/create-bestax/src/telemetry-core.ts b/create-bestax/src/telemetry-core.ts index 71a5ba77b..fde77a90a 100644 --- a/create-bestax/src/telemetry-core.ts +++ b/create-bestax/src/telemetry-core.ts @@ -46,7 +46,15 @@ async function readConfig(): Promise { const parsed: unknown = JSON.parse( await readFile(telemetryConfigPath(), 'utf-8') ); - if (parsed !== null && typeof parsed === 'object') { + // Consent fails closed: only a well-formed v1 record is trusted. A bare + // {"enabled":true}, an array, or an unknown future schema version reads + // as "never asked" (prompt again) — never as an opt-in. + if ( + parsed !== null && + typeof parsed === 'object' && + !Array.isArray(parsed) && + (parsed as { version?: unknown }).version === 1 + ) { return parsed as TelemetryConfig; } } catch { diff --git a/scripts/check-conformance.mjs b/scripts/check-conformance.mjs index 8270c2c47..fb89c20bb 100644 --- a/scripts/check-conformance.mjs +++ b/scripts/check-conformance.mjs @@ -2436,7 +2436,7 @@ function quotedStringsIn(block) { return [...block.matchAll(/'([^']+)'/g)].map(m => m[1]); } -function constStringArray(src, name) { +export function constStringArray(src, name) { const m = src.match( new RegExp(`const ${name} = \\[([\\s\\S]*?)\\] as const`) ); @@ -2464,7 +2464,7 @@ async function importProducer(relPath, pick) { // bestax-migrate/src/cli.ts stays regex-scraped: importing it would drag the // whole transform chain (jscodeshift included) into every conformance run. // The scrape fails loudly as null when the declaration stops matching. -function cssModes(src) { +export function cssModes(src) { const m = src.match(/const CSS_MODES: CssMode\[\] = \[([\s\S]*?)\];/); return m ? quotedStringsIn(m[1]) : null; } @@ -2476,8 +2476,9 @@ function cssModes(src) { * unchecked against the worker, and its production events would 400 at * ingest with every gate green. */ -async function migrateSourceNames() { - const dir = join(REPO, 'bestax-migrate/src/sources'); +export async function migrateSourceNames( + dir = join(REPO, 'bestax-migrate/src/sources') +) { const names = []; const unparsed = []; for (const entry of await readdir(dir, { withFileTypes: true })) { @@ -2495,7 +2496,7 @@ async function migrateSourceNames() { return { names, unparsed }; } -function missingFromWorker(label, producer, worker, workerFile) { +export function missingFromWorker(label, producer, worker, workerFile) { if (producer === null) { return [`could not parse producer values for ${label}`]; } @@ -2513,7 +2514,7 @@ function missingFromWorker(label, producer, worker, workerFile) { ); } -async function checkTelemetryAllowlists() { +export async function checkTelemetryAllowlists() { const workerFile = 'telemetry-worker/src/schema.ts'; const constantsFile = 'create-bestax/src/constants.ts'; const cliFile = 'bestax-migrate/src/cli.ts'; diff --git a/scripts/lib/skills.mjs b/scripts/lib/skills.mjs index d3492af00..9e06893bb 100644 --- a/scripts/lib/skills.mjs +++ b/scripts/lib/skills.mjs @@ -123,14 +123,16 @@ export function untrackedSkillPaths(skillsDir, names) { if (realpathSync(toplevel) !== realpathSync(resolve(dirname(skillsDir)))) { return []; } - const others = git(skillsDir, [ - 'ls-files', - '--others', - '--exclude-standard', - '--', - '.', - ]); - return pathsInsideSkills(others.split('\n'), names); + // No --exclude-standard: sync copies everything on disk, so gitignored + // content inside a skill (.env, *.log, a stray node_modules) would ship + // right past an exclude-standard gate. The one exemption is .DS_Store — + // Finder drops it everywhere and the sync scripts filter it out of the + // copy instead, so it neither blocks builds nor ships. + const others = git(skillsDir, ['ls-files', '--others', '--', '.']); + return pathsInsideSkills( + others.split('\n').filter(p => !p.endsWith('.DS_Store')), + names + ); } catch { return []; } diff --git a/scripts/telemetry-allowlists.test.mjs b/scripts/telemetry-allowlists.test.mjs new file mode 100644 index 000000000..34d62d16d --- /dev/null +++ b/scripts/telemetry-allowlists.test.mjs @@ -0,0 +1,82 @@ +/** + * Holds the `telemetry-allowlists` conformance check to its failure contract + * (#550 review): every parse/import failure must surface as a violation, and + * a CLI value the worker does not list must be named. Fixtures drive the + * branches the real (agreeing) tree never executes — the same reason + * skills-roster.test.mjs is shaped this way. + */ +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { + constStringArray, + cssModes, + missingFromWorker, + migrateSourceNames, + checkTelemetryAllowlists, +} from './check-conformance.mjs'; + +test('the real tree passes the whole check', async () => { + assert.deepEqual(await checkTelemetryAllowlists(), []); +}); + +test('a null producer is a loud violation, never a silent pass', () => { + const v = missingFromWorker('template', null, ['vite'], 'worker.ts'); + assert.equal(v.length, 1); + assert.match(v[0], /could not parse producer values for template/); +}); + +test('a missing or malformed worker array is a loud violation', () => { + const v = missingFromWorker('template', ['vite'], null, 'worker.ts'); + assert.equal(v.length, 1); + assert.match(v[0], /worker schema array is missing or malformed/); +}); + +test('a CLI value absent from the worker allowlist is named', () => { + const v = missingFromWorker('template', ['vite', 'next'], ['vite'], 'w.ts'); + assert.equal(v.length, 1); + assert.match(v[0], /'next'/); + assert.match(v[0], /silently dropped/); + assert.deepEqual(missingFromWorker('template', ['vite'], ['vite'], 'w'), []); +}); + +test('constStringArray parses the worker shape and nulls on absence', () => { + const src = "const TEMPLATE_VALUES = ['vite', 'vite-ts'] as const;"; + assert.deepEqual(constStringArray(src, 'TEMPLATE_VALUES'), [ + 'vite', + 'vite-ts', + ]); + assert.equal(constStringArray(src, 'MISSING_VALUES'), null); +}); + +test('cssModes parses the cli shape and nulls when it stops matching', () => { + assert.deepEqual( + cssModes("const CSS_MODES: CssMode[] = ['bestax', 'keep'];"), + ['bestax', 'keep'] + ); + assert.equal(cssModes('const CSS_MODES = reshaped;'), null); +}); + +test('an unparseable source declaration is reported, not skipped', async t => { + const dir = mkdtempSync(join(tmpdir(), 'bestax-sources-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + + mkdirSync(join(dir, 'good')); + writeFileSync( + join(dir, 'good', 'index.ts'), + "export const good: MigrationSource = {\n name: 'good-source',\n};\n" + ); + mkdirSync(join(dir, 'reshaped')); + writeFileSync( + join(dir, 'reshaped', 'index.ts'), + "export const bad = defineSource('bad-source');\n" + ); + mkdirSync(join(dir, 'not-a-source')); + + const { names, unparsed } = await migrateSourceNames(dir); + assert.deepEqual(names, ['good-source']); + assert.deepEqual(unparsed, ['reshaped']); +}); From ec0aae3825b0d6dd0312da662c877947527032b2 Mon Sep 17 00:00:00 2001 From: Alex Smith Date: Sun, 23 Aug 2026 16:33:12 -0400 Subject: [PATCH 29/29] ci: deploy wrangler from the workspace lockfile; dedupe worker tasks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit wrangler-action installed its own wrangler with npm at deploy time, resolving transitive ranges fresh and outside the lockfile, cooldown, audit, and blocked-install-script policy — the exact controls the repo builds around dependencies. The deploy job now installs with --frozen-lockfile and runs the workspace-pinned wrangler directly, which also removes wrangler-action from the supply chain. Root test/typecheck/all drop their explicit telemetry-worker invocations: turbo has owned those tasks since the package joined the workspace. --- .github/workflows/deploy-worker.yml | 39 +++++++++++++++++++---------- package.json | 6 ++--- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/.github/workflows/deploy-worker.yml b/.github/workflows/deploy-worker.yml index f715c60c0..4f8e3a5af 100644 --- a/.github/workflows/deploy-worker.yml +++ b/.github/workflows/deploy-worker.yml @@ -34,9 +34,11 @@ jobs: steps: # Rule 10 (.github/CLAUDE.md): new jobs ship with egress-policy: block — # this is the job holding the Cloudflare deploy token. Endpoints: - # checkout (github.com trio), wrangler install (npm registry), and the - # upload itself (api.cloudflare.com). Wrangler's own metrics beacon is - # disabled below instead of allow-listing its host. + # checkout (github.com trio), the pnpm/node toolchain (npm registry, + # nodejs.org + release assets for a toolcache miss), and the upload + # itself (api.cloudflare.com). Wrangler's own metrics beacon is disabled + # below instead of allow-listing its host. No actions-cache endpoints on + # purpose: setup-node runs cache-less here to keep this list short. - name: Harden runner uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: @@ -46,6 +48,8 @@ jobs: api.github.com:443 github.com:443 objects.githubusercontent.com:443 + release-assets.githubusercontent.com:443 + nodejs.org:443 registry.npmjs.org:443 - name: Checkout code @@ -55,16 +59,25 @@ jobs: # the local git config for the rest of the job. persist-credentials: false - # No pnpm/Node setup: the worker is stdlib-only TypeScript with no build - # step of its own — wrangler-action installs the pinned wrangler and its - # bundler does the rest. + - name: Setup pnpm + uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: '24' + + # Wrangler comes from the workspace lockfile — the same pinned version, + # cooldown, audit, and blocked-install-script policy that governs every + # other dependency. wrangler-action would instead `npm i` its own copy + # at deploy time, resolving wrangler's transitive ranges fresh and + # outside all of those controls. + - name: Install dependencies + run: pnpm install --frozen-lockfile + - name: Deploy with wrangler - uses: cloudflare/wrangler-action@ebbaa1584979971c8614a24965b4405ff95890e0 # v4 env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_WORKERS_DEPLOY_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} WRANGLER_SEND_METRICS: 'false' - with: - apiToken: ${{ secrets.CLOUDFLARE_WORKERS_DEPLOY_TOKEN }} - accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} - workingDirectory: telemetry-worker - wranglerVersion: '4.124.0' - command: deploy + run: pnpm --filter telemetry-worker exec wrangler deploy diff --git a/package.json b/package.json index 82b6d02b2..8f8a0dcdd 100644 --- a/package.json +++ b/package.json @@ -6,10 +6,10 @@ "scripts": { "build": "turbo run build", "dev": "turbo run dev", - "test": "turbo run test && node --test \"scripts/*.test.mjs\" && node --test \"telemetry-worker/src/__tests__/*.test.ts\"", + "test": "turbo run test && node --test \"scripts/*.test.mjs\"", "test:coverage": "turbo run test:coverage", "bundle:stats": "turbo run bundle:stats", - "typecheck": "turbo run typecheck && tsc -p telemetry-worker/tsconfig.json --noEmit", + "typecheck": "turbo run typecheck", "format": "prettier --write \"**/*.{ts,tsx,js,jsx,mjs,md,mdx}\"", "format:check": "prettier --check \"**/*.{ts,tsx,js,jsx,mjs,md,mdx}\"", "lint": "turbo run lint && eslint scripts docs/scripts", @@ -23,7 +23,7 @@ "gen:mcp": "node scripts/gen-mcp-index.mjs", "gen:mcp:check": "node scripts/gen-mcp-index.mjs && git add --intent-to-add --all bestax-mcp/data && git diff --exit-code -- bestax-mcp/data", "gen": "pnpm gen:api-docs && pnpm gen:catalog && pnpm gen:mcp", - "all": "turbo run build typecheck test test:coverage bundle:stats && node --test \"scripts/*.test.mjs\" && node --test \"telemetry-worker/src/__tests__/*.test.ts\" && tsc -p telemetry-worker/tsconfig.json --noEmit && pnpm run lint && pnpm run format:check && turbo run build-storybook --filter=@allxsmith/bestax-bulma", + "all": "turbo run build typecheck test test:coverage bundle:stats && node --test \"scripts/*.test.mjs\" && pnpm run lint && pnpm run format:check && turbo run build-storybook --filter=@allxsmith/bestax-bulma", "storybook": "turbo run storybook --filter=@allxsmith/bestax-bulma", "docs": "turbo run docs --filter=@allxsmith/bestax-docs", "release": "turbo run release",