diff --git a/src/server/index.ts b/src/server/index.ts index 4da7ad6..76dcbb9 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -354,10 +354,27 @@ app.post('/internal/menu/dashboard', async (c) => { recent.push({ ...h, id: String(m.member) }); } + // Dry-run results for the draft rules (written by /internal/scheduler/dry-run-replay). + const dryRunLines: string[] = []; + for (const r of draft?.rules ?? []) { + const raw = await redis.get(`${subredditName}:dryrun:${r.id}`); + if (!raw) continue; + const d = JSON.parse(raw) as DryRunResult; + if (d.status === 'ok') { + dryRunLines.push( + ` ${r.id}: would match ${d.matched.length}/${d.sampledPosts} recent post(s)` + + (d.matched.length ? ` → ${[...new Set(d.matched.flatMap((m) => m.would))].join(', ')}` : ''), + ); + } else { + dryRunLines.push(` ${r.id}: ${d.note ?? 'dry-run unavailable'}`); + } + } + const summary = [ `Active rules: ${active?.rules.length ?? 0}`, `Draft rules: ${draft?.rules.length ?? 0}`, `Recent actions: ${recent.length}`, + ...(dryRunLines.length ? ['', 'Dry-run preview (draft rules):', ...dryRunLines] : []), '', 'Recent actions:', ...recent.slice(0, 10).map((r) => ` ${r.action} (${r.outcome}) — ${(r.ruleSourceNL ?? '').slice(0, 60)}…`), @@ -598,10 +615,89 @@ app.post('/internal/scheduler/audit-retention', async (c) => { return c.json({ status: 'ok' }); }); +// Dry-run preview (hard lock #3 — forced before Activate). When a rule is +// compiled into the draft, this job runs immediately, replays the *last few +// posts* through the draft rule (no actions taken — pure evaluation), and writes +// a `${sub}:dryrun:${ruleId}` summary the Dashboard renders. v0.1 samples posts +// only (no `getNewComments` in the SDK); a comment-only rule gets a "shadow-mode +// it to see real comments" note. +const DRY_RUN_SAMPLE = 10; // recent posts to replay (×~3 Reddit API calls each via the fact bag) +const DRY_RUN_TTL_SECONDS = 7 * 24 * 60 * 60; + +interface DryRunResult { + ruleId: string; + ruleSourceNL: string; + ranAt: number; + status: 'ok' | 'unavailable'; + note?: string; + sampledPosts: number; + matched: Array<{ thingId: string; thingType: 'post'; authorName: string; would: string[] }>; +} + app.post('/internal/scheduler/dry-run-replay', async (c) => { - await c.req.json>(); - // v0.1: simplified — pull last 100 posts and log what would have happened. - // Full implementation reads rules:draft, builds factBags, evaluates, writes simulated audits. + const body = await c.req.json>(); + const ruleId = body.data?.ruleId; + const sub = getCurrentSubredditName(); + if (!ruleId) return c.json({ status: 'ok' }); + + const result: DryRunResult = { + ruleId, + ruleSourceNL: '', + ranAt: Date.now(), + status: 'ok', + sampledPosts: 0, + matched: [], + }; + try { + const draftJson = await redis.get(`${sub}:rules:draft`); + const rule = draftJson ? RuleBundle.parse(JSON.parse(draftJson)).rules.find((r) => r.id === ruleId) : undefined; + if (!rule) { + result.status = 'unavailable'; + result.note = `Rule ${ruleId} is no longer in the draft.`; + } else { + result.ruleSourceNL = rule.sourceNL; + const postTrigger = rule.on.includes('onPostSubmit') + ? 'onPostSubmit' + : rule.on.includes('onPostReport') + ? 'onPostReport' + : null; + if (!postTrigger) { + result.status = 'unavailable'; + result.note = + 'This rule listens to comment events. v0.1 dry-run replays recent posts only — activate it (shadow mode is ON by default) to see how it behaves on real comments.'; + } else { + const posts = await reddit.getNewPosts({ subredditName: sub, limit: DRY_RUN_SAMPLE }).all(); + for (const p of posts) { + result.sampledPosts++; + const facts = await buildPostFactBag( + { + id: p.id, + title: p.title, + body: p.body ?? '', + url: p.url, + authorId: p.authorId ?? 't2_unknown', + authorName: p.authorName, + }, + p.numberOfReports ?? 0, + ); + if (selectMatchingRules([rule], postTrigger, facts).length > 0) { + result.matched.push({ + thingId: p.id, + thingType: 'post', + authorName: p.authorName, + would: rule.then.map((a) => a.action), + }); + } + } + } + } + } catch (err) { + result.status = 'unavailable'; + result.note = `Dry-run replay couldn't complete (${String(err).slice(0, 120)}). Activate in shadow mode to observe live behaviour instead.`; + } + + await redis.set(`${sub}:dryrun:${ruleId}`, JSON.stringify(result)); + await redis.expire(`${sub}:dryrun:${ruleId}`, DRY_RUN_TTL_SECONDS); return c.json({ status: 'ok' }); }); diff --git a/src/server/routes-dashboard.test.ts b/src/server/routes-dashboard.test.ts index 9abdeaf..46f9167 100644 --- a/src/server/routes-dashboard.test.ts +++ b/src/server/routes-dashboard.test.ts @@ -73,6 +73,43 @@ describe('POST /internal/menu/dashboard', () => { expect(body.showForm.form.description).toContain('remove (shadow)'); expect(body.showForm.form.acceptLabel).toBe('Activate 5 draft rule(s)'); }); + + it('shows the dry-run preview for draft rules that have a stored result', async () => { + asMod(); + await fakeRedis.set('testsub:rules:draft', JSON.stringify(seedStarterRules(7))); + await fakeRedis.set( + 'testsub:dryrun:r_new_account_fast_post', + JSON.stringify({ + ruleId: 'r_new_account_fast_post', + ruleSourceNL: '…', + ranAt: Date.now(), + status: 'ok', + sampledPosts: 10, + matched: [{ thingId: 't3_x', thingType: 'post', authorName: 'newbie', would: ['modqueue'] }], + }), + ); + await fakeRedis.set( + 'testsub:dryrun:r_wall_of_caps_comment', + JSON.stringify({ + ruleId: 'r_wall_of_caps_comment', + ruleSourceNL: '…', + ranAt: Date.now(), + status: 'unavailable', + note: 'comment events; shadow mode it', + sampledPosts: 0, + matched: [], + }), + ); + + const body = await ( + await call('/internal/menu/dashboard', { location: 'subreddit', targetId: 't5_testsub' }) + ).json(); + expect(body.showForm.form.description).toContain('Dry-run preview (draft rules):'); + expect(body.showForm.form.description).toContain( + 'r_new_account_fast_post: would match 1/10 recent post(s) → modqueue', + ); + expect(body.showForm.form.description).toContain('r_wall_of_caps_comment: comment events; shadow mode it'); + }); }); describe('POST /internal/form/dashboard-action', () => { diff --git a/src/server/routes-scheduler.test.ts b/src/server/routes-scheduler.test.ts index bc6ad1e..202574c 100644 --- a/src/server/routes-scheduler.test.ts +++ b/src/server/routes-scheduler.test.ts @@ -1,10 +1,10 @@ // src/server/routes-scheduler.test.ts // Functional call-tests for the cron-scheduled jobs: audit retention, dry-run -// replay (stub), shadow→live promotion, and the actions/hour circuit breaker. +// replay, shadow→live promotion, and the actions/hour circuit breaker. import { describe, it, expect } from 'vitest'; import app from './index'; -import { fakeRedis, fakeReddit, fakeSettings } from '../../test/setup'; +import { fakeRedis, fakeReddit, fakeSettings, fakeListing } from '../../test/setup'; import { Rule, RuleBundle, type RuleType } from '../shared/rule-schema'; const call = (path: string, body: unknown = {}) => @@ -60,12 +60,84 @@ describe('POST /internal/scheduler/audit-retention', () => { }); describe('POST /internal/scheduler/dry-run-replay', () => { - it('acknowledges (v0.1 stub)', async () => { - expect( - await ( - await call('/internal/scheduler/dry-run-replay', { data: { ruleId: 'r_x', subredditName: 'testsub' } }) - ).json(), - ).toEqual({ status: 'ok' }); + const post = (id: string, over: Record = {}) => ({ + id, + title: 'a title', + body: 'a body', + url: 'https://example.com', + authorId: 't2_a', + authorName: 'alice', + numberOfReports: 0, + ...over, + }); + + it('writes an "unavailable" summary when the rule is no longer in the draft', async () => { + await call('/internal/scheduler/dry-run-replay', { data: { ruleId: 'r_gone', subredditName: 'testsub' } }); + const d = JSON.parse((await fakeRedis.get('testsub:dryrun:r_gone'))!); + expect(d.status).toBe('unavailable'); + expect(d.note).toMatch(/no longer in the draft/i); + }); + + it('replays recent posts through a draft post-rule and records which would match', async () => { + // rule: matches authors with <50 karma (the mocked author defaults to 0 karma → matches) + await fakeRedis.set( + 'testsub:rules:draft', + JSON.stringify( + bundleWith({ + id: 'r_lowk', + when: { fact: 'author.totalKarma', op: 'lt', value: 50 }, + then: [{ action: 'modqueue', params: { note: 'x' } }], + }), + ), + ); + fakeReddit.getNewPosts.mockReturnValue(fakeListing([post('t3_a'), post('t3_b')])); + + await call('/internal/scheduler/dry-run-replay', { data: { ruleId: 'r_lowk', subredditName: 'testsub' } }); + const d = JSON.parse((await fakeRedis.get('testsub:dryrun:r_lowk'))!); + expect(d.status).toBe('ok'); + expect(d.sampledPosts).toBe(2); + expect(d.matched.map((m: { thingId: string }) => m.thingId)).toEqual(['t3_a', 't3_b']); + expect(d.matched[0].would).toEqual(['modqueue']); + }); + + it('records zero matches when no recent post satisfies the rule', async () => { + await fakeRedis.set( + 'testsub:rules:draft', + JSON.stringify(bundleWith({ id: 'r_hik', when: { fact: 'author.totalKarma', op: 'gt', value: 1_000_000 } })), + ); + fakeReddit.getNewPosts.mockReturnValue(fakeListing([post('t3_x')])); + await call('/internal/scheduler/dry-run-replay', { data: { ruleId: 'r_hik', subredditName: 'testsub' } }); + const d = JSON.parse((await fakeRedis.get('testsub:dryrun:r_hik'))!); + expect(d.status).toBe('ok'); + expect(d.sampledPosts).toBe(1); + expect(d.matched).toEqual([]); + }); + + it('marks a comment-only rule "unavailable" with a shadow-mode hint', async () => { + await fakeRedis.set('testsub:rules:draft', JSON.stringify(bundleWith({ id: 'r_cmt', on: ['onCommentSubmit'] }))); + await call('/internal/scheduler/dry-run-replay', { data: { ruleId: 'r_cmt', subredditName: 'testsub' } }); + const d = JSON.parse((await fakeRedis.get('testsub:dryrun:r_cmt'))!); + expect(d.status).toBe('unavailable'); + expect(d.note).toMatch(/comment events.*shadow mode/is); + }); + + it('degrades gracefully (status unavailable, never throws) when the Reddit API fails', async () => { + await fakeRedis.set('testsub:rules:draft', JSON.stringify(bundleWith({ id: 'r_err' }))); + fakeReddit.getNewPosts.mockImplementation(() => { + throw new Error('reddit 503'); + }); + const res = await call('/internal/scheduler/dry-run-replay', { + data: { ruleId: 'r_err', subredditName: 'testsub' }, + }); + expect(await res.json()).toEqual({ status: 'ok' }); + const d = JSON.parse((await fakeRedis.get('testsub:dryrun:r_err'))!); + expect(d.status).toBe('unavailable'); + expect(d.note).toContain('503'); + }); + + it('is a no-op when no ruleId is supplied', async () => { + expect(await (await call('/internal/scheduler/dry-run-replay', { data: {} })).json()).toEqual({ status: 'ok' }); + expect(fakeReddit.getNewPosts).not.toHaveBeenCalled(); }); }); diff --git a/test/devvit-testkit.ts b/test/devvit-testkit.ts index 0dfc9c7..71243b6 100644 --- a/test/devvit-testkit.ts +++ b/test/devvit-testkit.ts @@ -140,6 +140,9 @@ export function makeFakeReddit(subName = 'testsub', subId = `t5_${subName}` as ` async () => ({ fromComments: 0, fromPosts: 0 }) as { fromComments?: number; fromPosts?: number }, ), getModerators: vi.fn(async (_opts: { subredditName: string }) => fakeListing([] as Array<{ username: string }>)), + getNewPosts: vi.fn((_opts: { subredditName?: string; limit?: number }) => + fakeListing([] as Array>), + ), getPostById: vi.fn(), getCommentById: vi.fn(), report: vi.fn(async () => ({}) as unknown), diff --git a/test/setup.ts b/test/setup.ts index 17b1579..4d13333 100644 --- a/test/setup.ts +++ b/test/setup.ts @@ -55,6 +55,7 @@ beforeEach(() => { fakeReddit.getUserByUsername.mockResolvedValue(null); fakeReddit.getUserKarmaFromCurrentSubreddit.mockResolvedValue({ fromComments: 0, fromPosts: 0 }); fakeReddit.getModerators.mockResolvedValue({ all: async () => [] }); + fakeReddit.getNewPosts.mockReturnValue({ all: async () => [] }); fakeReddit.report.mockResolvedValue({}); fakeReddit.setPostFlair.mockResolvedValue(undefined); fakeReddit.banUser.mockResolvedValue(undefined);