feat(dry-run): implement /internal/scheduler/dry-run-replay (was a v0.1 stub)

src/server/index.ts

@@ -354,10 +354,27 @@ app.post('/internal/menu/dashboard', async (c) => {
     recent.push({ ...h, id: String(m.member) });
   }
 
+  // Dry-run results for the draft rules (written by /internal/scheduler/dry-run-replay).
+  const dryRunLines: string[] = [];
+  for (const r of draft?.rules ?? []) {
+    const raw = await redis.get(`${subredditName}:dryrun:${r.id}`);
+    if (!raw) continue;
+    const d = JSON.parse(raw) as DryRunResult;
+    if (d.status === 'ok') {
+      dryRunLines.push(
+        `  ${r.id}: would match ${d.matched.length}/${d.sampledPosts} recent post(s)` +
+          (d.matched.length ? ` → ${[...new Set(d.matched.flatMap((m) => m.would))].join(', ')}` : ''),
+      );
+    } else {
+      dryRunLines.push(`  ${r.id}: ${d.note ?? 'dry-run unavailable'}`);
+    }
+  }
+
   const summary = [
     `Active rules: ${active?.rules.length ?? 0}`,
     `Draft rules: ${draft?.rules.length ?? 0}`,
     `Recent actions: ${recent.length}`,
+    ...(dryRunLines.length ? ['', 'Dry-run preview (draft rules):', ...dryRunLines] : []),
     '',
     'Recent actions:',
     ...recent.slice(0, 10).map((r) => `  ${r.action} (${r.outcome}) — ${(r.ruleSourceNL ?? '').slice(0, 60)}…`),
@@ -598,10 +615,89 @@ app.post('/internal/scheduler/audit-retention', async (c) => {
   return c.json<TaskResponse>({ status: 'ok' });
 });
 
+// Dry-run preview (hard lock #3 — forced before Activate). When a rule is
+// compiled into the draft, this job runs immediately, replays the *last few
+// posts* through the draft rule (no actions taken — pure evaluation), and writes
+// a `${sub}:dryrun:${ruleId}` summary the Dashboard renders. v0.1 samples posts
+// only (no `getNewComments` in the SDK); a comment-only rule gets a "shadow-mode
+// it to see real comments" note.
+const DRY_RUN_SAMPLE = 10; // recent posts to replay (×~3 Reddit API calls each via the fact bag)
+const DRY_RUN_TTL_SECONDS = 7 * 24 * 60 * 60;
+
+interface DryRunResult {
+  ruleId: string;
+  ruleSourceNL: string;
+  ranAt: number;
+  status: 'ok' | 'unavailable';
+  note?: string;
+  sampledPosts: number;
+  matched: Array<{ thingId: string; thingType: 'post'; authorName: string; would: string[] }>;
+}
+
 app.post('/internal/scheduler/dry-run-replay', async (c) => {
-  await c.req.json<TaskRequest<{ ruleId: string; subredditName: string }>>();
-  // v0.1: simplified — pull last 100 posts and log what would have happened.
-  // Full implementation reads rules:draft, builds factBags, evaluates, writes simulated audits.
+  const body = await c.req.json<TaskRequest<{ ruleId: string; subredditName: string }>>();
+  const ruleId = body.data?.ruleId;
+  const sub = getCurrentSubredditName();
+  if (!ruleId) return c.json<TaskResponse>({ status: 'ok' });
+
+  const result: DryRunResult = {
+    ruleId,
+    ruleSourceNL: '',
+    ranAt: Date.now(),
+    status: 'ok',
+    sampledPosts: 0,
+    matched: [],
+  };
+  try {
+    const draftJson = await redis.get(`${sub}:rules:draft`);
+    const rule = draftJson ? RuleBundle.parse(JSON.parse(draftJson)).rules.find((r) => r.id === ruleId) : undefined;
+    if (!rule) {
+      result.status = 'unavailable';
+      result.note = `Rule ${ruleId} is no longer in the draft.`;
+    } else {
+      result.ruleSourceNL = rule.sourceNL;
+      const postTrigger = rule.on.includes('onPostSubmit')
+        ? 'onPostSubmit'
+        : rule.on.includes('onPostReport')
+          ? 'onPostReport'
+          : null;
+      if (!postTrigger) {
+        result.status = 'unavailable';
+        result.note =
+          'This rule listens to comment events. v0.1 dry-run replays recent posts only — activate it (shadow mode is ON by default) to see how it behaves on real comments.';
+      } else {
+        const posts = await reddit.getNewPosts({ subredditName: sub, limit: DRY_RUN_SAMPLE }).all();
+        for (const p of posts) {
+          result.sampledPosts++;
+          const facts = await buildPostFactBag(
+            {
+              id: p.id,
+              title: p.title,
+              body: p.body ?? '',
+              url: p.url,
+              authorId: p.authorId ?? 't2_unknown',
+              authorName: p.authorName,
+            },
+            p.numberOfReports ?? 0,
+          );
+          if (selectMatchingRules([rule], postTrigger, facts).length > 0) {
+            result.matched.push({
+              thingId: p.id,
+              thingType: 'post',
+              authorName: p.authorName,
+              would: rule.then.map((a) => a.action),
+            });
+          }
+        }
+      }
+    }
+  } catch (err) {
+    result.status = 'unavailable';
+    result.note = `Dry-run replay couldn't complete (${String(err).slice(0, 120)}). Activate in shadow mode to observe live behaviour instead.`;
+  }
+
+  await redis.set(`${sub}:dryrun:${ruleId}`, JSON.stringify(result));
+  await redis.expire(`${sub}:dryrun:${ruleId}`, DRY_RUN_TTL_SECONDS);
   return c.json<TaskResponse>({ status: 'ok' });
 });
 

src/server/routes-dashboard.test.ts

@@ -73,6 +73,43 @@ describe('POST /internal/menu/dashboard', () => {
     expect(body.showForm.form.description).toContain('remove (shadow)');
     expect(body.showForm.form.acceptLabel).toBe('Activate 5 draft rule(s)');
   });
+
+  it('shows the dry-run preview for draft rules that have a stored result', async () => {
+    asMod();
+    await fakeRedis.set('testsub:rules:draft', JSON.stringify(seedStarterRules(7)));
+    await fakeRedis.set(
+      'testsub:dryrun:r_new_account_fast_post',
+      JSON.stringify({
+        ruleId: 'r_new_account_fast_post',
+        ruleSourceNL: '…',
+        ranAt: Date.now(),
+        status: 'ok',
+        sampledPosts: 10,
+        matched: [{ thingId: 't3_x', thingType: 'post', authorName: 'newbie', would: ['modqueue'] }],
+      }),
+    );
+    await fakeRedis.set(
+      'testsub:dryrun:r_wall_of_caps_comment',
+      JSON.stringify({
+        ruleId: 'r_wall_of_caps_comment',
+        ruleSourceNL: '…',
+        ranAt: Date.now(),
+        status: 'unavailable',
+        note: 'comment events; shadow mode it',
+        sampledPosts: 0,
+        matched: [],
+      }),
+    );
+
+    const body = await (
+      await call('/internal/menu/dashboard', { location: 'subreddit', targetId: 't5_testsub' })
+    ).json();
+    expect(body.showForm.form.description).toContain('Dry-run preview (draft rules):');
+    expect(body.showForm.form.description).toContain(
+      'r_new_account_fast_post: would match 1/10 recent post(s) → modqueue',
+    );
+    expect(body.showForm.form.description).toContain('r_wall_of_caps_comment: comment events; shadow mode it');
+  });
 });
 
 describe('POST /internal/form/dashboard-action', () => {

src/server/routes-scheduler.test.ts

@@ -1,10 +1,10 @@
 // src/server/routes-scheduler.test.ts
 // Functional call-tests for the cron-scheduled jobs: audit retention, dry-run
-// replay (stub), shadow→live promotion, and the actions/hour circuit breaker.
+// replay, shadow→live promotion, and the actions/hour circuit breaker.
 
 import { describe, it, expect } from 'vitest';
 import app from './index';
-import { fakeRedis, fakeReddit, fakeSettings } from '../../test/setup';
+import { fakeRedis, fakeReddit, fakeSettings, fakeListing } from '../../test/setup';
 import { Rule, RuleBundle, type RuleType } from '../shared/rule-schema';
 
 const call = (path: string, body: unknown = {}) =>
@@ -60,12 +60,84 @@ describe('POST /internal/scheduler/audit-retention', () => {
 });
 
 describe('POST /internal/scheduler/dry-run-replay', () => {
-  it('acknowledges (v0.1 stub)', async () => {
-    expect(
-      await (
-        await call('/internal/scheduler/dry-run-replay', { data: { ruleId: 'r_x', subredditName: 'testsub' } })
-      ).json(),
-    ).toEqual({ status: 'ok' });
+  const post = (id: string, over: Record<string, unknown> = {}) => ({
+    id,
+    title: 'a title',
+    body: 'a body',
+    url: 'https://example.com',
+    authorId: 't2_a',
+    authorName: 'alice',
+    numberOfReports: 0,
+    ...over,
+  });
+
+  it('writes an "unavailable" summary when the rule is no longer in the draft', async () => {
+    await call('/internal/scheduler/dry-run-replay', { data: { ruleId: 'r_gone', subredditName: 'testsub' } });
+    const d = JSON.parse((await fakeRedis.get('testsub:dryrun:r_gone'))!);
+    expect(d.status).toBe('unavailable');
+    expect(d.note).toMatch(/no longer in the draft/i);
+  });
+
+  it('replays recent posts through a draft post-rule and records which would match', async () => {
+    // rule: matches authors with <50 karma (the mocked author defaults to 0 karma → matches)
+    await fakeRedis.set(
+      'testsub:rules:draft',
+      JSON.stringify(
+        bundleWith({
+          id: 'r_lowk',
+          when: { fact: 'author.totalKarma', op: 'lt', value: 50 },
+          then: [{ action: 'modqueue', params: { note: 'x' } }],
+        }),
+      ),
+    );
+    fakeReddit.getNewPosts.mockReturnValue(fakeListing([post('t3_a'), post('t3_b')]));
+
+    await call('/internal/scheduler/dry-run-replay', { data: { ruleId: 'r_lowk', subredditName: 'testsub' } });
+    const d = JSON.parse((await fakeRedis.get('testsub:dryrun:r_lowk'))!);
+    expect(d.status).toBe('ok');
+    expect(d.sampledPosts).toBe(2);
+    expect(d.matched.map((m: { thingId: string }) => m.thingId)).toEqual(['t3_a', 't3_b']);
+    expect(d.matched[0].would).toEqual(['modqueue']);
+  });
+
+  it('records zero matches when no recent post satisfies the rule', async () => {
+    await fakeRedis.set(
+      'testsub:rules:draft',
+      JSON.stringify(bundleWith({ id: 'r_hik', when: { fact: 'author.totalKarma', op: 'gt', value: 1_000_000 } })),
+    );
+    fakeReddit.getNewPosts.mockReturnValue(fakeListing([post('t3_x')]));
+    await call('/internal/scheduler/dry-run-replay', { data: { ruleId: 'r_hik', subredditName: 'testsub' } });
+    const d = JSON.parse((await fakeRedis.get('testsub:dryrun:r_hik'))!);
+    expect(d.status).toBe('ok');
+    expect(d.sampledPosts).toBe(1);
+    expect(d.matched).toEqual([]);
+  });
+
+  it('marks a comment-only rule "unavailable" with a shadow-mode hint', async () => {
+    await fakeRedis.set('testsub:rules:draft', JSON.stringify(bundleWith({ id: 'r_cmt', on: ['onCommentSubmit'] })));
+    await call('/internal/scheduler/dry-run-replay', { data: { ruleId: 'r_cmt', subredditName: 'testsub' } });
+    const d = JSON.parse((await fakeRedis.get('testsub:dryrun:r_cmt'))!);
+    expect(d.status).toBe('unavailable');
+    expect(d.note).toMatch(/comment events.*shadow mode/is);
+  });
+
+  it('degrades gracefully (status unavailable, never throws) when the Reddit API fails', async () => {
+    await fakeRedis.set('testsub:rules:draft', JSON.stringify(bundleWith({ id: 'r_err' })));
+    fakeReddit.getNewPosts.mockImplementation(() => {
+      throw new Error('reddit 503');
+    });
+    const res = await call('/internal/scheduler/dry-run-replay', {
+      data: { ruleId: 'r_err', subredditName: 'testsub' },
+    });
+    expect(await res.json()).toEqual({ status: 'ok' });
+    const d = JSON.parse((await fakeRedis.get('testsub:dryrun:r_err'))!);
+    expect(d.status).toBe('unavailable');
+    expect(d.note).toContain('503');
+  });
+
+  it('is a no-op when no ruleId is supplied', async () => {
+    expect(await (await call('/internal/scheduler/dry-run-replay', { data: {} })).json()).toEqual({ status: 'ok' });
+    expect(fakeReddit.getNewPosts).not.toHaveBeenCalled();
   });
 });
 

test/devvit-testkit.ts

@@ -140,6 +140,9 @@ export function makeFakeReddit(subName = 'testsub', subId = `t5_${subName}` as `
       async () => ({ fromComments: 0, fromPosts: 0 }) as { fromComments?: number; fromPosts?: number },
     ),
     getModerators: vi.fn(async (_opts: { subredditName: string }) => fakeListing([] as Array<{ username: string }>)),
+    getNewPosts: vi.fn((_opts: { subredditName?: string; limit?: number }) =>
+      fakeListing([] as Array<Record<string, unknown>>),
+    ),
     getPostById: vi.fn(),
     getCommentById: vi.fn(),
     report: vi.fn(async () => ({}) as unknown),

test/setup.ts

@@ -55,6 +55,7 @@ beforeEach(() => {
   fakeReddit.getUserByUsername.mockResolvedValue(null);
   fakeReddit.getUserKarmaFromCurrentSubreddit.mockResolvedValue({ fromComments: 0, fromPosts: 0 });
   fakeReddit.getModerators.mockResolvedValue({ all: async () => [] });
+  fakeReddit.getNewPosts.mockReturnValue({ all: async () => [] });
   fakeReddit.report.mockResolvedValue({});
   fakeReddit.setPostFlair.mockResolvedValue(undefined);
   fakeReddit.banUser.mockResolvedValue(undefined);