feat: add public anonymous one-page check as the proof-to-repair entry path #66
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -21,6 +21,12 @@ import { | |
| supportHtml, | ||
| termsHtml | ||
| } from "../worker/routes/pages.js"; | ||
| import { | ||
| buildPublicCheckResponse, | ||
| checkHtml, | ||
| publicCheckQuotaChecks, | ||
| validatePublicCheckUrl | ||
| } from "../worker/routes/public-check.js"; | ||
| import { | ||
| backlinkRowsKey, | ||
| parseBacklinkRows | ||
|
|
@@ -43,6 +49,7 @@ import { | |
| buildCrawlInventory | ||
| } from "../shared/crawl-inventory.js"; | ||
| import { resolvesToPrivateAddress } from "../shared/url-safety.js"; | ||
| import { requestIpHash } from "../worker/lib/security.js"; | ||
| import { | ||
| agentActionResponse, | ||
| apiRepairQueueStatusResponse, | ||
|
|
@@ -130,6 +137,28 @@ const localRepairQueueRows = new Map(); | |
| const localRepairActionRows = new Map(); | ||
| const siteClaims = new Map(); | ||
| const fixRequests = []; | ||
| // In-memory rate-limit counters for the anonymous one-page check, mirroring | ||
| // the Worker's D1-backed buckets (worker/routes/public-check.js). Entries | ||
| // expire by age so a long-running dev server does not leak memory. | ||
| const publicCheckQuotaCounts = new Map(); | ||
|
|
||
| function localPublicCheckQuota(checks) { | ||
| const now = Date.now(); | ||
| for (const check of checks) { | ||
| const entry = publicCheckQuotaCounts.get(check.bucket) || { count: 0, createdAt: now }; | ||
| if (entry.count >= check.limit) { | ||
| return { ok: false, error: check.error, resetAt: check.resetAt.toISOString() }; | ||
| } | ||
| entry.count += 1; | ||
| publicCheckQuotaCounts.set(check.bucket, entry); | ||
| } | ||
| for (const [key, entry] of publicCheckQuotaCounts) { | ||
| if (now - entry.createdAt > 48 * 60 * 60 * 1000) { | ||
| publicCheckQuotaCounts.delete(key); | ||
| } | ||
| } | ||
| return { ok: true }; | ||
| } | ||
| const VERSION = "0.9.0"; | ||
| const SESSION_COOKIE = "sfk_beta_session"; | ||
| const ADMIN_SESSION_COOKIE = "sfk_admin_session"; | ||
|
|
@@ -1791,6 +1820,11 @@ app.get("/demo", (req, res) => { | |
| res.set("content-type", "text/html; charset=utf-8").send(demoHtml(origin)); | ||
| }); | ||
|
|
||
| app.get("/check", (req, res) => { | ||
| const origin = `http://${req.get("host")}`; | ||
| res.set("content-type", "text/html; charset=utf-8").send(checkHtml(origin)); | ||
| }); | ||
|
|
||
| app.get("/methodology", (req, res) => { | ||
| const origin = `http://${req.get("host")}`; | ||
| res.set("content-type", "text/html; charset=utf-8").send(methodologyHtml(origin)); | ||
|
|
@@ -1845,6 +1879,39 @@ app.get("/api/demo-audit", async (req, res) => { | |
| } | ||
| }); | ||
|
|
||
| app.post("/api/public-check", async (req, res) => { | ||
| try { | ||
| const body = req.body || {}; | ||
| const input = typeof body.url === "string" ? body.url : ""; | ||
| const validated = validatePublicCheckUrl(input); | ||
| if (!validated.ok) { | ||
| res.status(400).set("cache-control", "no-store").json({ error: validated.error }); | ||
| return; | ||
| } | ||
| const hostname = new URL(validated.url).hostname; | ||
| if (await resolvesToPrivateAddress(hostname)) { | ||
| res.status(400).set("cache-control", "no-store").json({ | ||
| error: "This URL points at a private or internal address and cannot be checked." | ||
| }); | ||
| return; | ||
| } | ||
| const ipHash = await requestIpHash({ | ||
| headers: { get: (name) => String(req.headers[name.toLowerCase()] || "") } | ||
| }); | ||
| const quota = localPublicCheckQuota(publicCheckQuotaChecks(ipHash, hostname)); | ||
| if (!quota.ok) { | ||
| res.status(429).set("cache-control", "no-store").json({ error: quota.error, resetAt: quota.resetAt }); | ||
| return; | ||
| } | ||
| const origin = `http://${req.get("host")}`; | ||
| const report = await auditUrl(validated.url, { maxPages: 1, appOrigin: origin }); | ||
| res.set("cache-control", "no-store").json(buildPublicCheckResponse(report)); | ||
| } catch (error) { | ||
| const message = String(error?.message || "The check failed. Try another public URL.").slice(0, 260); | ||
| res.status(422).set("cache-control", "no-store").json({ error: message }); | ||
| } | ||
| }); | ||
|
|
||
| app.post("/api/audit", async (req, res) => { | ||
| try { | ||
| const access = localBetaAccess(req); | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2874,7 +2874,7 @@ function wait(ms) { | |
| } | ||
|
|
||
| export function rootSitemap(origin) { | ||
| const urls = ["/", "/demo", "/check", "/methodology", "/packages", "/privacy", "/support", "/terms"]; | ||
| return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">${urls | ||
| .map((path) => `<url><loc>${origin}${path}</loc></url>`) | ||
| .join("")}</urlset>`; | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -109,7 +109,7 @@ test("README weekly monitor promise matches the schedule interval", () => { | |
|
|
||
| test("README public page promise matches Worker routing and copy", () => { | ||
| assert.match(liveSection, /Public `\/demo`, `\/methodology`, and `\/packages` pages/i); | ||
| for (const path of ["/demo", "/methodology", "/packages", "/check"]) { | ||
| assert.ok(workerIndex.includes(`url.pathname === "${path}"`), `Worker must route ${path}`); | ||
| } | ||
| assert.match(pagesSource, /FIX_PACK_PUBLIC_PRICE/, "packages page price constant exists"); | ||
|
|
@@ -156,6 +156,8 @@ test("README Cloudflare path routes are actually registered in the Worker", () = | |
| "/demo", | ||
| "/methodology", | ||
| "/packages", | ||
| "/check", | ||
| "/api/public-check", | ||
| "/beta" | ||
| ]; | ||
| for (const route of claimedRoutes) { | ||
|
|
@@ -165,12 +167,26 @@ test("README Cloudflare path routes are actually registered in the Worker", () = | |
|
|
||
| test("README abuse-control claim matches D1 buckets for every listed surface", () => { | ||
| assert.match(liveSection, /D1-backed abuse controls/i); | ||
| for (const bucket of ["waitlist:ip", "login:ip", "access:ip", "audit:ip", "audit:session", "audit:target", "audit:lite-day", "check:ip-hour", "check:target-hour"]) { | ||
| let source = auditsSource; | ||
| if (bucket.startsWith("audit")) source = auditsSource; | ||
| else if (bucket.startsWith("check")) source = readFileSync(new URL("../worker/routes/public-check.js", import.meta.url), "utf8"); | ||
| else source = readFileSync(new URL("../worker/routes/access.js", import.meta.url), "utf8"); | ||
| assert.ok(source.includes(`bucket: \`${bucket}`), `abuse control must cover ${bucket}`); | ||
| } | ||
| }); | ||
|
|
||
| test("README anonymous one-page check claim matches the Worker, page, and rate limits", () => { | ||
| assert.match(liveSection, /anonymous one-page URL check/i); | ||
| assert.ok(workerIndex.includes('url.pathname === "/check"'), "Worker must route /check"); | ||
| assert.ok(workerIndex.includes('url.pathname === "/api/public-check"'), "Worker must route /api/public-check"); | ||
| const publicCheckSource = readFileSync(new URL("../worker/routes/public-check.js", import.meta.url), "utf8"); | ||
| assert.match(publicCheckSource, /maxPages: 1/, "the public check is a one-page run"); | ||
| assert.match(publicCheckSource, /check:target-day/, "the check is rate-limited per site"); | ||
| assert.match(publicCheckSource, /does not guarantee rankings, traffic, indexing, revenue, AI citations/i, "the public check keeps the no-ranking boundary"); | ||
| assert.match(wranglerJsonc, /"\/check"/, "/check is served by the Worker before SPA assets"); | ||
| }); | ||
|
|
||
| // State-truthfulness pins. The numeric caps and routes above are locked to | ||
| // code; these claims describe what is NOT live or how limits are computed, so | ||
| // they were spot-checked once (PR #60) and are locked here so the README | ||
|
|
||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🔒 Security & Privacy | 🟠 Major | ⚡ Quick win
Narrow the storage claim.
The rate-limit design stores hashed network and target identifiers in
audit_usage, andREADME.md, Line 57 documents quota-bucket cleanup. The phrasenothing storedcan mislead users about retention. State that no report or page contents are stored. Apply the same wording topublic/.well-known/skill.md, Line 10, and the/checkcopy inworker/routes/public-check.js, Lines 192-351.🤖 Prompt for AI Agents