From 23ebd208ad511061c3613d999595e7d2b5ad71a1 Mon Sep 17 00:00:00 2001 From: Abhineet Jain Date: Tue, 6 Oct 2026 16:30:26 +0530 Subject: [PATCH 1/5] feat(marketing): add six browser-only website tools --- docs/frontend-architecture.md | 13 ++ .../apps/marketing/src/pages/sitemap.xml.ts | 7 + .../marketing/src/pages/tools/[slug].astro | 40 +++++ .../marketing/src/pages/tools/index.astro | 7 + .../components/marketing/chrome/footer.tsx | 1 + .../marketing/tools/markup-tools.tsx | 111 +++++++++++++ .../marketing/tools/robots-tools.tsx | 116 ++++++++++++++ .../marketing/tools/sitemap-tool.tsx | 97 ++++++++++++ .../marketing/tools/social-tool.tsx | 116 ++++++++++++++ .../components/marketing/tools/tool-form.tsx | 146 ++++++++++++++++++ .../marketing/tools/tools-index.tsx | 79 ++++++++++ .../components/marketing/tools/tools.test.tsx | 46 ++++++ frontend/e2e/marketing-pages.spec.ts | 17 ++ frontend/lib/config/free-tools.ts | 8 + frontend/lib/free-tools/generators.ts | 90 +++++++++++ frontend/lib/free-tools/input.ts | 33 ++++ frontend/lib/free-tools/markup.ts | 60 +++++++ frontend/lib/free-tools/robots.ts | 57 +++++++ frontend/lib/free-tools/sitemap.ts | 48 ++++++ frontend/lib/free-tools/tools.test.ts | 114 ++++++++++++++ frontend/lib/marketing-content/nav.ts | 5 + frontend/lib/marketing-content/tools.ts | 60 +++++++ frontend/package.json | 1 + frontend/pnpm-lock.yaml | 3 + 24 files changed, 1275 insertions(+) create mode 100644 frontend/apps/marketing/src/pages/tools/[slug].astro create mode 100644 frontend/apps/marketing/src/pages/tools/index.astro create mode 100644 frontend/components/marketing/tools/markup-tools.tsx create mode 100644 frontend/components/marketing/tools/robots-tools.tsx create mode 100644 frontend/components/marketing/tools/sitemap-tool.tsx create mode 100644 frontend/components/marketing/tools/social-tool.tsx create mode 100644 frontend/components/marketing/tools/tool-form.tsx create mode 100644 frontend/components/marketing/tools/tools-index.tsx create mode 100644 frontend/components/marketing/tools/tools.test.tsx create mode 100644 frontend/lib/config/free-tools.ts create mode 100644 frontend/lib/free-tools/generators.ts create mode 100644 frontend/lib/free-tools/input.ts create mode 100644 frontend/lib/free-tools/markup.ts create mode 100644 frontend/lib/free-tools/robots.ts create mode 100644 frontend/lib/free-tools/sitemap.ts create mode 100644 frontend/lib/free-tools/tools.test.ts create mode 100644 frontend/lib/marketing-content/tools.ts diff --git a/docs/frontend-architecture.md b/docs/frontend-architecture.md index 8d4ea06f0..f34776fac 100644 --- a/docs/frontend-architecture.md +++ b/docs/frontend-architecture.md @@ -60,6 +60,19 @@ The documentation Worker serves assets only, with no Worker script, so Astro emi ## Routes and shared shell +The marketing `/tools` hub links to six server-rendered `/tools/:slug` pages: +crawler rules testing, robots.txt generation, meta/directive inspection, +structured data generation, sitemap comparison and social previews. These are +browser-only utilities over bounded pasted inputs and local files; they do not +fetch sites, call providers, persist results or put submissions in URLs. +The existing crawler catalog supplies public bot metadata and rules use the same +`robots-parser` dependency as acquisition. Tool metadata lives in +`lib/marketing-content/tools.ts`, browser admission in `lib/config/free-tools.ts`, +and local transformations in `lib/free-tools/`. These reports are not persisted +Site Health evidence or claims of live access, indexability or citation eligibility. +Social previews read local raster images; published image URLs only enter exported +markup. Sitemap indexes compare declared child sitemap addresses without recursion. + | Surface | Browser location | Feature owner | |---|---|---| | Overview and company facts | /projects | [Onboarding](onboarding.md) | diff --git a/frontend/apps/marketing/src/pages/sitemap.xml.ts b/frontend/apps/marketing/src/pages/sitemap.xml.ts index ad02bf977..426b09cb2 100644 --- a/frontend/apps/marketing/src/pages/sitemap.xml.ts +++ b/frontend/apps/marketing/src/pages/sitemap.xml.ts @@ -2,6 +2,7 @@ import { POSTS } from '@/lib/marketing-content/blog'; import { COMPETITORS } from '@/lib/marketing-content/compare'; import { FOOTER_LEGAL_LINKS } from '@/lib/marketing-content/legal'; import { absoluteUrl } from '@/lib/seo/site'; +import { FREE_TOOLS } from '@/lib/marketing-content/tools'; type RouteEntry = { path: string; @@ -22,6 +23,12 @@ const staticRoutes: readonly RouteEntry[] = [ { path: '/entity-map', changeFrequency: 'monthly', priority: 0.5 }, { path: '/blog', changeFrequency: 'weekly', priority: 0.6 }, { path: '/compare', changeFrequency: 'monthly', priority: 0.6 }, + { path: '/tools', changeFrequency: 'monthly', priority: 0.8 }, + ...FREE_TOOLS.map((tool) => ({ + path: `/tools/${tool.slug}`, + changeFrequency: 'monthly' as const, + priority: 0.7, + })), ]; const escapeXml = (value: string) => diff --git a/frontend/apps/marketing/src/pages/tools/[slug].astro b/frontend/apps/marketing/src/pages/tools/[slug].astro new file mode 100644 index 000000000..b80422334 --- /dev/null +++ b/frontend/apps/marketing/src/pages/tools/[slug].astro @@ -0,0 +1,40 @@ +--- +import MarketingLayout from '../../layouts/MarketingLayout.astro'; +import { PageHero } from '@/components/marketing/primitives/page-hero'; +import { Section } from '@/components/marketing/primitives/section'; +import { FREE_TOOLS } from '@/lib/marketing-content/tools'; +import { CrawlerChecker, RobotsGenerator } from '@/components/marketing/tools/robots-tools'; +import { MetaChecker, StructuredDataBuilder } from '@/components/marketing/tools/markup-tools'; +import { SitemapComparison } from '@/components/marketing/tools/sitemap-tool'; +import { SocialPreview } from '@/components/marketing/tools/social-tool'; +import catalog from '@/services/api/src/config/crawlers.json'; + +const tool = FREE_TOOLS.find((entry) => entry.slug === Astro.params.slug); +if (!tool) return new Response('Tool not found', { status: 404 }); +// Project only the public catalog fields; no server configuration enters the browser. +const bots = catalog.bots.map((bot) => ({ label: bot.label, purpose: bot.purpose, tokens: bot.robots_tokens })); +--- + +
+ +
+ ← All free tools +

Your input is processed in this browser. Text and files are not uploaded or saved by this tool. Use up to 200,000 characters per text area.

+ + {tool.slug === 'ai-crawler-checker' && } + {tool.slug === 'robots-txt-generator' && } + {tool.slug === 'meta-directive-checker' && } + {tool.slug === 'structured-data-builder' && } + {tool.slug === 'sitemap-comparison' && } + {tool.slug === 'social-preview' && } +
+
+

What this tool can tell you

+

{tool.limitation}

+

Need evidence across your website? Explore CiteLadder’s site-wide workflow.

+ +
+
+ diff --git a/frontend/apps/marketing/src/pages/tools/index.astro b/frontend/apps/marketing/src/pages/tools/index.astro new file mode 100644 index 000000000..5ae045b40 --- /dev/null +++ b/frontend/apps/marketing/src/pages/tools/index.astro @@ -0,0 +1,7 @@ +--- +import MarketingLayout from '../../layouts/MarketingLayout.astro'; +import { ToolsIndex } from '@/components/marketing/tools/tools-index'; +--- + +
+ diff --git a/frontend/components/marketing/chrome/footer.tsx b/frontend/components/marketing/chrome/footer.tsx index 51d11455a..b0bd2abd0 100644 --- a/frontend/components/marketing/chrome/footer.tsx +++ b/frontend/components/marketing/chrome/footer.tsx @@ -42,6 +42,7 @@ const FOOTER_COLUMNS: readonly FooterColumn[] = [ links: [ { label: 'Docs', href: docsHref(), external: true }, { label: 'Blog', href: '/blog' }, + { label: 'Free tools', href: '/tools' }, { label: 'Changelog', href: docsHref('/changelog/'), external: true }, { label: 'FAQ', href: '/faq' }, ], diff --git a/frontend/components/marketing/tools/markup-tools.tsx b/frontend/components/marketing/tools/markup-tools.tsx new file mode 100644 index 000000000..e58e44424 --- /dev/null +++ b/frontend/components/marketing/tools/markup-tools.tsx @@ -0,0 +1,111 @@ +import { useState } from 'react'; +import { Field } from '@/components/ui/field'; +import { Select } from '@/components/ui/select'; +import { inspectMarkup } from '@/lib/free-tools/markup'; +import { buildSchema, type SchemaKind } from '@/lib/free-tools/generators'; +import { ToolForm, ToolInput } from './tool-form'; + +export function MetaChecker() { + const [html, setHtml] = useState(''); + const [headers, setHeaders] = useState(''); + return ( + inspectMarkup(html, headers)} + signature={JSON.stringify([html, headers])} + sample={() => { + setHtml( + 'Example guide\n\n', + ); + setHeaders('HTTP/2 200\nX-Robots-Tag: googlebot: noindex'); + }} + > + + + + ); +} + +export function StructuredDataBuilder() { + const [kind, setKind] = useState('Article'); + const [name, setName] = useState(''); + const [url, setUrl] = useState(''); + const [detail, setDetail] = useState(''); + const [date, setDate] = useState(''); + const breadcrumb = kind === 'BreadcrumbList'; + return ( + buildSchema(kind, name, url, detail, date)} + signature={JSON.stringify([kind, name, url, detail, date])} + sample={() => { + setKind('Article'); + setName('A practical guide to crawl access'); + setUrl('https://example.com/guide'); + setDetail('Alex Morgan'); + setDate('2026-10-06'); + }} + > + + {(props) => ( + { + const file = event.target.files?.[0]; + const current = ++generation.current; + if (!file) return; + event.target.value = ''; + try { + const text = await readToolFile(file); + if (generation.current === current) { + onChange(text); + setError(''); + } + } catch (cause) { + if (generation.current === current) + setError(cause instanceof Error ? cause.message : 'Could not read file.'); + } + }} + /> + )} + + { + generation.current++; + onChange(text); + setError(''); + }} + multiline + /> + + ); +} +const xml = (paths: string[]) => + `\n${paths.map((path) => ` https://example.com/${path}`).join('\n')}\n`; +export function SitemapComparison() { + const [before, setBefore] = useState(''); + const [after, setAfter] = useState(''); + return ( + { + setBefore(xml(['', 'old-guide'])); + setAfter(xml(['', 'new-guide'])); + }} + run={() => { + const result = compareSitemaps(before, after); + return [ + `Comparing ${result.kind === 'url' ? 'page URLs' : 'child sitemap URLs (not their contents)'}`, + `Duplicate entries ignored: ${result.duplicates}`, + '', + `Added (${result.added.length})`, + ...result.added, + '', + `Removed (${result.removed.length})`, + ...result.removed, + '', + `Unchanged (${result.kept.length})`, + ...result.kept, + ].join('\n'); + }} + > + + + + ); +} diff --git a/frontend/components/marketing/tools/social-tool.tsx b/frontend/components/marketing/tools/social-tool.tsx new file mode 100644 index 000000000..4fefed5a3 --- /dev/null +++ b/frontend/components/marketing/tools/social-tool.tsx @@ -0,0 +1,116 @@ +import { useEffect, useState } from 'react'; +import { Field } from '@/components/ui/field'; +import { Input } from '@/components/ui/input'; +import { FREE_TOOL_LIMITS } from '@/lib/config/free-tools'; +import { socialTags } from '@/lib/free-tools/generators'; +import { ToolForm, ToolInput } from './tool-form'; + +export function SocialPreview() { + const [title, setTitle] = useState(''); + const [description, setDescription] = useState(''); + const [url, setUrl] = useState(''); + const [image, setImage] = useState(''); + const [alt, setAlt] = useState(''); + const [file, setFile] = useState(null); + const [localImage, setLocalImage] = useState(''); + const [error, setError] = useState(''); + useEffect(() => { + if (!file) return; + const reader = new FileReader(); + reader.onload = () => setLocalImage(String(reader.result)); + reader.onerror = () => { + setLocalImage(''); + setError('Could not read that image.'); + }; + reader.readAsDataURL(file); + return () => reader.abort(); + }, [file]); + const preview = ( +
+ {file && localImage ? ( + {alt { + setFile(null); + setLocalImage(''); + setError('This image could not be decoded. Choose another PNG, JPEG or WebP.'); + }} + /> + ) : ( +
+ Select a local image to preview its crop +
+ )} +
+

{url || 'https://example.com'}

+

{title || 'Your page title'}

+

+ {description || 'Your page description appears here.'} +

+
+
+ ); + return ( + socialTags(title, description, url, image, alt)} + sample={() => { + setTitle('Make your next page easier to discover'); + setDescription( + 'A practical guide to crawler access, page directives and clear website evidence.', + ); + setUrl('https://example.com/guide'); + setImage('https://example.com/share.jpg'); + setAlt('A guide to website discovery'); + setFile(null); + setLocalImage(''); + setError(''); + }} + > + + + + + + + {(props) => ( + { + const next = event.target.files?.[0]; + event.target.value = ''; + if (!next) return; + if ( + !['image/png', 'image/jpeg', 'image/webp'].includes(next.type) || + next.size > FREE_TOOL_LIMITS.imageBytes + ) { + setError('Choose a PNG, JPEG or WebP smaller than 5 MB.'); + setFile(null); + return; + } + setFile(next); + setLocalImage(''); + setError(''); + }} + /> + )} + + + ); +} diff --git a/frontend/components/marketing/tools/tool-form.tsx b/frontend/components/marketing/tools/tool-form.tsx new file mode 100644 index 000000000..aec7c3ebb --- /dev/null +++ b/frontend/components/marketing/tools/tool-form.tsx @@ -0,0 +1,146 @@ +import { useState, type ReactNode, type FormEvent } from 'react'; +import { Button } from '@/components/ui/button'; +import { CopyButton } from '@/components/ui/copy-button'; +import { Field } from '@/components/ui/field'; +import { Input } from '@/components/ui/input'; +import { Textarea } from '@/components/ui/textarea'; +import { FREE_TOOL_LIMITS } from '@/lib/config/free-tools'; + +export function ToolInput({ + label, + value, + onChange, + multiline = false, + hint, + type = 'text', +}: { + label: string; + value: string; + onChange: (value: string) => void; + multiline?: boolean; + hint?: string; + type?: string; +}) { + return ( + + {(props) => + multiline ? ( +