Mirror: feat(docs): add dynamic sitemap.xml generation (#5728)

.devcontainer/devcontainer.json

@@ -5,7 +5,10 @@
 
 	"features": {
 		"ghcr.io/devcontainers/features/git:1": {},
-		"ghcr.io/devcontainers/features/github-cli:1": {}
+		"ghcr.io/devcontainers/features/github-cli:1": {},
+		"ghcr.io/devcontainers/features/sshd:1": {
+			"version": "latest"
+		}
 	},
 
 	"customizations": {
@@ -54,15 +57,12 @@
 	"remoteUser": "root",
 	"containerUser": "root",
 
-	// Mounts for persisting Kilo Code state across container rebuilds
-	// These mounts preserve threads, settings, and caches
 	"mounts": [
 		"source=${localWorkspaceFolder}/.git,target=/workspace/.git,type=bind,consistency=cached",
 		"source=kilocode-global-storage,target=/root/.vscode-remote/data/User/globalStorage/kilocode.kilo-code,type=volume",
 		"source=kilocode-settings,target=/root/.vscode-remote/data/User,type=volume"
 	],
 
-	// Configure custom properties for workspace storage
 	"workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind",
 	"workspaceFolder": "/workspace"
 }

.github/dependabot.yml

@@ -1,21 +1,12 @@
-# To get started with Dependabot version updates, you'll need to specify which
-# package ecosystems to update and where the package manifests are located.
-# Please see the documentation for all configuration options:
-# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
-
 version: 2
 updates:
   - package-ecosystem: "npm"
     directory: "/"
     schedule:
       interval: "weekly"
-
-  - package-ecosystem: "npm"
-    directory: "/cli"
-    schedule:
-      interval: "weekly"
-
+    open-pull-requests-limit: 5
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
       interval: "weekly"
+    open-pull-requests-limit: 3

.github/workflows/codeql.yml

@@ -0,0 +1,28 @@
+name: CodeQL
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  security-events: write
+  contents: read
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [javascript-typescript]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v3
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3

.github/workflows/pr-agent.yml

@@ -0,0 +1,24 @@
+name: PR Agent
+on:
+  pull_request:
+    types: [opened, reopened, ready_for_review]
+  issue_comment:
+    types: [created]
+
+permissions:
+  issues: write
+  pull-requests: write
+  contents: read
+
+jobs:
+  pr_agent:
+    if: ${{ github.event.sender.type != 'Bot' }}
+    runs-on: ubuntu-latest
+    name: Run PR Agent
+    steps:
+      - name: PR Agent action step
+        id: pragent
+        uses: qodo-ai/pr-agent@main
+        env:
+          OPENAI_KEY: ${{ secrets.OPENAI_API_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

apps/kilocode-docs/__tests__/sitemap.spec.ts

@@ -0,0 +1,162 @@
+/**
+ * Tests for the sitemap.xml API endpoint
+ *
+ * This test suite verifies that the sitemap generation works correctly:
+ * 1. Generates valid XML structure
+ * 2. Includes all markdown pages
+ * 3. Uses correct URL format with the docs basePath
+ */
+
+import { expect, describe, it, vi, beforeEach, afterEach } from "vitest"
+import type { NextApiRequest, NextApiResponse } from "next"
+
+// Mock fs module
+vi.mock("fs", () => ({
+	default: {
+		readdirSync: vi.fn(),
+		statSync: vi.fn(),
+	},
+	readdirSync: vi.fn(),
+	statSync: vi.fn(),
+}))
+
+import fs from "fs"
+import handler from "../pages/api/sitemap.xml"
+
+describe("sitemap.xml API", () => {
+	let mockReq: Partial<NextApiRequest>
+	let mockRes: Partial<NextApiResponse>
+	let responseData: string
+	let responseHeaders: Record<string, string>
+	let responseStatus: number
+
+	beforeEach(() => {
+		responseHeaders = {}
+		responseStatus = 200
+		responseData = ""
+
+		mockReq = {
+			method: "GET",
+		}
+
+		mockRes = {
+			status: vi.fn().mockReturnThis(),
+			send: vi.fn((data) => {
+				responseData = data
+				return mockRes
+			}),
+			json: vi.fn().mockReturnThis(),
+			setHeader: vi.fn((key, value) => {
+				responseHeaders[key as string] = value as string
+				return mockRes
+			}),
+		}
+
+		// Mock fs.readdirSync to return test files
+		vi.mocked(fs.readdirSync).mockImplementation((dir: any) => {
+			const dirStr = dir.toString()
+			if (dirStr.endsWith("pages")) {
+				return [
+					{ name: "index.md", isDirectory: () => false },
+					{ name: "getting-started", isDirectory: () => true },
+					{ name: "api", isDirectory: () => true },
+				] as any
+			}
+			if (dirStr.includes("getting-started")) {
+				return [
+					{ name: "index.md", isDirectory: () => false },
+					{ name: "quickstart.md", isDirectory: () => false },
+				] as any
+			}
+			return []
+		})
+
+		// Mock fs.statSync to return a fixed date
+		vi.mocked(fs.statSync).mockReturnValue({
+			mtime: new Date("2025-01-15T10:00:00Z"),
+		} as any)
+	})
+
+	afterEach(() => {
+		vi.clearAllMocks()
+	})
+
+	describe("HTTP method handling", () => {
+		it("should return 405 for non-GET requests", async () => {
+			mockReq.method = "POST"
+
+			await handler(mockReq as NextApiRequest, mockRes as NextApiResponse)
+
+			expect(mockRes.status).toHaveBeenCalledWith(405)
+			expect(mockRes.json).toHaveBeenCalledWith({ error: "Method not allowed" })
+		})
+
+		it("should accept GET requests", async () => {
+			await handler(mockReq as NextApiRequest, mockRes as NextApiResponse)
+
+			expect(mockRes.status).toHaveBeenCalledWith(200)
+		})
+	})
+
+	describe("sitemap generation", () => {
+		it("should generate valid XML with correct headers", async () => {
+			await handler(mockReq as NextApiRequest, mockRes as NextApiResponse)
+
+			expect(responseHeaders["Content-Type"]).toBe("application/xml; charset=utf-8")
+			expect(responseHeaders["Cache-Control"]).toBe("public, max-age=3600, s-maxage=3600")
+		})
+
+		it("should include XML declaration and urlset", async () => {
+			await handler(mockReq as NextApiRequest, mockRes as NextApiResponse)
+
+			expect(responseData).toContain('<?xml version="1.0" encoding="UTF-8"?>')
+			expect(responseData).toContain('<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">')
+			expect(responseData).toContain("</urlset>")
+		})
+
+		it("should include homepage with priority 1.0", async () => {
+			await handler(mockReq as NextApiRequest, mockRes as NextApiResponse)
+
+			expect(responseData).toContain("<loc>https://kilo.ai/docs</loc>")
+			expect(responseData).toContain("<priority>1.0</priority>")
+		})
+
+		it("should include markdown pages with correct URLs", async () => {
+			await handler(mockReq as NextApiRequest, mockRes as NextApiResponse)
+
+			expect(responseData).toContain("<loc>https://kilo.ai/docs/getting-started</loc>")
+			expect(responseData).toContain("<loc>https://kilo.ai/docs/getting-started/quickstart</loc>")
+		})
+
+		it("should include lastmod dates", async () => {
+			await handler(mockReq as NextApiRequest, mockRes as NextApiResponse)
+
+			expect(responseData).toContain("<lastmod>2025-01-15</lastmod>")
+		})
+
+		it("should include changefreq", async () => {
+			await handler(mockReq as NextApiRequest, mockRes as NextApiResponse)
+
+			expect(responseData).toContain("<changefreq>weekly</changefreq>")
+		})
+
+		it("should skip api directory", async () => {
+			await handler(mockReq as NextApiRequest, mockRes as NextApiResponse)
+
+			expect(responseData).not.toContain("/api/")
+		})
+	})
+
+	describe("error handling", () => {
+		it("should return 500 on filesystem errors", async () => {
+			vi.mocked(fs.readdirSync).mockImplementation(() => {
+				throw new Error("Filesystem error")
+			})
+
+			await handler(mockReq as NextApiRequest, mockRes as NextApiResponse)
+
+			expect(mockRes.status).toHaveBeenCalledWith(500)
+			expect(mockRes.json).toHaveBeenCalledWith({ error: "Internal server error" })
+		})
+	})
+})

apps/kilocode-docs/next.config.js

@@ -19,6 +19,11 @@ module.exports = withMarkdoc(/* config: https://markdoc.io/docs/nextjs#options *
 					source: "/llms.txt",
 					destination: "/api/llms.txt",
 				},
+				{
+					// Rewrite /docs/sitemap.xml to the API endpoint (internal to basePath)
+					source: "/sitemap.xml",
+					destination: "/api/sitemap.xml",
+				},
 			],
 		}
 	},

apps/kilocode-docs/pages/api/sitemap.xml.ts

@@ -0,0 +1,112 @@
+import type { NextApiRequest, NextApiResponse } from "next"
+import fs from "fs"
+import path from "path"
+
+const SITE_URL = "https://kilo.ai/docs"
+
+/**
+ * Recursively finds all markdown files in a directory
+ */
+function findMarkdownFiles(dir: string, baseDir: string = dir): string[] {
+	const files: string[] = []
+	const entries = fs.readdirSync(dir, { withFileTypes: true })
+
+	for (const entry of entries) {
+		const fullPath = path.join(dir, entry.name)
+
+		if (entry.isDirectory()) {
+			// Skip api directory
+			if (entry.name === "api") continue
+			files.push(...findMarkdownFiles(fullPath, baseDir))
+		} else if (entry.name.endsWith(".md")) {
+			files.push(fullPath)
+		}
+	}
+
+	return files
+}
+
+/**
+ * Converts a file path to a URL path
+ */
+function filePathToUrlPath(filePath: string, pagesDir: string): string {
+	let relativePath = path.relative(pagesDir, filePath)
+	// Remove .md extension
+	relativePath = relativePath.replace(/\.md$/, "")
+	// Handle index files
+	relativePath = relativePath.replace(/(^|\/)index$/, "")
+	// Convert to URL path
+	return "/" + relativePath.split(path.sep).join("/")
+}
+
+/**
+ * Gets the last modified date of a file
+ */
+function getLastModified(filePath: string): string {
+	const stats = fs.statSync(filePath)
+	return stats.mtime.toISOString().split("T")[0]
+}
+
+/**
+ * Escapes special XML characters
+ */
+function escapeXml(str: string): string {
+	return str
+		.replace(/&/g, "&")
+		.replace(/</g, "&lt;")
+		.replace(/>/g, "&gt;")
+		.replace(/"/g, "&quot;")
+		.replace(/'/g, "&apos;")
+}
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+	if (req.method !== "GET") {
+		return res.status(405).json({ error: "Method not allowed" })
+	}
+
+	try {
+		const pagesDir = path.join(process.cwd(), "pages")
+		const markdownFiles = findMarkdownFiles(pagesDir)
+
+		// Sort files for consistent output
+		markdownFiles.sort()
+
+		const urls: string[] = []
+
+		// Add homepage
+		urls.push(`  <url>
+    <loc>${SITE_URL}</loc>
+    <changefreq>weekly</changefreq>
+    <priority>1.0</priority>
+  </url>`)
+
+		for (const filePath of markdownFiles) {
+			const urlPath = filePathToUrlPath(filePath, pagesDir)
+			const lastMod = getLastModified(filePath)
+			const fullUrl = `${SITE_URL}${urlPath}`
+
+			// Determine priority based on path depth
+			const depth = urlPath.split("/").filter(Boolean).length
+			const priority = Math.max(0.5, 1.0 - depth * 0.1).toFixed(1)
+
+			urls.push(`  <url>
+    <loc>${escapeXml(fullUrl)}</loc>
+    <lastmod>${lastMod}</lastmod>
+    <changefreq>weekly</changefreq>
+    <priority>${priority}</priority>
+  </url>`)
+		}
+
+		const sitemap = `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${urls.join("\n")}
+</urlset>`
+
+		res.setHeader("Content-Type", "application/xml; charset=utf-8")
+		res.setHeader("Cache-Control", "public, max-age=3600, s-maxage=3600") // Cache for 1 hour
+		res.status(200).send(sitemap)
+	} catch (error) {
+		console.error("Error generating sitemap:", error)
+		res.status(500).json({ error: "Internal server error" })
+	}
+}