feat(token): use github proxy instead of retrieving raw tokens

.github/workflows/aqua.yml

@@ -17,9 +17,9 @@ permissions:
   id-token: write
 
 env:
-  GITHUB_API_TOKEN: ${{ secrets.MISE_GITHUB_TOKEN }}
-  GH_TOKEN: ${{ secrets.MISE_GITHUB_TOKEN }}
+  GITHUB_TOKEN: ${{ secrets.MISE_GITHUB_TOKEN }}
   DRY_RUN: 0
+
 jobs:
   update:
     runs-on: ubuntu-latest

.github/workflows/backfill.yml

@@ -32,8 +32,6 @@ permissions:
 
 env:
   GITHUB_TOKEN: ${{ secrets.MISE_GITHUB_TOKEN }}
-  TOKEN_MANAGER_URL: https://mise-tools.jdx.dev
-  TOKEN_MANAGER_SECRET: ${{ secrets.TOKEN_MANAGER_SECRET }}
 
 jobs:
   backfill:
@@ -73,6 +71,9 @@ jobs:
             args="$args --debug"
           fi
           node scripts/${{ github.event.inputs.script }} $args
+        env:
+          GITHUB_PROXY_URL: https://mise-tools.jdx.dev
+          API_SECRET: ${{ secrets.TOKEN_MANAGER_SECRET }}
 
       - name: Push any remaining changes
         if: ${{ github.event.inputs.dry_run != 'true' }}

.github/workflows/metadata.yml

@@ -11,8 +11,6 @@ concurrency:
 
 env:
   GITHUB_TOKEN: ${{ secrets.MISE_GITHUB_TOKEN }}
-  TOKEN_MANAGER_URL: https://mise-tools.jdx.dev
-  TOKEN_MANAGER_SECRET: ${{ secrets.TOKEN_MANAGER_SECRET }}
 
 jobs:
   fetch-metadata:
@@ -33,4 +31,5 @@ jobs:
         run: node scripts/fetch-metadata.js
         env:
           SYNC_API_URL: https://mise-tools.jdx.dev
+          GITHUB_PROXY_URL: https://mise-tools.jdx.dev
           API_SECRET: ${{ secrets.TOKEN_MANAGER_SECRET }}

.github/workflows/python.yml

@@ -14,9 +14,9 @@ permissions:
   id-token: write
 
 env:
-  GITHUB_API_TOKEN: ${{ secrets.MISE_GITHUB_TOKEN }}
-  GH_TOKEN: ${{ secrets.MISE_GITHUB_TOKEN }}
+  GITHUB_TOKEN: ${{ secrets.MISE_GITHUB_TOKEN }}
   DRY_RUN: 0
+
 jobs:
   update:
     runs-on: ubuntu-latest

.github/workflows/tool-analysis.yml

@@ -14,8 +14,7 @@ permissions:
   id-token: write
 
 env:
-  GITHUB_API_TOKEN: ${{ secrets.MISE_GITHUB_TOKEN }}
-  GH_TOKEN: ${{ secrets.MISE_GITHUB_TOKEN }}
+  GITHUB_TOKEN: ${{ secrets.MISE_GITHUB_TOKEN }}
 
 jobs:
   analyze:

.github/workflows/update.yml

@@ -5,7 +5,7 @@ on:
       full_sync:
         description: "Force full sync of all versions to D1"
         required: false
-        default: "false"
+        default: false
         type: boolean
   schedule:
     - cron: "*/15 * * * *"
@@ -20,11 +20,9 @@ permissions:
   id-token: write
 
 env:
-  GITHUB_API_TOKEN: ${{ secrets.MISE_GITHUB_TOKEN }}
-  GH_TOKEN: ${{ secrets.MISE_GITHUB_TOKEN }}
-  TOKEN_MANAGER_URL: https://mise-tools.jdx.dev
-  TOKEN_MANAGER_SECRET: ${{ secrets.TOKEN_MANAGER_SECRET }}
+  GITHUB_TOKEN: ${{ secrets.MISE_GITHUB_TOKEN }}
   DRY_RUN: 0
+
 jobs:
   update:
     runs-on: ubuntu-latest
@@ -43,6 +41,9 @@ jobs:
           restore-keys: |
             last-processed-tool-
       - run: ./scripts/update.sh "${{ github.event.schedule }}"
+        env:
+          GITHUB_PROXY_URL: https://mise-tools.jdx.dev
+          API_SECRET: ${{ secrets.TOKEN_MANAGER_SECRET }}
       - run: git checkout docs && git clean -df docs
       - name: Sync tools to D1
         run: node scripts/sync-to-d1.js

CLAUDE.md

@@ -95,6 +95,6 @@ The `ANALYTICS_DB` contains:
 
 The update workflow uses a token rotation system:
 
-- `TOKEN_MANAGER_URL` / `TOKEN_MANAGER_SECRET`: Cloudflare Worker API for token pool
-- `scripts/github-token.js`: Gets tokens, marks rate-limited tokens
-- Tokens rotate automatically when rate limited
+- `web/src/pages/gh/[...path].ts`: GitHub proxy that handles token rotation server-side
+- `GITHUB_PROXY_URL`, `API_SECRET`: Proxy URL and API secret for authentication
+- The proxy manages authentication and automatically rotates tokens when rate limits are hit, removing the need for clients to handle raw tokens

scripts/backfill-created-at.js

@@ -8,9 +8,9 @@
  * and updates TOML files that have placeholder timestamps.
  *
  * Environment variables:
- *   GITHUB_TOKEN - GitHub token for API access
- *   TOKEN_MANAGER_URL - URL of token manager service
- *   TOKEN_MANAGER_SECRET - Secret for token manager
+ *   GITHUB_PROXY_URL - URL of GitHub proxy (e.g., https://mise-tools.jdx.dev)
+ *   API_SECRET       - Secret for proxy authentication
+ *   GITHUB_TOKEN     - Fallback GitHub token (optional)
  */
 
 import { readFileSync, writeFileSync, readdirSync, existsSync } from "fs";
@@ -22,79 +22,53 @@ const DOCS_DIR = join(process.cwd(), "docs");
 const CONCURRENCY = 30; // Process 30 tools in parallel
 const COMMIT_INTERVAL = 100; // Commit every 100 tools
 
-// Get a random token from the token manager for each request
-async function getRandomToken() {
-  const baseUrl = process.env.TOKEN_MANAGER_URL;
-  const secret = process.env.TOKEN_MANAGER_SECRET;
+// Fetch versions with timestamps from mise
+async function fetchVersionsWithTimestamps(tool, debug = false) {
+  // Use GitHub Proxy if available, otherwise fall back to direct GitHub access
+  const proxyUrl = process.env.GITHUB_PROXY_URL; // e.g. https://mise-tools.jdx.dev
+  const apiSecret = process.env.API_SECRET;
+  const githubToken = process.env.GITHUB_TOKEN;
+
+  const env = {
+    ...process.env,
+    MISE_LIST_ALL_VERSIONS: "1", // Get all versions, not just first page
+    MISE_USE_VERSIONS_HOST: "0", // Bypass versions host to get real timestamps from GitHub
+  };
+
+  if (proxyUrl && apiSecret) {
+    env.MISE_URL_REPLACEMENTS = JSON.stringify({
+      "regex:^https://api\\.github\\.com": `${proxyUrl}/gh`,
+    });
+    env.MISE_GITHUB_TOKEN = apiSecret;
+  } else if (githubToken) {
+    env.MISE_GITHUB_TOKEN = githubToken;
+  }
 
-  if (!baseUrl || !secret) {
-    return process.env.GITHUB_TOKEN || null;
+  if (debug) {
+    env.MISE_DEBUG = "1";
   }
 
   try {
-    const response = await fetch(`${baseUrl}/api/token`, {
-      headers: {
-        Authorization: `Bearer ${secret}`,
-      },
+    const output = execSync(`mise ls-remote --json "${tool}"`, {
+      encoding: "utf-8",
+      stdio: ["pipe", "pipe", "pipe"],
+      env,
+      timeout: 60000,
     });
 
-    if (!response.ok) {
-      return process.env.GITHUB_TOKEN || null;
+    if (!output || !output.trim()) {
+      return null;
     }
 
-    const data = await response.json();
-    return data.token;
+    const data = JSON.parse(output);
+    return data;
   } catch (e) {
-    return process.env.GITHUB_TOKEN || null;
-  }
-}
-
-// Fetch versions with timestamps from mise
-async function fetchVersionsWithTimestamps(tool, retries = 2, debug = false) {
-  for (let attempt = 0; attempt <= retries; attempt++) {
-    // Get a fresh random token for each attempt
-    const token = await getRandomToken();
-    const env = {
-      ...process.env,
-      MISE_LIST_ALL_VERSIONS: "1", // Get all versions, not just first page
-      MISE_USE_VERSIONS_HOST: "0", // Bypass versions host to get real timestamps from GitHub
-    };
-    if (debug) {
-      env.MISE_DEBUG = "1";
-    }
-    if (token) {
-      env.GITHUB_TOKEN = token;
-    }
-
-    try {
-      const output = execSync(`mise ls-remote --json "${tool}"`, {
-        encoding: "utf-8",
-        stdio: ["pipe", "pipe", "pipe"],
-        env,
-        timeout: 60000,
-      });
-
-      if (!output || !output.trim()) {
-        return null;
-      }
-
-      const data = JSON.parse(output);
-      return data;
-    } catch (e) {
-      const stderr = e.stderr?.toString() || "";
-      // Retry on rate limiting with a new token
-      if (stderr.includes("rate limit") || stderr.includes("403")) {
-        continue;
-      }
-      if (attempt === retries) {
-        if (stderr) {
-          console.log(`  stderr: ${stderr.slice(0, 200)}`);
-        }
-        return null;
-      }
+    const stderr = e.stderr?.toString() || "";
+    if (stderr) {
+      console.log(`  stderr: ${stderr.slice(0, 200)}`);
     }
+    return null;
   }
-  return null;
 }
 
 // Parse command line arguments
@@ -156,7 +130,7 @@ async function processTool(tool, dryRun, debug = false) {
   }
 
   // Fetch real timestamps from mise
-  const versionData = await fetchVersionsWithTimestamps(tool, 2, debug);
+  const versionData = await fetchVersionsWithTimestamps(tool, debug);
   if (!versionData) {
     return { tool, status: "failed", error: "Failed to fetch versions" };
   }