diff --git a/.github/workflows/crowdin-ai-import.yml b/.github/workflows/crowdin-ai-import.yml new file mode 100644 index 00000000000..0f09c6a552e --- /dev/null +++ b/.github/workflows/crowdin-ai-import.yml @@ -0,0 +1,49 @@ +name: Import Crowdin AI Translations + +on: + workflow_dispatch: + inputs: + file_limit: + description: "Number of files to process (default: 100, use 1-10 for testing)" + required: false + default: "100" + type: string + target_languages: + description: "Comma-separated Crowdin language codes (default: es-EM)" + required: false + default: "es-EM" + type: string + base_branch: + description: "Base branch to create PR against (default: dev)" + required: false + default: "dev" + type: string + +jobs: + import_translations: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v5 + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + + - name: Set up Node.js + uses: actions/setup-node@v6 + with: + node-version: 20 + cache: "pnpm" + + - name: Install dependencies + run: pnpm install + + - name: Run Crowdin AI translation import + run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/i18n/main.ts + env: + I18N_CROWDIN_API_KEY: ${{ secrets.CROWDIN_API_KEY }} + I18N_GITHUB_API_KEY: ${{ secrets.I18N_GITHUB_TOKEN }} + FILE_LIMIT: ${{ github.event.inputs.file_limit }} + TARGET_LANGUAGES: ${{ github.event.inputs.target_languages }} + BASE_BRANCH: ${{ github.event.inputs.base_branch }} + GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/src/scripts/i18n/check-translation-status.ts b/src/scripts/i18n/check-translation-status.ts new file mode 100644 index 00000000000..5913e5457f1 --- /dev/null +++ b/src/scripts/i18n/check-translation-status.ts @@ -0,0 +1,118 @@ +/** + * Quick script to check translation status of a specific file in Crowdin + */ + +const CROWDIN_API_KEY = process.env.CROWDIN_TOKEN! +const PROJECT_ID = 834930 +const FILE_ID = 17434 // organizing/index.md +const LANGUAGE_ID = "es-EM" + +const headers = { + Authorization: `Bearer ${CROWDIN_API_KEY}`, + "Content-Type": "application/json", +} + +async function checkTranslationProgress() { + console.log("\n=== Checking Translation Progress ===") + console.log(`File ID: ${FILE_ID}`) + console.log(`Language: ${LANGUAGE_ID}`) + + // Get translation progress for the file + const url = `https://api.crowdin.com/api/v2/projects/${PROJECT_ID}/languages/${LANGUAGE_ID}/progress?fileIds=${FILE_ID}` + + try { + const res = await fetch(url, { headers }) + if (!res.ok) { + const text = await res.text() + throw new Error(`Failed to get progress (${res.status}): ${text}`) + } + + const json = await res.json() + console.log("\nTranslation Progress:") + console.log(JSON.stringify(json, null, 2)) + } catch (error) { + console.error("Error:", error) + } +} + +async function listStrings() { + console.log("\n=== Listing Strings in File ===") + + // Get strings from the file + const url = `https://api.crowdin.com/api/v2/projects/${PROJECT_ID}/strings?fileId=${FILE_ID}&limit=10` + + try { + const res = await fetch(url, { headers }) + if (!res.ok) { + const text = await res.text() + throw new Error(`Failed to list strings (${res.status}): ${text}`) + } + + const json = await res.json() + console.log(`\nFound ${json.data.length} strings (showing first 10):`) + for (const item of json.data) { + console.log(`\nString ID: ${item.data.id}`) + console.log(`Text: "${item.data.text.substring(0, 100)}..."`) + console.log(`Context: ${item.data.context || "none"}`) + } + } catch (error) { + console.error("Error:", error) + } +} + +async function checkStringTranslations() { + console.log("\n=== Checking String Translations ===") + + // First get a string ID + const stringsUrl = `https://api.crowdin.com/api/v2/projects/${PROJECT_ID}/strings?fileId=${FILE_ID}&limit=1` + + try { + const stringsRes = await fetch(stringsUrl, { headers }) + if (!stringsRes.ok) { + throw new Error(`Failed to get strings: ${stringsRes.status}`) + } + + const stringsJson = await stringsRes.json() + if (stringsJson.data.length === 0) { + console.log("❌ No strings found in file!") + return + } + + const stringId = stringsJson.data[0].data.id + console.log(`\nChecking translations for string ID: ${stringId}`) + console.log( + `String text: "${stringsJson.data[0].data.text.substring(0, 100)}..."` + ) + + // Get translations for this string + const translationsUrl = `https://api.crowdin.com/api/v2/projects/${PROJECT_ID}/translations?stringId=${stringId}&languageId=${LANGUAGE_ID}` + const transRes = await fetch(translationsUrl, { headers }) + + if (!transRes.ok) { + const text = await transRes.text() + console.log( + `\n⚠️ No translations found or error (${transRes.status}): ${text}` + ) + return + } + + const transJson = await transRes.json() + console.log(`\nTranslations found: ${transJson.data.length}`) + if (transJson.data.length > 0) { + console.log("First translation:") + console.log(JSON.stringify(transJson.data[0].data, null, 2)) + } else { + console.log("❌ String has NO translations in Spanish!") + } + } catch (error) { + console.error("Error:", error) + } +} + +async function main() { + await checkTranslationProgress() + await listStrings() + await checkStringTranslations() +} + +main() diff --git a/src/scripts/i18n/main.ts b/src/scripts/i18n/main.ts new file mode 100644 index 00000000000..27cfab83e25 --- /dev/null +++ b/src/scripts/i18n/main.ts @@ -0,0 +1,1349 @@ +import dotenv from "dotenv" + +import i18nConfig from "../../../i18n.config.json" + +import type { + BranchDetailsResponse, + BranchObject, + BuildProjectFileTranslationResponse, + ContentType, + CrowdinAddFileResponse, + CrowdinFileData, + CrowdinPreTranslateResponse, + GitHubCrowdinFileMetadata, + GitHubQueryResponseItem, +} from "./types" + +dotenv.config({ path: ".env.local" }) + +const crowdinToInternalCodeMapping: Record = i18nConfig.reduce( + (acc, { crowdinCode, code }) => { + acc[crowdinCode] = code + return acc + }, + {} as Record +) + +const gitHubApiKey = process.env.I18N_GITHUB_API_KEY || "" +if (!gitHubApiKey) { + console.error("[ERROR] Missing I18N_GITHUB_API_KEY environment variable") + console.error( + "[ERROR] Please set I18N_GITHUB_API_KEY in your .env.local file" + ) + throw new Error("No GitHub API Key found (I18N_GITHUB_API_KEY)") +} +console.log("[DEBUG] GitHub API key found ✓") +const gitHubBearerHeaders = { + Authorization: `Bearer ${gitHubApiKey}`, + Accept: "application/vnd.github.v3+json", +} + +const crowdinApiKey = process.env.I18N_CROWDIN_API_KEY || "" +if (!crowdinApiKey) { + console.error("[ERROR] Missing I18N_CROWDIN_API_KEY environment variable") + console.error( + "[ERROR] Please set I18N_CROWDIN_API_KEY in your .env.local file" + ) + throw new Error("No Crowdin API Key found (I18N_CROWDIN_API_KEY)") +} +console.log("[DEBUG] Crowdin API key found ✓") +const crowdinBearerHeaders = { Authorization: `Bearer ${crowdinApiKey}` } + +// Parse environment variables with defaults +const targetLanguages = process.env.TARGET_LANGUAGES + ? process.env.TARGET_LANGUAGES.split(",").map((lang) => lang.trim()) + : ["es-EM"] + +const baseBranch = process.env.BASE_BRANCH || "dev" + +const fileLimit = process.env.FILE_LIMIT + ? parseInt(process.env.FILE_LIMIT, 10) + : 100 + +// Parse GitHub repository from env (format: "owner/repo") +const githubRepo = + process.env.GITHUB_REPOSITORY || "ethereum/ethereum-org-website" +const [ghOrganization, ghRepo] = githubRepo.split("/") + +console.log("[DEBUG] Configuration:") +console.log(`[DEBUG] - Target languages: ${targetLanguages.join(", ")}`) +console.log(`[DEBUG] - Base branch: ${baseBranch}`) +console.log(`[DEBUG] - File limit: ${fileLimit}`) +console.log(`[DEBUG] - GitHub repo: ${ghOrganization}/${ghRepo}`) + +const env = { + projectId: 834930, + ghOrganization, + ghRepo, + jsonRoot: "src/intl/en", + mdRoot: "public/content", + preTranslatePromptId: 168584, + allCrowdinCodes: targetLanguages, + baseBranch, +} + +// --- Utilities: resilient fetch for GitHub calls --- +const delay = (ms: number) => new Promise((res) => setTimeout(res, ms)) + +type RetryOptions = { + retries?: number + timeoutMs?: number + backoffMs?: number + retryOnStatuses?: number[] +} + +const fetchWithRetry = async ( + url: string, + init?: RequestInit, + options?: RetryOptions +) => { + const retries = options?.retries ?? 3 + const timeoutMs = options?.timeoutMs ?? 30000 + const backoffMs = options?.backoffMs ?? 1000 + const retryOnStatuses = options?.retryOnStatuses ?? [ + 408, 429, 500, 502, 503, 504, + ] + + for (let attempt = 0; attempt <= retries; attempt++) { + const controller = new AbortController() + const id = setTimeout(() => controller.abort(), timeoutMs) + try { + const res = await fetch(url, { + ...(init || {}), + signal: controller.signal, + }) + clearTimeout(id) + if ( + !res.ok && + retryOnStatuses.includes(res.status) && + attempt < retries + ) { + const wait = backoffMs * Math.pow(2, attempt) + console.warn( + `[RETRY] ${url} -> ${res.status}. Attempt ${attempt + 1}/${retries}. Waiting ${wait}ms.` + ) + await delay(wait) + continue + } + return res + } catch (err: unknown) { + clearTimeout(id) + const errObj = err as { name?: string; code?: string } + const isAbort = errObj?.name === "AbortError" + const isConnectTimeout = errObj?.code === "UND_ERR_CONNECT_TIMEOUT" + if ((isAbort || isConnectTimeout) && attempt < retries) { + const wait = backoffMs * Math.pow(2, attempt) + console.warn( + `[RETRY] ${url} -> ${isAbort ? "AbortError" : errObj?.code}. Attempt ${ + attempt + 1 + }/${retries}. Waiting ${wait}ms.` + ) + await delay(wait) + continue + } + throw err + } + } + // Unreachable, but TS wants a return + throw new Error("fetchWithRetry: exhausted retries") +} + +/** + * Get all files, using perPage to limit amount fetched + */ +const getAllEnglishFiles = async ( + perPage = 100 +): Promise => { + const ghSearchEndpointBase = "https://api.github.com/search/code" + const query = `repo:${env.ghOrganization}/${env.ghRepo} extension:md path:"${env.mdRoot}" -path:"${env.mdRoot}/translations" OR repo:${env.ghOrganization}/${env.ghRepo} extension:json path:"${env.jsonRoot}"` + + const url = new URL(ghSearchEndpointBase) + url.searchParams.set("q", query) + url.searchParams.set("per_page", perPage.toString()) + url.searchParams.set("page", "1") + + console.log(`[DEBUG] GitHub search query: ${query}`) + console.log(`[DEBUG] GitHub search URL: ${url.toString()}`) + + try { + const res = await fetchWithRetry(url.toString(), { + headers: gitHubBearerHeaders, + }) + + if (!res.ok) { + console.warn(`[ERROR] GitHub API response not OK: ${res.status}`) + const body = await res.text().catch(() => "") + console.error(`[ERROR] Response body:`, body) + throw new Error(`GitHub getAllEnglishFiles (${res.status}): ${body}`) + } + + type JsonResponse = { items: GitHubQueryResponseItem[] } + const json: JsonResponse = await res.json() + + console.log(`[DEBUG] Found ${json.items.length} files from GitHub`) + console.log(`[DEBUG] First GitHub file:`, json.items[0]) + return json.items + } catch (error) { + console.error(`[ERROR] Failed to get English files from GitHub:`, error) + process.exit(1) + } +} + +const getFileMetadata = async ( + items: GitHubQueryResponseItem[] +): Promise => { + if (!items.length) return [] + + const owner = items[0].repository.owner.login + const repo = items[0].repository.name + + const englishFileMetadata = items.map((item) => { + // https://raw.githubusercontent.com/:owner/:repo/:ref/:path + const download_url = `https://raw.githubusercontent.com/${owner}/${repo}/${env.baseBranch}/${item.path}` + const filePath = item.path + const filePathSplit = filePath.split("/") + const fileName = filePathSplit[filePathSplit.length - 1] + const contentType: ContentType = fileName?.endsWith(".json") + ? "application/json" + : "text/markdown" + + return { + "Crowdin-API-FileName": fileName, + filePath: filePath, + download_url: download_url, + "Content-Type": contentType, + } + }) + return englishFileMetadata +} + +const getCrowdinProjectFiles = async (): Promise => { + const url = new URL( + `https://api.crowdin.com/api/v2/projects/${env.projectId}/files` + ) + url.searchParams.set("limit", "500") + + console.log(`[DEBUG] Fetching Crowdin project files from: ${url.toString()}`) + + try { + const res = await fetch(url.toString(), { headers: crowdinBearerHeaders }) + + if (!res.ok) { + console.warn(`[ERROR] Crowdin API response not OK: ${res.status}`) + const body = await res.text().catch(() => "") + console.error(`[ERROR] Response body:`, body) + throw new Error( + `Crowdin getCrowdinProjectFiles failed (${res.status}): ${body}` + ) + } + + type JsonResponse = { data: { data: CrowdinFileData }[] } + const json: JsonResponse = await res.json() + + const mappedData = json.data.map(({ data }) => data) + + console.log( + `[DEBUG] Successfully fetched ${mappedData.length} Crowdin files` + ) + console.log(`[DEBUG] First Crowdin file:`, mappedData[0]) + return mappedData + } catch (error) { + console.error(`[ERROR] Failed to fetch Crowdin project files:`, error) + process.exit(1) + } +} + +const findCrowdinFile = ( + targetFile: GitHubCrowdinFileMetadata, + crowdinFiles: CrowdinFileData[] +): CrowdinFileData => { + console.log( + `[DEBUG] Looking for Crowdin file matching: ${targetFile.filePath}` + ) + console.log(`[DEBUG] Target file name: ${targetFile["Crowdin-API-FileName"]}`) + + // Log first few Crowdin files for comparison + console.log(`[DEBUG] Total Crowdin files found: ${crowdinFiles.length}`) + console.log( + `[DEBUG] First 3 Crowdin file paths:`, + crowdinFiles.slice(0, 3).map((f) => f.path) + ) + + const found = crowdinFiles.find(({ path }) => + path.endsWith(targetFile.filePath) + ) + + if (!found) { + console.error( + `[ERROR] No matching Crowdin project file found for: ${targetFile.filePath}` + ) + console.error( + `[ERROR] Available Crowdin file paths:`, + crowdinFiles.map((f) => f.path) + ) + throw new Error( + `No matching Crowdin project file found for: ${targetFile.filePath}` + ) + } + + console.log( + `[DEBUG] Successfully matched with Crowdin file: ${found.path} (ID: ${found.id})` + ) + return found +} + +/** + * Unhides all hidden strings in a Crowdin file. + * Hidden strings (often marked as duplicates) cannot be translated. + * This function makes them visible so they can be processed by pre-translation. + */ +const unhideStringsInFile = async (fileId: number): Promise => { + console.log(`[UNHIDE] Checking for hidden strings in fileId=${fileId}`) + + // Get all strings from the file + const listUrl = `https://api.crowdin.com/api/v2/projects/${env.projectId}/strings?fileId=${fileId}&limit=500` + + try { + const listRes = await fetch(listUrl, { headers: crowdinBearerHeaders }) + if (!listRes.ok) { + const text = await listRes.text().catch(() => "") + console.warn( + `[UNHIDE] Failed to list strings for fileId=${fileId}: ${text}` + ) + return 0 + } + + const listJson = await listRes.json() + const strings = listJson.data || [] + + let unhiddenCount = 0 + + for (const item of strings) { + const stringId = item.data.id + const isHidden = item.data.isHidden + + if (!isHidden) continue + + // Unhide the string using PATCH + const patchUrl = `https://api.crowdin.com/api/v2/projects/${env.projectId}/strings/${stringId}` + + try { + const patchRes = await fetch(patchUrl, { + method: "PATCH", + headers: { + ...crowdinBearerHeaders, + "Content-Type": "application/json", + }, + body: JSON.stringify([ + { + op: "replace", + path: "/isHidden", + value: false, + }, + ]), + }) + + if (patchRes.ok) { + unhiddenCount++ + } else { + const text = await patchRes.text().catch(() => "") + console.warn(`[UNHIDE] Failed to unhide string ${stringId}: ${text}`) + } + } catch (err) { + console.warn(`[UNHIDE] Error unhiding string ${stringId}:`, err) + } + } + + if (unhiddenCount > 0) { + console.log( + `[UNHIDE] ✓ Unhidden ${unhiddenCount} strings in fileId=${fileId}` + ) + } else { + console.log(`[UNHIDE] No hidden strings found in fileId=${fileId}`) + } + + return unhiddenCount + } catch (error) { + console.error(`[UNHIDE] Error processing fileId=${fileId}:`, error) + return 0 + } +} + +/** + * Lists all Crowdin directories in the project. + */ +const getCrowdinProjectDirectories = async (): Promise< + { id: number; name: string; directoryId?: number }[] +> => { + const url = new URL( + `https://api.crowdin.com/api/v2/projects/${env.projectId}/directories` + ) + url.searchParams.set("limit", "500") + + console.log(`[DEBUG] Fetching Crowdin directories: ${url.toString()}`) + + try { + const res = await fetch(url.toString(), { headers: crowdinBearerHeaders }) + if (!res.ok) { + const body = await res.text().catch(() => "") + throw new Error( + `Crowdin getCrowdinProjectDirectories failed (${res.status}): ${body}` + ) + } + type DirJson = { + data: { data: { id: number; name: string; directoryId?: number } }[] + } + const json: DirJson = await res.json() + const dirs = json.data.map(({ data }) => data) + console.log(`[DEBUG] Loaded ${dirs.length} directories`) + return dirs + } catch (error) { + console.error("[ERROR] getCrowdinProjectDirectories:", error) + throw error + } +} + +/** + * Creates a single Crowdin directory (one segment). Parent may be undefined for root. + */ +const postCrowdinDirectory = async ( + name: string, + parentDirectoryId?: number +): Promise => { + const url = new URL( + `https://api.crowdin.com/api/v2/projects/${env.projectId}/directories` + ) + + const body: Record = { name } + if (parentDirectoryId) body.directoryId = parentDirectoryId + + console.log( + `[DEBUG] Creating directory segment "${name}" parent=${parentDirectoryId ?? "ROOT"}` + ) + + try { + const res = await fetch(url.toString(), { + method: "POST", + headers: { + ...crowdinBearerHeaders, + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(body), + }) + + if (!res.ok) { + const text = await res.text().catch(() => "") + // 409 = already exists race condition + throw new Error( + `Crowdin postCrowdinDirectory failed (${res.status}): ${text}` + ) + } + + type JsonResponse = { data: { id: number } } + const json: JsonResponse = await res.json() + console.log(`[DEBUG] Created directory id=${json.data.id} name="${name}"`) + return json.data.id + } catch (error) { + console.error("[ERROR] postCrowdinDirectory:", error) + throw error + } +} + +/** + * Ensures a nested path of directories exists. + * Example path: "public/content/community/events/organizing" + * Returns the final (deepest) directory id. + * + * - Splits path on "/" ignoring empty segments. + * - Reuses existing segments (matched by name + parent). + * - Creates missing segments sequentially. + */ +const createCrowdinDirectory = async (fullPath: string): Promise => { + if (!fullPath || typeof fullPath !== "string") { + throw new Error("createCrowdinDirectory: path must be a non-empty string") + } + console.log(`[DEBUG] Ensuring Crowdin directory path: "${fullPath}"`) + + const segments = fullPath + .split("/") + .map((s) => s.trim()) + .filter(Boolean) + if (!segments.length) throw new Error("No valid path segments") + + const invalidChars = /[\\:*?"<>|]/ // Disallowed per Crowdin docs for directory name (exclude forward slash which is path separator) + for (const segment of segments) { + if (invalidChars.test(segment)) { + throw new Error( + `createCrowdinDirectory: segment "${segment}" contains invalid characters in path "${fullPath}"` + ) + } + } + + // Load existing directories once + const existing = await getCrowdinProjectDirectories() + + // Build quick lookup: parentId|name -> id (root parentId = 0 sentinel) + const key = (parentId: number | undefined, name: string) => + `${parentId || 0}|${name}` + + const directoryIndex = new Map() + for (const dir of existing) { + directoryIndex.set(key(dir.directoryId, dir.name), dir.id) + } + + let currentParentId: number | undefined + for (const segment of segments) { + const k = key(currentParentId, segment) + let dirId = directoryIndex.get(k) + if (dirId) { + console.log( + `[DEBUG] Reusing existing directory "${segment}" id=${dirId} parent=${currentParentId ?? "ROOT"}` + ) + currentParentId = dirId + continue + } + // Create + dirId = await postCrowdinDirectory(segment, currentParentId) + directoryIndex.set(k, dirId) + currentParentId = dirId + } + + if (!currentParentId) + throw new Error("Failed to resolve final directory id (unexpected)") + + console.log( + `[DEBUG] Final directory id for path "${fullPath}" = ${currentParentId}` + ) + return currentParentId +} + +const postCrowdinFile = async ( + storageId: number, + name: string, + dir: string +): Promise => { + const directoryId = await createCrowdinDirectory(dir) + const url = new URL( + `https://api.crowdin.com/api/v2/projects/${env.projectId}/files` + ) + + try { + const res = await fetch(url.toString(), { + method: "POST", + headers: { + ...crowdinBearerHeaders, + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify({ storageId, name, directoryId }), + }) + + if (!res.ok) { + console.warn("Res not OK") + const body = await res.text().catch(() => "") + throw new Error(`Crowdin postCrowdinFile failed (${res.status}): ${body}`) + } + + type JsonResponse = { data: CrowdinAddFileResponse } + const json: JsonResponse = await res.json() + console.log("Updated file:", json.data) + return json.data + } catch (error) { + console.error(error) + process.exit(1) + } +} + +const downloadGitHubFile = async (download_url: string): Promise => { + try { + // const res = await fetch(download_url, { headers: gitHubBearerHeaders }) + const res = await fetch(download_url) + if (!res.ok) { + const body = await res.text().catch(() => "") + throw new Error(`Failed to download from GitHub (${res.status}): ${body}`) + } + const arrayBuffer = await res.arrayBuffer() + return Buffer.from(arrayBuffer) + } catch (error) { + console.error("downloadGitHubFile error:", error) + throw error + } +} + +const postFileToStorage = async (fileBuffer: Buffer, fileName: string) => { + const url = new URL("https://api.crowdin.com/api/v2/storages") + + try { + const res = await fetch(url.toString(), { + method: "POST", + headers: { + ...crowdinBearerHeaders, + // Crowdin expects raw bytes for storages endpoint; use octet-stream. + "Content-Type": "application/octet-stream", + "Crowdin-API-FileName": fileName, + }, + body: fileBuffer, + }) + + if (!res.ok) { + const text = await res.text().catch(() => "") + throw new Error( + `Crowdin postFileToStorage failed (${res.status}): ${text}` + ) + } + + type JsonResponse = { + data: { + id: number + fileName: string + } + } + const json: JsonResponse = await res.json() + console.log("Uploaded storage:", json.data) + return json.data + } catch (error) { + console.error("postFileToStorage error:", error) + throw error + } +} + +const postApplyPreTranslation = async ( + fileIds: number[], + languageIds?: string[] +): Promise => { + const url = new URL( + `https://api.crowdin.com/api/v2/projects/${env.projectId}/pre-translations` + ) + try { + const res = await fetch(url.toString(), { + method: "POST", + headers: { + ...crowdinBearerHeaders, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + languageIds: languageIds || env.allCrowdinCodes, // ["es-EM"], // TODO: All languages + fileIds, + method: "ai", + aiPromptId: env.preTranslatePromptId, + }), + }) + + if (!res.ok) { + const text = await res.text().catch(() => "") + throw new Error( + `Crowdin postApplyPreTranslation failed (${res.status}): ${text}` + ) + } + + type JsonResponse = { + data: CrowdinPreTranslateResponse + } + const json: JsonResponse = await res.json() + + return json.data + } catch (error) { + console.error("postApplyPreTranslation error:", error) + throw error + } +} + +const getPreTranslationStatus = async ( + preTranslationId: string +): Promise => { + const url = new URL( + `https://api.crowdin.com/api/v2/projects/${env.projectId}/pre-translations/${preTranslationId}` + ) + try { + const res = await fetch(url.toString(), { headers: crowdinBearerHeaders }) + + if (!res.ok) { + const text = await res.text().catch(() => "") + throw new Error( + `Crowdin getPreTranslationStatus failed (${res.status}): ${text}` + ) + } + + type JsonResponse = { + data: CrowdinPreTranslateResponse + } + const json: JsonResponse = await res.json() + + return json.data + } catch (error) { + console.error("postApplyPreTranslation error:", error) + throw error + } +} + +/** + * Polls Crowdin for the status of a pre-translation job and resolves when it finishes. + * + * This function repeatedly calls `getPreTranslationStatus` for the given + * pre-translation ID until the job is no longer in progress. It polls at a + * fixed interval (10 seconds) and will abort with an error if the operation + * does not complete within the configured timeout (30 minutes). + * + * @param preTranslationId - The identifier of the Crowdin pre-translation job to monitor. + * + * @returns A promise that resolves with the final CrowdinPreTranslateResponse when the + * job status becomes "finished". + * + * @throws {Error} If the wait times out (after 30 minutes). + * @throws {Error} If the pre-translation completes with an unexpected status + * (i.e., any status other than "finished"). + * @throws {Error} If an error is thrown while fetching the pre-translation status + * (errors from `getPreTranslationStatus` are propagated). + * + * @remarks + * - Polling interval: 10,000 ms (10 seconds). + * - Timeout: 30 minutes. + * + * @example + * // Wait for a pre-translation to complete + * const result = await awaitPreTranslationCompleted("abc123") + */ +const awaitPreTranslationCompleted = async ( + preTranslationId: string, + options?: { intervalMs?: number; timeoutMs?: number } +): Promise => { + const intervalMs = options?.intervalMs ?? 10_000 + const timeoutMs = options?.timeoutMs ?? 30 /* min */ * 60 * 1000 + + return await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject(new Error("Timed out waiting for pre-translation to finish")) + }, timeoutMs) + + const poll = async () => { + try { + const res = await getPreTranslationStatus(preTranslationId) + if (res.status !== "in_progress") { + clearTimeout(timeout) + if (res.status === "finished") { + resolve(res) + } else { + reject( + new Error( + `Pre-translation ended with unexpected status: ${res.status}` + ) + ) + } + } else { + setTimeout(poll, intervalMs) + } + } catch (err) { + clearTimeout(timeout) + reject(err) + } + } + + void poll() + }) +} + +/** + * Method: POST + * https://support.crowdin.com/developer/api/v2/#tag/Translations/operation/api.projects.translations.builds.directories.post + * @param fileId + * @param targetLanguageId + * @param projectId + * @returns { url: string; expireIn: string; etag: string; } + */ +const postBuildProjectFileTranslation = async ( + fileId: number, + targetLanguageId: string, + projectId = env.projectId +): Promise => { + const url = new URL( + `https://api.crowdin.com/api/v2/projects/${projectId}/translations/builds/files/${fileId}` + ) + + const res = await fetch(url.toString(), { + method: "POST", + headers: { + ...crowdinBearerHeaders, + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify({ targetLanguageId }), + }) + + if (!res.ok) { + console.warn("Res not OK") + const body = await res.text().catch(() => "") + throw new Error( + `Crowdin postBuildProjectFileTranslation failed (${res.status}): ${body}` + ) + } + + type JsonResponse = { data: BuildProjectFileTranslationResponse } + const json: JsonResponse = await res.json() + console.log("Built file:", json.data) + return json.data +} + +/** + * method: GET + * @param downloadUrl + * @returns { buffer: Buffer } + */ +const getBuiltFile = async ( + downloadUrl: string + // ): Promise<{ buffer: Buffer; fileName: string; contentType: string }> => { +): Promise<{ buffer: Buffer }> => { + try { + const res = await fetch(downloadUrl) + + if (!res.ok) { + const body = await res.text().catch(() => "") + throw new Error(`Failed to download built file (${res.status}): ${body}`) + } + + const arrayBuffer = await res.arrayBuffer() + const buffer = Buffer.from(arrayBuffer) + + return { buffer } + } catch (error) { + console.error("getBuiltFile error:", error) + throw error + } +} + +/** + * Retrieves the Git object for a branch from the GitHub API and returns its underlying BranchObject. + * + * Fetches the ref for the given branch name from: + * https://api.github.com/repos/{env.ghOrganization}/{env.ghRepo}/git/ref/heads/{branch} + * using the preconfigured `gitHubBearerHeaders`. + * + * @param branch - The branch name to look up (for example "main" or "dev"). + * @returns A promise that resolves to the BranchObject extracted from the GitHub API response. + * + * @throws {Error} If the HTTP response is not OK (non-2xx). The thrown error includes the HTTP status + * and the response body text (when available). + * @throws {SyntaxError} If the response body cannot be parsed as JSON. + * + * @remarks + * - This function expects `env.ghOrganization`, `env.ghRepo`, and `gitHubBearerHeaders` to be available + * in the enclosing scope and correctly configured. + * - The function returns the `.object` property of the BranchDetailsResponse returned by GitHub. + * - Network errors (e.g. connectivity issues) will propagate as rejected promises from `fetch`. + * + * @example + * ```ts + * // resolves to the branch's object (sha, type, url) + * const obj = await getBranchObject("dev"); + * ``` + */ +const getBranchObject = async (branch: string): Promise => { + // https://api.github.com/repos/{{ $('env').item.json.ghOrganization }}/{{ $('env').item.json.ghRepo }}/git/ref/heads/dev + const url = new URL( + `https://api.github.com/repos/${env.ghOrganization}/${env.ghRepo}/git/ref/heads/${branch}` + ) + + const res = await fetchWithRetry(url.toString(), { + headers: gitHubBearerHeaders, + }) + + if (!res.ok) { + console.warn("Res not OK") + const body = await res.text().catch(() => "") + throw new Error(`GitHub getBranchObject (${res.status}): ${body}`) + } + + type JsonResponse = BranchDetailsResponse + const json: JsonResponse = await res.json() + // console.log("getBranchDetails results", json) + return json.object +} + +const createBranchName = () => { + const ts = new Date().toISOString().replace(/\..*$/, "").replace(/[:]/g, "-") // e.g., 2025-11-10T04-20-13 + return "i18n/import/" + ts +} + +const getDestinationFromPath = ( + crowdinFilePath: string, // e.g. src/intl/en/page-foo.json OR public/content/.../index.md + internalLanguageCode: string +) => { + const normalized = crowdinFilePath.replace(/^\//, "") + const isJson = normalized.toLowerCase().endsWith(".json") + const isMarkdown = normalized.toLowerCase().endsWith(".md") + + let destinationPath = normalized + + if (isJson) { + // JSON: src/intl/en/*.json -> src/intl//*.json + if (normalized.startsWith("src/intl/en/")) { + destinationPath = normalized.replace( + /^src\/intl\/en\//, + `src/intl/${internalLanguageCode}/` + ) + } else if (normalized.startsWith("src/intl/")) { + // Fallback: if for some reason "en" segment is missing, inject lang after src/intl/ + const parts = normalized.split("/") + // parts: [src, intl, ...] + parts.splice(2, 0, internalLanguageCode) + destinationPath = parts.join("/") + } + } else if (isMarkdown) { + // Markdown: public/content//index.md -> public/content/translations///index.md + if (normalized.startsWith("public/content/")) { + const rel = normalized.replace(/^public\/content\//, "") + // If already inside translations/, avoid duplicating; rewrite to current lang + const relParts = rel.split("/").filter(Boolean) + if (relParts[0] === "translations") { + // Drop existing translations// + const rest = relParts.slice(2).join("/") + destinationPath = `public/content/translations/${internalLanguageCode}/${rest}` + } else { + destinationPath = `public/content/translations/${internalLanguageCode}/${rel}` + } + } + } + + console.log( + `[DEBUG] Destination mapping: ${crowdinFilePath} -> ${destinationPath} (lang=${internalLanguageCode})` + ) + return destinationPath +} + +/** + * method: PUT + */ +const postCreateBranchFrom = async (ref = env.baseBranch) => { + const { sha } = await getBranchObject(ref) + + const branch = createBranchName() + + const url = new URL( + `https://api.github.com/repos/${env.ghOrganization}/${env.ghRepo}/git/refs` + ) + + try { + console.log( + `[DEBUG] Creating branch from base="${ref}" sha=${sha} -> new branch="${branch}"` + ) + const res = await fetchWithRetry(url.toString(), { + method: "POST", + headers: { + ...gitHubBearerHeaders, + "Content-Type": "application/json", + }, + body: JSON.stringify({ ref: `refs/heads/${branch}`, sha }), + }) + + if (!res.ok) { + console.warn("Res not OK") + const body = await res.text().catch(() => "") + console.error( + `[ERROR] Failed to create branch. URL=${url.toString()} status=${res.status}` + ) + throw new Error(`GitHub createBranchFrom (${res.status}): ${body}`) + } + + return { branch, sha } + } catch (error) { + console.error(error) + process.exit(1) + } +} + +const getPathSha = async (path: string, branch: string) => { + const url = new URL( + `https://api.github.com/repos/${env.ghOrganization}/${env.ghRepo}/contents/${path}?ref=${branch}` + ) + + const res = await fetchWithRetry(url.toString(), { + headers: gitHubBearerHeaders, + }) + + if (!res.ok) { + console.warn("Res not OK") + const body = await res.text().catch(() => "") + throw new Error(`GitHub getPathSha (${res.status}): ${body}`) + } + + type JsonResponse = { sha: string } + const { sha }: JsonResponse = await res.json() + + return { sha } +} +const putCommitFile = async ( + buffer: Buffer, + destinationPath: string, + branch: string, + sha?: string, + attempt = 0 +): Promise => { + const url = `https://api.github.com/repos/${env.ghOrganization}/${env.ghRepo}/contents/${destinationPath}` + + try { + // Use the buffer contents as base64-encoded content for the commit + const contentBase64 = buffer.toString("base64") + + const body = { + message: `update(i18n): ${destinationPath}`, + content: contentBase64, + branch, + } + + if (sha) body["sha"] = sha + + const res = await fetchWithRetry(url.toString(), { + method: "PUT", + headers: { + ...gitHubBearerHeaders, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }) + + if (res.status === 422) { + const { sha: fileSha } = await getPathSha(destinationPath, branch) + console.warn( + `[RETRY] 422 Unprocessable for ${destinationPath}. Retrying with existing SHA ${fileSha}` + ) + return await putCommitFile( + buffer, + destinationPath, + branch, + fileSha, + attempt + ) + } + + if (res.status === 409) { + if (attempt >= 5) { + const bodyText = await res.text().catch(() => "") + throw new Error( + `GitHub putCommitFile conflict persists after ${attempt} retries (${res.status}): ${bodyText}` + ) + } + const backoff = 500 * Math.pow(2, attempt) // 500ms, 1s, 2s, 4s, 8s + console.warn( + `[RETRY] 409 Conflict for ${destinationPath}. Attempt ${attempt + 1}. Waiting ${backoff}ms before retry.` + ) + await delay(backoff) + const { sha: latestSha } = await getPathSha(destinationPath, branch) + return await putCommitFile( + buffer, + destinationPath, + branch, + latestSha, + attempt + 1 + ) + } + + if (!res.ok) { + console.warn("Res not OK") + const body = await res.text().catch(() => "") + throw new Error(`GitHub putCommitFile (${res.status}): ${body}`) + } + } catch (error) { + console.error(error) + process.exit(1) + } +} + +const postPullRequest = async (head: string, base = env.baseBranch) => { + const url = new URL( + `https://api.github.com/repos/${env.ghOrganization}/${env.ghRepo}/pulls` + ) + + const body = { + title: "i18n: automated Crowdin translation import", + head, + base, + body: "Automated Crowdin translation import", + } + + const res = await fetchWithRetry(url.toString(), { + method: "POST", + headers: { + ...gitHubBearerHeaders, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }) + + if (!res.ok) { + console.warn("Res not OK") + const body = await res.text().catch(() => "") + throw new Error(`Crowdin postPullRequest failed (${res.status}): ${body}`) + } + + const json = await res.json() + return json +} + +async function main(options?: { allLangs: boolean }) { + console.log(`[DEBUG] Starting main function with options:`, options) + console.log(`[DEBUG] Environment config:`, { + projectId: env.projectId, + baseBranch: env.baseBranch, + jsonRoot: env.jsonRoot, + mdRoot: env.mdRoot, + allCrowdinCodes: env.allCrowdinCodes, + }) + + // Fetch English files with the configured file limit + const allEnglishFiles = await getAllEnglishFiles(fileLimit) + console.log( + `[DEBUG] Found ${allEnglishFiles.length} English files from GitHub` + ) + + // TODO: Add filter here to select specific files + const fileMetadata = await getFileMetadata(allEnglishFiles) + console.log(`[DEBUG] Generated metadata for ${fileMetadata.length} files`) + console.log(`[DEBUG] First file metadata:`, fileMetadata[0]) + + const crowdinProjectFiles = await getCrowdinProjectFiles() // *** + console.log( + `[DEBUG] Found ${crowdinProjectFiles.length} files in Crowdin project` + ) + + /** + * Iterate through each file and upload + */ + const fileIdsSet = new Set() + // Maintain authoritative mapping of processed Crowdin fileId -> path (including newly added files this run) + const processedFileIdToPath: Record = {} + // Keep original English buffers to detect untranslated outputs + const englishBuffers: Record = {} + for (const file of fileMetadata) { + console.log(`[DEBUG] Processing file: ${file.filePath}`) + await (async () => { + let foundFile: CrowdinFileData | undefined + try { + foundFile = findCrowdinFile(file, crowdinProjectFiles) + } catch { + console.log("File not found in Crowdin, attempting to add new file") + } + + let crowdinFileResponse: CrowdinAddFileResponse | undefined + let effectiveFileId: number + let effectivePath: string + + if (foundFile) { + // File exists - DO NOT update to preserve parsed string structure + console.log( + `[SKIP-UPDATE] File already exists in Crowdin with ID: ${foundFile.id}, using existing structure` + ) + console.log( + `[SKIP-UPDATE] Skipping upload/update to preserve existing parsed strings` + ) + effectiveFileId = foundFile.id + effectivePath = foundFile.path + + // Still download English for buffer comparison later + console.log( + `[DOWNLOAD] Downloading English source for buffer comparison: ${file.download_url}` + ) + const fileBuffer = await downloadGitHubFile(file.download_url) + englishBuffers[effectiveFileId] = fileBuffer + } else { + // File doesn't exist - create it + console.log(`[UPLOAD] File NOT found in Crowdin, creating new file`) + console.log( + `[UPLOAD] Downloading English source from: ${file.download_url}` + ) + const fileBuffer = await downloadGitHubFile(file.download_url) + console.log(`[UPLOAD] Downloaded ${fileBuffer.length} bytes`) + + const storageInfo = await postFileToStorage( + fileBuffer, + file["Crowdin-API-FileName"] + ) + console.log( + `[UPLOAD] Uploaded to Crowdin storage with ID: ${storageInfo.id}` + ) + + // Derive full parent directory path (exclude filename) + const parts = file.filePath.split("/").filter(Boolean) + parts.pop() // remove filename + const parentDirPath = parts.join("/") || "/" + console.log( + `[UPLOAD] Creating new Crowdin file in directory path: ${parentDirPath}` + ) + crowdinFileResponse = await postCrowdinFile( + storageInfo.id, + file["Crowdin-API-FileName"], + parentDirPath + ) + console.log( + `[UPLOAD] ✓ Created new Crowdin file with ID: ${crowdinFileResponse.id}` + ) + + effectiveFileId = crowdinFileResponse.id + effectivePath = crowdinFileResponse.path + englishBuffers[effectiveFileId] = fileBuffer + + // Wait for new file parsing + const delayMs = 10000 + console.log( + `[UPLOAD] ⏱️ Waiting ${delayMs / 1000}s for Crowdin to parse new file...` + ) + await delay(delayMs) + console.log(`[UPLOAD] ✓ Parsing delay complete`) + } + + fileIdsSet.add(effectiveFileId) + // Record path for destination mapping later (Crowdin returns leading slash paths) + if (effectivePath) processedFileIdToPath[effectiveFileId] = effectivePath + })() + } + + // Unhide any hidden/duplicate strings before pre-translation + console.log( + `\n[UNHIDE] ========== Unhiding strings in ${fileIdsSet.size} files ==========` + ) + for (const fileId of fileIdsSet) { + await unhideStringsInFile(fileId) + } + + console.log( + `\n[PRE-TRANSLATE] ========== Requesting AI Pre-Translation ==========` + ) + console.log(`[PRE-TRANSLATE] FileIds to translate:`, Array.from(fileIdsSet)) + console.log(`[PRE-TRANSLATE] Target languages:`, env.allCrowdinCodes) + console.log(`[PRE-TRANSLATE] AI Prompt ID:`, env.preTranslatePromptId) + + const applyPreTranslationResponse = await postApplyPreTranslation( + Array.from(fileIdsSet), + options?.allLangs ? env.allCrowdinCodes : env.allCrowdinCodes + ) + console.log( + `[PRE-TRANSLATE] ✓ Pre-translation job created with ID: ${applyPreTranslationResponse.identifier}` + ) + console.log( + `[PRE-TRANSLATE] Initial status:`, + applyPreTranslationResponse.status + ) + + console.log(`\n[PRE-TRANSLATE] Waiting for job to complete...`) + const preTranslateJobCompletedResponse = await awaitPreTranslationCompleted( + applyPreTranslationResponse.identifier + ) + + if (preTranslateJobCompletedResponse.status !== "finished") { + console.error( + "[PRE-TRANSLATE] ❌ Pre-translation did not finish successfully. Full response:", + preTranslateJobCompletedResponse + ) + throw new Error( + `Pre-translation ended with unexpected status: ${preTranslateJobCompletedResponse.status}` + ) + } + + console.log(`[PRE-TRANSLATE] ✓ Job completed successfully!`) + console.log( + `[PRE-TRANSLATE] Progress: ${preTranslateJobCompletedResponse.progress}%` + ) + console.log( + `[PRE-TRANSLATE] Full response:`, + JSON.stringify(preTranslateJobCompletedResponse, null, 2) + ) + + const { languageIds, fileIds } = preTranslateJobCompletedResponse.attributes + + // Build mapping for commit phase. Prefer processed mapping (includes newly added files); fall back to existing Crowdin snapshot for any missed IDs. + const fileIdToPathMapping: Record = {} + for (const fid of fileIds) { + if (processedFileIdToPath[fid]) { + fileIdToPathMapping[fid] = processedFileIdToPath[fid] + } else { + const existing = crowdinProjectFiles.find((f) => f.id === fid) + if (existing) fileIdToPathMapping[fid] = existing.path + } + if (!fileIdToPathMapping[fid]) { + console.warn( + `[WARN] Missing path mapping for fileId=${fid} (may impact destination path calculation)` + ) + } + } + // Build mapping between Crowdin IDs (e.g. "es-EM") and internal codes (e.g. "es") + const languagePairs = languageIds.map((crowdinId) => ({ + crowdinId, + internalLanguageCode: crowdinToInternalCodeMapping[crowdinId], + })) + + const { branch } = await postCreateBranchFrom(env.baseBranch) + console.log(`\n[BRANCH] ✓ Created branch: ${branch}`) + + // For each language + for (const { crowdinId, internalLanguageCode } of languagePairs) { + console.log( + `\n[BUILD] ========== Building translations for language: ${crowdinId} (internal: ${internalLanguageCode}) ==========` + ) + + // Build, download and commit each file updated + for (const fileId of fileIds) { + console.log(`\n[BUILD] --- Processing fileId: ${fileId} ---`) + const crowdinPath = fileIdToPathMapping[fileId] + console.log(`[BUILD] Crowdin path: ${crowdinPath}`) + + // 1- Build + console.log( + `[BUILD] Requesting build for fileId=${fileId}, language=${crowdinId}` + ) + const { url: downloadUrl } = await postBuildProjectFileTranslation( + fileId, + crowdinId, // Crowdin expects the Crowdin language ID here (e.g., "es-EM") + env.projectId + ) + console.log(`[BUILD] ✓ Build complete, download URL: ${downloadUrl}`) + + // 2- Download + console.log(`[BUILD] Downloading translated file...`) + const { buffer } = await getBuiltFile(downloadUrl) + console.log(`[BUILD] Downloaded ${buffer.length} bytes`) + + // Check if translation differs from English + const originalEnglish = englishBuffers[fileId] + if (originalEnglish) { + console.log( + `[BUILD] Original English size: ${originalEnglish.length} bytes` + ) + if (originalEnglish.compare(buffer) === 0) { + console.warn( + `[BUILD] ⚠️ Skipping commit - content identical to English (no translation occurred)` + ) + continue + } else { + console.log(`[BUILD] ✓ Translation differs from English, will commit`) + } + } + + // 3a- Get destination path + const destinationPath = getDestinationFromPath( + crowdinPath, + internalLanguageCode // Use internal code (e.g., "es") for repo path replacement + ) + console.log(`[BUILD] Destination path: ${destinationPath}`) + + // 3b- Commit + console.log(`[BUILD] Committing to branch: ${branch}`) + await putCommitFile(buffer, destinationPath, branch) + console.log(`[BUILD] ✓ Committed successfully`) + } + } + + console.log(`\n[PR] ========== Creating Pull Request ==========`) + console.log(`[PR] Head branch: ${branch}`) + console.log(`[PR] Base branch: ${env.baseBranch}`) + + const pr = await postPullRequest(branch, env.baseBranch) + + console.log(`\n[SUCCESS] ========== Translation import complete! ==========`) + console.log(`[SUCCESS] Pull Request URL: ${pr.html_url}`) + console.log(`[SUCCESS] PR Number: #${pr.number}`) + console.log(pr) +} + +main().catch((err) => { + console.error("Fatal error:", err) + process.exit(1) +}) diff --git a/src/scripts/i18n/types.ts b/src/scripts/i18n/types.ts new file mode 100644 index 00000000000..8c81a295668 --- /dev/null +++ b/src/scripts/i18n/types.ts @@ -0,0 +1,236 @@ +/** + * GET https://api.github.com/search/code + */ +export type GHOwner = { + login: string + id: number + node_id: string + avatar_url: string + gravatar_id: string + url: string + html_url: string + followers_url: string + following_url: string + gists_url: string + starred_url: string + subscriptions_url: string + organizations_url: string + repos_url: string + events_url: string + received_events_url: string + type: string + user_view_type: string + site_admin: boolean +} + +export type GHRepository = { + id: number + node_id: string + name: string + full_name: string + private: boolean + owner: GHOwner + html_url: string + description: string | null + fork: boolean + url: string + forks_url: string + keys_url: string + collaborators_url: string + teams_url: string + hooks_url: string + issue_events_url: string + events_url: string + assignees_url: string + branches_url: string + tags_url: string + blobs_url: string + git_tags_url: string + git_refs_url: string + trees_url: string + statuses_url: string + languages_url: string + stargazers_url: string + contributors_url: string + subscribers_url: string + subscription_url: string + commits_url: string + git_commits_url: string + comments_url: string + issue_comment_url: string + contents_url: string + compare_url: string + merges_url: string + archive_url: string + downloads_url: string + issues_url: string + pulls_url: string + milestones_url: string + notifications_url: string + labels_url: string + releases_url: string + deployments_url: string +} + +export type GitHubQueryResponseItem = { + name: string + path: string + sha: string + url: string + git_url: string + html_url: string + repository: GHRepository + score: number +} + +// Optional: the whole response is an array of items +export type GitHubQueryResponse = GitHubQueryResponseItem[] + +/** + * getFileMetadata + */ +export type ContentType = + | "application/json" + | "text/markdown" + | "application/octet-stream" + +export type GitHubCrowdinFileMetadata = { + "Crowdin-API-FileName": string + filePath: string // e.g., src/intl/en/page-layer-2-networks.json (no leading slash) + download_url: string + "Content-Type": ContentType +} + +/** + * GET https://api.crowdin.com/api/v2/projects/${env.projectID}/files + */ +export type CrowdinImportOptions = { + contentSegmentation: boolean + customSegmentation: boolean + excludeCodeBlocks: boolean + excludedFrontMatterElements: string[] + inlineTags: string[] +} + +export type CrowdinExportOptions = { + exportPattern: string | null + strongMarker: string + emphasisMarker: string + unorderedListBullet: string + tableColumnWidth: string + frontMatterQuotes: string +} + +export type CrowdinFileData = { + id: number // fileId + projectId: number + branchId: number | null + directoryId: number + name: string + title: string | null + context: string | null + type: "md" | "json" // string + path: string // e.g., /public/content/about/index.md (with leading slash) + status: string + revisionId: number + priority: string + importOptions: CrowdinImportOptions + exportOptions: CrowdinExportOptions + excludedTargetLanguages: string[] | null + parserVersion: number + createdAt: string + updatedAt: string +} + +/** + * PUT https://api.crowdin.com/api/v2/projects/${projectId}/files/${fileId} + * https://support.crowdin.com/developer/api/v2/#tag/Source-Files/operation/api.projects.files.put + */ +export type CrowdinFileInfoResponseModel = { + id: number + projectId: number + branchId: number | null + directoryId: number | null + name: string + title: string | null + context: string | null + type: string + path: string + status: string + revisionId: number + priority: string + importOptions: Record | null + exportOptions: Record | null + excludedTargetLanguages: string[] | null + parserVersion: number | null + createdAt: string | null + updatedAt: string | null +} + +export type CrowdinPreTranslateAttributes = { + languageIds: string[] + fileIds: number[] + method: string + autoApproveOption: string + duplicateTranslations: boolean + skipApprovedTranslations: boolean + labelIds: number[] + aiPromptId: number | null + excludeLabelIds: number[] + sourceLanguageId: string | null + fallbackLanguages: string[] | null + translateUntranslatedOnly: boolean + translateWithPerfectMatchOnly: boolean +} + +export type CrowdinPreTranslateResponse = { + identifier: string + status: "created" | "in_progress" | "canceled" | "failed" | "finished" + progress: number // In percentages + attributes: CrowdinPreTranslateAttributes + createdAt: string + updatedAt: string + startedAt: string | null + finishedAt: string | null + eta: string | null +} + +export type BuildProjectFileTranslationResponse = { + url: string + expireIn: string + etag: string +} + +export type BranchObject = { + sha: string + type: string // e.g. "commit" + url: string +} + +export type BranchDetailsResponse = { + ref: string // e.g. "refs/heads/dev" + node_id: string + url: string + object: BranchObject +} + +export type CrowdinAddFileResponse = { + id: number + projectId: number + branchId: number | null + directoryId: number | null + name: string + title: string | null + context: string | null + type: string + path: string + status: string + revisionId: number + priority: string + importOptions: Record | null + exportOptions: Record | null + excludedTargetLanguages: string[] | null + parserVersion: number | null + createdAt: string | null + updatedAt: string | null +} diff --git a/src/scripts/i18n/unhide-strings.ts b/src/scripts/i18n/unhide-strings.ts new file mode 100644 index 00000000000..40aed1b9298 --- /dev/null +++ b/src/scripts/i18n/unhide-strings.ts @@ -0,0 +1,72 @@ +/** + * Unhide all hidden/duplicate strings in a Crowdin file + */ + +import dotenv from "dotenv" + +dotenv.config({ path: ".env.local" }) + +const API_KEY = process.env.I18N_CROWDIN_API_KEY! +const PROJ_ID = 834930 +const TARGET_FILE_ID = 17434 // organizing/index.md + +const requestHeaders = { + Authorization: `Bearer ${API_KEY}`, + "Content-Type": "application/json", +} + +async function unhideAllStrings() { + console.log(`\n=== Unhiding strings in file ${TARGET_FILE_ID} ===`) + + // Get all strings from the file + const listUrl = `https://api.crowdin.com/api/v2/projects/${PROJ_ID}/strings?fileId=${TARGET_FILE_ID}&limit=500` + + const listRes = await fetch(listUrl, { headers: requestHeaders }) + if (!listRes.ok) { + throw new Error(`Failed to list strings: ${listRes.status}`) + } + + const listJson = await listRes.json() + console.log(`Found ${listJson.data.length} strings`) + + let unhiddenCount = 0 + + for (const item of listJson.data) { + const stringId = item.data.id + const isHidden = item.data.isHidden + + if (!isHidden) { + continue + } + + // Unhide the string using PATCH + const patchUrl = `https://api.crowdin.com/api/v2/projects/${PROJ_ID}/strings/${stringId}` + + const patchRes = await fetch(patchUrl, { + method: "PATCH", + headers: requestHeaders, + body: JSON.stringify([ + { + op: "replace", + path: "/isHidden", + value: false, + }, + ]), + }) + + if (!patchRes.ok) { + const text = await patchRes.text() + console.error(`Failed to unhide string ${stringId}: ${text}`) + continue + } + + unhiddenCount++ + if (unhiddenCount % 10 === 0) { + console.log(`Unhidden ${unhiddenCount} strings...`) + } + } + + console.log(`\n✅ Successfully unhidden ${unhiddenCount} strings!`) +} + +unhideAllStrings().catch(console.error)