diff --git a/.github/workflows/crowdin-ci.yml b/.github/workflows/crowdin-ci.yml deleted file mode 100644 index 6f9fcf48d54..00000000000 --- a/.github/workflows/crowdin-ci.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: Crowdin CI - -on: - schedule: - - cron: "20 4 1 * *" # Runs at 4:20 AM on the first day of every month - workflow_dispatch: # Can be dispatched manually - -jobs: - create_approved_language_bucket_prs: - runs-on: ubuntu-latest - env: - CROWDIN_API_KEY: ${{ secrets.CROWDIN_API_KEY }} - CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - steps: - # Set up environment - - name: Check out code - uses: actions/checkout@v6 - - - name: Setup pnpm - uses: pnpm/action-setup@v4 - - - name: Set up Node.js - uses: actions/setup-node@v6 - with: - node-version: 20 - cache: 'pnpm' - - - name: Install dependencies - run: pnpm install - - - name: Install ts-node - run: pnpm add -g ts-node - - - name: Set up git - run: | - git config --global user.email "actions@github.com" - git config --global user.name "GitHub Action" - - - name: Fetch latest dev - run: git fetch origin dev - - # Build translations - - name: Build Crowdin project - id: build-crowdin - run: | - npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/translations/triggerBuild.ts; - grep BUILD_ID output.env >> $GITHUB_ENV; - - - name: Await latest build to finish - run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/translations/awaitLatestBuild.ts - - - name: Check build success - run: | - if [ $(grep BUILD_SUCCESS output.env | cut -d'=' -f2) = false ]; then - echo "Build timed out, exiting" - exit 1 - fi - shell: bash - - # Prepare bucket ids - - name: Get latest translation bucket directory ids - run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/translations/getBucketDirectoryIds.ts - - # Import approved translations - - name: Get translations - run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/translations/getTranslations.ts - - # Post updates as language-specific PRs - - name: Process commits and post PRs by language - run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/translations/postLangPRs.ts diff --git a/.github/workflows/generate-review-report.yml b/.github/workflows/generate-review-report.yml deleted file mode 100644 index 0e4765b455d..00000000000 --- a/.github/workflows/generate-review-report.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Generate Crowdin translation review report - -on: - workflow_dispatch: - -jobs: - generate_report: - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v6 - - - name: Setup pnpm - uses: pnpm/action-setup@v4 - - - name: Set up Node.js - uses: actions/setup-node@v6 - with: - node-version: 20 - cache: 'pnpm' - - - name: Install dependencies - run: pnpm install - - - name: Install ts-node - run: pnpm add -g ts-node - - - name: Run script - run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/reports/generateReviewReport.ts - env: - CROWDIN_API_KEY: ${{ secrets.CROWDIN_API_KEY }} - CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }} - - - name: Upload output as artifact - uses: actions/upload-artifact@v5 - with: - name: output - path: ./src/data/crowdin/bucketsAwaitingReviewReport.csv diff --git a/.github/workflows/get-crowdin-contributors.yml b/.github/workflows/get-crowdin-contributors.yml deleted file mode 100644 index 3add4e20731..00000000000 --- a/.github/workflows/get-crowdin-contributors.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: Update Crowdin Contributors - -on: - schedule: - - cron: "0 0 * * SUN" # Runs every Sunday at midnight - workflow_dispatch: - -jobs: - get_data_and_create_pr: - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v6 - - - name: Setup pnpm - uses: pnpm/action-setup@v4 - - - name: Set up Node.js - uses: actions/setup-node@v6 - with: - node-version: 20 - cache: 'pnpm' - - - name: Install dependencies - run: pnpm install - - - name: Install ts-node - run: pnpm add -g ts-node - - - name: Set up git - run: | - git config --global user.email "actions@github.com" - git config --global user.name "GitHub Action" - - - name: Generate timestamp and readable date - id: date - run: | - echo "TIMESTAMP=$(date +'%Y%m%d%H%M%S')" >> $GITHUB_ENV - echo "READABLE_DATE=$(date +'%B %-d')" >> $GITHUB_ENV - - - name: Fetch latest dev and create new branch - run: | - git fetch origin dev - git checkout -b "automated-update-${{ env.TIMESTAMP }}" origin/dev - - - name: Run script - run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/getCrowdinContributors.ts - env: - CROWDIN_API_KEY: ${{ secrets.CROWDIN_API_KEY }} - - - name: Commit and push - run: | - git add -A - git commit -m "Update Crowdin contributors" - git push origin "automated-update-${{ env.TIMESTAMP }}" - - - name: Create PR body - run: | - echo "This PR was automatically created to update Crowdin contributors." > pr_body.txt - echo "This workflows runs every Sunday at 00:00 (UTC)." >> pr_body.txt - echo "" >> pr_body.txt - echo "Thank you to everyone contributing to translate ethereum.org ❤️" >> pr_body.txt - - - name: Create Pull Request - run: | - echo "${{ secrets.GITHUB_TOKEN }}" | gh auth login --with-token - gh pr create --base dev --head "automated-update-${{ env.TIMESTAMP }}" --title "Update translation contributors from Crowdin - ${{ env.READABLE_DATE }}" --body-file pr_body.txt diff --git a/.github/workflows/get-leaderboard-reports.yml b/.github/workflows/get-leaderboard-reports.yml deleted file mode 100644 index 4798e0a4076..00000000000 --- a/.github/workflows/get-leaderboard-reports.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Update Crowdin leaderboard data - -on: - schedule: - - cron: "20 16 1 * *" - workflow_dispatch: - -jobs: - get_data_and_create_pr: - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v6 - - - name: Setup pnpm - uses: pnpm/action-setup@v4 - - - name: Set up Node.js - uses: actions/setup-node@v6 - with: - node-version: 20 - cache: 'pnpm' - - - name: Install dependencies - run: pnpm install - - - name: Install ts-node - run: pnpm add -g ts-node - - - name: Set up git - run: | - git config --global user.email "actions@github.com" - git config --global user.name "GitHub Action" - - - name: Generate timestamp and readable date - id: date - run: | - echo "TIMESTAMP=$(date +'%Y%m%d%H%M%S')" >> $GITHUB_ENV - echo "READABLE_DATE=$(date +'%B %-d')" >> $GITHUB_ENV - - - name: Fetch latest dev and create new branch - run: | - git fetch origin dev - git checkout -b "automated-update-${{ env.TIMESTAMP }}" origin/dev - - - name: Run script - run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/leaderboard/getLeaderboardReports.ts - env: - CROWDIN_API_KEY: ${{ secrets.CROWDIN_API_KEY }} - CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }} - - - name: Commit and push - run: | - git add -A - git commit -m "Update Crowdin leaderboard data" - git push origin "automated-update-${{ env.TIMESTAMP }}" - - - name: Create PR body - run: | - echo "This PR was automatically created to update Crowdin leaderboard data." > pr_body.txt - echo "This workflows runs on the first of each month at 16:20 (UTC)." >> pr_body.txt - echo "" >> pr_body.txt - echo "Thank you to everyone contributing to translate ethereum.org ❤️" >> pr_body.txt - - - name: Create Pull Request - run: | - gh auth login --with-token ${{ secrets.GITHUB_TOKEN }} - gh pr create --base dev --head "automated-update-${{ env.TIMESTAMP }}" --title "Update translation leaderboard data from Crowdin - ${{ env.READABLE_DATE }}" --body-file pr_body.txt diff --git a/.github/workflows/get-translation-progress.yml b/.github/workflows/get-translation-progress.yml deleted file mode 100644 index a16d9f957f3..00000000000 --- a/.github/workflows/get-translation-progress.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Update Crowdin translation progression - -on: - schedule: - - cron: "20 16 * * FRI" - workflow_dispatch: - -jobs: - get_data_and_create_pr: - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v6 - - - name: Setup pnpm - uses: pnpm/action-setup@v4 - - - name: Set up Node.js - uses: actions/setup-node@v6 - with: - node-version: 20 - cache: 'pnpm' - - - name: Install dependencies - run: pnpm install - - - name: Install ts-node - run: pnpm add -g ts-node - - - name: Set up git - run: | - git config --global user.email "actions@github.com" - git config --global user.name "GitHub Action" - - - name: Generate timestamp and readable date - id: date - run: | - echo "TIMESTAMP=$(date +'%Y%m%d%H%M%S')" >> $GITHUB_ENV - echo "READABLE_DATE=$(date +'%B %-d')" >> $GITHUB_ENV - - - name: Fetch latest dev and create new branch - run: | - git fetch origin dev - git checkout -b "automated-update-${{ env.TIMESTAMP }}" origin/dev - - - name: Run script - run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/getTranslationProgress.ts - env: - CROWDIN_API_KEY: ${{ secrets.CROWDIN_API_KEY }} - CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }} - - - name: Commit and push - run: | - git add -A - git commit -m "Update Crowdin translation progress" - git push origin "automated-update-${{ env.TIMESTAMP }}" - - - name: Create PR body - run: | - echo "This PR was automatically created to update Crowdin translation progress." > pr_body.txt - echo "This workflows runs every Friday at 16:20 (UTC)." >> pr_body.txt - echo "" >> pr_body.txt - echo "Thank you to everyone contributing to translate ethereum.org ❤️" >> pr_body.txt - - - name: Create Pull Request - run: | - echo "${{ secrets.GITHUB_TOKEN }}" | gh auth login --with-token - gh pr create --base dev --head "automated-update-${{ env.TIMESTAMP }}" --title "Update translation progress from Crowdin - ${{ env.READABLE_DATE }}" --body-file pr_body.txt diff --git a/.gitignore b/.gitignore index bd0197d2f42..93385efd35e 100644 --- a/.gitignore +++ b/.gitignore @@ -53,9 +53,6 @@ robots.txt # Local Netlify folder .netlify -# .crowdin folder used as temp forlder for crowdin-import script -.crowdin - # workplace configuration .vscode .idea diff --git a/AGENTS.md b/AGENTS.md index e9ecc6edf0c..5effa3ccd2b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -100,7 +100,6 @@ pnpm build-storybook # Build Storybook pnpm chromatic # Run Chromatic visual tests # Content Management -pnpm crowdin-import # Import translations from Crowdin pnpm markdown-checker # Validate markdown content pnpm events-import # Import community events ``` diff --git a/package.json b/package.json index bcbd3d6d473..5fa6137c45d 100644 --- a/package.json +++ b/package.json @@ -11,15 +11,11 @@ "lint:fix": "next lint --fix", "format": "prettier \"**/*.{js,jsx,ts,tsx}\" --write", "preversion": "bash ./src/scripts/updatePublishDate.sh", - "crowdin-contributors": "ts-node -O '{ \"module\": \"commonjs\" }' src/scripts/crowdin/getCrowdinContributors.ts", "storybook": "storybook dev -p 6006", "build-storybook": "storybook build", "build-storybook:chromatic": "storybook build --test", "chromatic": "chromatic --project-token fee8e66c9916", - "crowdin-clean": "rm -rf .crowdin && mkdir .crowdin", - "crowdin-import": "ts-node src/scripts/crowdin-import.ts", "markdown-checker": "ts-node -O '{ \"module\": \"commonjs\" }' src/scripts/markdownChecker.ts", - "crowdin-needs-review": "ts-node -O '{ \"module\": \"commonjs\" }' src/scripts/crowdin/reports/generateReviewReport.ts", "update-tutorials": "ts-node -O '{ \"module\": \"commonjs\" }' src/scripts/update-tutorials-list.ts", "prepare": "husky", "test": "pnpm test:unit && pnpm test:e2e", @@ -33,7 +29,6 @@ }, "dependencies": { "@aws-sdk/client-ses": "^3.859.0", - "@crowdin/crowdin-api-client": "^1.25.0", "@docsearch/react": "^3.5.2", "@hookform/resolvers": "^3.8.0", "@netlify/blobs": "^10.4.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6d89a30cb08..f979e3299b2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -16,9 +16,6 @@ importers: '@aws-sdk/client-ses': specifier: ^3.859.0 version: 3.859.0 - '@crowdin/crowdin-api-client': - specifier: ^1.25.0 - version: 1.44.0 '@docsearch/react': specifier: ^3.5.2 version: 3.9.0(@algolia/client-search@5.25.0)(@types/react@18.2.57)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(search-insights@2.17.3) @@ -1182,10 +1179,6 @@ packages: '@coinbase/wallet-sdk@4.3.0': resolution: {integrity: sha512-T3+SNmiCw4HzDm4we9wCHCxlP0pqCiwKe4sOwPH3YAK2KSKjxPRydKu6UQJrdONFVLG7ujXvbd/6ZqmvJb8rkw==} - '@crowdin/crowdin-api-client@1.44.0': - resolution: {integrity: sha512-mDfow8999uC0jxoQ57yJACx6gYZohvrgbXN3/vW2E/sdrrnvYNOaYGG1o/QdNy9qq3PyKBMhc3SED7tRejigZw==} - engines: {node: '>=12.9.0'} - '@cspotcode/source-map-support@0.8.1': resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} engines: {node: '>=12'} @@ -4950,9 +4943,6 @@ packages: async-mutex@0.2.6: resolution: {integrity: sha512-Hs4R+4SPgamu6rSGW8C7cV9gaWUKEHykfzCCvIRuaVv636Ju10ZdeUbvb4TBEW0INuq2DHZqXbK4Nd3yG4RaRw==} - asynckit@0.4.0: - resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} - atomic-sleep@1.0.0: resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==} engines: {node: '>=8.0.0'} @@ -4972,9 +4962,6 @@ packages: resolution: {integrity: sha512-Xm7bpRXnDSX2YE2YFfBk2FnF0ep6tmG7xPh8iHee8MIcrgq762Nkce856dYtJYLkuIoYZvGfTs/PbZhideTcEg==} engines: {node: '>=4'} - axios@1.9.0: - resolution: {integrity: sha512-re4CqKTJaURpzbLHtIi6XpDv20/CnpXOtjRY5/CU32L8gU8ek9UIivcfvSWvmKEngmVbrUtPpdDwWDWL7DNHvg==} - axobject-query@4.1.0: resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==} engines: {node: '>= 0.4'} @@ -5318,10 +5305,6 @@ packages: colorette@2.0.20: resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} - combined-stream@1.0.8: - resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} - engines: {node: '>= 0.8'} - comma-separated-tokens@2.0.3: resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} @@ -5678,10 +5661,6 @@ packages: defu@6.1.4: resolution: {integrity: sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==} - delayed-stream@1.0.0: - resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} - engines: {node: '>=0.4.0'} - dequal@2.0.3: resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} engines: {node: '>=6'} @@ -6330,15 +6309,6 @@ packages: flatted@3.3.3: resolution: {integrity: sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==} - follow-redirects@1.15.9: - resolution: {integrity: sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==} - engines: {node: '>=4.0'} - peerDependencies: - debug: '*' - peerDependenciesMeta: - debug: - optional: true - for-each@0.3.5: resolution: {integrity: sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==} engines: {node: '>= 0.4'} @@ -6354,10 +6324,6 @@ packages: typescript: '>3.6.0' webpack: ^5.11.0 - form-data@4.0.5: - resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} - engines: {node: '>= 6'} - forwarded-parse@2.1.2: resolution: {integrity: sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==} @@ -11029,12 +10995,6 @@ snapshots: eventemitter3: 5.0.1 preact: 10.26.8 - '@crowdin/crowdin-api-client@1.44.0': - dependencies: - axios: 1.9.0 - transitivePeerDependencies: - - debug - '@cspotcode/source-map-support@0.8.1': dependencies: '@jridgewell/trace-mapping': 0.3.9 @@ -16061,8 +16021,6 @@ snapshots: dependencies: tslib: 2.8.1 - asynckit@0.4.0: {} - atomic-sleep@1.0.0: {} autoprefixer@10.4.21(postcss@8.5.4): @@ -16081,14 +16039,6 @@ snapshots: axe-core@4.10.3: {} - axios@1.9.0: - dependencies: - follow-redirects: 1.15.9 - form-data: 4.0.5 - proxy-from-env: 1.1.0 - transitivePeerDependencies: - - debug - axobject-query@4.1.0: {} babel-loader@9.2.1(@babel/core@7.27.4)(webpack@5.99.9(esbuild@0.25.12)): @@ -16443,10 +16393,6 @@ snapshots: colorette@2.0.20: {} - combined-stream@1.0.8: - dependencies: - delayed-stream: 1.0.0 - comma-separated-tokens@2.0.3: {} commander@13.1.0: {} @@ -16802,8 +16748,6 @@ snapshots: defu@6.1.4: {} - delayed-stream@1.0.0: {} - dequal@2.0.3: {} derive-valtio@0.1.0(valtio@1.13.2(@types/react@18.2.57)(react@18.3.1)): @@ -17704,8 +17648,6 @@ snapshots: flatted@3.3.3: {} - follow-redirects@1.15.9: {} - for-each@0.3.5: dependencies: is-callable: 1.2.7 @@ -17732,14 +17674,6 @@ snapshots: typescript: 5.8.3 webpack: 5.99.9(esbuild@0.25.12) - form-data@4.0.5: - dependencies: - asynckit: 0.4.0 - combined-stream: 1.0.8 - es-set-tostringtag: 2.1.0 - hasown: 2.0.2 - mime-types: 2.1.35 - forwarded-parse@2.1.2: {} fraction.js@4.3.7: {} diff --git a/src/data/crowdin/translation-buckets-dirs.json b/src/data/crowdin/translation-buckets-dirs.json deleted file mode 100644 index 9995068573e..00000000000 --- a/src/data/crowdin/translation-buckets-dirs.json +++ /dev/null @@ -1,114 +0,0 @@ -[ - { - "id": 5497, - "name": "01)" - }, - { - "id": 5499, - "name": "05)" - }, - { - "id": 5501, - "name": "04)" - }, - { - "id": 5503, - "name": "13)" - }, - { - "id": 5505, - "name": "18)" - }, - { - "id": 5507, - "name": "21)" - }, - { - "id": 5509, - "name": "23)" - }, - { - "id": 5513, - "name": "22)" - }, - { - "id": 6189, - "name": "14)" - }, - { - "id": 6197, - "name": "11)" - }, - { - "id": 6386, - "name": "27)" - }, - { - "id": 6534, - "name": "25)" - }, - { - "id": 7290, - "name": "07)" - }, - { - "id": 7292, - "name": "03)" - }, - { - "id": 7296, - "name": "09)" - }, - { - "id": 7300, - "name": "06)" - }, - { - "id": 7517, - "name": "10)" - }, - { - "id": 7819, - "name": "15)" - }, - { - "id": 7821, - "name": "16)" - }, - { - "id": 7823, - "name": "17)" - }, - { - "id": 7825, - "name": "19)" - }, - { - "id": 7827, - "name": "20)" - }, - { - "id": 7829, - "name": "24)" - }, - { - "id": 7831, - "name": "26)" - }, - { - "id": 11134, - "name": "02)" - }, - { - "id": 11136, - "name": "08)" - }, - { - "id": 11138, - "name": "12)" - }, - { - "id": 11140, - "name": "28)" - } -] \ No newline at end of file diff --git a/src/lib/constants.ts b/src/lib/constants.ts index 4459f348a4f..d9a0098c3bd 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -64,11 +64,6 @@ export const TOTAL_QUIZ_RETRY_RATE = 15.6 // Crowdin export const CROWDIN_PROJECT_URL = "https://crowdin.com/project/ethereum-org" -export const CROWDIN_PROJECT_ID = 363359 -export const CROWDIN_API_MAX_LIMIT = 500 -export const FIRST_CROWDIN_CONTRIBUTION_DATE = "2019-07-01T00:00:00+00:00" - -export const languagePathRootRegExp = /^.+\/content\/translations\/[a-z-]*\// // Metrics export const DAYS_TO_FETCH = 1 diff --git a/src/scripts/crowdin-import.ts b/src/scripts/crowdin-import.ts deleted file mode 100644 index 5fa9b9eeba8..00000000000 --- a/src/scripts/crowdin-import.ts +++ /dev/null @@ -1,414 +0,0 @@ -/* eslint-disable @typescript-eslint/no-var-requires */ -// Library requires -const i18Config = require("../../i18n.config.json") -const { - copyFileSync, - existsSync, - mkdirSync, - readdirSync, - readFileSync, -} = require("fs") -const { resolve, join } = require("path") -const argv = require("minimist")(process.argv.slice(2)) - -/****************************** - * Console flags * - ******************************/ - -/** - * -b,--buckets Prints buckets overview and exits - * -v,--verbose Prints verbose console logs - * -f,--full Prints full name of buckets in summary - */ - -/****************************** - * Instructions for use * - ******************************/ - -/** - * 1. Run `pnpm crowdin-clean` to initialize fresh ./.crowdin folder. This can also be used to erase contents when finished. - * - * 2a. Export/import CSV of languages ready for review: - * 1. Open "Website translation board" document in ethereum.org Notion (internal only) - * 2. Switch view of "Translation status by language" table to "All reviewed" - * 3. Click triple-dot (...) menu in TOP right corner of the entire app - * 4. Select "Export" > "Export as CSV" - * Export format: Markdown & CSV - * Include databases: Current view - * Include content: No files or images - * Include subpages: Off - * Click "Export" > Save zip file - * 5. Unzip contents into (or copy into) ./.crowdin folder in the root of this repo - * - * 2b. Alternatively, you can manually add buckets to import to the USER_OVERRIDE object below. - * 1. Add the number of the corresponding content bucket to the chosen language array below - * i.e., `es: [1, 10],` would import the "Homepage" and "Learn" buckets for Spanish - * 2. Save file without committing* - * - * Optionally: To view summary of buckets from CSV, run `pnpm crowdin-import --buckets` or `pnpm crowdin-import -b` - * Any items in USER_OVERRIDE will override the CSV import - * - * 3. Export translated content from Crowdin and import into ./.crowdin folder: - * 1. Export latest translated content from Crowdin and unzip - * 2. Copy languages folder from Crowdin export to ./.crowdin - * i.e., ./.crowdin/{lang-codes} - * - * 4. Execute script: - * 1. Execute script by running `pnpm crowdin-import` - * 2. If successful, copy `BUILD_LOCALES={langs}` output and paste in - * your `.env`, then build site to test results. - * - * *Remember: Revert any working changes to this file before committing Crowdin import - */ - -type BucketsList = { [key: string]: Array } -const USER_OVERRIDE: BucketsList = { - // FORMAT: lang_code: [bucket_number, bucket_number, ...], - // EXAMPLE: es: [1, 10, 12, 14], -} - -/****************************** - * Primary script logic below * - ******************************/ - -/** - * Exported languages from Crowdin come as a .zip file, when unzipped contains - * one folder for each language, using Crowdin language codes (which differ - * slight from those used in the repo). These folders must be copied into the - * root `.crowdin` folder of this repo. - * - * A CSV containing the language buckets that have been "Reviewed" can be exported - * from Crowdin to automate the process of importing the needed buckets. See - * "Instructions for use" above. - * - * You can alternatively use the USER_OVERRIDE object above to manually select buckets. - * - * The script iterates through each language chosen, using the dictionary object - * below to convert the repo lang code to the code used by Crowdin (only if - * needed, defaults to same). `fs` is used to find matching language folder. - * - * The "buckets" chosen (type number[]) are then iterated over, opening the - * corresponding folder that begins with the same number string (formatted 00). - * - * Each selected bucket folder is then iterated over, calling the - * `scrapeDirectory(_path, contentSubpath, repoLangCode)` function. This function - * iterates over every file contained in the working directory. If the filetype - * is `.json` the file is moved to the `/src/intl/{lang}/` directory. If the - * filetype is `.md` the `contentSubpath` is used to copy the file to its - * correct place in `/src/content/translations/{lang}/{contentSubpath}. - * - * If the directory item is another directory, `scrapeDirectory` is called - * recursively, passing the new `_path` and `contentSubpath`. The base case for - * this function is no more directory items remaining, and returns void. - */ - -// Initialize console arguments -const VERBOSE = Boolean(argv.v || argv.verbose) -const BUCKET_GENERATION_ONLY = Boolean(argv.b || argv.buckets) -const FULL_BUCKET_NAME_SUMMARY = Boolean(argv.f || argv.full) - -// Initialize root paths -const repoRoot: string = resolve("./") -const crowdinRoot: string = join(repoRoot, ".crowdin") -// If first time, create directory for user -if (!existsSync(crowdinRoot)) mkdirSync(crowdinRoot) - -/** - * Some language codes used in the repo differ from those used by Crowdin. - * This is used to convert any codes that may differ when performing folder lookup. - */ -const getCrowdinCode = (code: string): string => - i18Config.filter((lang) => lang.code === code)?.[0]?.crowdinCode || code - -/** - * Names for each bucket in order, zero indexed. - * Used for lookup in summary if FULL_BUCKET_NAME_SUMMARY (-f,--full) flag enabled. - */ -const BUCKET_NAMES: Array = [ - "Homepage", - "Use Ethereum Pages", - "Use Case Pages", - "Upgrades", - "Community Pages", - "Docs - Foundational Pages", - "Docs - Tech Stack Pages", - "Whitepaper", - "Docs - Advanced Pages", - "Learn Pages", - "Research Documentation", - "Contributing", - "Developer Tutorials I", - "Developer Tutorials II", - "Developer Tutorials III", -] - -// Initialize trackers object for summary -type LangTrackerEntry = { - buckets: Array - jsonCopyCount: number - mdCopyCount: number - error: string -} -type TrackerObject = { - emptyBuckets: number - langs: { - [key: string]: LangTrackerEntry - } -} -const trackers: TrackerObject = { - emptyBuckets: 0, - langs: {}, -} - -// Functions -/** - * Wrapper function to call console.log() only if VERBOSE (-v, --verbose) flag is enabled - * - * @param message Any arbitrary message - * @param optionalParams Any additional arbitrary messages - */ -const log = (message: unknown, ...optionalParams: unknown[]): void => { - VERBOSE && console.log(message, ...optionalParams) -} - -/** - * Fetches CSV exported from Notion "Website translation board" table - * See above for details on how to export CSV and import into repo - * @returns Object containing language codes as keys, and an array of bucket numbers to be imported - */ -const fetchReviewedCsv = (): BucketsList => { - const csvDir: string = readdirSync(crowdinRoot).filter((dir: string) => - dir.startsWith("Website translation board") - )[0] - if (!csvDir) return {} - const path = join(crowdinRoot, csvDir) - const reviewedCsvPath: Array = readdirSync(path).filter( - (file: string) => { - const fileParts: Array = file.split(".") - return ( - fileParts[0].startsWith("https") && - !fileParts[0].endsWith("all") && - fileParts[1] === "csv" - ) - } - )[0] - const bucketsList: BucketsList = {} - const csvFile = readFileSync(join(path, reviewedCsvPath), "utf8") - if (!csvFile) return {} - const data = csvFile.split("\n").map((row: string) => { - const quotePair = /"([^"]+)"/g - const sanitized = row.replaceAll(quotePair, (match) => - match.replace(",", " ").replace(/"/g, "") - ) - return sanitized.split(",") - }) - const headings = data.shift() - const langCodeIndex = headings.indexOf("code") - const firstBucketIndex = headings.findIndex((item: string) => - item.startsWith("1)") - ) - data.forEach((rowItems: Array) => { - const langCode = rowItems[langCodeIndex].split(" ").at(-1) // "es-EM → es" parses to "es" - if (!langCode) return - const bucketsForLang: Array = [] - rowItems.forEach((item: string, idx: number) => { - if (item.includes("Reviewed")) - bucketsForLang.push(idx - firstBucketIndex + 1) - }) - bucketsList[langCode] = bucketsForLang - }) - return bucketsList -} - -/** - * If any buckets are selected in USER_OVERRIDE, use those instead of importing from CSV. - */ -const useUserOverRide = - Object.values(USER_OVERRIDE).filter((buckets) => buckets.length > 0).length > - 0 - -const bucketsToImport: BucketsList = useUserOverRide - ? USER_OVERRIDE - : fetchReviewedCsv() - -const highestBucketNumber: number = Object.values(bucketsToImport).reduce( - (prev: number, buckets: Array): number => - buckets[buckets.length - 1] > prev ? buckets[buckets.length - 1] : prev, - 0 -) - -/** - * If BUCKET_GENERATION_ONLY (-b, --buckets) flag is enabled, show overview - * of all langs and buckets to be imported. Also print a copy/paste ready - * object for USER_OVERRIDE, then exit the script early. - */ -if (BUCKET_GENERATION_ONLY) { - const bucketsOverview = {} - Object.entries(bucketsToImport).forEach(([langCode, buckets]) => { - bucketsOverview[langCode] = Array(highestBucketNumber - 1) - .fill(0) - .map((_, i) => (buckets.includes(i + 1) ? i + 1 : "")) - }) - console.table(bucketsOverview) - console.log("const USER_OVERRIDE: BucketsList =", bucketsToImport) - process.exit(0) -} - -/** - * Reads `ls` file contents of `_path`, moving .md and .json files - * to their corresponding destinations in the repo. Function is called - * again recursively for subdirectories. - * - * @param _path An absolute path to the directory being scraped. - * @param contentSubpath The subpath deep to the lang-code directory, - * used to construct destination for markdown content files - * @param repoLangCode Language code used within the repo - * @returns void - */ -const scrapeDirectory = ( - _path: string, - contentSubpath: string, - repoLangCode: string -): void => { - if (!existsSync(_path)) return - const ls: Array = readdirSync(_path).filter( - (dir: string) => !dir.startsWith(".") - ) - ls.forEach((item: string) => { - const source: string = join(_path, item) - if (item.endsWith(".json")) { - const jsonDestDirPath: string = join( - repoRoot, - "src", - "intl", - repoLangCode - ) - if (!existsSync(jsonDestDirPath)) - mkdirSync(jsonDestDirPath, { recursive: true }) - const jsonDestinationPath: string = join(jsonDestDirPath, item) - log("Copy .json from", source, "to", jsonDestinationPath) - copyFileSync(source, jsonDestinationPath) - // Update .json tracker - trackers.langs[repoLangCode].jsonCopyCount++ - } else if (item.endsWith(".md")) { - const mdDestDirPath: string = join( - repoRoot, - "public", - "content", - "translations", - repoLangCode, - contentSubpath - ) - if (!existsSync(mdDestDirPath)) - mkdirSync(mdDestDirPath, { recursive: true }) - const mdDestinationPath: string = join(mdDestDirPath, item) - log("Copy .md from", source, "to", mdDestinationPath) - copyFileSync(source, mdDestinationPath) - // Update .md tracker - trackers.langs[repoLangCode].mdCopyCount++ - } else { - log(`Entering ${_path}/${item}`) - // If another directory, recursively call `scrapeDirectory` - scrapeDirectory( - `${_path}/${item}`, - `${contentSubpath}/${item}`, - repoLangCode - ) - } - }) -} - -// Filter out empty requests and map selection to usable array -type SelectionItem = { - repoLangCode: string - crowdinLangCode: string - buckets: Array -} -const importSelection: Array = Object.keys(bucketsToImport) - .filter((repoLangCode: string): boolean => { - if (!bucketsToImport[repoLangCode].length) trackers.emptyBuckets++ - return !!bucketsToImport[repoLangCode].length - }) - .map( - (repoLangCode: string): SelectionItem => ({ - repoLangCode, - crowdinLangCode: getCrowdinCode(repoLangCode), - buckets: bucketsToImport[repoLangCode], - }) - ) - -// Iterate through each selected language -importSelection.forEach( - ({ repoLangCode, crowdinLangCode, buckets }: SelectionItem): void => { - // Initialize tracker for language - trackers.langs[repoLangCode] = { - buckets: [], - jsonCopyCount: 0, - mdCopyCount: 0, - error: "", - } - // Initialize working directory and check for existence - const _path: string = join(crowdinRoot, crowdinLangCode) - if (!existsSync(_path)) { - trackers.langs[repoLangCode].error = - `Path doesn't exist for lang ${crowdinLangCode}` - return - } - const langLs: Array = readdirSync(_path) - // Iterate over each selected bucket, scraping contents with `scrapeDirectory` - buckets.forEach((bucket: number): void => { - const paddedBucket: string = bucket.toString().padStart(2, "0") - let bucketDirName = "" - langLs.forEach((bucketName: string) => { - bucketDirName += bucketName.startsWith(paddedBucket) ? bucketName : "" - }) - const bucketDirectoryPath: string = `${crowdinRoot}/${crowdinLangCode}/${bucketDirName}` - // Initial scrapeDirectory function call - scrapeDirectory(bucketDirectoryPath, ".", repoLangCode) - // Update tracker - trackers.langs[repoLangCode].buckets.push(BUCKET_NAMES[bucket - 1]) - }) - } -) - -// Construct console summary -type SummaryItem = { - repoLangCode: string - buckets: Array | Array - jsonCopyCount: number - mdCopyCount: number - error?: string -} -const summary: Array = importSelection.map( - (item: SelectionItem): SummaryItem => { - const { buckets: bucketNumbers, repoLangCode } = item - const { - buckets: bucketNames, - jsonCopyCount, - mdCopyCount, - error, - } = trackers.langs[repoLangCode] - return { - repoLangCode, - buckets: FULL_BUCKET_NAME_SUMMARY ? bucketNames : bucketNumbers, - jsonCopyCount, - mdCopyCount, - error, - } - } -) -const langsSummary: string = summary.reduce( - (prev: string, { repoLangCode }: { repoLangCode: string }): string => - `${prev},${repoLangCode}`, - "" -) - -// Print summary logs -log("Empty buckets:", trackers.emptyBuckets) -if (summary.length) { - console.table(summary) - console.log("Langs to test:", `\nNEXT_PUBLIC_BUILD_LOCALES=en${langsSummary}`) - console.log("🎉 Crowdin import complete.") -} else { - console.warn("Nothing imported, see instruction at top of crowdin-imports.ts") -} diff --git a/src/scripts/crowdin/api-client/crowdinClient.ts b/src/scripts/crowdin/api-client/crowdinClient.ts deleted file mode 100644 index 19cad7f7ba9..00000000000 --- a/src/scripts/crowdin/api-client/crowdinClient.ts +++ /dev/null @@ -1,9 +0,0 @@ -import crowdin from "@crowdin/crowdin-api-client" - -import "dotenv/config" - -const crowdinClient = new crowdin({ - token: process.env.CROWDIN_API_KEY || "", -}) - -export default crowdinClient diff --git a/src/scripts/crowdin/getCrowdinContributors.ts b/src/scripts/crowdin/getCrowdinContributors.ts deleted file mode 100644 index e5da700f572..00000000000 --- a/src/scripts/crowdin/getCrowdinContributors.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { getTranslatedMarkdownPaths } from "../markdownChecker" - -import getTranslationCostsReports from "./reports/getTranslationCostsReports" -import getAndSaveDirectories from "./source-files/fetchAndSaveDirectories" -import fetchAndSaveFileIds from "./source-files/fetchAndSaveFileIds" -import getDirectoryIds from "./utils" - -async function main() { - await getAndSaveDirectories() - const directoryIds = getDirectoryIds() - await fetchAndSaveFileIds(directoryIds) - const translatedMarkdownPaths = await getTranslatedMarkdownPaths() - await getTranslationCostsReports(translatedMarkdownPaths) -} - -main() - -export default main diff --git a/src/scripts/crowdin/getTranslationProgress.ts b/src/scripts/crowdin/getTranslationProgress.ts deleted file mode 100644 index 24fbbf72dbd..00000000000 --- a/src/scripts/crowdin/getTranslationProgress.ts +++ /dev/null @@ -1,48 +0,0 @@ -import fs from "fs" - -import { CROWDIN_API_MAX_LIMIT } from "../../lib/constants" -import type { ProjectProgressData } from "../../lib/types" - -import crowdin from "./api-client/crowdinClient" - -import "dotenv/config" - -async function main() { - const projectId = Number(process.env.CROWDIN_PROJECT_ID) || 363359 - - try { - const response = await crowdin.translationStatusApi.getProjectProgress( - projectId, - { - limit: CROWDIN_API_MAX_LIMIT, - } - ) - - if (!response) - throw new Error( - "Error fetching Crowdin translation progress. Check your environment variables for a working API key." - ) - - const progress = response.data.map( - ({ data }) => - ({ - languageId: data.languageId, - words: { - approved: data.words.approved, - total: data.words.total, - }, - }) as ProjectProgressData - ) - - fs.writeFileSync( - "src/data/translationProgress.json", - JSON.stringify(progress, null, 2) - ) - } catch (error: unknown) { - console.error((error as Error).message) - } -} - -main() - -export default main diff --git a/src/scripts/crowdin/import/main.ts b/src/scripts/crowdin/import/main.ts deleted file mode 100644 index 71e1714042b..00000000000 --- a/src/scripts/crowdin/import/main.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { existsSync, mkdirSync } from "fs" - -import { DOT_CROWDIN } from "../translations/constants" - -import type { BucketsList, SelectionItem, TrackerObject } from "./types" -import { getImportSelection, handleSummary, processLanguage } from "./utils" - -const main = (bucketList: BucketsList) => { - console.log("Bucket list:", bucketList) - - // If first time, create directory for user - if (!existsSync(DOT_CROWDIN)) mkdirSync(DOT_CROWDIN) - - // Initialize trackers object for summary - const trackers: TrackerObject = { emptyBuckets: 0, langs: {} } - - // Filter out empty requests and map selection to usable array - const importSelection: SelectionItem[] = getImportSelection( - bucketList, - trackers - ) - - // Iterate through each selected language - importSelection.forEach((item) => processLanguage(item, trackers)) - - handleSummary(importSelection, trackers) -} - -export default main diff --git a/src/scripts/crowdin/import/types.ts b/src/scripts/crowdin/import/types.ts deleted file mode 100644 index 5b0727c8575..00000000000 --- a/src/scripts/crowdin/import/types.ts +++ /dev/null @@ -1,27 +0,0 @@ -export type BucketsList = Record - -export type LangTrackerEntry = { - buckets: number[] - jsonCopyCount: number - mdCopyCount: number - error: string -} - -export type TrackerObject = { - emptyBuckets: number - langs: Record -} - -export type SelectionItem = { - repoLangCode: string - crowdinLangCode: string - buckets: number[] -} - -export type SummaryItem = { - repoLangCode: string - buckets: string[] | number[] - jsonCopyCount: number - mdCopyCount: number - error?: string -} diff --git a/src/scripts/crowdin/import/utils.ts b/src/scripts/crowdin/import/utils.ts deleted file mode 100644 index 22abdd53de2..00000000000 --- a/src/scripts/crowdin/import/utils.ts +++ /dev/null @@ -1,166 +0,0 @@ -import { copyFileSync, existsSync, mkdirSync, readdirSync, statSync } from "fs" -import { join } from "path" - -import i18Config from "../../../../i18n.config.json" -import { INTL_JSON_DIR, TRANSLATIONS_DIR } from "../../../lib/constants" -import { DOT_CROWDIN } from "../translations/constants" - -import { BucketsList, SelectionItem, SummaryItem, TrackerObject } from "./types" - -/** - * Some language codes used in the repo differ from those used by Crowdin. - * This is used to convert any codes that may differ when performing folder lookup. - */ -export const getCrowdinCode = (code: string): string => - i18Config.filter((lang) => lang.code === code)?.[0]?.crowdinCode || code - -/** - * Reads `ls` file contents of `path`, moving .md and .json files - * to their corresponding destinations in the repo. Function is called - * again recursively for subdirectories. - * - * @param path An absolute path to the directory being scraped. - * @param contentSubpath The subpath deep to the lang-code directory, - * used to construct destination for markdown content files - * @param repoLangCode Language code used within the repo - * @returns void - */ -export const scrapeDirectory = ( - path: string, - contentSubpath: string, - repoLangCode: string, - trackers: TrackerObject -): void => { - if (!existsSync(path)) return - const ls: string[] = readdirSync(path).filter( - (dir: string) => !dir.startsWith(".") - ) - ls.forEach((item: string) => { - const source: string = join(path, item) - if (item.endsWith(".json")) { - const jsonDestDirPath: string = join(INTL_JSON_DIR, repoLangCode) - if (!existsSync(jsonDestDirPath)) - mkdirSync(jsonDestDirPath, { recursive: true }) - const jsonDestinationPath: string = join(jsonDestDirPath, item) - copyFileSync(source, jsonDestinationPath) - // Update .json tracker - trackers.langs[repoLangCode].jsonCopyCount++ - } else if (item.endsWith(".md")) { - const mdDestDirPath: string = join( - TRANSLATIONS_DIR, - repoLangCode, - contentSubpath - ) - if (!existsSync(mdDestDirPath)) - mkdirSync(mdDestDirPath, { recursive: true }) - const mdDestinationPath: string = join(mdDestDirPath, item) - copyFileSync(source, mdDestinationPath) - // Update .md tracker - trackers.langs[repoLangCode].mdCopyCount++ - } else { - if (!statSync(source).isDirectory()) return - // If another directory, recursively call `scrapeDirectory` - scrapeDirectory( - `${path}/${item}`, - `${contentSubpath}/${item}`, - repoLangCode, - trackers - ) - } - }) -} - -export const getImportSelection = ( - buckets: BucketsList, - trackers: TrackerObject -): SelectionItem[] => - Object.keys(buckets) - .filter((repoLangCode: string): boolean => { - if (!buckets[repoLangCode].length) trackers.emptyBuckets++ - return !!buckets[repoLangCode].length - }) - .map( - (repoLangCode: string): SelectionItem => ({ - repoLangCode, - crowdinLangCode: getCrowdinCode(repoLangCode), - buckets: buckets[repoLangCode], - }) - ) - -/** - * ./postLangPRs.ts - */ - -export const processBucket = ( - bucket: number, - crowdinLangCode: string, - repoLangCode: string, - langLs: string[], - trackers: TrackerObject -): void => { - const paddedBucket: string = bucket.toString().padStart(2, "0") - let bucketDirName = "" - langLs.forEach((bucketName: string) => { - bucketDirName += bucketName.startsWith(paddedBucket) ? bucketName : "" - }) - const bucketDirectoryPath: string = `${DOT_CROWDIN}/${crowdinLangCode}/${bucketDirName}` - // Initial scrapeDirectory function call - scrapeDirectory(bucketDirectoryPath, ".", repoLangCode, trackers) - // Update tracker - trackers.langs[repoLangCode].buckets.push(bucket) -} - -export const processLanguage = ( - { repoLangCode, crowdinLangCode, buckets }: SelectionItem, - trackers: TrackerObject -): void => { - // Initialize tracker for language - trackers.langs[repoLangCode] = { - buckets: [], - jsonCopyCount: 0, - mdCopyCount: 0, - error: "", - } - // Initialize working directory and check for existence - const path: string = join(DOT_CROWDIN, crowdinLangCode) - if (!existsSync(path)) { - trackers.langs[repoLangCode].error = - `Path doesn't exist for lang ${crowdinLangCode}` - return - } - const langLs: string[] = readdirSync(path) - // Iterate over each selected bucket, scraping contents with `scrapeDirectory` - buckets.forEach((bucket) => - processBucket(bucket, crowdinLangCode, repoLangCode, langLs, trackers) - ) -} - -export const handleSummary = ( - selection: SelectionItem[], - trackers: TrackerObject -) => { - // Construct console summary - const summary: SummaryItem[] = selection.map( - (item: SelectionItem): SummaryItem => { - const { buckets, repoLangCode } = item - const { jsonCopyCount, mdCopyCount, error } = trackers.langs[repoLangCode] - return { - repoLangCode, - buckets, - jsonCopyCount, - mdCopyCount, - error, - } - } - ) - - // Print summary logs - if (!summary.length) { - console.warn( - "Nothing imported, see instruction at top of crowdin-imports.ts" - ) - return - } - console.table(summary) - console.log("🎉 Crowdin import complete.") -} diff --git a/src/scripts/crowdin/leaderboard/constants.ts b/src/scripts/crowdin/leaderboard/constants.ts deleted file mode 100644 index 644e8708964..00000000000 --- a/src/scripts/crowdin/leaderboard/constants.ts +++ /dev/null @@ -1,5 +0,0 @@ -export const ONE = 0.999 // Requires a non-integer float, represents 1 -export const ZERO = 0.001 // Requires a non-integer float, represent 0 -export const TIMEOUT_DURATION_MS = 10 * 60 * 1e3 // 10 minutes -export const DATA_SAVE_PATH = "src/data/translation-reports" -export const ALL_TIME_START = "2019-06-27T00:00:00Z" diff --git a/src/scripts/crowdin/leaderboard/getLeaderboardReports.ts b/src/scripts/crowdin/leaderboard/getLeaderboardReports.ts deleted file mode 100644 index e5b1fd8156f..00000000000 --- a/src/scripts/crowdin/leaderboard/getLeaderboardReports.ts +++ /dev/null @@ -1,95 +0,0 @@ -import fs from "fs" -import { join } from "path" - -import { TranslationCostReport } from "@/lib/types" - -import { ALL_TIME_START, DATA_SAVE_PATH } from "./constants" -import { - awaitFinishedReport, - downloadJsonReport, - generateReport, - getLastMonth, - getLastQuarter, - getUrlFromReport, - parseData, -} from "./utils" - -import "dotenv/config" - -const projectId = Number(process.env.CROWDIN_PROJECT_ID) || 363359 - -const dates = { - month: { - from: getLastMonth().from, - to: getLastMonth().to, - }, - quarter: { - from: getLastQuarter().from, - to: getLastQuarter().to, - }, - allTime: { - from: ALL_TIME_START, - to: new Date().toISOString(), - }, -} - -const reports = Object.keys(dates) - -async function main() { - // Log calculated dates being used for report generation - console.table(dates) - - const summary = {} - - for (const report of reports) { - // Generate a report for the given date range - const reportId = await generateReport( - projectId, - dates[report].from, - dates[report].to - ) - - // Log reportId's for each report - console.log({ report, reportId }) - - // Wait for report to finish generating - await awaitFinishedReport(projectId, reportId) - - // Read data url from report - const url = await getUrlFromReport(projectId, reportId) - - // Fetch JSON data from url - const json: TranslationCostReport = await downloadJsonReport(url) - - // Parse data - const data = parseData(json) - - // Write parsed data to file system - const filePath = join( - DATA_SAVE_PATH, - report, - `${report}-data.json` - ).toLowerCase() - fs.writeFileSync(filePath, JSON.stringify(data, null, 2)) - - // Add report to summary table - summary[report] = { - reportId, - from: dates[report].from, - to: dates[report].to, - filePath, - } - } - - // Write summary table - fs.writeFileSync( - join(DATA_SAVE_PATH, "leaderboard-import-summary.json"), - JSON.stringify(summary, null, 2) - ) - - console.table(summary, ["filePath"]) -} - -main() - -export default main diff --git a/src/scripts/crowdin/leaderboard/utils.ts b/src/scripts/crowdin/leaderboard/utils.ts deleted file mode 100644 index 8d08bad03b6..00000000000 --- a/src/scripts/crowdin/leaderboard/utils.ts +++ /dev/null @@ -1,172 +0,0 @@ -import { CostLeaderboardData, TranslationCostReport } from "@/lib/types" - -import crowdin from "../api-client/crowdinClient" - -import { ONE, TIMEOUT_DURATION_MS, ZERO } from "./constants" - -export const getLastQuarter = (): { from: string; to: string } => { - const now = new Date() - - // Months per quarter - const MPQ = 3 - - // Get current quarter - const currentQ = Math.floor(now.getUTCMonth() / MPQ) - - // Get start of current quarter - const currentQStart = new Date(now.getUTCFullYear(), currentQ * MPQ) - currentQStart.setUTCHours(0) - - // Get start of last quarter - const lastQStart = new Date(currentQStart) - lastQStart.setUTCMonth(lastQStart.getUTCMonth() - MPQ) - - // Return ISO strings of start and end of last quarter - return { - from: lastQStart.toISOString(), - to: currentQStart.toISOString(), - } -} - -export const getLastMonth = (): { from: string; to: string } => { - const now = new Date() - // Get current month - const currentM = now.getUTCMonth() - // Get start of current month - const currentMStart = new Date(now.getUTCFullYear(), currentM) - currentMStart.setUTCDate(1) - currentMStart.setUTCHours(0) - - // Get start of last month - const lastMStart = new Date(currentMStart) - lastMStart.setUTCMonth(lastMStart.getUTCMonth() - 1) - - // Return ISO strings of start and end of last month - return { - from: lastMStart.toISOString(), - to: currentMStart.toISOString(), - } -} - -export const generateReport = async ( - projectId: number, - from: string, - to: string -): Promise => { - const generatedReport = await crowdin.reportsApi.generateReport(projectId, { - name: "translation-costs-pe", - schema: { - unit: "words", - format: "json", - dateFrom: from, - dateTo: to, - baseRates: { - fullTranslation: ONE, - proofread: ONE, - }, - individualRates: [], - netRateSchemes: { - tmMatch: [ - { matchType: "perfect", price: ZERO }, - { matchType: "100", price: ZERO }, - ], - mtMatch: [{ matchType: "100", price: ONE }], - suggestionMatch: [{ matchType: "100", price: ONE }], - }, - groupBy: "user", - }, - }) - if (!generatedReport) - throw new Error( - "getLeaderboardCosts.ts > Error generating report for translation costs" - ) - return generatedReport.data.identifier -} - -/** - * Checks the status of a report in the Crowdin API. - * @param projectId - The ID of the project. - * @param reportId - The ID of the report. - * @returns A boolean indicating whether the report status is "finished". - * @throws An error if the report status is not acceptable. - */ -export const checkReportStatus = async ( - projectId: number, - reportId: string -) => { - const [FINISHED, IN_PROGRESS] = ["finished", "in_progress"] - const checkReport = await crowdin.reportsApi.checkReportStatus( - projectId, - reportId - ) - - if (![FINISHED, IN_PROGRESS].includes(checkReport.data.status)) - throw new Error(`Status not acceptable. ReportId: ${reportId}`) - - return checkReport.data.status === FINISHED -} - -export const getUrlFromReport = async (projectId: number, reportId: string) => { - const readReport = await crowdin.reportsApi.downloadReport( - projectId, - reportId - ) - return readReport.data.url -} - -export const downloadJsonReport = async ( - url: string -): Promise => { - const reportResponse = await fetch(url) - - if (!reportResponse.ok) - throw new Error( - `getLeaderboardCosts.ts > Error fetching report from report url: ${url}` - ) - - const reportData = (await reportResponse.json()) as TranslationCostReport - - return reportData -} - -export const parseData = (json: TranslationCostReport): CostLeaderboardData[] => - json.data - .map(({ user, languages, totalCosts }) => { - const { username, fullName, avatarUrl } = user - const langs = languages - .sort((a, b) => b.totalCosts - a.totalCosts) - .map(({ language: { name } }) => name) - const _totalCosts = Math.floor(totalCosts) - return { username, fullName, avatarUrl, totalCosts: _totalCosts, langs } - }) - .filter(({ username, fullName }) => { - // TODO: Remove specific user checks once Acolad has updated their usernames - const lUser = username.toLowerCase() - const lFull = (username + fullName).toLowerCase() - const isBlocked = - lUser.includes("lqs_") || - lUser.includes("removed_user") || - lFull.includes("aco_") || - lFull.includes("acc_") || - [ - "ethdotorg", - "finnish_sandberg", - "norwegian_sandberg", - "swedish_sandberg", - ].includes(lUser) - return !isBlocked - }) - -export const awaitFinishedReport = async ( - projectId: number, - reportId: string -) => { - const start = new Date().getTime() - let now = start - while ( - !(await checkReportStatus(projectId, reportId)) && - now - start < TIMEOUT_DURATION_MS - ) { - now = new Date().getTime() - } -} diff --git a/src/scripts/crowdin/reports/dataHelpers.ts b/src/scripts/crowdin/reports/dataHelpers.ts deleted file mode 100644 index a6ff6bc3a9f..00000000000 --- a/src/scripts/crowdin/reports/dataHelpers.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { loadExcludedTranslators, UserData } from "./fileHelpers" - -export async function filterAndFormatData(data: UserData[]) { - const excludedTranslators = await loadExcludedTranslators() - - return data - .filter((userObj) => { - const fullName = userObj.user.fullName?.toLowerCase() - const username = userObj.user.username?.toLowerCase() - - return ( - fullName && - !excludedTranslators.excludedNames.includes(fullName) && - username && - !excludedTranslators.excludedUsernames.includes(username) && - !excludedTranslators.excludedPhrases.some( - (phrase) => fullName.includes(phrase) || username.includes(phrase) - ) - ) - }) - .map((userObj) => ({ - id: userObj.user.id, - username: userObj.user.username, - totalCosts: userObj.user.totalCosts, - avatarUrl: userObj.user.avatarUrl, - })) - .sort((a, b) => b.totalCosts - a.totalCosts) // sort users by totalCosts, highest first -} diff --git a/src/scripts/crowdin/reports/fileHelpers.ts b/src/scripts/crowdin/reports/fileHelpers.ts deleted file mode 100644 index 752b1167cdf..00000000000 --- a/src/scripts/crowdin/reports/fileHelpers.ts +++ /dev/null @@ -1,116 +0,0 @@ -import fs from "fs" -import path from "path" - -import { getLangCodeFromCrowdinCode } from "../utils" - -import { filterAndFormatData } from "./dataHelpers" - -export type CombinedData = { - lang: string - data: { - fileId: string - contributors: User[] - }[] -} - -export interface User { - id: number - username: string - fullName?: string - totalCosts: number - avatarUrl: string -} - -export interface UserData { - user: User -} - -export interface ReportData { - data: UserData[] -} - -const combinedFilePath = path.join( - __dirname, - "../../../data/crowdin/combined-translators.json" -) - -export async function saveReportDataToJson( - reportData: ReportData, - fileId: number, - language: string -): Promise { - const repoLangCode = await getLangCodeFromCrowdinCode(language) - const combinedData = await loadCombinedTranslators() - const formattedData = await filterAndFormatData(reportData.data) - const languageData = combinedData.find((data) => data.lang === repoLangCode) - - if (languageData) { - const existingData = languageData.data.find( - (data) => data.fileId === fileId.toString() - ) - if (existingData) { - existingData.contributors = formattedData - } else { - languageData.data.push({ - fileId: fileId.toString(), - contributors: formattedData, - }) - } - } else { - combinedData.push({ - lang: repoLangCode, - data: [ - { - fileId: fileId.toString(), - contributors: formattedData, - }, - ], - }) - } - - try { - await fs.promises.writeFile( - combinedFilePath, - JSON.stringify(combinedData, null, 2) - ) - } catch (error: unknown) { - if (error instanceof Error) { - console.log(`Error writing to ${combinedFilePath}:`, error.message) - } - } -} - -interface ExcludedTranslatorsData { - excludedNames: string[] - excludedUsernames: string[] - excludedPhrases: string[] -} - -export async function loadExcludedTranslators(): Promise { - const filePath = path.join( - __dirname, - "../../../data/crowdin/excluded-translators.json" - ) - let excludedTranslators = { - excludedNames: [], - excludedUsernames: [], - excludedPhrases: [], - } - if (fs.existsSync(filePath)) { - const rawData = fs.readFileSync(filePath, "utf8") - excludedTranslators = rawData - ? JSON.parse(rawData) - : { excludedNames: [], excludedUsernames: [], excludedPhrases: [] } - } - - return excludedTranslators -} - -export async function loadCombinedTranslators(): Promise { - let combinedData: CombinedData[] = [] - if (fs.existsSync(combinedFilePath)) { - const rawData = fs.readFileSync(combinedFilePath, "utf8") - combinedData = rawData ? JSON.parse(rawData) : [] - } - return combinedData -} diff --git a/src/scripts/crowdin/reports/generateReviewReport.ts b/src/scripts/crowdin/reports/generateReviewReport.ts deleted file mode 100644 index f142e3bce17..00000000000 --- a/src/scripts/crowdin/reports/generateReviewReport.ts +++ /dev/null @@ -1,85 +0,0 @@ -import fs from "fs" -import path from "path" - -import i18n from "../../../../i18n.config.json" -import dirs from "../../../data/crowdin/translation-buckets-dirs.json" -import { CROWDIN_API_MAX_LIMIT } from "../../../lib/constants" -import crowdinClient from "../api-client/crowdinClient" - -type SummaryItem = [code: string, bucket: string, needsReview: number] - -/** - * Generates a report of words needing review for each content bucket in all languages. - * Report in CSV format with columns: Language, Bucket Name, Words needing review. - * To run: - * - Ensure CROWDIN_API_KEY is set in the .env file (.env.local will not work) - * 1. https://crowdin.com/settings#api-key - * 2. Click: "New token" - * 3. Give token a name - * 4. Select "Translation Status" under "Projects" for scope - * 5. Click: "Create" and authenticate - * 6. Copy the token to the .env file - * - Can be run with `pnpm crowdin-needs-review` - * - Results are saved to src/data/crowdin/bucketsAwaitingReviewReport.csv - * - Report is git ignored, and should not be committed - */ -async function main() { - const projectId = Number(process.env.CROWDIN_PROJECT_ID) || 363359 - - const reportSummary = [] as SummaryItem[] - - const directories = dirs.sort((a, b) => a.name.localeCompare(b.name)) - - // Loop through list of content buckets (dirs) - for (const dir of directories) { - console.log(`Processing: ${dir.name}...`) - - // Get translation progress for bucket (dir.id) in all languages - const { data } = - await crowdinClient.translationStatusApi.getDirectoryProgress( - projectId, - dir.id, - { limit: CROWDIN_API_MAX_LIMIT } - ) - - // Loop through supported languages - i18n.forEach(({ crowdinCode }) => { - const match = data.find( - ({ data: { languageId } }) => languageId === crowdinCode - ) - if (!match) return - const { words, translationProgress } = match.data - if (translationProgress < 100) return - const needsReview = words.translated - words.approved - if (needsReview === 0) return - // If match, 100% translation progress, and not full reviewed, add to summary - reportSummary.push([crowdinCode, dir.name, needsReview]) - }) - } - - // Sort first by language code, then by bucket name - const sorted = reportSummary.sort((a, b) => - a[0] === b[0] ? a[1].localeCompare(b[1]) : a[0].localeCompare(b[0]) - ) - // Transform to çsv string - const csvArray = sorted.map((item) => item.join(",")) - // Insert header names at beginning of csv array - csvArray.unshift("Language,Bucket Name,Words needing review") - const csv = csvArray.join("\n") - - // Write csv to file to fs - const csvPath = path.resolve( - process.cwd(), - "src/data/crowdin/bucketsAwaitingReviewReport.csv" - ) - fs.writeFileSync(csvPath, csv) - - // Log summary - console.log("\nReport summary:\n") - console.log(csv) - console.log(`\n✅ Report saved to ${csvPath}`) -} - -main() - -export default main diff --git a/src/scripts/crowdin/reports/getTranslationCostsReports.ts b/src/scripts/crowdin/reports/getTranslationCostsReports.ts deleted file mode 100644 index 64f7ab5f03a..00000000000 --- a/src/scripts/crowdin/reports/getTranslationCostsReports.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { findFileIdsByPaths } from "../utils" -import { getCrowdinCode } from "../utils" - -import { fetchTranslationCostsReport } from "./reportsHelpers" - -async function getTranslationCostsReports(translatedMarkdownPaths) { - for (const lang in translatedMarkdownPaths) { - const fileIds = await findFileIdsByPaths( - translatedMarkdownPaths[lang], - lang - ) - console.log(`Getting reports for ${lang} for ${fileIds.length} files`) - // The CrowdinCode is often different from what we use in our repo - const crowdinLangCode = await getCrowdinCode(lang) - - for (const fileId of fileIds) { - if (!fileId) { - console.log("Error: No file ID found for one of the paths") - continue - } - await fetchTranslationCostsReport(fileId, crowdinLangCode) - } - } -} - -export default getTranslationCostsReports diff --git a/src/scripts/crowdin/reports/reportsHelpers.ts b/src/scripts/crowdin/reports/reportsHelpers.ts deleted file mode 100644 index affd5e90cb9..00000000000 --- a/src/scripts/crowdin/reports/reportsHelpers.ts +++ /dev/null @@ -1,141 +0,0 @@ -import { ReportsModel } from "@crowdin/crowdin-api-client" - -import { - CROWDIN_PROJECT_ID, - FIRST_CROWDIN_CONTRIBUTION_DATE, -} from "../../../lib/constants" -import crowdinClient from "../api-client/crowdinClient" - -import { ReportData, saveReportDataToJson } from "./fileHelpers" - -const { reportsApi } = crowdinClient - -function getPreviousDayISOString(): string { - const now = new Date() - now.setDate(now.getDate() - 1) - return now.toISOString() -} - -export async function fetchTranslationCostsReport( - fileId: number, - crowdinLangCode: string -): Promise { - const dateTo = getPreviousDayISOString() - - // https://github.com/crowdin/crowdin-api-client-js/pull/282 - const schema: ReportsModel.TranslationCostsPostEndingSchema = { - unit: "words", - currency: "USD", - format: "json", - baseRates: { - fullTranslation: 0.1, - proofread: 0.12, - }, - individualRates: [], - netRateSchemes: { - tmMatch: [ - { - matchType: "perfect", - price: 0.1, - }, - ], - mtMatch: [ - { - matchType: "perfect", - price: 0.05, - }, - ], - suggestionMatch: [ - { - matchType: "perfect", - price: 0.08, - }, - ], - }, - groupBy: "user", - dateFrom: FIRST_CROWDIN_CONTRIBUTION_DATE, - dateTo, - languageId: crowdinLangCode, - fileIds: [fileId], - } - - const reportRequest: ReportsModel.GenerateReportRequest = { - name: "translation-costs-pe", - schema: schema, - } - - try { - const response = await reportsApi.generateReport( - CROWDIN_PROJECT_ID, - reportRequest - ) - if (response.data.status === "created") { - await checkReportStatus(response.data.identifier, fileId, crowdinLangCode) - } - } catch (error: unknown) { - if (error instanceof Error) { - console.log( - `There was a problem generating the report for file ID ${fileId}: ${error.message}` - ) - } - } -} - -export async function checkReportStatus( - identifier: string, - fileId: number, - crowdinLangCode: string -): Promise { - try { - const response = await reportsApi.checkReportStatus( - CROWDIN_PROJECT_ID, - identifier - ) - if (response.data.status === "finished" && response.data.progress === 100) { - console.log(`Report for identifier ${identifier} is finished`) - await downloadReport(identifier, fileId, crowdinLangCode) - } else { - console.log("Not ready") - await new Promise((resolve) => setTimeout(resolve, 1000)) - await checkReportStatus(identifier, fileId, crowdinLangCode) - } - } catch (error: unknown) { - if (error instanceof Error) { - console.log( - `There was a problem checking the report status for identifier ${identifier}: ${error.message}` - ) - } - } -} - -export async function downloadReport( - identifier: string, - fileId: number, - crowdinLangCode: string -): Promise { - console.log(`Starting download of report for file ID ${fileId}`) - - try { - const response = await reportsApi.downloadReport( - CROWDIN_PROJECT_ID, - identifier - ) - const jsonUrl = response.data.url - console.log( - `${crowdinLangCode}—Retrieved JSON URL for report of file ID ${fileId}` - ) - - const reportReq = await fetch(jsonUrl) - console.log(`Downloaded report data for file ID ${fileId}`) - const reportData: ReportData = await reportReq.json() - - await saveReportDataToJson(reportData, fileId, crowdinLangCode) - console.log(`Saved report data for file ID ${fileId} to JSON file`) - } catch (error: unknown) { - if (error instanceof Error) { - console.log( - `There was a problem downloading the report for identifier ${identifier}: ${error.message}` - ) - } - } -} diff --git a/src/scripts/crowdin/source-files/fetchAndSaveDirectories.ts b/src/scripts/crowdin/source-files/fetchAndSaveDirectories.ts deleted file mode 100644 index a4cb68f87e8..00000000000 --- a/src/scripts/crowdin/source-files/fetchAndSaveDirectories.ts +++ /dev/null @@ -1,74 +0,0 @@ -import fs from "fs" -import path from "path" - -import { ResponseList, SourceFilesModel } from "@crowdin/crowdin-api-client" - -import { - CROWDIN_API_MAX_LIMIT, - CROWDIN_PROJECT_ID, -} from "../../../lib/constants" -import crowdinClient from "../api-client/crowdinClient" - -const { sourceFilesApi } = crowdinClient - -async function getDirectories(): Promise< - ResponseList -> { - try { - const response = await sourceFilesApi.listProjectDirectories( - CROWDIN_PROJECT_ID, - { limit: CROWDIN_API_MAX_LIMIT } - ) - return response - } catch (error: unknown) { - if (error instanceof Error) { - throw new Error( - `There was a problem fetching the directories: ${error.message}` - ) - } - throw new Error("An unknown error occurred while fetching the directories.") - } -} - -async function filterDirectoriesAndSave( - directoriesData: ResponseList -): Promise { - try { - if (directoriesData.data.length === 0) { - console.log("No data received from the API. Keeping the existing data.") - return - } - - const filteredData = directoriesData.data - .filter((item) => item.data.directoryId === null) - .map((item) => ({ id: item.data.id, name: item.data.name })) // Extract only id and name - - const dir = "./src/data/crowdin" - const outputFilePath = path.join(dir, "translation-buckets-dirs.json") - - // Create the directory if it doesn't exist - fs.mkdirSync(dir, { recursive: true }) - fs.writeFileSync(outputFilePath, JSON.stringify(filteredData, null, 2)) - - console.log( - "Filtered directory data saved to translation-buckets-dirs.json" - ) - } catch (error: unknown) { - if (error instanceof Error) { - console.error("Error filtering and saving directories:", error.message) - } - } -} - -async function getAndSaveDirectories(): Promise { - try { - const directoriesData = await getDirectories() - await filterDirectoriesAndSave(directoriesData) - } catch (error: unknown) { - if (error instanceof Error) { - console.error("Error in getAndSaveDirectories:", error.message) - } - } -} - -export default getAndSaveDirectories diff --git a/src/scripts/crowdin/source-files/fetchAndSaveFileIds.ts b/src/scripts/crowdin/source-files/fetchAndSaveFileIds.ts deleted file mode 100644 index 246b076dce4..00000000000 --- a/src/scripts/crowdin/source-files/fetchAndSaveFileIds.ts +++ /dev/null @@ -1,126 +0,0 @@ -import fs from "fs" -import path from "path" - -import { ResponseObject, SourceFilesModel } from "@crowdin/crowdin-api-client" - -import { - CROWDIN_API_MAX_LIMIT, - CROWDIN_PROJECT_ID, -} from "../../../lib/constants" -import crowdinClient from "../api-client/crowdinClient" - -const { sourceFilesApi } = crowdinClient - -interface FileItem { - id: number - path: string -} - -async function fetchFileIdsForDirectory( - directoryId: number -): Promise { - try { - const response = await sourceFilesApi.listProjectFiles(CROWDIN_PROJECT_ID, { - limit: CROWDIN_API_MAX_LIMIT, - directoryId, - recursion: true, - }) - - if (!response.data || !Array.isArray(response.data)) { - console.error(`Invalid response data structure.`, response.data) - return [] - } - - return response.data - .map( - (item: ResponseObject): FileItem => ({ - id: item.data.id, - path: item.data.path, - }) - ) - .filter((file: FileItem) => file.path.endsWith(".md")) // filter out non-md files - } catch (error: unknown) { - if (error instanceof Error) { - console.error( - `There was a problem with the fetch operation for directory ${directoryId}: ${error.message}` - ) - } - return [] - } -} - -async function fetchFileIdsForMultipleDirectories( - directoryIds: number[] -): Promise { - const promises = directoryIds.map(fetchFileIdsForDirectory) - const results = await Promise.allSettled(promises) - - const successfulResults: FileItem[][] = results - .filter( - (result): result is PromiseFulfilledResult => - result.status === "fulfilled" - ) - .map((result) => result.value) - - if (successfulResults.length === 0) { - console.log("No successful fetch operations.") - return - } - - const combinedData: FileItem[] = successfulResults - .flat() - .map(transformResponseData) - - return combinedData -} - -// Convert path by removing everything before the second forward slash -// Before: '/28) Developer Tutorials IV/developers/tutorials/testing-erc-20-tokens-with-waffle/index.md' -// After: '/developers/tutorials/testing-erc-20-tokens-with-waffle/index.md' -function transformResponseData(item: FileItem): FileItem { - const pathSegments = item.path.split("/") - const newPath = "/" + pathSegments.slice(2).join("/") - - return { - id: item.id, - path: newPath, - } -} - -function saveFileIdsToJSON(combinedData: FileItem[]): void { - const dir = "./src/data/crowdin" - const outputFilePath = path.join(dir, "file-ids.json") - - try { - // Do not overwrite the file if there's no data to save - if (combinedData.length === 0) { - console.log("No data to save. Keeping the existing data.") - return - } - - // TODO: Remove if check is redundant (will this always follow getAndSaveDirectories.ts??) - fs.mkdirSync(dir, { recursive: true }) - fs.writeFileSync(outputFilePath, JSON.stringify(combinedData, null, 2)) - console.log(`File id data saved to ${outputFilePath}`) - } catch (error: unknown) { - if (error instanceof Error) { - console.error( - "There was a problem saving the data to the file:", - error.message - ) - } - } -} - -async function fetchAndSaveFileIds(directoryIds: number[]): Promise { - const transformedFileData = - await fetchFileIdsForMultipleDirectories(directoryIds) - - if (transformedFileData) { - saveFileIdsToJSON(transformedFileData) - } else { - console.log("No data to save. Keeping the existing data.") - } -} - -export default fetchAndSaveFileIds diff --git a/src/scripts/crowdin/translations/awaitLatestBuild.ts b/src/scripts/crowdin/translations/awaitLatestBuild.ts deleted file mode 100644 index 8ecdc8144be..00000000000 --- a/src/scripts/crowdin/translations/awaitLatestBuild.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { writeFileSync } from "fs" -import { join } from "path" - -import crowdin from "../api-client/crowdinClient" - -const FINISHED = "finished" -const TIMEOUT = 2 * 60 * 60 * 1000 // Timeout after 2 hours -const INTERVAL = 10 * 1000 // 10 seconds between checks - -const OUTPUT_PATH = join(process.env["GITHUB_WORKSPACE"] || "", "output.env") - -async function awaitLatestBuild() { - const projectId = Number(process.env.CROWDIN_PROJECT_ID) || 363359 - - // BUILD_ID is provided by the triggerBuild script run in the same workflow prior to this script - const buildId = process.env.BUILD_ID - - console.log("Build ID provided:", buildId) - const initialResponse = await crowdin.translationsApi.checkBuildStatus( - projectId, - Number(buildId) - ) - let data = initialResponse.data - - let isFinished = data.status === FINISHED - - const timeoutTime = Date.now() + TIMEOUT - let tryAgainTime = Date.now() - 1 - while (!isFinished && Date.now() < timeoutTime) { - if (Date.now() < tryAgainTime) continue - tryAgainTime = Date.now() + INTERVAL - - const repeatCheck = await crowdin.translationsApi.checkBuildStatus( - projectId, - Number(buildId) - ) - data = repeatCheck.data - isFinished = data.status === FINISHED - console.log( - `id: ${buildId}, status: ${data.status}, progress ${data.progress}` - ) - } - - if (data.status !== FINISHED) { - writeFileSync(OUTPUT_PATH, `BUILD_SUCCESS=false\n`, { flag: "a" }) - throw new Error( - `Timeout: Build did not finish in ${TIMEOUT / 1000 / 60} minutes` - ) - } - - console.log("Latest build data:", data) - writeFileSync(OUTPUT_PATH, `BUILD_SUCCESS=true\n`, { flag: "a" }) -} - -awaitLatestBuild() - -export default awaitLatestBuild diff --git a/src/scripts/crowdin/translations/constants.ts b/src/scripts/crowdin/translations/constants.ts deleted file mode 100644 index 6f0756df4fc..00000000000 --- a/src/scripts/crowdin/translations/constants.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { join } from "path" - -export const DOT_CROWDIN = ".crowdin" - -export const CROWDIN_DATA_DIR = "src/data/crowdin" -export const SAVE_FILE = "download.zip" -export const FILE_PATH = join(CROWDIN_DATA_DIR, SAVE_FILE) - -export const SUMMARY_SAVE_FILE = "import-summary.json" -export const SUMMARY_PATH = join(CROWDIN_DATA_DIR, SUMMARY_SAVE_FILE) - -export const BUCKETS_IMPORTED_FILE = "buckets-imported.json" -export const BUCKETS_PATH = join(CROWDIN_DATA_DIR, BUCKETS_IMPORTED_FILE) - -export const APPROVAL_THRESHOLD = 100 diff --git a/src/scripts/crowdin/translations/getApprovedBuckets.ts b/src/scripts/crowdin/translations/getApprovedBuckets.ts deleted file mode 100644 index bcc12ad29ba..00000000000 --- a/src/scripts/crowdin/translations/getApprovedBuckets.ts +++ /dev/null @@ -1,43 +0,0 @@ -import i18n from "../../../../i18n.config.json" -import bucketDirs from "../../../data/crowdin/translation-buckets-dirs.json" -import { CROWDIN_API_MAX_LIMIT } from "../../../lib/constants" -import crowdin from "../api-client/crowdinClient" -import type { BucketsList } from "../import/types" - -import { APPROVAL_THRESHOLD } from "./constants" - -async function getApprovedBuckets(): Promise { - console.log("⏳ Getting approved buckets...") - const projectId = Number(process.env.CROWDIN_PROJECT_ID) || 363359 - - const bucketsList: BucketsList = {} - - for (const bucketDir of bucketDirs) { - const directoryProgress = - await crowdin.translationStatusApi.getDirectoryProgress( - projectId, - bucketDir.id, - { limit: CROWDIN_API_MAX_LIMIT } - ) - - const onlyApproved = directoryProgress.data.filter( - ({ data: { approvalProgress } }) => approvalProgress >= APPROVAL_THRESHOLD - ) - - for (const { code, crowdinCode } of i18n) { - const match = onlyApproved.find( - ({ data: { languageId } }) => languageId === crowdinCode - ) - if (!match) continue - - if (!bucketsList[code]) bucketsList[code] = [] - - const n = parseInt(bucketDir.name.substring(0, 2)) - bucketsList[code].push(n) - } - } - - return bucketsList -} - -export default getApprovedBuckets diff --git a/src/scripts/crowdin/translations/getBucketDirectoryIds.ts b/src/scripts/crowdin/translations/getBucketDirectoryIds.ts deleted file mode 100644 index 43846cdccaa..00000000000 --- a/src/scripts/crowdin/translations/getBucketDirectoryIds.ts +++ /dev/null @@ -1,9 +0,0 @@ -import getAndSaveDirectories from "../source-files/fetchAndSaveDirectories" - -async function main() { - await getAndSaveDirectories() -} - -main() - -export default main diff --git a/src/scripts/crowdin/translations/getTranslations.ts b/src/scripts/crowdin/translations/getTranslations.ts deleted file mode 100644 index ad6fc93f723..00000000000 --- a/src/scripts/crowdin/translations/getTranslations.ts +++ /dev/null @@ -1,52 +0,0 @@ -import fs from "fs" - -import crowdin from "../api-client/crowdinClient" -import crowdinImport from "../import/main" -import type { BucketsList } from "../import/types" - -import { BUCKETS_PATH, DOT_CROWDIN, FILE_PATH } from "./constants" -import getApprovedBuckets from "./getApprovedBuckets" -import { decompressFile, downloadFile } from "./utils" - -import "dotenv/config" - -async function main() { - const projectId = Number(process.env.CROWDIN_PROJECT_ID) || 363359 - - try { - const listProjectBuilds = - await crowdin.translationsApi.listProjectBuilds(projectId) - - const latestId = listProjectBuilds.data - .filter(({ data }) => data.status === "finished") - .reverse()[0].data.id - - const downloadTranslations = - await crowdin.translationsApi.downloadTranslations(projectId, latestId) - const { url } = downloadTranslations.data - - // Download ZIP file - await downloadFile(url, FILE_PATH) - - // Unzip file to .crowdin/ - await decompressFile(FILE_PATH, DOT_CROWDIN) - - // Delete .zip file once decompressed - fs.rmSync(FILE_PATH) - console.log("🗑️ Removed download from:", FILE_PATH) - - const buckets = (await getApprovedBuckets()) as BucketsList - - // Save buckets for use in PR body later - fs.writeFileSync(BUCKETS_PATH, JSON.stringify(buckets, null, 2)) - - // Run Crowdin import script with buckets from Notion - crowdinImport(buckets) - } catch (error: unknown) { - console.error((error as Error).message) - } -} - -main() - -export default main diff --git a/src/scripts/crowdin/translations/postLangPRs.ts b/src/scripts/crowdin/translations/postLangPRs.ts deleted file mode 100644 index abd1e6eda44..00000000000 --- a/src/scripts/crowdin/translations/postLangPRs.ts +++ /dev/null @@ -1,23 +0,0 @@ -import fs from "fs" - -import { LOCALES_CODES } from "../../../lib/constants" -import type { BucketsList } from "../import/types" - -import { BUCKETS_PATH } from "./constants" -import { createLocaleTranslationPR } from "./utils" - -function postLangPRs() { - const bucketsListRead = fs.readFileSync(BUCKETS_PATH, "utf-8") - const bucketsList = JSON.parse(bucketsListRead) as BucketsList - - if (!bucketsList) throw new Error("Failed to read buckets list.") - - for (const locale of LOCALES_CODES) { - if (!bucketsList[locale]) continue - createLocaleTranslationPR(locale, bucketsList[locale]) - } -} - -postLangPRs() - -export default postLangPRs diff --git a/src/scripts/crowdin/translations/triggerBuild.ts b/src/scripts/crowdin/translations/triggerBuild.ts deleted file mode 100644 index f3e54dfd403..00000000000 --- a/src/scripts/crowdin/translations/triggerBuild.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { writeFileSync } from "fs" -import { join } from "path" - -import crowdin from "../api-client/crowdinClient" - -const OUTPUT_PATH = join(process.env["GITHUB_WORKSPACE"] || "", "output.env") - -async function triggerBuild() { - const projectId = Number(process.env.CROWDIN_PROJECT_ID) || 363359 - - try { - const response = await crowdin.translationsApi.buildProject(projectId) - const { id, status } = response.data - const isAlreadyFinished = status === "finished" - console.log( - `Build ${isAlreadyFinished ? "already finished" : "triggered"} id:`, - id - ) - writeFileSync(OUTPUT_PATH, `BUILD_ID=${id}\n`, { flag: "a" }) - } catch (error: unknown) { - console.error((error as Error).message) - } -} - -triggerBuild() - -export default triggerBuild diff --git a/src/scripts/crowdin/translations/types.ts b/src/scripts/crowdin/translations/types.ts deleted file mode 100644 index d1608054c43..00000000000 --- a/src/scripts/crowdin/translations/types.ts +++ /dev/null @@ -1,7 +0,0 @@ -export type DirectoryItem = { - id: number - path: string - updatedAt: string -} - -export type QASummary = Record diff --git a/src/scripts/crowdin/translations/utils.ts b/src/scripts/crowdin/translations/utils.ts deleted file mode 100644 index fdb59a1827d..00000000000 --- a/src/scripts/crowdin/translations/utils.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { execSync } from "child_process" -import fs, { unlinkSync, writeFileSync } from "fs" -import path from "path" -import { Readable } from "stream" -import { finished } from "stream/promises" -import ReadableStream from "stream/web" - -import decompress from "decompress" - -import { INTL_JSON_DIR, TRANSLATIONS_DIR } from "../../../lib/constants" - -export const downloadFile = async (url: string, writePath: string) => { - // Get directory from writePath and ensure it exists - const dir = writePath.substring(0, writePath.lastIndexOf("/")) - if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }) - - // Remove old file at writePath if it exists - if (fs.existsSync(writePath)) fs.rmSync(writePath) - - // Fetch data - const res = await fetch(url) - if (!res.ok) throw new Error(`Failed to fetch ${url}`) - - const body = res.body as ReadableStream.ReadableStream - - const fileStream = fs.createWriteStream(writePath, { flags: "wx" }) - - console.log("⬇ Downloading latest finished build to", writePath) - - await finished(Readable.fromWeb(body).pipe(fileStream)) - console.log("✅ Download complete") -} - -export const decompressFile = async (filePath: string, targetDir: string) => { - console.log(`🥡 Decompressing ${filePath} to ${targetDir}`) - await decompress(filePath, targetDir) - console.log("✅ Decompression complete.") -} - -export const createLocaleTranslationPR = ( - locale: string, - buckets: number[] -) => { - const gitStatus = execSync(`git status -s | grep -E "/${locale}/" | wc -l`, { - encoding: "utf-8", - }).trim() - if (+gitStatus === 0) return - - const month = new Date() - .toLocaleString("default", { month: "long" }) - .toLowerCase() - const timestamp = new Date().toISOString().replace(/[^0-9]/g, "") - - const branchName = `crowdin-${month}-${locale}-${timestamp}` - const message = `chore: import translations for ${locale}` - const startingBranch = execSync("git branch --show-current", { - encoding: "utf-8", - }).trim() - execSync(`git checkout -b ${branchName}`) - execSync("git reset .") - - // Check if the translations directory exists and contains files - const translationsDir = path.join(TRANSLATIONS_DIR, locale) - if ( - fs.existsSync(translationsDir) && - fs.readdirSync(translationsDir).length > 0 - ) { - execSync(`git add ${translationsDir}`) - } - - // Check if the intl JSON directory exists and contains files - const intlJsonDir = path.join(INTL_JSON_DIR, locale) - if (fs.existsSync(intlJsonDir) && fs.readdirSync(intlJsonDir).length > 0) { - execSync(`git add ${intlJsonDir}`) - } - - execSync(`git commit -m "${message}"`) - execSync(`git push origin ${branchName}`) - - const prBody = `## Description - This PR was automatically created to import Crowdin translations. - This workflows runs on the first of every month at 16:20 (UTC). - - Thank you to everyone contributing to translate ethereum.org ❤️ - - ## Content buckets imported - ${buckets.sort((a, b) => a - b).join(", ")} - ` - - const bodyWritePath = path.resolve(process.cwd(), "body.txt") - writeFileSync(bodyWritePath, prBody) - - // Create GitHub PR and get the PR URL from the output - const prUrl = execSync( - `gh pr create \ - --base dev \ - --head ${branchName} \ - --title "${message}" \ - --body-file body.txt`, - { encoding: "utf-8" } - ).trim() - console.log(`PR created: ${prUrl}`) - - // Update the PR that was just posted to include the Netlify preview link - const prNumber = prUrl.split("/").pop() - const previewUrl = `https://deploy-preview-${prNumber}--ethereumorg.netlify.app/${locale}/` - const updatedBody = - prBody + - `\n\n## Preview link -[${previewUrl}](${previewUrl}) - ` - // Update PR with new body - writeFileSync(bodyWritePath, updatedBody) - execSync(`gh pr edit ${prNumber} --body-file ${bodyWritePath}`) - - unlinkSync(bodyWritePath) - - execSync(`git checkout ${startingBranch}`) -} diff --git a/src/scripts/crowdin/utils.ts b/src/scripts/crowdin/utils.ts index 654c527c383..afb73ed7cf7 100644 --- a/src/scripts/crowdin/utils.ts +++ b/src/scripts/crowdin/utils.ts @@ -1,66 +1,7 @@ import fs from "fs" -import path from "path" -import { languagePathRootRegExp } from "../../lib/constants" import type { I18nLocale } from "../../lib/types" -export function getDirectoryIdsFromJson() { - try { - const filePath = path.join( - __dirname, - "../../data/crowdin/translation-buckets-dirs.json" - ) - const directoriesData = fs.readFileSync(filePath) - const directories = JSON.parse(directoriesData.toString()) - return directories.map((directory) => directory.id) - } catch (error) { - console.error(`Error reading translation-buckets-dirs.json: ${error}`) - return [] - } -} - -export async function findFileIdsByPaths(paths, lang) { - const filePath = path.join(__dirname, "../../data/crowdin/file-ids.json") - const fileData = JSON.parse(fs.readFileSync(filePath, "utf8")) - - const pathToIdMap = fileData.reduce((map, item) => { - // Normalize the item path: remove leading and trailing slashes - const normalizedItemPath = item.path.replace(/^\/+|\/+$/g, "") - map[normalizedItemPath] = item.id - return map - }, {}) - - return paths - .map((path) => { - const normalizedPath = path.replace(languagePathRootRegExp, "") - - if (!pathToIdMap[normalizedPath]) { - console.warn(`Lang ${lang}, NULL ID:`, normalizedPath) - return null - } - - return pathToIdMap[normalizedPath] - }) - .filter(Boolean) // filter falsy values -} - -export async function getCrowdinCode(langCode: string): Promise { - try { - const data = await fs.promises.readFile("i18n.config.json", "utf-8") - const langs: I18nLocale[] = JSON.parse(data) - const lang = langs.find((lang) => lang.code === langCode) - - if (!lang) { - throw new Error(`Language code ${langCode} not found`) - } - - return lang.crowdinCode - } catch (error: unknown) { - if (error instanceof Error) throw new Error(`Error: ${error.message}`) - return "" - } -} - export async function getLangCodeFromCrowdinCode( crowdinCode: string ): Promise { @@ -79,5 +20,3 @@ export async function getLangCodeFromCrowdinCode( return "" } } - -export default getDirectoryIdsFromJson