From eb18aa8d233acdbe65f9df2fb889f1d163214ea1 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Fri, 24 Nov 2023 14:36:07 +0400 Subject: [PATCH 1/3] Drop separate download fallback --- .../src/locales/scripts/get-translations.js | 20 +++---- .../src/locales/scripts/separate-download.js | 57 ------------------- 2 files changed, 7 insertions(+), 70 deletions(-) delete mode 100644 frontend/src/locales/scripts/separate-download.js diff --git a/frontend/src/locales/scripts/get-translations.js b/frontend/src/locales/scripts/get-translations.js index 67c64fd95f1..b1faf01d377 100644 --- a/frontend/src/locales/scripts/get-translations.js +++ b/frontend/src/locales/scripts/get-translations.js @@ -11,7 +11,6 @@ const chokidar = require("chokidar") const { parseJson } = require("./read-i18n") const bulkDownload = require("./bulk-download") -const separateDownload = require("./separate-download") /** * Write `en.json` from `en.json5`. @@ -37,16 +36,11 @@ if (process.argv.includes("--watch")) { } if (!process.argv.includes("--en-only")) { - bulkDownload() - .catch((err) => { - console.error(err) - return separateDownload() - }) - .catch((err) => { - console.error(err) - console.error(":'-( Downloading translations failed.") - if (process.argv.includes("--require-complete")) { - process.exitCode = 1 - } - }) + bulkDownload().catch((err) => { + console.error(err) + console.error(":'-( Downloading translations failed.") + if (process.argv.includes("--require-complete")) { + process.exitCode = 1 + } + }) } diff --git a/frontend/src/locales/scripts/separate-download.js b/frontend/src/locales/scripts/separate-download.js deleted file mode 100644 index a4ca219afae..00000000000 --- a/frontend/src/locales/scripts/separate-download.js +++ /dev/null @@ -1,57 +0,0 @@ -const axios = require("./axios") -const { writeLocaleFile } = require("./utils") -const jed1xJsonToJson = require("./jed1x-json-to-json") - -const DOWNLOAD_BASE_URL = - "https://translate.wordpress.org/projects/meta/openverse" - -const getTranslationUrl = (locale) => - `${DOWNLOAD_BASE_URL}/${locale}/default/export-translations/` - -const fetchJed1x = async (locale) => { - try { - const res = await axios.get(getTranslationUrl(locale), { - params: { format: "jed1x" }, - }) - return res.data - } catch (err) { - return err.response.status - } -} - -const isEmpty = (obj) => Object.values(obj).every((x) => x === null) - -const fetchJed1xAll = async (locales) => { - const results = await Promise.allSettled(locales.map(fetchJed1x)) - - let succeeded = {} - let failed = {} - results.forEach(({ status, value }, index) => { - if (status === "fulfilled" && !isEmpty(value)) { - succeeded[locales[index]] = jed1xJsonToJson(value) - } else { - failed[locales[index]] = value - } - }) - await Promise.all( - Object.entries(succeeded).map((args) => writeLocaleFile(...args)) - ) - - return [Object.keys(succeeded).length, Object.keys(failed).length] -} - -const separateDownload = async () => { - console.log("Performing parallel download.") - - const localeJson = require("./wp-locales.json") - const locales = Object.values(localeJson).map((item) => item.slug) - const [succeeded, failed] = await fetchJed1xAll(locales) - - console.log(`Successfully downloaded ${succeeded} translations.`) - if (failed) { - console.log(`Failed to download ${failed} translations.`) - throw new Error("Parallel download partially failed") - } -} - -module.exports = separateDownload From 708159802aaf43e7c50145ddec5abd5863aa7aa1 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Fri, 24 Nov 2023 14:38:04 +0400 Subject: [PATCH 2/3] Document the overview of the bulk download process --- documentation/frontend/reference/i18n.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/documentation/frontend/reference/i18n.md b/documentation/frontend/reference/i18n.md index 1fd695c8270..36293cceb0d 100644 --- a/documentation/frontend/reference/i18n.md +++ b/documentation/frontend/reference/i18n.md @@ -51,8 +51,9 @@ processed and loaded into Nuxt via the Nuxt i18n module. JED 1.x (derived from the flattened POT files) files are converted back into the nested JSON as expected by Nuxt i18n. - This script uses the `wp-locales.json` file for the list of locales, if - downloading each locale separately and in parallel. + This script downloads all available translations in bulk as a ZIP file and + then extracts JSON files from the ZIP file. This prevents excessive calls to + GlotPress, which can be throttled and cause some locales to be missed. **Script:** `i18n:get-translations` From 99455e7ea02d2e15ba6c40db32831652ccdb6c0f Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Fri, 24 Nov 2023 14:38:40 +0400 Subject: [PATCH 3/3] Get rid of rate-limiting that's not needed anymore --- frontend/package.json | 1 - frontend/src/locales/scripts/axios.js | 12 +++--------- pnpm-lock.yaml | 14 ++++---------- 3 files changed, 7 insertions(+), 20 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index 0249d2300b1..07f7836d783 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -118,7 +118,6 @@ "@vue/test-utils": "^1.1.3", "adm-zip": "^0.5.10", "autoprefixer": "^10.4.16", - "axios-rate-limit": "^1.3.0", "babel-core": "^7.0.0-bridge.0", "babel-jest": "^26.6.3", "babel-loader": "8.2.5", diff --git a/frontend/src/locales/scripts/axios.js b/frontend/src/locales/scripts/axios.js index 0aceed7a858..f92100fca69 100644 --- a/frontend/src/locales/scripts/axios.js +++ b/frontend/src/locales/scripts/axios.js @@ -1,13 +1,7 @@ const axios = require("axios") -const rateLimit = require("axios-rate-limit") const { userAgent } = require("../../constants/user-agent") -module.exports = rateLimit( - axios.create({ - headers: { "User-Agent": userAgent }, - }), - { - maxRPS: 50, // limit GlotPress calls to 50 requests per second - } -) +module.exports = module.exports = axios.create({ + headers: { "User-Agent": userAgent }, +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 95ad0a88bd0..0aa60bca7ac 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -76,7 +76,6 @@ importers: autoprefixer: ^10.4.16 axios: ^1.0.0 axios-mock-adapter: ^1.20.0 - axios-rate-limit: ^1.3.0 babel-core: ^7.0.0-bridge.0 babel-jest: ^26.6.3 babel-loader: 8.2.5 @@ -186,7 +185,6 @@ importers: '@vue/test-utils': 1.1.3_5bwbnhtkovckcydjgad3t2muke adm-zip: 0.5.10 autoprefixer: 10.4.16_postcss@8.4.31 - axios-rate-limit: 1.3.0_axios@1.6.1 babel-core: 7.0.0-bridge.0_@babel+core@7.22.5 babel-jest: 26.6.3_@babel+core@7.22.5 babel-loader: 8.2.5_rf5mwho5nu3s3spznxs3423x5y @@ -9131,14 +9129,6 @@ packages: is-buffer: 2.0.5 dev: false - /axios-rate-limit/1.3.0_axios@1.6.1: - resolution: {integrity: sha512-cKR5wTbU/CeeyF1xVl5hl6FlYsmzDVqxlN4rGtfO5x7J83UxKDckudsW0yW21/ZJRcO0Qrfm3fUFbhEbWTLayw==} - peerDependencies: - axios: '*' - dependencies: - axios: 1.6.1 - dev: true - /axios/1.6.1: resolution: {integrity: sha512-vfBmhDpKafglh0EldBEbVuoe7DyAavGSLWhuSm5ZSEKQnHhBf0xAAwybbNH1IkrJNGnS/VG4I5yxig1pCEXE4g==} dependencies: @@ -9147,6 +9137,7 @@ packages: proxy-from-env: 1.1.0 transitivePeerDependencies: - debug + dev: false /babel-code-frame/6.26.0: resolution: {integrity: sha512-XqYMR2dfdGMW+hd0IUZ2PwK+fGeFkOxZJ0wY+JaQAHzt1Zx8LcvpiZD2NiGkEG8qx0CfkAOr5xt76d1e8vG90g==} @@ -13014,6 +13005,7 @@ packages: peerDependenciesMeta: debug: optional: true + dev: false /for-each/0.3.3: resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==} @@ -13100,6 +13092,7 @@ packages: asynckit: 0.4.0 combined-stream: 1.0.8 mime-types: 2.1.35 + dev: false /forwarded/0.2.0: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} @@ -18957,6 +18950,7 @@ packages: /proxy-from-env/1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + dev: false /prr/1.0.1: resolution: {integrity: sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw==}