From 5c26892cdb40e4e74dfb01edd1e8470556b0ae4c Mon Sep 17 00:00:00 2001 From: Dillon Hardy Date: Thu, 18 Sep 2025 11:50:24 -0400 Subject: [PATCH 1/3] Create language-data.json GitHub action This action will periodically download the latest version of `language-data.json`, and replace the existing file in the repository. If the file has been updated, it will create a pull request to recommend merging the new changes. Bug: T404882 --- .github/workflows/language-data.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/language-data.yml diff --git a/.github/workflows/language-data.yml b/.github/workflows/language-data.yml new file mode 100644 index 0000000000..d5c895b279 --- /dev/null +++ b/.github/workflows/language-data.yml @@ -0,0 +1,25 @@ +name: Update language data + +on: + schedule: + - cron: 0 0 * * * + workflow_dispatch: + +jobs: + update: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Download + run: curl -o locale/language-data.json https://raw.githubusercontent.com/wikimedia/language-data/refs/heads/master/data/language-data.json + + - name: Pull Request + uses: peter-evans/create-pull-request@v7 + with: + add-paths: locale/language-data.json + commit-message: Update language-data.json + branch: actions/language-data + title: Update language data + body: An upstream change to `locale/language-data.json` was detected in the [language-data repository](https://github.com/wikimedia/language-data). From 0500598a5172ff973e8d4282904410247aea1ffc Mon Sep 17 00:00:00 2001 From: Dillon Hardy Date: Fri, 19 Sep 2025 15:38:09 -0400 Subject: [PATCH 2/3] Override Django settings instead of using custom function We can leverage the existing Django variables and functions instead of having to replace them all with custom ones in the models, views, and controllers. --- TWLight/resources/helpers.py | 41 +++++++----------------------------- TWLight/settings/base.py | 36 ++++++++++++++++++++----------- 2 files changed, 32 insertions(+), 45 deletions(-) diff --git a/TWLight/resources/helpers.py b/TWLight/resources/helpers.py index f84f5b5576..ac38895fba 100644 --- a/TWLight/resources/helpers.py +++ b/TWLight/resources/helpers.py @@ -1,38 +1,9 @@ from django.conf import settings from numbers import Number import json -import logging import os -LANGS_DIRECTION = {} - - -def get_language_direction(language_code: str): - """ - Caches the direction (LTR/RTL) of each language by checking if the - first list value (script type) is defined in the RTL list (rtlscripts) - """ - if not LANGS_DIRECTION: - try: - with open( - os.path.join(settings.LOCALE_PATHS[0], "language-data.json") - ) as file: - data = json.load(file) - languages = data["languages"] - rtlscripts = data["rtlscripts"] - LANGS_DIRECTION.update( - { - lang: "rtl" if meta[0] in rtlscripts else "ltr" - for lang, meta in languages.items() - } - ) - except: - logging.getLogger(__name__).exception("Failed to load language data") - - return LANGS_DIRECTION.get(language_code, "ltr") - - def get_partner_description_json_schema(): """ JSON Schema for partner description translations @@ -93,8 +64,10 @@ def get_partner_description( descriptions["short_description_language"] = ( "en" if short_description["is_default"] else language_code ) - descriptions["short_description_direction"] = get_language_direction( - descriptions["short_description_language"] + descriptions["short_description_direction"] = ( + "rtl" + if descriptions["short_description_language"] in settings.LANGUAGES_BIDI + else "ltr" ) description = _get_any_description( @@ -107,8 +80,10 @@ def get_partner_description( descriptions["description_language"] = ( "en" if description["is_default"] else language_code ) - descriptions["description_direction"] = get_language_direction( - descriptions["description_language"] + descriptions["description_direction"] = ( + "rtl" + if descriptions["description_language"] in settings.LANGUAGES_BIDI + else "ltr" ) return descriptions diff --git a/TWLight/settings/base.py b/TWLight/settings/base.py index 44c31ecd52..ad5437b4f6 100644 --- a/TWLight/settings/base.py +++ b/TWLight/settings/base.py @@ -66,17 +66,29 @@ # https://raw.githubusercontent.com/wikimedia/language-data/master/data/language-data.json # into locale/language-data.json def get_languages_from_locale_subdirectories(dir): - current_languages = [] - language_data_json = open(os.path.join(dir, "language-data.json")) - languages = json.loads(language_data_json.read())["languages"] - language_data_json.close() - for locale_dir in os.listdir(dir): - if os.path.isdir(os.path.join(dir, locale_dir)): - for lang_code, lang_data in languages.items(): - autonym = lang_data[-1] - if to_language(locale_dir) == lang_code: - current_languages += [(lang_code, autonym)] - return sorted(set(current_languages)) + try: + with open(os.path.join(dir, "language-data.json")) as file: + data = json.load(file) + languages = data["languages"] + rtlscripts = data["rtlscripts"] + + current_languages = set() + current_bidi = set() + + for locale_dir in os.listdir(dir): + if os.path.isdir(os.path.join(dir, locale_dir)): + lang_code = to_language(locale_dir) + lang_data = languages.get(lang_code) + + if lang_data and len(lang_data) == 3: + current_languages.add((lang_code, lang_data[2])) + + if lang_data[0] in rtlscripts: + current_bidi.add(lang_code) + + return sorted(current_languages), sorted(current_bidi) + except: + logging.getLogger(__name__).exception("Failed to load language data") # Get the intersection of available Faker locales and the specified language set. @@ -310,7 +322,7 @@ def show_toolbar(request): # available to the system. This keeps our column and index count for db-stored # translations as low as possible while allowing translatewiki contributions to # be used without reconfiguring the site. -LANGUAGES = get_languages_from_locale_subdirectories(LOCALE_PATHS[0]) +LANGUAGES, LANGUAGES_BIDI = get_languages_from_locale_subdirectories(LOCALE_PATHS[0]) FAKER_LOCALES = get_django_faker_languages_intersection(LANGUAGES) TIME_ZONE = "UTC" From a9c24cf730c8dad6bf61c09f35494f2e8d8fec62 Mon Sep 17 00:00:00 2001 From: Dillon Hardy Date: Fri, 19 Sep 2025 15:40:04 -0400 Subject: [PATCH 3/3] Change language data update from daily to monthly The GitHub Action doesn't need to run as periodically anymore since the upstream `langugage-data.json` file doesn't seem to change that often. --- .github/workflows/language-data.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/language-data.yml b/.github/workflows/language-data.yml index d5c895b279..dfe8a4f49c 100644 --- a/.github/workflows/language-data.yml +++ b/.github/workflows/language-data.yml @@ -2,7 +2,7 @@ name: Update language data on: schedule: - - cron: 0 0 * * * + - cron: 0 0 1 * * workflow_dispatch: jobs: