Improve language data handling

.github/workflows/language-data.yml

@@ -2,7 +2,7 @@ name: Update language data
 
 on:
   schedule:
-    - cron: 0 0 * * *
+    - cron: 0 0 1 * *
   workflow_dispatch:
 
 jobs:
@@ -22,4 +22,4 @@ jobs:
           commit-message: Update language-data.json
           branch: actions/language-data
           title: Update language data
-          body: An upstream change was detected in the [language-data repository](https://github.com/wikimedia/language-data).
+          body: An upstream change to `locale/language-data.json` was detected in the [language-data repository](https://github.com/wikimedia/language-data).

TWLight/resources/helpers.py

@@ -1,38 +1,9 @@
 from django.conf import settings
 from numbers import Number
 import json
-import logging
 import os
 
 
-LANGS_DIRECTION = {}
-
-
-def get_language_direction(language_code: str):
-    """
-    Caches the direction (LTR/RTL) of each language by checking if the
-    first list value (script type) is defined in the RTL list (rtlscripts)
-    """
-    if not LANGS_DIRECTION:
-        try:
-            with open(
-                os.path.join(settings.LOCALE_PATHS[0], "language-data.json")
-            ) as file:
-                data = json.load(file)
-                languages = data["languages"]
-                rtlscripts = data["rtlscripts"]
-                LANGS_DIRECTION.update(
-                    {
-                        lang: "rtl" if meta[0] in rtlscripts else "ltr"
-                        for lang, meta in languages.items()
-                    }
-                )
-        except:
-            logging.getLogger(__name__).exception("Failed to load language data")
-
-    return LANGS_DIRECTION.get(language_code, "ltr")
-
-
 def get_partner_description_json_schema():
     """
     JSON Schema for partner description translations
@@ -93,8 +64,10 @@ def get_partner_description(
     descriptions["short_description_language"] = (
         "en" if short_description["is_default"] else language_code
     )
-    descriptions["short_description_direction"] = get_language_direction(
-        descriptions["short_description_language"]
+    descriptions["short_description_direction"] = (
+        "rtl"
+        if descriptions["short_description_language"] in settings.LANGUAGES_BIDI
+        else "ltr"
     )
 
     description = _get_any_description(
@@ -107,8 +80,10 @@ def get_partner_description(
     descriptions["description_language"] = (
         "en" if description["is_default"] else language_code
     )
-    descriptions["description_direction"] = get_language_direction(
-        descriptions["description_language"]
+    descriptions["description_direction"] = (
+        "rtl"
+        if descriptions["description_language"] in settings.LANGUAGES_BIDI
+        else "ltr"
     )
 
     return descriptions

TWLight/settings/base.py

@@ -66,17 +66,29 @@
 # https://raw.githubusercontent.com/wikimedia/language-data/master/data/language-data.json
 # into locale/language-data.json
 def get_languages_from_locale_subdirectories(dir):
-    current_languages = []
-    language_data_json = open(os.path.join(dir, "language-data.json"))
-    languages = json.loads(language_data_json.read())["languages"]
-    language_data_json.close()
-    for locale_dir in os.listdir(dir):
-        if os.path.isdir(os.path.join(dir, locale_dir)):
-            for lang_code, lang_data in languages.items():
-                autonym = lang_data[-1]
-                if to_language(locale_dir) == lang_code:
-                    current_languages += [(lang_code, autonym)]
-    return sorted(set(current_languages))
+    try:
+        with open(os.path.join(dir, "language-data.json")) as file:
+            data = json.load(file)
+            languages = data["languages"]
+            rtlscripts = data["rtlscripts"]
+
+            current_languages = set()
+            current_bidi = set()
+
+            for locale_dir in os.listdir(dir):
+                if os.path.isdir(os.path.join(dir, locale_dir)):
+                    lang_code = to_language(locale_dir)
+                    lang_data = languages.get(lang_code)
+
+                    if lang_data and len(lang_data) == 3:
+                        current_languages.add((lang_code, lang_data[2]))
+
+                        if lang_data[0] in rtlscripts:
+                            current_bidi.add(lang_code)
+
+            return sorted(current_languages), sorted(current_bidi)
+    except:
+        logging.getLogger(__name__).exception("Failed to load language data")
 
 
 # Get the intersection of available Faker locales and the specified language set.
@@ -310,7 +322,7 @@ def show_toolbar(request):
 # available to the system. This keeps our column and index count for db-stored
 # translations as low as possible while allowing translatewiki contributions to
 # be used without reconfiguring the site.
-LANGUAGES = get_languages_from_locale_subdirectories(LOCALE_PATHS[0])
+LANGUAGES, LANGUAGES_BIDI = get_languages_from_locale_subdirectories(LOCALE_PATHS[0])
 FAKER_LOCALES = get_django_faker_languages_intersection(LANGUAGES)
 
 TIME_ZONE = "UTC"