From 0715d41006d7f4cc7e354ae00edec5a763d27a49 Mon Sep 17 00:00:00 2001 From: openhands Date: Sat, 24 May 2025 17:55:26 +0000 Subject: [PATCH 1/2] Implement recursive translations folder support (fixes #67) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add load_translation_folder() function to support translations/ folder structure - Modify extension loading to check for both translations.yml and translations/ folder - Maintain backward compatibility with existing translations.yml files - Support recursive folder structure with dot-notation keys for strings - Add comprehensive documentation for the new feature Features: - Recursive processing of .yml/.yaml files in translations/ folder - Automatic separation of commands and strings based on folder structure - Dot-notation keys for nested string organization (e.g., strings.general.welcome) - Priority given to translations.yml for backward compatibility - Support for both .yml and .yaml file extensions This allows extensions to organize translations in a more structured way: translations/ ├── commands/ │ ├── command1.yaml │ └── command2.yaml └── strings/ └── messages.yaml --- RECURSIVE_TRANSLATIONS.md | 118 ++++++++++++++++++++++++++++++++++++++ src/i18n/__init__.py | 4 +- src/i18n/utils.py | 102 ++++++++++++++++++++++++++++++++ src/start.py | 19 +++++- 4 files changed, 238 insertions(+), 5 deletions(-) create mode 100644 RECURSIVE_TRANSLATIONS.md diff --git a/RECURSIVE_TRANSLATIONS.md b/RECURSIVE_TRANSLATIONS.md new file mode 100644 index 0000000..04fdc81 --- /dev/null +++ b/RECURSIVE_TRANSLATIONS.md @@ -0,0 +1,118 @@ +# Recursive Translations Folder Support + +This document describes the new recursive translations folder support feature implemented for issue #67. + +## Overview + +Botkit now supports loading translations from a folder structure in addition to the traditional single `translations.yml` file. This allows for better organization of translations, especially for extensions with many commands and strings. + +## Backward Compatibility + +The existing `translations.yml` file format is still fully supported. Extensions can continue to use the single file approach without any changes. + +## New Folder Structure + +Extensions can now use a `translations/` folder instead of `translations.yml`. The folder structure is recursively processed, and subfolder names become keys in the translation structure. + +### Example Structure + +``` +src/extensions/my_extension/ +├── __init__.py +└── translations/ + ├── commands/ + │ ├── hello.yaml + │ └── goodbye.yaml + └── strings/ + └── messages.yaml +``` + +### File Contents + +**commands/hello.yaml:** +```yaml +name: + en-US: hello + fr: bonjour +description: + en-US: Say hello + fr: Dire bonjour +strings: + response: + en-US: Hello, {user}! + fr: Bonjour, {user}! +``` + +**commands/goodbye.yaml:** +```yaml +name: + en-US: goodbye + fr: au revoir +description: + en-US: Say goodbye + fr: Dire au revoir +strings: + response: + en-US: Goodbye, {user}! + fr: Au revoir, {user}! +``` + +**strings/messages.yaml:** +```yaml +welcome: + en-US: Welcome to the bot! + fr: Bienvenue dans le bot! +error: + en-US: An error occurred + fr: Une erreur s'est produite +``` + +## How It Works + +1. **Priority**: If both `translations.yml` and `translations/` exist, the file takes precedence for backward compatibility. + +2. **Recursive Processing**: The system recursively processes all `.yml` and `.yaml` files in the translations folder. + +3. **Key Generation**: + - Files in the `commands/` subfolder become command definitions + - Files in other subfolders become strings with dot-notation keys (e.g., `strings.messages.welcome`) + +4. **Structure Preservation**: The folder structure is preserved in the translation keys, allowing for organized access to translations. + +## Loading Logic + +The extension loading system in `src/start.py` now follows this logic: + +1. Check if `translations.yml` exists → load it using `load_translation()` +2. If not, check if `translations/` folder exists → load it using `load_translation_folder()` +3. If neither exists → log a warning + +## Implementation Details + +- **New Functions**: + - `load_translation_folder()` in `src/i18n/utils.py` + - `_load_recursive_translations()` helper function +- **Modified Files**: + - `src/i18n/utils.py` - Added recursive loading functions + - `src/i18n/__init__.py` - Exported new function + - `src/start.py` - Updated extension loading logic + +## Benefits + +1. **Better Organization**: Large extensions can organize translations by command or category +2. **Easier Maintenance**: Smaller files are easier to edit and maintain +3. **Team Collaboration**: Multiple translators can work on different files simultaneously +4. **Backward Compatibility**: Existing extensions continue to work without changes +5. **Flexible Structure**: Extensions can organize translations however makes sense for their use case + +## Migration + +Existing extensions do not need to migrate. However, if you want to use the new folder structure: + +1. Create a `translations/` folder in your extension +2. Create subfolders as needed (e.g., `commands/`, `strings/`) +3. Move command definitions to individual files in `commands/` +4. Move other strings to organized files in appropriate subfolders +5. Remove the old `translations.yml` file + +The new system will automatically detect and load the folder structure. \ No newline at end of file diff --git a/src/i18n/__init__.py b/src/i18n/__init__.py index f52ffb7..5bc59fe 100644 --- a/src/i18n/__init__.py +++ b/src/i18n/__init__.py @@ -2,6 +2,6 @@ # SPDX-License-Identifier: MIT from .classes import apply_locale -from .utils import apply, load_translation +from .utils import apply, load_translation, load_translation_folder -__all__ = ["apply", "apply_locale", "load_translation"] +__all__ = ["apply", "apply_locale", "load_translation", "load_translation_folder"] diff --git a/src/i18n/utils.py b/src/i18n/utils.py index 15630e8..98c8a1d 100644 --- a/src/i18n/utils.py +++ b/src/i18n/utils.py @@ -3,6 +3,9 @@ from typing import TYPE_CHECKING, TypeVar +import os +from pathlib import Path + import discord import yaml from discord.ext import commands as prefixed @@ -182,6 +185,105 @@ def load_translation(path: str) -> ExtensionTranslation: return ExtensionTranslation(**data) +def _load_recursive_translations(folder_path: Path, prefix: str = "") -> dict: + """Recursively load translations from a folder structure. + + Args: + ---- + folder_path (Path): The path to the translations folder. + prefix (str): The current prefix for nested keys. + + Returns: + ------- + dict: A flattened dictionary with dot-notation keys for nested structure. + + """ + result = {} + + if not folder_path.exists() or not folder_path.is_dir(): + return result + + for item in folder_path.iterdir(): + if item.is_file() and item.suffix in ('.yml', '.yaml'): + # Load YAML file content + try: + with open(item, encoding="utf-8") as f: + file_data = yaml.safe_load(f) + if file_data is not None: + # Use filename without extension as key + key = item.stem + full_key = f"{prefix}.{key}" if prefix else key + + # If the file contains a flat structure of translations, add them with dot notation + if isinstance(file_data, dict): + for sub_key, sub_value in file_data.items(): + nested_key = f"{full_key}.{sub_key}" + result[nested_key] = sub_value + else: + result[full_key] = file_data + except yaml.YAMLError as e: + logger.warning(f"Error loading translation file {item}: {e}") + elif item.is_dir() and not item.name.startswith('.'): + # Recursively process subdirectories + new_prefix = f"{prefix}.{item.name}" if prefix else item.name + subdir_data = _load_recursive_translations(item, new_prefix) + result.update(subdir_data) + + return result + + +def load_translation_folder(folder_path: str) -> ExtensionTranslation: + """Load translations from a folder structure. + + Args: + ---- + folder_path (str): The path to the translations folder. + + Returns: + ------- + ExtensionTranslation: The loaded translation with nested structure. + + Raises: + ------ + yaml.YAMLError: If any YAML file is not valid. + + """ + path = Path(folder_path) + flattened_data = _load_recursive_translations(path) + + # Separate commands and strings based on the structure + result_data = {} + commands_data = {} + strings_data = {} + + for key, value in flattened_data.items(): + if key.startswith('commands.'): + # Extract command structure + parts = key.split('.', 2) # ['commands', 'command_name', 'rest'] + if len(parts) >= 2: + command_name = parts[1] + if command_name not in commands_data: + commands_data[command_name] = {} + + if len(parts) == 2: + # Direct command data + commands_data[command_name].update(value if isinstance(value, dict) else {}) + else: + # Nested command data (e.g., commands.ping.name) + rest_key = parts[2] + commands_data[command_name][rest_key] = value + else: + # Everything else goes to strings with dot notation + strings_data[key] = value + + if commands_data: + result_data['commands'] = commands_data + if strings_data: + result_data['strings'] = strings_data + + return ExtensionTranslation(**result_data) + + def apply( bot: "custom.Bot", translations: list[ExtensionTranslation], diff --git a/src/start.py b/src/start.py index fc0e63d..bde15a9 100644 --- a/src/start.py +++ b/src/start.py @@ -128,12 +128,25 @@ def load_extensions() -> tuple[ logger.info(f"Loading extension {name}") translation: ExtensionTranslation | None = None - if (translation_path := (extension / "translations.yml")).exists(): + + # Check for translations.yml file first (backward compatibility) + translation_file = extension / "translations.yml" + translation_folder = extension / "translations" + + if translation_file.exists(): try: - translation = i18n.load_translation(str(translation_path)) + translation = i18n.load_translation(str(translation_file)) translations.append(translation) + logger.debug(f"Loaded translation file for extension {name}") except yaml.YAMLError as e: - logger.error(f"Error loading translation {translation_path}: {e}") + logger.error(f"Error loading translation file {translation_file}: {e}") + elif translation_folder.exists() and translation_folder.is_dir(): + try: + translation = i18n.load_translation_folder(str(translation_folder)) + translations.append(translation) + logger.debug(f"Loaded translation folder for extension {name}") + except yaml.YAMLError as e: + logger.error(f"Error loading translation folder {translation_folder}: {e}") else: logger.warning(f"No translation found for extension {name}") From 78c79382aa24f347dc3798de340bd839ba6f7b43 Mon Sep 17 00:00:00 2001 From: openhands Date: Sat, 24 May 2025 18:03:28 +0000 Subject: [PATCH 2/2] feat: enhance recursive translations with shortcuts and dot notation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add folder name shortcuts: c. → commands., t. → translations., s. → strings. - Support dots in folder names for nested key structures - Update readme.md with comprehensive folder structure documentation - Remove separate RECURSIVE_TRANSLATIONS.md file as requested - Fix command structure loading for folder-based translations - Maintain backward compatibility with existing translation files Resolves #67 --- RECURSIVE_TRANSLATIONS.md | 118 -------------------------------------- readme.md | 67 +++++++++++++++++++++- src/i18n/utils.py | 52 ++++++++++++++--- 3 files changed, 109 insertions(+), 128 deletions(-) delete mode 100644 RECURSIVE_TRANSLATIONS.md diff --git a/RECURSIVE_TRANSLATIONS.md b/RECURSIVE_TRANSLATIONS.md deleted file mode 100644 index 04fdc81..0000000 --- a/RECURSIVE_TRANSLATIONS.md +++ /dev/null @@ -1,118 +0,0 @@ -# Recursive Translations Folder Support - -This document describes the new recursive translations folder support feature implemented for issue #67. - -## Overview - -Botkit now supports loading translations from a folder structure in addition to the traditional single `translations.yml` file. This allows for better organization of translations, especially for extensions with many commands and strings. - -## Backward Compatibility - -The existing `translations.yml` file format is still fully supported. Extensions can continue to use the single file approach without any changes. - -## New Folder Structure - -Extensions can now use a `translations/` folder instead of `translations.yml`. The folder structure is recursively processed, and subfolder names become keys in the translation structure. - -### Example Structure - -``` -src/extensions/my_extension/ -├── __init__.py -└── translations/ - ├── commands/ - │ ├── hello.yaml - │ └── goodbye.yaml - └── strings/ - └── messages.yaml -``` - -### File Contents - -**commands/hello.yaml:** -```yaml -name: - en-US: hello - fr: bonjour -description: - en-US: Say hello - fr: Dire bonjour -strings: - response: - en-US: Hello, {user}! - fr: Bonjour, {user}! -``` - -**commands/goodbye.yaml:** -```yaml -name: - en-US: goodbye - fr: au revoir -description: - en-US: Say goodbye - fr: Dire au revoir -strings: - response: - en-US: Goodbye, {user}! - fr: Au revoir, {user}! -``` - -**strings/messages.yaml:** -```yaml -welcome: - en-US: Welcome to the bot! - fr: Bienvenue dans le bot! -error: - en-US: An error occurred - fr: Une erreur s'est produite -``` - -## How It Works - -1. **Priority**: If both `translations.yml` and `translations/` exist, the file takes precedence for backward compatibility. - -2. **Recursive Processing**: The system recursively processes all `.yml` and `.yaml` files in the translations folder. - -3. **Key Generation**: - - Files in the `commands/` subfolder become command definitions - - Files in other subfolders become strings with dot-notation keys (e.g., `strings.messages.welcome`) - -4. **Structure Preservation**: The folder structure is preserved in the translation keys, allowing for organized access to translations. - -## Loading Logic - -The extension loading system in `src/start.py` now follows this logic: - -1. Check if `translations.yml` exists → load it using `load_translation()` -2. If not, check if `translations/` folder exists → load it using `load_translation_folder()` -3. If neither exists → log a warning - -## Implementation Details - -- **New Functions**: - - `load_translation_folder()` in `src/i18n/utils.py` - - `_load_recursive_translations()` helper function -- **Modified Files**: - - `src/i18n/utils.py` - Added recursive loading functions - - `src/i18n/__init__.py` - Exported new function - - `src/start.py` - Updated extension loading logic - -## Benefits - -1. **Better Organization**: Large extensions can organize translations by command or category -2. **Easier Maintenance**: Smaller files are easier to edit and maintain -3. **Team Collaboration**: Multiple translators can work on different files simultaneously -4. **Backward Compatibility**: Existing extensions continue to work without changes -5. **Flexible Structure**: Extensions can organize translations however makes sense for their use case - -## Migration - -Existing extensions do not need to migrate. However, if you want to use the new folder structure: - -1. Create a `translations/` folder in your extension -2. Create subfolders as needed (e.g., `commands/`, `strings/`) -3. Move command definitions to individual files in `commands/` -4. Move other strings to organized files in appropriate subfolders -5. Remove the old `translations.yml` file - -The new system will automatically detect and load the folder structure. \ No newline at end of file diff --git a/readme.md b/readme.md index efed327..29769ba 100644 --- a/readme.md +++ b/readme.md @@ -236,8 +236,16 @@ your extensions: ### Translation File Structure -Each extension can have its own `translations.yml` file located at -`src/extensions/EXT_NAME/translations.yml`. This file follows a specific structure: +Each extension can have translations in one of two formats: + +1. **Single File**: A `translations.yml` file located at `src/extensions/EXT_NAME/translations.yml` +2. **Folder Structure**: A `translations/` folder located at `src/extensions/EXT_NAME/translations/` + +If both exist, the single file takes precedence for backward compatibility. + +#### Single File Format + +The traditional `translations.yml` file follows this structure: ```yaml commands: @@ -279,6 +287,61 @@ strings: > `config["translations"]`. This section is for general strings not directly tied to > specific commands. +#### Folder Structure Format + +The new folder structure allows for better organization of translations, especially for extensions with many commands and strings. Here's how it works: + +``` +src/extensions/my_extension/ +├── __init__.py +└── translations/ + ├── commands/ + │ ├── hello.yaml + │ └── goodbye.yaml + ├── strings/ + │ ├── general.yaml + │ └── errors.yaml + └── c.help/ # Shortcut for commands.help/ + └── subcommand.yml +``` + +**Folder Name Features:** +- **Dot Notation**: Dots in folder names create nested structures (e.g., `strings.general` becomes a nested key) +- **Shortcuts**: Use `c.`, `t.`, and `s.` as shortcuts for `commands.`, `translations.`, and `strings.` respectively (only in folder names, not in YAML files) +- **File Extensions**: Both `.yml` and `.yaml` files are supported + +**Example file contents:** + +**commands/hello.yaml:** +```yaml +name: + en-US: hello + fr: bonjour +description: + en-US: Say hello + fr: Dire bonjour +strings: + response: + en-US: Hello, {user}! + fr: Bonjour, {user}! +``` + +**strings/general.yaml:** +```yaml +welcome: + en-US: Welcome to the bot! + fr: Bienvenue dans le bot! +help: + en-US: Need help? Use /help + fr: Besoin d'aide? Utilisez /help +``` + +**How it maps:** +- Files in `commands/` become command definitions +- Files in `strings/` become general strings with dot-notation keys (e.g., `strings.general.welcome`) +- Folder shortcuts expand: `c.help/` becomes `commands.help/` +- Nested folders create nested keys: `strings.errors.not_found` + ### Nested Commands and Sub-commands For command groups and sub-commands, you can nest the structure using the `commands` diff --git a/src/i18n/utils.py b/src/i18n/utils.py index 98c8a1d..776faab 100644 --- a/src/i18n/utils.py +++ b/src/i18n/utils.py @@ -185,6 +185,33 @@ def load_translation(path: str) -> ExtensionTranslation: return ExtensionTranslation(**data) +def _expand_folder_name(folder_name: str) -> str: + """Expand folder name shortcuts and handle dots as path separators. + + Args: + ---- + folder_name (str): The folder name to expand. + + Returns: + ------- + str: The expanded folder name with shortcuts replaced and dots converted to path separators. + """ + # Handle shortcuts (only in folder names) + shortcuts = { + 'c.': 'commands.', + 't.': 'translations.', + 's.': 'strings.' + } + + for shortcut, expansion in shortcuts.items(): + if folder_name.startswith(shortcut): + folder_name = expansion + folder_name[len(shortcut):] + break + + # Convert dots to path separators for nested structure + return folder_name.replace('.', '.') + + def _load_recursive_translations(folder_path: Path, prefix: str = "") -> dict: """Recursively load translations from a folder structure. @@ -214,8 +241,13 @@ def _load_recursive_translations(folder_path: Path, prefix: str = "") -> dict: key = item.stem full_key = f"{prefix}.{key}" if prefix else key - # If the file contains a flat structure of translations, add them with dot notation - if isinstance(file_data, dict): + # For command files, keep the entire structure intact + # For string files, flatten with dot notation + if prefix.startswith('commands.') or (prefix == '' and key == 'commands'): + # This is a command file - keep structure intact + result[full_key] = file_data + elif isinstance(file_data, dict): + # This is a strings file - flatten with dot notation for sub_key, sub_value in file_data.items(): nested_key = f"{full_key}.{sub_key}" result[nested_key] = sub_value @@ -224,8 +256,9 @@ def _load_recursive_translations(folder_path: Path, prefix: str = "") -> dict: except yaml.YAMLError as e: logger.warning(f"Error loading translation file {item}: {e}") elif item.is_dir() and not item.name.startswith('.'): - # Recursively process subdirectories - new_prefix = f"{prefix}.{item.name}" if prefix else item.name + # Expand folder name with shortcuts and dot notation + expanded_name = _expand_folder_name(item.name) + new_prefix = f"{prefix}.{expanded_name}" if prefix else expanded_name subdir_data = _load_recursive_translations(item, new_prefix) result.update(subdir_data) @@ -266,12 +299,15 @@ def load_translation_folder(folder_path: str) -> ExtensionTranslation: commands_data[command_name] = {} if len(parts) == 2: - # Direct command data + # Direct command data (file named after command group) commands_data[command_name].update(value if isinstance(value, dict) else {}) else: - # Nested command data (e.g., commands.ping.name) - rest_key = parts[2] - commands_data[command_name][rest_key] = value + # This is a file inside a command folder (e.g., commands.admin.ban) + # The file name becomes a sub-command + subcommand_name = parts[2] + if 'commands' not in commands_data[command_name]: + commands_data[command_name]['commands'] = {} + commands_data[command_name]['commands'][subcommand_name] = value else: # Everything else goes to strings with dot notation strings_data[key] = value