FEATURE: Translate all new posts automatically

Adds a new site setting 'translate_posts_to_languages'

app/jobs/regular/translate_translatable.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Jobs
+  class TranslateTranslatable < ::Jobs::Base
+    def execute(args)
+      return unless SiteSetting.translator_enabled
+      return if SiteSetting.automatic_translation_target_languages.blank?
+
+      translatable = args[:type].constantize.find_by(id: args[:translatable_id])
+      return if translatable.blank?
+
+      target_locales = SiteSetting.automatic_translation_target_languages.split("|")
+      target_locales.each do |target_locale|
+        # 1. no special retry, job will be automatically retried with backoff
+        # 2. translate function will handle cases where translation is not needed or not possible
+        "DiscourseTranslator::#{SiteSetting.translator}".constantize.translate(
+          translatable,
+          target_locale.to_sym,
+        )
+      end
+
+      topic_id = translatable.is_a?(Post) ? translatable.topic.id : translatable.id
+      post_id = translatable.is_a?(Post) ? translatable.id : 1
+      MessageBus.publish("/topic/#{topic_id}", type: :revised, id: post_id)
+    end
+  end
+end

app/jobs/scheduled/automatic_translation_backfill.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Jobs
+  class AutomaticTranslationBackfill < ::Jobs::Scheduled
+    every 5.minutes
+
+    BACKFILL_LOCK_KEY = "discourse_translator_backfill_lock"
+
+    def execute(args = nil)
+      return unless SiteSetting.translator_enabled
+      return unless should_backfill?
+      return unless secure_backfill_lock
+
+      begin
+        process_batch
+      ensure
+        Discourse.redis.del(BACKFILL_LOCK_KEY)
+      end
+    end
+
+    def fetch_untranslated_model_ids(model = Post, limit = 100, target_locales = backfill_locales)
+      m = model.name.downcase
+      DB.query_single(<<~SQL, target_locales: target_locales, limit: limit)
+        SELECT m.id
+        FROM #{m}s m
+        LEFT JOIN discourse_translator_#{m}_locales dl ON dl.#{m}_id = m.id
+        LEFT JOIN LATERAL (
+          SELECT array_agg(DISTINCT locale)::text[] as locales
+          FROM discourse_translator_#{m}_translations dt
+          WHERE dt.#{m}_id = m.id
+        ) translations ON true
+        WHERE NOT (
+          ARRAY[:target_locales]::text[] <@
+            (COALESCE(
+              array_cat(
+                ARRAY[COALESCE(dl.detected_locale, '')]::text[],
+                COALESCE(translations.locales, ARRAY[]::text[])
+              ),
+              ARRAY[]::text[]
+            ))
+        )
+        ORDER BY m.id DESC
+        LIMIT :limit
+      SQL
+    end
+
+    private
+
+    def should_backfill?
+      return false if SiteSetting.automatic_translation_target_languages.blank?
+      return false if SiteSetting.automatic_translation_backfill_maximum_translations_per_hour == 0
+      true
+    end
+
+    def secure_backfill_lock
+      Discourse.redis.set(BACKFILL_LOCK_KEY, "1", ex: 5.minutes.to_i, nx: true)
+    end
+
+    def translations_per_run
+      [
+        (SiteSetting.automatic_translation_backfill_maximum_translations_per_hour / 12) /
+          backfill_locales.size,
+        1,
+      ].max
+    end
+
+    def backfill_locales
+      @backfill_locales ||= SiteSetting.automatic_translation_target_languages.split("|")
+    end
+
+    def translator
+      @translator_klass ||= "DiscourseTranslator::#{SiteSetting.translator}".constantize
+    end
+
+    def translate_records(type, record_ids)
+      record_ids.each do |id|
+        record = type.find(id)
+        backfill_locales.each do |target_locale|
+          begin
+            translator.translate(record, target_locale.to_sym)
+          rescue => e
+            # continue with other locales even if one fails
+            Rails.logger.warn(
+              "Failed to machine-translate #{type.name}##{id} to #{target_locale}: #{e.message}\n#{e.backtrace.join("\n")}",
+            )
+            next
+          end
+        end
+      end
+    end
+
+    def process_batch
+      models_translated = [Post, Topic].size
+      translations_to_run = [translations_per_run / models_translated, 1].max
+      topic_ids = fetch_untranslated_model_ids(Topic, translations_to_run)
+      translations_to_run = translations_per_run if topic_ids.empty?
+      post_ids = fetch_untranslated_model_ids(Post, translations_to_run)
+      return if topic_ids.empty? && post_ids.empty?
+
+      translate_records(Topic, topic_ids)
+      translate_records(Post, post_ids)
+    end
+  end
+end

app/services/discourse_translator/amazon.rb

@@ -126,7 +126,7 @@ def self.detect!(topic_or_post)
     def self.translate!(translatable, target_locale_sym = I18n.locale)
       detected_lang = detect(translatable)
 
-      save_translation(translatable) do
+      save_translation(translatable, target_locale_sym) do
         begin
           client.translate_text(
             {

app/services/discourse_translator/base.rb

@@ -48,7 +48,7 @@ def self.translate(translatable, target_locale_sym = I18n.locale)
                 ),
               )
       end
-      [detected_lang, translate!(translatable)]
+      [detected_lang, translate!(translatable, target_locale_sym)]
     end
 
     # Subclasses must implement this method to translate the text of a post or topic
@@ -77,9 +77,9 @@ def self.access_token
       raise "Not Implemented"
     end
 
-    def self.save_translation(translatable)
+    def self.save_translation(translatable, target_locale_sym = I18n.locale)
       translation = yield
-      translatable.set_translation(I18n.locale, translation)
+      translatable.set_translation(target_locale_sym, translation)
       translation
     end
 

app/services/discourse_translator/discourse_ai.rb

@@ -21,7 +21,7 @@ def self.detect!(topic_or_post)
 
     def self.translate!(translatable, target_locale_sym = I18n.locale)
       return unless required_settings_enabled
-      save_translation(translatable) do
+      save_translation(translatable, target_locale_sym) do
         ::DiscourseAi::Translator.new(
           text_for_translation(translatable),
           target_locale_sym,

app/services/discourse_translator/google.rb

@@ -91,7 +91,7 @@ def self.translate_supported?(source, target)
 
     def self.translate!(translatable, target_locale_sym = I18n.locale)
       detected_locale = detect(translatable)
-      save_translation(translatable) do
+      save_translation(translatable, target_locale_sym) do
         res =
           result(
             TRANSLATE_URI,

app/services/discourse_translator/libre_translate.rb

@@ -98,7 +98,7 @@ def self.translate_supported?(source, target)
     def self.translate!(translatable, target_locale_sym = I18n.locale)
       detected_lang = detect(translatable)
 
-      save_translation(translatable) do
+      save_translation(translatable, target_locale_sym) do
         res =
           result(
             translate_uri,

app/services/discourse_translator/microsoft.rb

@@ -166,7 +166,7 @@ def self.translate!(translatable, target_locale_sym = I18n.locale)
       locale =
         SUPPORTED_LANG_MAPPING[target_locale_sym] || (raise I18n.t("translator.not_supported"))
 
-      save_translation(translatable) do
+      save_translation(translatable, target_locale_sym) do
         query = default_query.merge("from" => detected_lang, "to" => locale, "textType" => "html")
 
         body = [{ "Text" => text_for_translation(translatable) }].to_json

app/services/discourse_translator/yandex.rb

@@ -137,7 +137,7 @@ def self.translate!(translatable, target_locale_sym = I18n.locale)
       locale =
         SUPPORTED_LANG_MAPPING[target_locale_sym] || (raise I18n.t("translator.not_supported"))
 
-      save_translation(translatable) do
+      save_translation(translatable, target_locale_sym) do
         query =
           default_query.merge(
             "lang" => "#{detected_lang}-#{locale}",

assets/javascripts/discourse/initializers/extend-for-translate-button.js

@@ -29,17 +29,6 @@ function initializeTranslation(api) {
     (currentUser || siteSettings.experimental_anon_language_switcher)
   ) {
     api.renderInOutlet("topic-navigation", ShowOriginalContent);
-    api.decorateCookedElement((cookedElement, helper) => {
-      if (helper) {
-        const translatedCooked = helper.getModel().get("translated_cooked");
-        if (translatedCooked) {
-          cookedElement.innerHTML = translatedCooked;
-        } else {
-          // this experimental feature does not yet support
-          // translating individual untranslated posts
-        }
-      }
-    });
 
     api.registerModelTransformer("topic", (topics) => {
       topics.forEach((topic) => {
@@ -48,6 +37,12 @@ function initializeTranslation(api) {
         }
       });
     });
+
+    api.registerModelTransformer("post", (post) => {
+      if (post.translated_cooked) {
+        post.set("cooked", post.translated_cooked);
+      }
+    });
   }
 
   if (!siteSettings.experimental_topic_translation) {

config/locales/server.en.yml

@@ -18,6 +18,7 @@ en:
     restrict_translation_by_group: "Only allowed groups can translate"
     restrict_translation_by_poster_group: "Only allow translation of posts made by users in allowed groups. If empty, allow translations of posts from all users."
     experimental_anon_language_switcher: "Enable experimental language switcher for anonymous users. This will allow anonymous users to switch between translated versions of Discourse and user-contributed content in topics."
+    translate_posts_to_languages: "Translate posts to languages"
     errors:
       set_locale_cookie_requirements: "The experimental language switcher for anonymous users requires the `set locale from cookie` site setting to be enabled."
     experimental_topic_translation: "Enable experimental topic translation feature. This replaces existing post in-line translation with a button that allows users to translate the entire topic."

config/settings.yml

@@ -109,3 +109,12 @@ discourse_translator:
   experimental_topic_translation:
     default: false
     client: true
+  automatic_translation_target_languages:
+    default: ""
+    type: list
+    list_type: named
+    choices: "DiscourseTranslator::TranslatableLanguagesSetting.values"
+    allow_any: false
+  automatic_translation_backfill_maximum_translations_per_hour:
+    default: 0
+    client: false

lib/discourse_translator/translatable_languages_setting.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module DiscourseTranslator
+  class TranslatableLanguagesSetting < LocaleSiteSetting
+    def self.printable_values
+      values.map { |v| v[:value] }
+    end
+
+    @lock = Mutex.new
+  end
+end

plugin.rb

@@ -38,6 +38,19 @@ module ::DiscourseTranslator
     end
   end
 
+  on(:post_process_cooked) do |_, post|
+    return if SiteSetting.automatic_translation_target_languages.blank?
+    Jobs.enqueue(:translate_translatable, type: Post, translatable_id: post.id)
+  end
+
+  on(:topic_created) do |topic|
+    Jobs.enqueue(:translate_translatable, type: Topic, translatable_id: topic.id)
+  end
+
+  on(:topic_edited) do |topic|
+    Jobs.enqueue(:translate_translatable, type: Topic, translatable_id: topic.id)
+  end
+
   add_to_serializer :post, :can_translate do
     scope.can_translate?(object)
   end