feat: Add translation language utilities

This commit adds a new file, `translation-language-utils.h`, which contains utility functions for handling translation languages. The `remove_start_punctuation` function removes any leading punctuation from a given string. This utility will be used in the translation process to improve the quality of the translated output.

Author: royshil

CMakeLists.txt

@@ -96,6 +96,11 @@ else()
   include(cmake/FetchOnnxruntime.cmake)
 endif()
 
+include(cmake/BuildICU.cmake)
+# Add ICU to the target
+target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE ICU::ICU)
+target_include_directories(${CMAKE_PROJECT_NAME} SYSTEM PUBLIC ${ICU_INCLUDE_DIR})
+
 target_sources(
   ${CMAKE_PROJECT_NAME}
   PRIVATE src/plugin-main.c
@@ -118,6 +123,7 @@ target_sources(
           src/translation/language_codes.cpp
           src/translation/translation.cpp
           src/translation/translation-utils.cpp
+          src/translation/translation-language-utils.cpp
           src/ui/filter-replace-dialog.cpp)
 
 set_target_properties_plugin(${CMAKE_PROJECT_NAME} PROPERTIES OUTPUT_NAME ${_name})
@@ -140,11 +146,12 @@ if(ENABLE_TESTS)
             src/whisper-utils/token-buffer-thread.cpp
             src/whisper-utils/vad-processing.cpp
             src/translation/language_codes.cpp
-            src/translation/translation.cpp)
+            src/translation/translation.cpp
+            src/translation/translation-language-utils.cpp)
 
   find_libav(${CMAKE_PROJECT_NAME}-tests)
 
-  target_link_libraries(${CMAKE_PROJECT_NAME}-tests PRIVATE ct2 sentencepiece Whispercpp Ort OBS::libobs)
+  target_link_libraries(${CMAKE_PROJECT_NAME}-tests PRIVATE ct2 sentencepiece Whispercpp Ort OBS::libobs ICU::ICU)
   target_include_directories(${CMAKE_PROJECT_NAME}-tests PRIVATE src)
 
   # install the tests to the release/test directory

cmake/BuildICU.cmake

@@ -0,0 +1,112 @@
+include(FetchContent)
+include(ExternalProject)
+
+set(ICU_VERSION "75.1")
+set(ICU_VERSION_UNDERSCORE "75_1")
+set(ICU_VERSION_DASH "75-1")
+set(ICU_VERSION_NO_MINOR "75")
+
+if(WIN32)
+  set(ICU_URL
+      "https://github.com/unicode-org/icu/releases/download/release-${ICU_VERSION_DASH}/icu4c-${ICU_VERSION_UNDERSCORE}-Win64-MSVC2022.zip"
+  )
+  set(ICU_HASH "SHA256=7ac9c0dc6ccc1ec809c7d5689b8d831c5b8f6b11ecf70fdccc55f7ae8731ac8f")
+
+  FetchContent_Declare(
+    ICU
+    URL ${ICU_URL}
+    URL_HASH ${ICU_HASH})
+
+  FetchContent_MakeAvailable(ICU)
+
+  # Assuming the ZIP structure, adjust paths as necessary
+  set(ICU_INCLUDE_DIR "${icu_SOURCE_DIR}/include")
+  set(ICU_LIBRARY_DIR "${icu_SOURCE_DIR}/lib64")
+  set(ICU_BINARY_DIR "${icu_SOURCE_DIR}/bin64")
+
+  # Add ICU libraries
+  find_library(
+    ICU_DATA_LIBRARY
+    NAMES icudt
+    PATHS ${ICU_LIBRARY_DIR}
+    NO_DEFAULT_PATH)
+  find_library(
+    ICU_UC_LIBRARY
+    NAMES icuuc
+    PATHS ${ICU_LIBRARY_DIR}
+    NO_DEFAULT_PATH)
+  find_library(
+    ICU_IN_LIBRARY
+    NAMES icuin
+    PATHS ${ICU_LIBRARY_DIR}
+    NO_DEFAULT_PATH)
+
+  # find the dlls
+  find_file(
+    ICU_DATA_DLL
+    NAMES icudt${ICU_VERSION_NO_MINOR}.dll
+    PATHS ${ICU_BINARY_DIR}
+    NO_DEFAULT_PATH)
+  find_file(
+    ICU_UC_DLL
+    NAMES icuuc${ICU_VERSION_NO_MINOR}.dll
+    PATHS ${ICU_BINARY_DIR}
+    NO_DEFAULT_PATH)
+  find_file(
+    ICU_IN_DLL
+    NAMES icuin${ICU_VERSION_NO_MINOR}.dll
+    PATHS ${ICU_BINARY_DIR}
+    NO_DEFAULT_PATH)
+
+  # Copy the DLLs to the output directory
+  install(FILES ${ICU_DATA_DLL} DESTINATION "obs-plugins/64bit")
+  install(FILES ${ICU_UC_DLL} DESTINATION "obs-plugins/64bit")
+  install(FILES ${ICU_IN_DLL} DESTINATION "obs-plugins/64bit")
+
+else() # Mac and Linux
+  set(ICU_URL
+      "https://github.com/unicode-org/icu/releases/download/release-${ICU_VERSION_UNDERSCORE}/icu4c-${ICU_VERSION_UNDERSCORE}-src.tgz"
+  )
+  set(ICU_HASH "SHA256=94bb97d88f13bb74ec0168446a845511bd92c1c49ee8e63df646a48c38dfde6d")
+
+  set(ICU_INSTALL_DIR "${CMAKE_BINARY_DIR}/icu-install")
+
+  ExternalProject_Add(
+    ICU
+    URL ${ICU_URL}
+    URL_HASH ${ICU_HASH}
+    CONFIGURE_COMMAND <SOURCE_DIR>/source/runConfigureICU Linux --prefix=${ICU_INSTALL_DIR}
+    BUILD_COMMAND make -j4
+    INSTALL_COMMAND make install
+    BUILD_IN_SOURCE 1)
+
+  set(ICU_INCLUDE_DIR "${ICU_INSTALL_DIR}/include")
+  set(ICU_LIBRARY_DIR "${ICU_INSTALL_DIR}/lib")
+
+  # Add ICU libraries
+  find_library(
+    ICU_DATA_LIBRARY
+    NAMES icudata
+    PATHS ${ICU_LIBRARY_DIR}
+    NO_DEFAULT_PATH)
+  find_library(
+    ICU_UC_LIBRARY
+    NAMES icuuc
+    PATHS ${ICU_LIBRARY_DIR}
+    NO_DEFAULT_PATH)
+  find_library(
+    ICU_I18N_LIBRARY
+    NAMES icui18n
+    PATHS ${ICU_LIBRARY_DIR}
+    NO_DEFAULT_PATH)
+endif()
+
+# Create an interface target for ICU
+add_library(ICU::ICU INTERFACE IMPORTED GLOBAL)
+target_include_directories(ICU::ICU INTERFACE ${ICU_INCLUDE_DIR})
+target_link_libraries(ICU::ICU INTERFACE ${ICU_DATA_LIBRARY} ${ICU_UC_LIBRARY}
+                                         $<IF:$<BOOL:${WIN32}>,${ICU_IN_LIBRARY},${ICU_I18N_LIBRARY}>)
+
+if(NOT WIN32)
+  add_dependencies(ICU::ICU ICU)
+endif()

src/transcription-filter-callbacks.cpp

@@ -281,7 +281,11 @@ void set_text_callback(struct transcription_filter_data *gf,
 		gf->cleared_last_sub = false;
 		if (result.result == DETECTION_RESULT_SPEECH) {
 			// save the last subtitle if it was a full sentence
-			gf->last_transcription_sentence = result.text;
+			gf->last_transcription_sentence.push_back(result.text);
+			// remove the oldest sentence if the buffer is too long
+			while (gf->last_transcription_sentence.size() > gf->n_context_sentences) {
+				gf->last_transcription_sentence.pop_front();
+			}
 		}
 	}
 };
@@ -330,7 +334,7 @@ void reset_caption_state(transcription_filter_data *gf_)
 	gf_->last_text_translation = "";
 	gf_->translation_ctx.last_input_tokens.clear();
 	gf_->translation_ctx.last_translation_tokens.clear();
-	gf_->last_transcription_sentence = "";
+	gf_->last_transcription_sentence.clear();
 	// flush the buffer
 	{
 		std::lock_guard<std::mutex> lock(gf_->whisper_buf_mutex);

src/transcription-filter-data.h

@@ -93,7 +93,9 @@ struct transcription_filter_data {
 	std::string last_text_for_translation;
 	std::string last_text_translation;
 
-	std::string last_transcription_sentence;
+	// Transcription context sentences
+	int n_context_sentences;
+	std::deque<std::string> last_transcription_sentence;
 
 	// Text source to output the subtitles
 	std::string text_source_name;

src/transcription-filter-properties.cpp

@@ -414,6 +414,10 @@ void add_whisper_params_group_properties(obs_properties_t *ppts)
 				  WHISPER_SAMPLING_BEAM_SEARCH);
 	obs_property_list_add_int(whisper_sampling_method_list, "Greedy", WHISPER_SAMPLING_GREEDY);
 
+	// add int slider for context sentences
+	obs_properties_add_int_slider(whisper_params_group, "n_context_sentences",
+				      MT_("n_context_sentences"), 0, 5, 1);
+
 	// int n_threads;
 	obs_properties_add_int_slider(whisper_params_group, "n_threads", MT_("n_threads"), 1, 8, 1);
 	// int n_max_text_ctx;     // max tokens to use from past text as prompt for the decoder
@@ -600,6 +604,7 @@ void transcription_filter_defaults(obs_data_t *s)
 
 	// Whisper parameters
 	obs_data_set_default_int(s, "whisper_sampling_method", WHISPER_SAMPLING_BEAM_SEARCH);
+	obs_data_set_default_int(s, "n_context_sentences", 0);
 	obs_data_set_default_string(s, "initial_prompt", "");
 	obs_data_set_default_int(s, "n_threads", 4);
 	obs_data_set_default_int(s, "n_max_text_ctx", 16384);

src/transcription-filter.cpp

@@ -346,6 +346,8 @@ void transcription_filter_update(void *data, obs_data_t *s)
 	{
 		std::lock_guard<std::mutex> lock(gf->whisper_ctx_mutex);
 
+		gf->n_context_sentences = (int)obs_data_get_int(s, "n_context_sentences");
+
 		gf->sentence_psum_accept_thresh =
 			(float)obs_data_get_double(s, "sentence_psum_accept_thresh");
 

src/translation/translation-language-utils.cpp

@@ -0,0 +1,33 @@
+#include "translation-language-utils.h"
+
+#include <unicode/unistr.h>
+#include <unicode/uchar.h>
+
+std::string remove_start_punctuation(const std::string &text)
+{
+	if (text.empty()) {
+		return text;
+	}
+
+	// Convert the input string to ICU's UnicodeString
+	icu::UnicodeString ustr = icu::UnicodeString::fromUTF8(text);
+
+	// Find the index of the first non-punctuation character
+	int32_t start = 0;
+	while (start < ustr.length()) {
+		UChar32 ch = ustr.char32At(start);
+		if (!u_ispunct(ch)) {
+			break;
+		}
+		start += U16_LENGTH(ch);
+	}
+
+	// Create a new UnicodeString with punctuation removed from the start
+	icu::UnicodeString result = ustr.tempSubString(start);
+
+	// Convert the result back to UTF-8
+	std::string output;
+	result.toUTF8String(output);
+
+	return output;
+}

src/translation/translation-language-utils.h

@@ -0,0 +1,8 @@
+#ifndef TRANSLATION_LANGUAGE_UTILS_H
+#define TRANSLATION_LANGUAGE_UTILS_H
+
+#include <string>
+
+std::string remove_start_punctuation(const std::string &text);
+
+#endif // TRANSLATION_LANGUAGE_UTILS_H

src/translation/translation.cpp

@@ -3,6 +3,7 @@
 #include "model-utils/model-find-utils.h"
 #include "transcription-filter-data.h"
 #include "language_codes.h"
+#include "translation-language-utils.h"
 
 #include <ctranslate2/translator.h>
 #include <sentencepiece_processor.h>
@@ -201,7 +202,8 @@ int translate(struct translation_context &translation_ctx, const std::string &te
 			(int)translation_ctx.last_translation_tokens.size());
 
 		// detokenize
-		result = translation_ctx.detokenizer(translation_tokens);
+		const std::string result_ = translation_ctx.detokenizer(translation_tokens);
+		result = remove_start_punctuation(result_);
 	} catch (std::exception &e) {
 		obs_log(LOG_ERROR, "Error: %s", e.what());
 		return OBS_POLYGLOT_TRANSLATION_FAIL;

src/whisper-utils/whisper-processing.cpp

@@ -181,8 +181,15 @@ struct DetectionResultWithText run_whisper_inference(struct transcription_filter
 		return {DETECTION_RESULT_UNKNOWN, "", t0, t1, {}, ""};
 	}
 
-	// obs_log(LOG_INFO, "initial prompt: %s", gf->last_transcription_sentence.c_str());
-	// gf->whisper_params.initial_prompt = gf->last_transcription_sentence.c_str();
+	if (gf->n_context_sentences > 0 && !gf->last_transcription_sentence.empty()) {
+		// set the initial prompt to the last transcription sentences (concatenated)
+		std::string initial_prompt = gf->last_transcription_sentence[0];
+		for (int i = 1; i < gf->last_transcription_sentence.size(); ++i) {
+			initial_prompt += " " + gf->last_transcription_sentence[i];
+		}
+		gf->whisper_params.initial_prompt = initial_prompt.c_str();
+		obs_log(gf->log_level, "Initial prompt: %s", gf->whisper_params.initial_prompt);
+	}
 
 	// run the inference
 	int whisper_full_result = -1;