Offline transcription accuracy tests

CMakeLists.txt

@@ -6,6 +6,7 @@ project(${_name} VERSION ${_version})
 
 option(ENABLE_FRONTEND_API "Use obs-frontend-api for UI functionality" ON)
 option(ENABLE_QT "Use Qt functionality" ON)
+option(ENABLE_TESTS "Enable tests" OFF)
 
 include(compilerconfig)
 include(defaults)
@@ -90,11 +91,41 @@ target_sources(
           src/model-utils/model-downloader.cpp
           src/model-utils/model-downloader-ui.cpp
           src/model-utils/model-infos.cpp
+          src/model-utils/model-find-utils.cpp
           src/whisper-utils/whisper-processing.cpp
           src/whisper-utils/whisper-utils.cpp
+          src/whisper-utils/whisper-model-utils.cpp
           src/whisper-utils/silero-vad-onnx.cpp
           src/whisper-utils/token-buffer-thread.cpp
           src/translation/translation.cpp
+          src/translation/translation-utils.cpp
           src/utils.cpp)
 
 set_target_properties_plugin(${CMAKE_PROJECT_NAME} PROPERTIES OUTPUT_NAME ${_name})
+
+if(ENABLE_TESTS)
+  add_executable(${CMAKE_PROJECT_NAME}-tests)
+
+  include(cmake/FindLibAvObs.cmake)
+
+  target_sources(
+    ${CMAKE_PROJECT_NAME}-tests
+    PRIVATE src/tests/localvocal-offline-test.cpp
+            src/transcription-utils.cpp
+            src/model-utils/model-infos.cpp
+            src/model-utils/model-find-utils.cpp
+            src/whisper-utils/whisper-processing.cpp
+            src/whisper-utils/whisper-utils.cpp
+            src/whisper-utils/silero-vad-onnx.cpp
+            src/whisper-utils/token-buffer-thread.cpp
+            src/translation/translation.cpp
+            src/utils.cpp)
+
+  find_libav(${CMAKE_PROJECT_NAME}-tests)
+
+  target_link_libraries(${CMAKE_PROJECT_NAME}-tests PRIVATE ct2 sentencepiece Whispercpp Ort OBS::libobs)
+  target_include_directories(${CMAKE_PROJECT_NAME}-tests PRIVATE src)
+
+  # install the tests to the release/test directory
+  install(TARGETS ${CMAKE_PROJECT_NAME}-tests DESTINATION test)
+endif()

cmake/BuildCTranslate2.cmake

@@ -7,15 +7,15 @@ if(APPLE)
 
   FetchContent_Declare(
     ctranslate2_fetch
-    URL https://github.com/occ-ai/obs-ai-ctranslate2-dep/releases/download/1.1.1/libctranslate2-macos-Release-1.1.1.tar.gz
-    URL_HASH SHA256=da04d88ecc1ea105f8ee672e4eab33af96e50c999c5cc8170e105e110392182b)
+    URL https://github.com/occ-ai/obs-ai-ctranslate2-dep/releases/download/1.2.0/libctranslate2-macos-Release-1.2.0.tar.gz
+    URL_HASH SHA256=9029F19B0F50E5EDC14473479EDF0A983F7D6FA00BE61DC1B01BF8AA7F1CDB1B)
   FetchContent_MakeAvailable(ctranslate2_fetch)
 
   add_library(ct2 INTERFACE)
   target_link_libraries(ct2 INTERFACE "-framework Accelerate" ${ctranslate2_fetch_SOURCE_DIR}/lib/libctranslate2.a
                                       ${ctranslate2_fetch_SOURCE_DIR}/lib/libcpu_features.a)
   set_target_properties(ct2 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${ctranslate2_fetch_SOURCE_DIR}/include)
-  target_compile_options(ct2 INTERFACE -Wno-shorten-64-to-32)
+  target_compile_options(ct2 INTERFACE -Wno-shorten-64-to-32 -Wno-comma)
 
 elseif(WIN32)
 

cmake/FindLibAvObs.cmake

@@ -0,0 +1,48 @@
+# Find LibAV from the OBS dependencies
+
+function(find_libav TARGET)
+  if(UNIX AND NOT APPLE)
+    find_package(PkgConfig REQUIRED)
+    pkg_check_modules(
+      FFMPEG
+      REQUIRED
+      IMPORTED_TARGET
+      libavformat
+      libavcodec
+      libavutil
+      libswresample)
+    if(FFMPEG_FOUND)
+      target_link_libraries(${TARGET} PRIVATE PkgConfig::FFMPEG)
+    else()
+      message(FATAL_ERROR "FFMPEG not found!")
+    endif()
+    return()
+  endif()
+
+  if(NOT buildspec)
+    file(READ "${CMAKE_CURRENT_SOURCE_DIR}/buildspec.json" buildspec)
+  endif()
+  string(
+    JSON
+    version
+    GET
+    ${buildspec}
+    dependencies
+    prebuilt
+    version)
+
+  if(MSVC)
+    set(arch ${CMAKE_GENERATOR_PLATFORM})
+  elseif(APPLE)
+    set(arch universal)
+  endif()
+  set(deps_root "${CMAKE_CURRENT_SOURCE_DIR}/.deps/obs-deps-${version}-${arch}")
+
+  target_include_directories(${TARGET} PRIVATE "${deps_root}/include")
+  target_link_libraries(
+    ${TARGET}
+    PRIVATE "${deps_root}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}avcodec${CMAKE_STATIC_LIBRARY_SUFFIX}"
+            "${deps_root}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}avformat${CMAKE_STATIC_LIBRARY_SUFFIX}"
+            "${deps_root}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}avutil${CMAKE_STATIC_LIBRARY_SUFFIX}"
+            "${deps_root}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}swresample${CMAKE_STATIC_LIBRARY_SUFFIX}")
+endfunction(find_libav)

data/locale/en-US.ini

@@ -55,3 +55,5 @@ suppress_sentences="Suppress sentences (each line)"
 translate_output="Translation output"
 dtw_token_timestamps="DTW token timestamps"
 buffered_output="Buffered output (Experimental)"
+translate_model="Translation Model"
+Whisper-Based-Translation="Whisper-Based Translation"

src/model-utils/model-downloader.cpp

@@ -1,44 +1,11 @@
 #include "model-downloader.h"
 #include "plugin-support.h"
 #include "model-downloader-ui.h"
+#include "model-find-utils.h"
 
 #include <obs-module.h>
 #include <obs-frontend-api.h>
 
-#include <filesystem>
-#include <fstream>
-#include <iostream>
-#include <string>
-
-#include <curl/curl.h>
-
-std::string find_file_in_folder_by_name(const std::string &folder_path,
-					const std::string &file_name)
-{
-	for (const auto &entry : std::filesystem::directory_iterator(folder_path)) {
-		if (entry.path().filename() == file_name) {
-			return entry.path().string();
-		}
-	}
-	return "";
-}
-
-std::string find_bin_file_in_folder(const std::string &model_local_folder_path)
-{
-	// find .bin file in folder
-	for (const auto &entry : std::filesystem::directory_iterator(model_local_folder_path)) {
-		if (entry.path().extension() == ".bin") {
-			const std::string bin_file_path = entry.path().string();
-			obs_log(LOG_INFO, "Model bin file found in folder: %s",
-				bin_file_path.c_str());
-			return bin_file_path;
-		}
-	}
-	obs_log(LOG_ERROR, "Model bin file not found in folder: %s",
-		model_local_folder_path.c_str());
-	return "";
-}
-
 std::string find_model_folder(const ModelInfo &model_info)
 {
 	char *data_folder_models = obs_module_file("models");

src/model-utils/model-downloader.h

@@ -2,13 +2,9 @@
 #define MODEL_DOWNLOADER_H
 
 #include <string>
-#include <functional>
 
 #include "model-downloader-types.h"
 
-std::string find_file_in_folder_by_name(const std::string &folder_path,
-					const std::string &file_name);
-std::string find_bin_file_in_folder(const std::string &path);
 std::string find_model_folder(const ModelInfo &model_info);
 std::string find_model_bin_file(const ModelInfo &model_info);
 

src/model-utils/model-find-utils.cpp

@@ -0,0 +1,50 @@
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <string>
+#include <regex>
+
+#include <obs-module.h>
+
+#include "model-find-utils.h"
+#include "plugin-support.h"
+
+std::string find_file_in_folder_by_name(const std::string &folder_path,
+					const std::string &file_name)
+{
+	for (const auto &entry : std::filesystem::directory_iterator(folder_path)) {
+		if (entry.path().filename() == file_name) {
+			return entry.path().string();
+		}
+	}
+	return "";
+}
+
+// Find a file in a folder by expression
+std::string find_file_in_folder_by_regex_expression(const std::string &folder_path,
+						    const std::string &file_name_regex)
+{
+	for (const auto &entry : std::filesystem::directory_iterator(folder_path)) {
+		if (std::regex_match(entry.path().filename().string(),
+				     std::regex(file_name_regex))) {
+			return entry.path().string();
+		}
+	}
+	return "";
+}
+
+std::string find_bin_file_in_folder(const std::string &model_local_folder_path)
+{
+	// find .bin file in folder
+	for (const auto &entry : std::filesystem::directory_iterator(model_local_folder_path)) {
+		if (entry.path().extension() == ".bin") {
+			const std::string bin_file_path = entry.path().string();
+			obs_log(LOG_INFO, "Model bin file found in folder: %s",
+				bin_file_path.c_str());
+			return bin_file_path;
+		}
+	}
+	obs_log(LOG_ERROR, "Model bin file not found in folder: %s",
+		model_local_folder_path.c_str());
+	return "";
+}

src/model-utils/model-find-utils.h

@@ -0,0 +1,14 @@
+#ifndef MODEL_FIND_UTILS_H
+#define MODEL_FIND_UTILS_H
+
+#include <string>
+
+#include "model-downloader-types.h"
+
+std::string find_file_in_folder_by_name(const std::string &folder_path,
+					const std::string &file_name);
+std::string find_bin_file_in_folder(const std::string &path);
+std::string find_file_in_folder_by_regex_expression(const std::string &folder_path,
+						    const std::string &file_name_regex);
+
+#endif // MODEL_FIND_UTILS_H

src/model-utils/model-infos.cpp

@@ -23,6 +23,28 @@ std::map<std::string, ModelInfo> models_info = {{
 	    "B6E77E474AEEA8F441363ACA7614317C06381F3EACFE10FB9856D5081D1074CC"},
 	   {"https://huggingface.co/jncraton/m2m100_418M-ct2-int8/resolve/main/sentencepiece.bpe.model?download=true",
 	    "D8F7C76ED2A5E0822BE39F0A4F95A55EB19C78F4593CE609E2EDBC2AEA4D380A"}}}},
+	{"M2M-100 1.2B (1.25Gb)",
+	 {"M2M-100 1.2BM",
+	  "m2m-100-1_2B",
+	  MODEL_TYPE_TRANSLATION,
+	  {{"https://huggingface.co/jncraton/m2m100_1.2B-ct2-int8/resolve/main/model.bin?download=true",
+	    "C97DF052A558895317312470E1FF7CB8EAE5416F7AE16214A2983C6853DD3CE5"},
+	   {
+		   "https://huggingface.co/jncraton/m2m100_1.2B-ct2-int8/resolve/main/config.json?download=true",
+		   "4244772990E30069563E3DDFB4AD6DC95BDFD2AC3DE667EA8858C9B0A8433FA8",
+	   },
+	   {"https://huggingface.co/jncraton/m2m100_1.2B-ct2-int8/resolve/main/generation_config.json?download=true",
+	    "AED76366507333DDBB8BD49960F23C82FE6446B3319A46A54BEFDB45324CCF61"},
+	   {"https://huggingface.co/jncraton/m2m100_1.2B-ct2-int8/resolve/main/shared_vocabulary.json?download=true",
+	    "7EB5D0FF184C6095C7C10F9911C0AEA492250ABD12854F9C3D787C64B1C6397E"},
+	   {"https://huggingface.co/jncraton/m2m100_1.2B-ct2-int8/resolve/main/special_tokens_map.json?download=true",
+	    "C1A4F86C3874D279AE1B2A05162858DB5DD6C61665D84223ED886CBCFF08FDA6"},
+	   {"https://huggingface.co/jncraton/m2m100_1.2B-ct2-int8/resolve/main/tokenizer_config.json?download=true",
+	    "1566A6CFA4F541A55594C9D5E090F530812D5DE7C94882EA3AF156962D9933AE"},
+	   {"https://huggingface.co/jncraton/m2m100_1.2B-ct2-int8/resolve/main/vocab.json?download=true",
+	    "B6E77E474AEEA8F441363ACA7614317C06381F3EACFE10FB9856D5081D1074CC"},
+	   {"https://huggingface.co/jncraton/m2m100_1.2B-ct2-int8/resolve/main/sentencepiece.bpe.model?download=true",
+	    "D8F7C76ED2A5E0822BE39F0A4F95A55EB19C78F4593CE609E2EDBC2AEA4D380A"}}}},
 	{"Whisper Base q5 (57Mb)",
 	 {"Whisper Base q5",
 	  "whisper-base-q5",