From 59407edcad3a4a26342cee7dc7f0fac6d1ff50b4 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 9 Oct 2024 12:01:20 +0800 Subject: [PATCH 001/183] C++ API for speaker diarization (#1396) --- .github/scripts/test-speaker-diarization.sh | 41 ++ .../export-pyannote-segmentation-to-onnx.yaml | 2 +- .github/workflows/linux.yaml | 11 + .github/workflows/macos.yaml | 11 + .github/workflows/speaker-diarization.yaml | 2 +- .github/workflows/windows-x64.yaml | 11 + .github/workflows/windows-x86.yaml | 11 + ...s-spotter-buffered-tokens-keywords-c-api.c | 2 +- .../streaming-ctc-buffered-tokens-c-api.c | 2 +- ...reaming-paraformer-buffered-tokens-c-api.c | 2 +- ...zipformer-buffered-tokens-hotwords-c-api.c | 2 +- cmake/cmake_extension.py | 1 + scripts/pyannote/segmentation/README.md | 9 +- scripts/pyannote/segmentation/export-onnx.py | 2 +- sherpa-onnx/csrc/CMakeLists.txt | 16 + sherpa-onnx/csrc/fast-clustering-config.cc | 22 +- sherpa-onnx/csrc/macros.h | 3 + sherpa-onnx/csrc/offline-sense-voice-model.cc | 1 + .../csrc/offline-speaker-diarization-impl.cc | 26 + .../csrc/offline-speaker-diarization-impl.h | 31 + ...ffline-speaker-diarization-pyannote-impl.h | 644 ++++++++++++++++++ .../offline-speaker-diarization-result.cc | 110 +++ .../csrc/offline-speaker-diarization-result.h | 65 ++ .../csrc/offline-speaker-diarization.cc | 79 +++ .../csrc/offline-speaker-diarization.h | 73 ++ ...fline-speaker-segmentation-model-config.cc | 57 ++ ...ffline-speaker-segmentation-model-config.h | 40 ++ ...aker-segmentation-pyannote-model-config.cc | 38 ++ ...eaker-segmentation-pyannote-model-config.h | 30 + ...er-segmentation-pyannote-model-meta-data.h | 29 + ...ine-speaker-segmentation-pyannote-model.cc | 108 +++ ...line-speaker-segmentation-pyannote-model.h | 40 ++ sherpa-onnx/csrc/provider-config.cc | 6 +- sherpa-onnx/csrc/session.cc | 42 +- sherpa-onnx/csrc/session.h | 45 +- ...sherpa-onnx-offline-speaker-diarization.cc | 133 ++++ sherpa-onnx/csrc/sherpa-onnx-offline-tts.cc | 7 +- .../csrc/sherpa-onnx-online-punctuation.cc | 2 +- .../csrc/speaker-embedding-extractor.cc | 4 +- 39 files changed, 1652 insertions(+), 108 deletions(-) create mode 100755 .github/scripts/test-speaker-diarization.sh create mode 100644 sherpa-onnx/csrc/offline-speaker-diarization-impl.cc create mode 100644 sherpa-onnx/csrc/offline-speaker-diarization-impl.h create mode 100644 sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h create mode 100644 sherpa-onnx/csrc/offline-speaker-diarization-result.cc create mode 100644 sherpa-onnx/csrc/offline-speaker-diarization-result.h create mode 100644 sherpa-onnx/csrc/offline-speaker-diarization.cc create mode 100644 sherpa-onnx/csrc/offline-speaker-diarization.h create mode 100644 sherpa-onnx/csrc/offline-speaker-segmentation-model-config.cc create mode 100644 sherpa-onnx/csrc/offline-speaker-segmentation-model-config.h create mode 100644 sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model-config.cc create mode 100644 sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model-config.h create mode 100644 sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model-meta-data.h create mode 100644 sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.cc create mode 100644 sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.h create mode 100644 sherpa-onnx/csrc/sherpa-onnx-offline-speaker-diarization.cc diff --git a/.github/scripts/test-speaker-diarization.sh b/.github/scripts/test-speaker-diarization.sh new file mode 100755 index 000000000..6d7b2effd --- /dev/null +++ b/.github/scripts/test-speaker-diarization.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +set -ex + +log() { + # This function is from espnet + local fname=${BASH_SOURCE[1]##*/} + echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" +} + +echo "EXE is $EXE" +echo "PATH: $PATH" + +which $EXE + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 +tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 +rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/0-four-speakers-zh.wav + +log "specify number of clusters" +$EXE \ + --clustering.num-clusters=4 \ + --segmentation.pyannote-model=./sherpa-onnx-pyannote-segmentation-3-0/model.onnx \ + --embedding.model=./3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx \ + ./0-four-speakers-zh.wav + +log "specify threshold for clustering" + +$EXE \ + --clustering.cluster-threshold=0.90 \ + --segmentation.pyannote-model=./sherpa-onnx-pyannote-segmentation-3-0/model.onnx \ + --embedding.model=./3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx \ + ./0-four-speakers-zh.wav + +rm -rf sherpa-onnx-pyannote-* +rm -fv *.onnx +rm -fv *.wav diff --git a/.github/workflows/export-pyannote-segmentation-to-onnx.yaml b/.github/workflows/export-pyannote-segmentation-to-onnx.yaml index 300aca500..ece0ffa28 100644 --- a/.github/workflows/export-pyannote-segmentation-to-onnx.yaml +++ b/.github/workflows/export-pyannote-segmentation-to-onnx.yaml @@ -29,7 +29,7 @@ jobs: - name: Install pyannote shell: bash run: | - pip install pyannote.audio onnx onnxruntime + pip install pyannote.audio onnx==1.15.0 onnxruntime==1.16.3 - name: Run shell: bash diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml index 0e1eca099..1d3e8dc7b 100644 --- a/.github/workflows/linux.yaml +++ b/.github/workflows/linux.yaml @@ -18,6 +18,7 @@ on: - '.github/scripts/test-audio-tagging.sh' - '.github/scripts/test-offline-punctuation.sh' - '.github/scripts/test-online-punctuation.sh' + - '.github/scripts/test-speaker-diarization.sh' - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' @@ -38,6 +39,7 @@ on: - '.github/scripts/test-audio-tagging.sh' - '.github/scripts/test-offline-punctuation.sh' - '.github/scripts/test-online-punctuation.sh' + - '.github/scripts/test-speaker-diarization.sh' - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' @@ -143,6 +145,15 @@ jobs: name: release-${{ matrix.build_type }}-with-shared-lib-${{ matrix.shared_lib }}-with-tts-${{ matrix.with_tts }} path: install/* + - name: Test offline speaker diarization + shell: bash + run: | + du -h -d1 . + export PATH=$PWD/build/bin:$PATH + export EXE=sherpa-onnx-offline-speaker-diarization + + .github/scripts/test-speaker-diarization.sh + - name: Test offline transducer shell: bash run: | diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml index 084531e4a..f3d70f583 100644 --- a/.github/workflows/macos.yaml +++ b/.github/workflows/macos.yaml @@ -18,6 +18,7 @@ on: - '.github/scripts/test-audio-tagging.sh' - '.github/scripts/test-offline-punctuation.sh' - '.github/scripts/test-online-punctuation.sh' + - '.github/scripts/test-speaker-diarization.sh' - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' @@ -37,6 +38,7 @@ on: - '.github/scripts/test-audio-tagging.sh' - '.github/scripts/test-offline-punctuation.sh' - '.github/scripts/test-online-punctuation.sh' + - '.github/scripts/test-speaker-diarization.sh' - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' @@ -115,6 +117,15 @@ jobs: otool -L build/bin/sherpa-onnx otool -l build/bin/sherpa-onnx + - name: Test offline speaker diarization + shell: bash + run: | + du -h -d1 . + export PATH=$PWD/build/bin:$PATH + export EXE=sherpa-onnx-offline-speaker-diarization + + .github/scripts/test-speaker-diarization.sh + - name: Test offline transducer shell: bash run: | diff --git a/.github/workflows/speaker-diarization.yaml b/.github/workflows/speaker-diarization.yaml index 0bd6a575c..ab2a4f090 100644 --- a/.github/workflows/speaker-diarization.yaml +++ b/.github/workflows/speaker-diarization.yaml @@ -67,7 +67,7 @@ jobs: curl -SL -O https://huggingface.co/csukuangfj/pyannote-models/resolve/main/segmentation-3.0/pytorch_model.bin test_wavs=( - 0-two-speakers-zh.wav + 0-four-speakers-zh.wav 1-two-speakers-en.wav 2-two-speakers-en.wav 3-two-speakers-en.wav diff --git a/.github/workflows/windows-x64.yaml b/.github/workflows/windows-x64.yaml index 2d2811c31..c67f3e0b5 100644 --- a/.github/workflows/windows-x64.yaml +++ b/.github/workflows/windows-x64.yaml @@ -17,6 +17,7 @@ on: - '.github/scripts/test-audio-tagging.sh' - '.github/scripts/test-offline-punctuation.sh' - '.github/scripts/test-online-punctuation.sh' + - '.github/scripts/test-speaker-diarization.sh' - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' @@ -34,6 +35,7 @@ on: - '.github/scripts/test-audio-tagging.sh' - '.github/scripts/test-offline-punctuation.sh' - '.github/scripts/test-online-punctuation.sh' + - '.github/scripts/test-speaker-diarization.sh' - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' @@ -87,6 +89,15 @@ jobs: name: release-windows-x64-${{ matrix.shared_lib }}-${{ matrix.with_tts }} path: build/install/* + - name: Test offline speaker diarization + shell: bash + run: | + du -h -d1 . + export PATH=$PWD/build/bin:$PATH + export EXE=sherpa-onnx-offline-speaker-diarization.exe + + .github/scripts/test-speaker-diarization.sh + - name: Test online punctuation shell: bash run: | diff --git a/.github/workflows/windows-x86.yaml b/.github/workflows/windows-x86.yaml index 316cef626..30394e90e 100644 --- a/.github/workflows/windows-x86.yaml +++ b/.github/workflows/windows-x86.yaml @@ -17,6 +17,7 @@ on: - '.github/scripts/test-audio-tagging.sh' - '.github/scripts/test-offline-punctuation.sh' - '.github/scripts/test-online-punctuation.sh' + - '.github/scripts/test-speaker-diarization.sh' - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' @@ -34,6 +35,7 @@ on: - '.github/scripts/test-audio-tagging.sh' - '.github/scripts/test-offline-punctuation.sh' - '.github/scripts/test-online-punctuation.sh' + - '.github/scripts/test-speaker-diarization.sh' - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' @@ -87,6 +89,15 @@ jobs: name: release-windows-x86-${{ matrix.shared_lib }}-${{ matrix.with_tts }} path: build/install/* + - name: Test offline speaker diarization + shell: bash + run: | + du -h -d1 . + export PATH=$PWD/build/bin:$PATH + export EXE=sherpa-onnx-offline-speaker-diarization.exe + + .github/scripts/test-speaker-diarization.sh + - name: Test online punctuation shell: bash run: | diff --git a/c-api-examples/keywords-spotter-buffered-tokens-keywords-c-api.c b/c-api-examples/keywords-spotter-buffered-tokens-keywords-c-api.c index ec8be3b07..45a0bb87a 100644 --- a/c-api-examples/keywords-spotter-buffered-tokens-keywords-c-api.c +++ b/c-api-examples/keywords-spotter-buffered-tokens-keywords-c-api.c @@ -36,7 +36,7 @@ static size_t ReadFile(const char *filename, const char **buffer_out) { fprintf(stderr, "Memory error\n"); return -1; } - size_t read_bytes = fread(*buffer_out, 1, size, file); + size_t read_bytes = fread((void *)*buffer_out, 1, size, file); if (read_bytes != size) { printf("Errors occured in reading the file %s\n", filename); free((void *)*buffer_out); diff --git a/c-api-examples/streaming-ctc-buffered-tokens-c-api.c b/c-api-examples/streaming-ctc-buffered-tokens-c-api.c index 3223772a8..33690e008 100644 --- a/c-api-examples/streaming-ctc-buffered-tokens-c-api.c +++ b/c-api-examples/streaming-ctc-buffered-tokens-c-api.c @@ -36,7 +36,7 @@ static size_t ReadFile(const char *filename, const char **buffer_out) { fprintf(stderr, "Memory error\n"); return -1; } - size_t read_bytes = fread(*buffer_out, 1, size, file); + size_t read_bytes = fread((void *)*buffer_out, 1, size, file); if (read_bytes != size) { printf("Errors occured in reading the file %s\n", filename); free((void *)*buffer_out); diff --git a/c-api-examples/streaming-paraformer-buffered-tokens-c-api.c b/c-api-examples/streaming-paraformer-buffered-tokens-c-api.c index cd87177b5..a597374df 100644 --- a/c-api-examples/streaming-paraformer-buffered-tokens-c-api.c +++ b/c-api-examples/streaming-paraformer-buffered-tokens-c-api.c @@ -36,7 +36,7 @@ static size_t ReadFile(const char *filename, const char **buffer_out) { fprintf(stderr, "Memory error\n"); return -1; } - size_t read_bytes = fread(*buffer_out, 1, size, file); + size_t read_bytes = fread((void *)*buffer_out, 1, size, file); if (read_bytes != size) { printf("Errors occured in reading the file %s\n", filename); free((void *)*buffer_out); diff --git a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c index d5092c5cc..c991d4999 100644 --- a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c +++ b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c @@ -36,7 +36,7 @@ static size_t ReadFile(const char *filename, const char **buffer_out) { fprintf(stderr, "Memory error\n"); return -1; } - size_t read_bytes = fread(*buffer_out, 1, size, file); + size_t read_bytes = fread((void *)*buffer_out, 1, size, file); if (read_bytes != size) { printf("Errors occured in reading the file %s\n", filename); free((void *)*buffer_out); diff --git a/cmake/cmake_extension.py b/cmake/cmake_extension.py index 672e3d17a..c49c32555 100644 --- a/cmake/cmake_extension.py +++ b/cmake/cmake_extension.py @@ -55,6 +55,7 @@ def get_binaries(): "sherpa-onnx-offline-audio-tagging", "sherpa-onnx-offline-language-identification", "sherpa-onnx-offline-punctuation", + "sherpa-onnx-offline-speaker-diarization", "sherpa-onnx-offline-tts", "sherpa-onnx-offline-tts-play", "sherpa-onnx-offline-websocket-server", diff --git a/scripts/pyannote/segmentation/README.md b/scripts/pyannote/segmentation/README.md index a2e35b2de..a9c5230d1 100644 --- a/scripts/pyannote/segmentation/README.md +++ b/scripts/pyannote/segmentation/README.md @@ -3,12 +3,9 @@ Please download test wave files from https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models -## 0-two-speakers-zh.wav +## 0-four-speakers-zh.wav -This file is from -https://www.modelscope.cn/models/iic/speech_campplus_speaker-diarization_common/file/view/master?fileName=examples%252F2speakers_example.wav&status=0 - -Note that we have renamed it from `2speakers_example.wav` to `0-two-speakers-zh.wav`. +It is recorded by @csukuangfj ## 1-two-speakers-en.wav @@ -40,5 +37,5 @@ commands to convert it to `3-two-speakers-en.wav` ```bash -sox ML16091-Audio.mp3 3-two-speakers-en.wav +sox ML16091-Audio.mp3 -r 16k 3-two-speakers-en.wav ``` diff --git a/scripts/pyannote/segmentation/export-onnx.py b/scripts/pyannote/segmentation/export-onnx.py index 5f6e79c7e..feb241a26 100755 --- a/scripts/pyannote/segmentation/export-onnx.py +++ b/scripts/pyannote/segmentation/export-onnx.py @@ -72,7 +72,7 @@ def main(): model.receptive_field.duration * 16000 ) - opset_version = 18 + opset_version = 13 filename = "model.onnx" torch.onnx.export( diff --git a/sherpa-onnx/csrc/CMakeLists.txt b/sherpa-onnx/csrc/CMakeLists.txt index e49fdeed4..3e6526563 100644 --- a/sherpa-onnx/csrc/CMakeLists.txt +++ b/sherpa-onnx/csrc/CMakeLists.txt @@ -164,6 +164,12 @@ if(SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION) list(APPEND sources fast-clustering-config.cc fast-clustering.cc + offline-speaker-diarization-impl.cc + offline-speaker-diarization-result.cc + offline-speaker-diarization.cc + offline-speaker-segmentation-model-config.cc + offline-speaker-segmentation-pyannote-model-config.cc + offline-speaker-segmentation-pyannote-model.cc ) endif() @@ -260,6 +266,10 @@ if(SHERPA_ONNX_ENABLE_BINARY) add_executable(sherpa-onnx-offline-tts sherpa-onnx-offline-tts.cc) endif() + if(SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION) + add_executable(sherpa-onnx-offline-speaker-diarization sherpa-onnx-offline-speaker-diarization.cc) + endif() + set(main_exes sherpa-onnx sherpa-onnx-keyword-spotter @@ -276,6 +286,12 @@ if(SHERPA_ONNX_ENABLE_BINARY) ) endif() + if(SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION) + list(APPEND main_exes + sherpa-onnx-offline-speaker-diarization + ) + endif() + foreach(exe IN LISTS main_exes) target_link_libraries(${exe} sherpa-onnx-core) endforeach() diff --git a/sherpa-onnx/csrc/fast-clustering-config.cc b/sherpa-onnx/csrc/fast-clustering-config.cc index e8382e598..e4f64fbbb 100644 --- a/sherpa-onnx/csrc/fast-clustering-config.cc +++ b/sherpa-onnx/csrc/fast-clustering-config.cc @@ -21,18 +21,16 @@ std::string FastClusteringConfig::ToString() const { } void FastClusteringConfig::Register(ParseOptions *po) { - std::string prefix = "ctc"; - ParseOptions p(prefix, po); - - p.Register("num-clusters", &num_clusters, - "Number of cluster. If greater than 0, then --cluster-thresold is " - "ignored. Please provide it if you know the actual number of " - "clusters in advance."); - - p.Register("cluster-threshold", &threshold, - "If --num-clusters is not specified, then it specifies the " - "distance threshold for clustering. smaller value -> more " - "clusters. larger value -> fewer clusters"); + po->Register( + "num-clusters", &num_clusters, + "Number of cluster. If greater than 0, then cluster threshold is " + "ignored. Please provide it if you know the actual number of " + "clusters in advance."); + + po->Register("cluster-threshold", &threshold, + "If num_clusters is not specified, then it specifies the " + "distance threshold for clustering. smaller value -> more " + "clusters. larger value -> fewer clusters"); } bool FastClusteringConfig::Validate() const { diff --git a/sherpa-onnx/csrc/macros.h b/sherpa-onnx/csrc/macros.h index b5dfb99e3..6bd6f62a6 100644 --- a/sherpa-onnx/csrc/macros.h +++ b/sherpa-onnx/csrc/macros.h @@ -5,6 +5,7 @@ #ifndef SHERPA_ONNX_CSRC_MACROS_H_ #define SHERPA_ONNX_CSRC_MACROS_H_ #include +#include #if __ANDROID_API__ >= 8 #include "android/log.h" @@ -169,4 +170,6 @@ } \ } while (0) +#define SHERPA_ONNX_EXIT(code) exit(code) + #endif // SHERPA_ONNX_CSRC_MACROS_H_ diff --git a/sherpa-onnx/csrc/offline-sense-voice-model.cc b/sherpa-onnx/csrc/offline-sense-voice-model.cc index 1d2a14ef5..24903a41a 100644 --- a/sherpa-onnx/csrc/offline-sense-voice-model.cc +++ b/sherpa-onnx/csrc/offline-sense-voice-model.cc @@ -9,6 +9,7 @@ #include #include "sherpa-onnx/csrc/macros.h" +#include "sherpa-onnx/csrc/onnx-utils.h" #include "sherpa-onnx/csrc/session.h" #include "sherpa-onnx/csrc/text-utils.h" diff --git a/sherpa-onnx/csrc/offline-speaker-diarization-impl.cc b/sherpa-onnx/csrc/offline-speaker-diarization-impl.cc new file mode 100644 index 000000000..e41a7767a --- /dev/null +++ b/sherpa-onnx/csrc/offline-speaker-diarization-impl.cc @@ -0,0 +1,26 @@ +// sherpa-onnx/csrc/offline-speaker-diarization-impl.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include "sherpa-onnx/csrc/offline-speaker-diarization-impl.h" + +#include + +#include "sherpa-onnx/csrc/macros.h" +#include "sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h" + +namespace sherpa_onnx { + +std::unique_ptr +OfflineSpeakerDiarizationImpl::Create( + const OfflineSpeakerDiarizationConfig &config) { + if (!config.segmentation.pyannote.model.empty()) { + return std::make_unique(config); + } + + SHERPA_ONNX_LOGE("Please specify a speaker segmentation model."); + + return nullptr; +} + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-speaker-diarization-impl.h b/sherpa-onnx/csrc/offline-speaker-diarization-impl.h new file mode 100644 index 000000000..f7fe39499 --- /dev/null +++ b/sherpa-onnx/csrc/offline-speaker-diarization-impl.h @@ -0,0 +1,31 @@ +// sherpa-onnx/csrc/offline-speaker-diarization-impl.h +// +// Copyright (c) 2024 Xiaomi Corporation + +#ifndef SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_DIARIZATION_IMPL_H_ +#define SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_DIARIZATION_IMPL_H_ + +#include +#include + +#include "sherpa-onnx/csrc/offline-speaker-diarization.h" +namespace sherpa_onnx { + +class OfflineSpeakerDiarizationImpl { + public: + static std::unique_ptr Create( + const OfflineSpeakerDiarizationConfig &config); + + virtual ~OfflineSpeakerDiarizationImpl() = default; + + virtual int32_t SampleRate() const = 0; + + virtual OfflineSpeakerDiarizationResult Process( + const float *audio, int32_t n, + OfflineSpeakerDiarizationProgressCallback callback = nullptr, + void *callback_arg = nullptr) const = 0; +}; + +} // namespace sherpa_onnx + +#endif // SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_DIARIZATION_IMPL_H_ diff --git a/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h b/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h new file mode 100644 index 000000000..bcd0c93a4 --- /dev/null +++ b/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h @@ -0,0 +1,644 @@ +// sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h +// +// Copyright (c) 2024 Xiaomi Corporation +#ifndef SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_DIARIZATION_PYANNOTE_IMPL_H_ +#define SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_DIARIZATION_PYANNOTE_IMPL_H_ + +#include +#include +#include +#include + +#include "Eigen/Dense" +#include "sherpa-onnx/csrc/fast-clustering.h" +#include "sherpa-onnx/csrc/math.h" +#include "sherpa-onnx/csrc/offline-speaker-diarization-impl.h" +#include "sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.h" +#include "sherpa-onnx/csrc/speaker-embedding-extractor.h" + +namespace sherpa_onnx { + +namespace { // NOLINT + +// copied from https://github.com/k2-fsa/k2/blob/master/k2/csrc/host/util.h#L41 +template +inline void hash_combine(std::size_t *seed, const T &v) { // NOLINT + std::hash hasher; + *seed ^= hasher(v) + 0x9e3779b9 + ((*seed) << 6) + ((*seed) >> 2); // NOLINT +} + +// copied from https://github.com/k2-fsa/k2/blob/master/k2/csrc/host/util.h#L47 +struct PairHash { + template + std::size_t operator()(const std::pair &pair) const { + std::size_t result = 0; + hash_combine(&result, pair.first); + hash_combine(&result, pair.second); + return result; + } +}; +} // namespace + +using Matrix2D = + Eigen::Matrix; + +using Matrix2DInt32 = + Eigen::Matrix; + +using FloatRowVector = Eigen::Matrix; +using Int32RowVector = Eigen::Matrix; + +using Int32Pair = std::pair; + +class OfflineSpeakerDiarizationPyannoteImpl + : public OfflineSpeakerDiarizationImpl { + public: + ~OfflineSpeakerDiarizationPyannoteImpl() override = default; + + explicit OfflineSpeakerDiarizationPyannoteImpl( + const OfflineSpeakerDiarizationConfig &config) + : config_(config), + segmentation_model_(config_.segmentation), + embedding_extractor_(config_.embedding), + clustering_(config_.clustering) { + Init(); + } + + int32_t SampleRate() const override { + const auto &meta_data = segmentation_model_.GetModelMetaData(); + + return meta_data.sample_rate; + } + + OfflineSpeakerDiarizationResult Process( + const float *audio, int32_t n, + OfflineSpeakerDiarizationProgressCallback callback = nullptr, + void *callback_arg = nullptr) const override { + std::vector segmentations = RunSpeakerSegmentationModel(audio, n); + // segmentations[i] is for chunk_i + // Each matrix is of shape (num_frames, num_powerset_classes) + if (segmentations.empty()) { + return {}; + } + + std::vector labels; + labels.reserve(segmentations.size()); + + for (const auto &m : segmentations) { + labels.push_back(ToMultiLabel(m)); + } + + segmentations.clear(); + + // labels[i] is a 0-1 matrix of shape (num_frames, num_speakers) + + // speaker count per frame + Int32RowVector speakers_per_frame = ComputeSpeakersPerFrame(labels); + + if (speakers_per_frame.maxCoeff() == 0) { + SHERPA_ONNX_LOGE("No speakers found in the audio samples"); + return {}; + } + + auto chunk_speaker_samples_list_pair = GetChunkSpeakerSampleIndexes(labels); + Matrix2D embeddings = + ComputeEmbeddings(audio, n, chunk_speaker_samples_list_pair.second, + callback, callback_arg); + + std::vector cluster_labels = clustering_.Cluster( + &embeddings(0, 0), embeddings.rows(), embeddings.cols()); + + int32_t max_cluster_index = + *std::max_element(cluster_labels.begin(), cluster_labels.end()); + + auto chunk_speaker_to_cluster = ConvertChunkSpeakerToCluster( + chunk_speaker_samples_list_pair.first, cluster_labels); + + auto new_labels = + ReLabel(labels, max_cluster_index, chunk_speaker_to_cluster); + + Matrix2DInt32 speaker_count = ComputeSpeakerCount(new_labels, n); + + Matrix2DInt32 final_labels = + FinalizeLabels(speaker_count, speakers_per_frame); + + auto result = ComputeResult(final_labels); + + return result; + } + + private: + void Init() { InitPowersetMapping(); } + + // see also + // https://github.com/pyannote/pyannote-audio/blob/develop/pyannote/audio/utils/powerset.py#L68 + void InitPowersetMapping() { + const auto &meta_data = segmentation_model_.GetModelMetaData(); + int32_t num_classes = meta_data.num_classes; + int32_t powerset_max_classes = meta_data.powerset_max_classes; + int32_t num_speakers = meta_data.num_speakers; + + powerset_mapping_ = Matrix2DInt32(num_classes, num_speakers); + powerset_mapping_.setZero(); + + int32_t k = 1; + for (int32_t i = 1; i <= powerset_max_classes; ++i) { + if (i == 1) { + for (int32_t j = 0; j != num_speakers; ++j, ++k) { + powerset_mapping_(k, j) = 1; + } + } else if (i == 2) { + for (int32_t j = 0; j != num_speakers; ++j) { + for (int32_t m = j + 1; m < num_speakers; ++m, ++k) { + powerset_mapping_(k, j) = 1; + powerset_mapping_(k, m) = 1; + } + } + } else { + SHERPA_ONNX_LOGE( + "powerset_max_classes = %d is currently not supported!", i); + SHERPA_ONNX_EXIT(-1); + } + } + } + + std::vector RunSpeakerSegmentationModel(const float *audio, + int32_t n) const { + std::vector ans; + + const auto &meta_data = segmentation_model_.GetModelMetaData(); + int32_t window_size = meta_data.window_size; + int32_t window_shift = meta_data.window_shift; + + if (n <= 0) { + SHERPA_ONNX_LOGE( + "number of audio samples is %d (<= 0). Please provide a positive " + "number", + n); + return {}; + } + + if (n <= window_size) { + std::vector buf(window_size); + // NOTE: buf is zero initialized by default + + std::copy(audio, audio + n, buf.data()); + + Matrix2D m = ProcessChunk(buf.data()); + + ans.push_back(std::move(m)); + + return ans; + } + + int32_t num_chunks = (n - window_size) / window_shift + 1; + bool has_last_chunk = (n - window_size) % window_shift > 0; + + ans.reserve(num_chunks + has_last_chunk); + + const float *p = audio; + + for (int32_t i = 0; i != num_chunks; ++i, p += window_shift) { + Matrix2D m = ProcessChunk(p); + + ans.push_back(std::move(m)); + } + + if (has_last_chunk) { + std::vector buf(window_size); + std::copy(p, audio + n, buf.data()); + + Matrix2D m = ProcessChunk(buf.data()); + + ans.push_back(std::move(m)); + } + + return ans; + } + + Matrix2D ProcessChunk(const float *p) const { + const auto &meta_data = segmentation_model_.GetModelMetaData(); + int32_t window_size = meta_data.window_size; + + auto memory_info = + Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault); + + std::array shape = {1, 1, window_size}; + + Ort::Value x = + Ort::Value::CreateTensor(memory_info, const_cast(p), + window_size, shape.data(), shape.size()); + + Ort::Value out = segmentation_model_.Forward(std::move(x)); + std::vector out_shape = out.GetTensorTypeAndShapeInfo().GetShape(); + Matrix2D m(out_shape[1], out_shape[2]); + std::copy(out.GetTensorData(), out.GetTensorData() + m.size(), + &m(0, 0)); + return m; + } + + Matrix2DInt32 ToMultiLabel(const Matrix2D &m) const { + int32_t num_rows = m.rows(); + Matrix2DInt32 ans(num_rows, powerset_mapping_.cols()); + + std::ptrdiff_t col_id; + + for (int32_t i = 0; i != num_rows; ++i) { + m.row(i).maxCoeff(&col_id); + ans.row(i) = powerset_mapping_.row(col_id); + } + + return ans; + } + + // See also + // https://github.com/pyannote/pyannote-audio/blob/develop/pyannote/audio/pipelines/utils/diarization.py#L122 + Int32RowVector ComputeSpeakersPerFrame( + const std::vector &labels) const { + const auto &meta_data = segmentation_model_.GetModelMetaData(); + int32_t window_size = meta_data.window_size; + int32_t window_shift = meta_data.window_shift; + int32_t receptive_field_shift = meta_data.receptive_field_shift; + + int32_t num_chunks = labels.size(); + + int32_t num_frames = (window_size + (num_chunks - 1) * window_shift) / + receptive_field_shift + + 1; + + FloatRowVector count(num_frames); + FloatRowVector weight(num_frames); + count.setZero(); + weight.setZero(); + + for (int32_t i = 0; i != num_chunks; ++i) { + int32_t start = + static_cast(i) * window_shift / receptive_field_shift + 0.5; + + auto seq = Eigen::seqN(start, labels[i].rows()); + + count(seq).array() += labels[i].rowwise().sum().array().cast(); + + weight(seq).array() += 1; + } + + return ((count.array() / (weight.array() + 1e-12f)) + 0.5).cast(); + } + + // ans.first: a list of (chunk_id, speaker_id) + // ans.second: a list of list of (start_sample_index, end_sample_index) + // + // ans.first[i] corresponds to ans.second[i] + std::pair, std::vector>> + GetChunkSpeakerSampleIndexes(const std::vector &labels) const { + auto new_labels = ExcludeOverlap(labels); + + std::vector chunk_speaker_list; + std::vector> samples_index_list; + + const auto &meta_data = segmentation_model_.GetModelMetaData(); + int32_t window_size = meta_data.window_size; + int32_t window_shift = meta_data.window_shift; + int32_t receptive_field_shift = meta_data.receptive_field_shift; + int32_t num_speakers = meta_data.num_speakers; + + int32_t chunk_index = 0; + for (const auto &label : new_labels) { + Matrix2DInt32 tmp = label.transpose(); + // tmp: (num_speakers, num_frames) + int32_t num_frames = tmp.cols(); + + int32_t sample_offset = chunk_index * window_shift; + + for (int32_t speaker_index = 0; speaker_index != num_speakers; + ++speaker_index) { + auto d = tmp.row(speaker_index); + if (d.sum() < 10) { + // skip segments less than 10 frames + continue; + } + + Int32Pair this_chunk_speaker = {chunk_index, speaker_index}; + std::vector this_speaker_samples; + + bool is_active = false; + int32_t start_index; + + for (int32_t k = 0; k != num_frames; ++k) { + if (d[k] != 0) { + if (!is_active) { + is_active = true; + start_index = k; + } + } else if (is_active) { + is_active = false; + + int32_t start_samples = + static_cast(start_index) / num_frames * window_size + + sample_offset; + int32_t end_samples = + static_cast(k) / num_frames * window_size + + sample_offset; + + this_speaker_samples.emplace_back(start_samples, end_samples); + } + } + + if (is_active) { + int32_t start_samples = + static_cast(start_index) / num_frames * window_size + + sample_offset; + int32_t end_samples = + static_cast(num_frames - 1) / num_frames * window_size + + sample_offset; + this_speaker_samples.emplace_back(start_samples, end_samples); + } + + chunk_speaker_list.push_back(std::move(this_chunk_speaker)); + samples_index_list.push_back(std::move(this_speaker_samples)); + } // for (int32_t speaker_index = 0; + chunk_index += 1; + } // for (const auto &label : new_labels) + + return {chunk_speaker_list, samples_index_list}; + } + + // If there are multiple speakers at a frame, then this frame is excluded. + std::vector ExcludeOverlap( + const std::vector &labels) const { + int32_t num_chunks = labels.size(); + std::vector ans; + ans.reserve(num_chunks); + + for (const auto &label : labels) { + Matrix2DInt32 new_label(label.rows(), label.cols()); + new_label.setZero(); + Int32RowVector v = label.rowwise().sum(); + + for (int32_t i = 0; i != v.cols(); ++i) { + if (v[i] < 2) { + new_label.row(i) = label.row(i); + } + } + + ans.push_back(std::move(new_label)); + } + + return ans; + } + + /** + * @param sample_indexes[i] contains the sample segment start and end indexes + * for the i-th (chunk, speaker) pair + * @return Return a matrix of shape (sample_indexes.size(), embedding_dim) + * where ans.row[i] contains the embedding for the + * i-th (chunk, speaker) pair + */ + Matrix2D ComputeEmbeddings( + const float *audio, int32_t n, + const std::vector> &sample_indexes, + OfflineSpeakerDiarizationProgressCallback callback, + void *callback_arg) const { + const auto &meta_data = segmentation_model_.GetModelMetaData(); + int32_t sample_rate = meta_data.sample_rate; + Matrix2D ans(sample_indexes.size(), embedding_extractor_.Dim()); + + int32_t k = 0; + for (const auto &v : sample_indexes) { + auto stream = embedding_extractor_.CreateStream(); + for (const auto &p : v) { + int32_t end = (p.second <= n) ? p.second : n; + int32_t num_samples = end - p.first; + + if (num_samples > 0) { + stream->AcceptWaveform(sample_rate, audio + p.first, num_samples); + } + } + + stream->InputFinished(); + if (!embedding_extractor_.IsReady(stream.get())) { + SHERPA_ONNX_LOGE( + "This segment is too short, which should not happen since we have " + "already filtered short segments"); + SHERPA_ONNX_EXIT(-1); + } + + std::vector embedding = embedding_extractor_.Compute(stream.get()); + + std::copy(embedding.begin(), embedding.end(), &ans(k, 0)); + + k += 1; + + if (callback) { + callback(k, ans.rows(), callback_arg); + } + } + + return ans; + } + + std::unordered_map ConvertChunkSpeakerToCluster( + const std::vector &chunk_speaker_pair, + const std::vector &cluster_labels) const { + std::unordered_map ans; + + int32_t k = 0; + for (const auto &p : chunk_speaker_pair) { + ans[p] = cluster_labels[k]; + k += 1; + } + + return ans; + } + + std::vector ReLabel( + const std::vector &labels, int32_t max_cluster_index, + std::unordered_map chunk_speaker_to_cluster) + const { + std::vector new_labels; + new_labels.reserve(labels.size()); + + int32_t chunk_index = 0; + for (const auto &label : labels) { + Matrix2DInt32 new_label(label.rows(), max_cluster_index + 1); + new_label.setZero(); + + Matrix2DInt32 t = label.transpose(); + // t: (num_speakers, num_frames) + + for (int32_t speaker_index = 0; speaker_index != t.rows(); + ++speaker_index) { + if (chunk_speaker_to_cluster.count({chunk_index, speaker_index}) == 0) { + continue; + } + + int32_t new_speaker_index = + chunk_speaker_to_cluster.at({chunk_index, speaker_index}); + + for (int32_t k = 0; k != t.cols(); ++k) { + if (t(speaker_index, k) == 1) { + new_label(k, new_speaker_index) = 1; + } + } + } + + new_labels.push_back(std::move(new_label)); + + chunk_index += 1; + } + + return new_labels; + } + + Matrix2DInt32 ComputeSpeakerCount(const std::vector &labels, + int32_t num_samples) const { + const auto &meta_data = segmentation_model_.GetModelMetaData(); + int32_t window_size = meta_data.window_size; + int32_t window_shift = meta_data.window_shift; + int32_t receptive_field_shift = meta_data.receptive_field_shift; + + int32_t num_chunks = labels.size(); + + int32_t num_frames = (window_size + (num_chunks - 1) * window_shift) / + receptive_field_shift + + 1; + + Matrix2DInt32 count(num_frames, labels[0].cols()); + count.setZero(); + + for (int32_t i = 0; i != num_chunks; ++i) { + int32_t start = + static_cast(i) * window_shift / receptive_field_shift + 0.5; + + auto seq = Eigen::seqN(start, labels[i].rows()); + + count(seq, Eigen::all).array() += labels[i].array(); + } + + bool has_last_chunk = (num_samples - window_size) % window_shift > 0; + + if (has_last_chunk) { + return count; + } + + int32_t last_frame = num_samples / receptive_field_shift; + return count(Eigen::seq(0, last_frame), Eigen::all); + } + + Matrix2DInt32 FinalizeLabels(const Matrix2DInt32 &count, + const Int32RowVector &speakers_per_frame) const { + int32_t num_rows = count.rows(); + int32_t num_cols = count.cols(); + + Matrix2DInt32 ans(num_rows, num_cols); + ans.setZero(); + + for (int32_t i = 0; i != num_rows; ++i) { + int32_t k = speakers_per_frame[i]; + if (k == 0) { + continue; + } + auto top_k = TopkIndex(&count(i, 0), num_cols, k); + + for (int32_t m : top_k) { + ans(i, m) = 1; + } + } + + return ans; + } + + OfflineSpeakerDiarizationResult ComputeResult( + const Matrix2DInt32 &final_labels) const { + Matrix2DInt32 final_labels_t = final_labels.transpose(); + int32_t num_speakers = final_labels_t.rows(); + int32_t num_frames = final_labels_t.cols(); + + const auto &meta_data = segmentation_model_.GetModelMetaData(); + int32_t window_size = meta_data.window_size; + int32_t window_shift = meta_data.window_shift; + int32_t receptive_field_shift = meta_data.receptive_field_shift; + int32_t receptive_field_size = meta_data.receptive_field_size; + int32_t sample_rate = meta_data.sample_rate; + + float scale = static_cast(receptive_field_shift) / sample_rate; + float scale_offset = 0.5 * receptive_field_size / sample_rate; + + OfflineSpeakerDiarizationResult ans; + + for (int32_t speaker_index = 0; speaker_index != num_speakers; + ++speaker_index) { + std::vector this_speaker; + + bool is_active = final_labels_t(speaker_index, 0) > 0; + int32_t start_index = is_active ? 0 : -1; + + for (int32_t frame_index = 1; frame_index != num_frames; ++frame_index) { + if (is_active) { + if (final_labels_t(speaker_index, frame_index) == 0) { + float start_time = start_index * scale + scale_offset; + float end_time = frame_index * scale + scale_offset; + + OfflineSpeakerDiarizationSegment segment(start_time, end_time, + speaker_index); + this_speaker.push_back(segment); + + is_active = false; + } + } else if (final_labels_t(speaker_index, frame_index) == 1) { + is_active = true; + start_index = frame_index; + } + } + + if (is_active) { + float start_time = start_index * scale + scale_offset; + float end_time = (num_frames - 1) * scale + scale_offset; + + OfflineSpeakerDiarizationSegment segment(start_time, end_time, + speaker_index); + this_speaker.push_back(segment); + } + + // merge segments if the gap between them is less than min_duration_off + MergeSegments(&this_speaker); + + for (const auto &seg : this_speaker) { + if (seg.Duration() > config_.min_duration_on) { + ans.Add(seg); + } + } + } // for (int32_t speaker_index = 0; speaker_index != num_speakers; + + return ans; + } + + void MergeSegments( + std::vector *segments) const { + float min_duration_off = config_.min_duration_off; + bool changed = true; + while (changed) { + changed = false; + for (int32_t i = 0; i < static_cast(segments->size()) - 1; ++i) { + auto s = (*segments)[i].Merge((*segments)[i + 1], min_duration_off); + if (s) { + (*segments)[i] = s.value(); + segments->erase(segments->begin() + i + 1); + + changed = true; + break; + } + } + } + } + + private: + OfflineSpeakerDiarizationConfig config_; + OfflineSpeakerSegmentationPyannoteModel segmentation_model_; + SpeakerEmbeddingExtractor embedding_extractor_; + FastClustering clustering_; + Matrix2DInt32 powerset_mapping_; +}; + +} // namespace sherpa_onnx +#endif // SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_DIARIZATION_PYANNOTE_IMPL_H_ diff --git a/sherpa-onnx/csrc/offline-speaker-diarization-result.cc b/sherpa-onnx/csrc/offline-speaker-diarization-result.cc new file mode 100644 index 000000000..8bf83f5d9 --- /dev/null +++ b/sherpa-onnx/csrc/offline-speaker-diarization-result.cc @@ -0,0 +1,110 @@ +// sherpa-onnx/csrc/offline-speaker-diarization-result.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include "sherpa-onnx/csrc/offline-speaker-diarization-result.h" + +#include +#include +#include +#include +#include + +#include "sherpa-onnx/csrc/macros.h" + +namespace sherpa_onnx { + +OfflineSpeakerDiarizationSegment::OfflineSpeakerDiarizationSegment( + float start, float end, int32_t speaker, const std::string &text /*= {}*/) { + if (start > end) { + SHERPA_ONNX_LOGE("start %.3f should be less than end %.3f", start, end); + SHERPA_ONNX_EXIT(-1); + } + + start_ = start; + end_ = end; + speaker_ = speaker; + text_ = text; +} + +std::optional +OfflineSpeakerDiarizationSegment::Merge( + const OfflineSpeakerDiarizationSegment &other, float gap) const { + if (other.speaker_ != speaker_) { + SHERPA_ONNX_LOGE( + "The two segments should have the same speaker. this->speaker: %d, " + "other.speaker: %d", + speaker_, other.speaker_); + return std::nullopt; + } + + if (end_ < other.start_ && end_ + gap >= other.start_) { + return OfflineSpeakerDiarizationSegment(start_, other.end_, speaker_); + } else if (other.end_ < start_ && other.end_ + gap >= start_) { + return OfflineSpeakerDiarizationSegment(other.start_, end_, speaker_); + } else { + return std::nullopt; + } +} + +std::string OfflineSpeakerDiarizationSegment::ToString() const { + char s[128]; + snprintf(s, sizeof(s), "%.3f -- %.3f speaker_%02d", start_, end_, speaker_); + + std::ostringstream os; + os << s; + + if (!text_.empty()) { + os << " " << text_; + } + + return os.str(); +} + +void OfflineSpeakerDiarizationResult::Add( + const OfflineSpeakerDiarizationSegment &segment) { + segments_.push_back(segment); +} + +int32_t OfflineSpeakerDiarizationResult::NumSpeakers() const { + std::unordered_set count; + for (const auto &s : segments_) { + count.insert(s.Speaker()); + } + + return count.size(); +} + +int32_t OfflineSpeakerDiarizationResult::NumSegments() const { + return segments_.size(); +} + +// Return a list of segments sorted by segment.start time +std::vector +OfflineSpeakerDiarizationResult::SortByStartTime() const { + auto ans = segments_; + std::sort(ans.begin(), ans.end(), [](const auto &a, const auto &b) { + return (a.Start() < b.Start()) || + ((a.Start() == b.Start()) && (a.Speaker() < b.Speaker())); + }); + + return ans; +} + +std::vector> +OfflineSpeakerDiarizationResult::SortBySpeaker() const { + auto tmp = segments_; + std::sort(tmp.begin(), tmp.end(), [](const auto &a, const auto &b) { + return (a.Speaker() < b.Speaker()) || + ((a.Speaker() == b.Speaker()) && (a.Start() < b.Start())); + }); + + std::vector> ans(NumSpeakers()); + for (auto &s : tmp) { + ans[s.Speaker()].push_back(std::move(s)); + } + + return ans; +} + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-speaker-diarization-result.h b/sherpa-onnx/csrc/offline-speaker-diarization-result.h new file mode 100644 index 000000000..e71d054e5 --- /dev/null +++ b/sherpa-onnx/csrc/offline-speaker-diarization-result.h @@ -0,0 +1,65 @@ +// sherpa-onnx/csrc/offline-speaker-diarization-result.h +// +// Copyright (c) 2024 Xiaomi Corporation + +#ifndef SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_DIARIZATION_RESULT_H_ +#define SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_DIARIZATION_RESULT_H_ + +#include +#include +#include +#include + +namespace sherpa_onnx { + +class OfflineSpeakerDiarizationSegment { + public: + OfflineSpeakerDiarizationSegment(float start, float end, int32_t speaker, + const std::string &text = {}); + + // If the gap between the two segments is less than the given gap, then we + // merge them and return a new segment. Otherwise, it returns null. + std::optional Merge( + const OfflineSpeakerDiarizationSegment &other, float gap) const; + + float Start() const { return start_; } + float End() const { return end_; } + int32_t Speaker() const { return speaker_; } + const std::string &Text() const { return text_; } + float Duration() const { return end_ - start_; } + + std::string ToString() const; + + private: + float start_; // in seconds + float end_; // in seconds + int32_t speaker_; // ID of the speaker, starting from 0 + std::string text_; // If not empty, it contains the speech recognition result + // of this segment +}; + +class OfflineSpeakerDiarizationResult { + public: + // Add a new segment + void Add(const OfflineSpeakerDiarizationSegment &segment); + + // Number of distinct speakers contained in this object at this point + int32_t NumSpeakers() const; + + int32_t NumSegments() const; + + // Return a list of segments sorted by segment.start time + std::vector SortByStartTime() const; + + // ans.size() == NumSpeakers(). + // ans[i] is for speaker_i and is sorted by start time + std::vector> SortBySpeaker() + const; + + public: + std::vector segments_; +}; + +} // namespace sherpa_onnx + +#endif // SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_DIARIZATION_RESULT_H_ diff --git a/sherpa-onnx/csrc/offline-speaker-diarization.cc b/sherpa-onnx/csrc/offline-speaker-diarization.cc new file mode 100644 index 000000000..aeff9b42d --- /dev/null +++ b/sherpa-onnx/csrc/offline-speaker-diarization.cc @@ -0,0 +1,79 @@ +// sherpa-onnx/csrc/offline-speaker-diarization.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include "sherpa-onnx/csrc/offline-speaker-diarization.h" + +#include + +#include "sherpa-onnx/csrc/offline-speaker-diarization-impl.h" + +namespace sherpa_onnx { + +void OfflineSpeakerDiarizationConfig::Register(ParseOptions *po) { + ParseOptions po_segmentation("segmentation", po); + segmentation.Register(&po_segmentation); + + ParseOptions po_embedding("embedding", po); + embedding.Register(&po_embedding); + + ParseOptions po_clustering("clustering", po); + clustering.Register(&po_clustering); + + po->Register("min-duration-on", &min_duration_on, + "if a segment is less than this value, then it is discarded. " + "Set it to 0 so that no segment is discarded"); + + po->Register("min-duration-off", &min_duration_off, + "if the gap between to segments of the same speaker is less " + "than this value, then these two segments are merged into a " + "single segment. We do it recursively."); +} + +bool OfflineSpeakerDiarizationConfig::Validate() const { + if (!segmentation.Validate()) { + return false; + } + + if (!embedding.Validate()) { + return false; + } + + if (!clustering.Validate()) { + return false; + } + + return true; +} + +std::string OfflineSpeakerDiarizationConfig::ToString() const { + std::ostringstream os; + + os << "OfflineSpeakerDiarizationConfig("; + os << "segmentation=" << segmentation.ToString() << ", "; + os << "embedding=" << embedding.ToString() << ", "; + os << "clustering=" << clustering.ToString() << ", "; + os << "min_duration_on=" << min_duration_on << ", "; + os << "min_duration_off=" << min_duration_off << ")"; + + return os.str(); +} + +OfflineSpeakerDiarization::OfflineSpeakerDiarization( + const OfflineSpeakerDiarizationConfig &config) + : impl_(OfflineSpeakerDiarizationImpl::Create(config)) {} + +OfflineSpeakerDiarization::~OfflineSpeakerDiarization() = default; + +int32_t OfflineSpeakerDiarization::SampleRate() const { + return impl_->SampleRate(); +} + +OfflineSpeakerDiarizationResult OfflineSpeakerDiarization::Process( + const float *audio, int32_t n, + OfflineSpeakerDiarizationProgressCallback callback /*= nullptr*/, + void *callback_arg /*= nullptr*/) const { + return impl_->Process(audio, n, callback, callback_arg); +} + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-speaker-diarization.h b/sherpa-onnx/csrc/offline-speaker-diarization.h new file mode 100644 index 000000000..ab9a440aa --- /dev/null +++ b/sherpa-onnx/csrc/offline-speaker-diarization.h @@ -0,0 +1,73 @@ +// sherpa-onnx/csrc/offline-speaker-diarization.h +// +// Copyright (c) 2024 Xiaomi Corporation + +#ifndef SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_DIARIZATION_H_ +#define SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_DIARIZATION_H_ + +#include +#include +#include + +#include "sherpa-onnx/csrc/fast-clustering-config.h" +#include "sherpa-onnx/csrc/offline-speaker-diarization-result.h" +#include "sherpa-onnx/csrc/offline-speaker-segmentation-model-config.h" +#include "sherpa-onnx/csrc/speaker-embedding-extractor.h" + +namespace sherpa_onnx { + +struct OfflineSpeakerDiarizationConfig { + OfflineSpeakerSegmentationModelConfig segmentation; + SpeakerEmbeddingExtractorConfig embedding; + FastClusteringConfig clustering; + + // if a segment is less than this value, then it is discarded + float min_duration_on = 0.3; // in seconds + + // if the gap between to segments of the same speaker is less than this value, + // then these two segments are merged into a single segment. + // We do this recursively. + float min_duration_off = 0.5; // in seconds + + OfflineSpeakerDiarizationConfig() = default; + + OfflineSpeakerDiarizationConfig( + const OfflineSpeakerSegmentationModelConfig &segmentation, + const SpeakerEmbeddingExtractorConfig &embedding, + const FastClusteringConfig &clustering) + : segmentation(segmentation), + embedding(embedding), + clustering(clustering) {} + + void Register(ParseOptions *po); + bool Validate() const; + std::string ToString() const; +}; + +class OfflineSpeakerDiarizationImpl; + +using OfflineSpeakerDiarizationProgressCallback = std::function; + +class OfflineSpeakerDiarization { + public: + explicit OfflineSpeakerDiarization( + const OfflineSpeakerDiarizationConfig &config); + + ~OfflineSpeakerDiarization(); + + // Expected sample rate of the input audio samples + int32_t SampleRate() const; + + OfflineSpeakerDiarizationResult Process( + const float *audio, int32_t n, + OfflineSpeakerDiarizationProgressCallback callback = nullptr, + void *callback_arg = nullptr) const; + + private: + std::unique_ptr impl_; +}; + +} // namespace sherpa_onnx + +#endif // SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_DIARIZATION_H_ diff --git a/sherpa-onnx/csrc/offline-speaker-segmentation-model-config.cc b/sherpa-onnx/csrc/offline-speaker-segmentation-model-config.cc new file mode 100644 index 000000000..f1c9f7d4a --- /dev/null +++ b/sherpa-onnx/csrc/offline-speaker-segmentation-model-config.cc @@ -0,0 +1,57 @@ +// sherpa-onnx/csrc/offline-speaker-segmentation-model-config.cc +// +// Copyright (c) 2024 Xiaomi Corporation +#include "sherpa-onnx/csrc/offline-speaker-segmentation-model-config.h" + +#include +#include + +#include "sherpa-onnx/csrc/macros.h" + +namespace sherpa_onnx { + +void OfflineSpeakerSegmentationModelConfig::Register(ParseOptions *po) { + pyannote.Register(po); + + po->Register("num-threads", &num_threads, + "Number of threads to run the neural network"); + + po->Register("debug", &debug, + "true to print model information while loading it."); + + po->Register("provider", &provider, + "Specify a provider to use: cpu, cuda, coreml"); +} + +bool OfflineSpeakerSegmentationModelConfig::Validate() const { + if (num_threads < 1) { + SHERPA_ONNX_LOGE("num_threads should be > 0. Given %d", num_threads); + return false; + } + + if (!pyannote.model.empty()) { + return pyannote.Validate(); + } + + if (pyannote.model.empty()) { + SHERPA_ONNX_LOGE( + "You have to provide at least one speaker segmentation model"); + return false; + } + + return true; +} + +std::string OfflineSpeakerSegmentationModelConfig::ToString() const { + std::ostringstream os; + + os << "OfflineSpeakerSegmentationModelConfig("; + os << "pyannote=" << pyannote.ToString() << ", "; + os << "num_threads=" << num_threads << ", "; + os << "debug=" << (debug ? "True" : "False") << ", "; + os << "provider=\"" << provider << "\")"; + + return os.str(); +} + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-speaker-segmentation-model-config.h b/sherpa-onnx/csrc/offline-speaker-segmentation-model-config.h new file mode 100644 index 000000000..8e9e4a96e --- /dev/null +++ b/sherpa-onnx/csrc/offline-speaker-segmentation-model-config.h @@ -0,0 +1,40 @@ +// sherpa-onnx/csrc/offline-speaker-segmentation-model-config.h +// +// Copyright (c) 2024 Xiaomi Corporation +#ifndef SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_SEGMENTATION_MODEL_CONFIG_H_ +#define SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_SEGMENTATION_MODEL_CONFIG_H_ + +#include + +#include "sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model-config.h" +#include "sherpa-onnx/csrc/parse-options.h" + +namespace sherpa_onnx { + +struct OfflineSpeakerSegmentationModelConfig { + OfflineSpeakerSegmentationPyannoteModelConfig pyannote; + + int32_t num_threads = 1; + bool debug = false; + std::string provider = "cpu"; + + OfflineSpeakerSegmentationModelConfig() = default; + + explicit OfflineSpeakerSegmentationModelConfig( + const OfflineSpeakerSegmentationPyannoteModelConfig &pyannote, + int32_t num_threads, bool debug, const std::string &provider) + : pyannote(pyannote), + num_threads(num_threads), + debug(debug), + provider(provider) {} + + void Register(ParseOptions *po); + + bool Validate() const; + + std::string ToString() const; +}; + +} // namespace sherpa_onnx + +#endif // SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_SEGMENTATION_MODEL_CONFIG_H_ diff --git a/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model-config.cc b/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model-config.cc new file mode 100644 index 000000000..f7417ea83 --- /dev/null +++ b/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model-config.cc @@ -0,0 +1,38 @@ +// sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model-config.cc +// +// Copyright (c) 2024 Xiaomi Corporation +#include "sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model-config.h" + +#include +#include + +#include "sherpa-onnx/csrc/file-utils.h" +#include "sherpa-onnx/csrc/macros.h" + +namespace sherpa_onnx { + +void OfflineSpeakerSegmentationPyannoteModelConfig::Register(ParseOptions *po) { + po->Register("pyannote-model", &model, + "Path to model.onnx of the Pyannote segmentation model."); +} + +bool OfflineSpeakerSegmentationPyannoteModelConfig::Validate() const { + if (!FileExists(model)) { + SHERPA_ONNX_LOGE("Pyannote segmentation model: '%s' does not exist", + model.c_str()); + return false; + } + + return true; +} + +std::string OfflineSpeakerSegmentationPyannoteModelConfig::ToString() const { + std::ostringstream os; + + os << "OfflineSpeakerSegmentationPyannoteModelConfig("; + os << "model=\"" << model << "\")"; + + return os.str(); +} + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model-config.h b/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model-config.h new file mode 100644 index 000000000..fb5ca4a48 --- /dev/null +++ b/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model-config.h @@ -0,0 +1,30 @@ +// sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model-config.h +// +// Copyright (c) 2024 Xiaomi Corporation + +#ifndef SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_SEGMENTATION_PYANNOTE_MODEL_CONFIG_H_ +#define SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_SEGMENTATION_PYANNOTE_MODEL_CONFIG_H_ +#include + +#include "sherpa-onnx/csrc/parse-options.h" + +namespace sherpa_onnx { + +struct OfflineSpeakerSegmentationPyannoteModelConfig { + std::string model; + + OfflineSpeakerSegmentationPyannoteModelConfig() = default; + + explicit OfflineSpeakerSegmentationPyannoteModelConfig( + const std::string &model) + : model(model) {} + + void Register(ParseOptions *po); + bool Validate() const; + + std::string ToString() const; +}; + +} // namespace sherpa_onnx + +#endif // SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_SEGMENTATION_PYANNOTE_MODEL_CONFIG_H_ diff --git a/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model-meta-data.h b/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model-meta-data.h new file mode 100644 index 000000000..728ed7ff4 --- /dev/null +++ b/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model-meta-data.h @@ -0,0 +1,29 @@ +// sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model-meta-data.h +// +// Copyright (c) 2024 Xiaomi Corporation + +#ifndef SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_SEGMENTATION_PYANNOTE_MODEL_META_DATA_H_ +#define SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_SEGMENTATION_PYANNOTE_MODEL_META_DATA_H_ + +#include +#include + +namespace sherpa_onnx { + +// If you are not sure what each field means, please +// have a look of the Python file in the model directory that +// you have downloaded. +struct OfflineSpeakerSegmentationPyannoteModelMetaData { + int32_t sample_rate = 0; + int32_t window_size = 0; // in samples + int32_t window_shift = 0; // in samples + int32_t receptive_field_size = 0; // in samples + int32_t receptive_field_shift = 0; // in samples + int32_t num_speakers = 0; + int32_t powerset_max_classes = 0; + int32_t num_classes = 0; +}; + +} // namespace sherpa_onnx + +#endif // SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_SEGMENTATION_PYANNOTE_MODEL_META_DATA_H_ diff --git a/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.cc b/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.cc new file mode 100644 index 000000000..3f3323698 --- /dev/null +++ b/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.cc @@ -0,0 +1,108 @@ +// sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include "sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.h" + +#include +#include +#include + +#include "sherpa-onnx/csrc/onnx-utils.h" +#include "sherpa-onnx/csrc/session.h" + +namespace sherpa_onnx { + +class OfflineSpeakerSegmentationPyannoteModel::Impl { + public: + explicit Impl(const OfflineSpeakerSegmentationModelConfig &config) + : config_(config), + env_(ORT_LOGGING_LEVEL_ERROR), + sess_opts_(GetSessionOptions(config)), + allocator_{} { + auto buf = ReadFile(config_.pyannote.model); + Init(buf.data(), buf.size()); + } + + const OfflineSpeakerSegmentationPyannoteModelMetaData &GetModelMetaData() + const { + return meta_data_; + } + + Ort::Value Forward(Ort::Value x) { + auto out = sess_->Run({}, input_names_ptr_.data(), &x, 1, + output_names_ptr_.data(), output_names_ptr_.size()); + + return std::move(out[0]); + } + + private: + void Init(void *model_data, size_t model_data_length) { + sess_ = std::make_unique(env_, model_data, model_data_length, + sess_opts_); + + GetInputNames(sess_.get(), &input_names_, &input_names_ptr_); + + GetOutputNames(sess_.get(), &output_names_, &output_names_ptr_); + + // get meta data + Ort::ModelMetadata meta_data = sess_->GetModelMetadata(); + if (config_.debug) { + std::ostringstream os; + PrintModelMetadata(os, meta_data); + SHERPA_ONNX_LOGE("%s\n", os.str().c_str()); + } + + Ort::AllocatorWithDefaultOptions allocator; // used in the macro below + SHERPA_ONNX_READ_META_DATA(meta_data_.sample_rate, "sample_rate"); + SHERPA_ONNX_READ_META_DATA(meta_data_.window_size, "window_size"); + + meta_data_.window_shift = + static_cast(0.1 * meta_data_.window_size); + + SHERPA_ONNX_READ_META_DATA(meta_data_.receptive_field_size, + "receptive_field_size"); + SHERPA_ONNX_READ_META_DATA(meta_data_.receptive_field_shift, + "receptive_field_shift"); + SHERPA_ONNX_READ_META_DATA(meta_data_.num_speakers, "num_speakers"); + SHERPA_ONNX_READ_META_DATA(meta_data_.powerset_max_classes, + "powerset_max_classes"); + SHERPA_ONNX_READ_META_DATA(meta_data_.num_classes, "num_classes"); + } + + private: + OfflineSpeakerSegmentationModelConfig config_; + Ort::Env env_; + Ort::SessionOptions sess_opts_; + Ort::AllocatorWithDefaultOptions allocator_; + + std::unique_ptr sess_; + + std::vector input_names_; + std::vector input_names_ptr_; + + std::vector output_names_; + std::vector output_names_ptr_; + + OfflineSpeakerSegmentationPyannoteModelMetaData meta_data_; +}; + +OfflineSpeakerSegmentationPyannoteModel:: + OfflineSpeakerSegmentationPyannoteModel( + const OfflineSpeakerSegmentationModelConfig &config) + : impl_(std::make_unique(config)) {} + +OfflineSpeakerSegmentationPyannoteModel:: + ~OfflineSpeakerSegmentationPyannoteModel() = default; + +const OfflineSpeakerSegmentationPyannoteModelMetaData & +OfflineSpeakerSegmentationPyannoteModel::GetModelMetaData() const { + return impl_->GetModelMetaData(); +} + +Ort::Value OfflineSpeakerSegmentationPyannoteModel::Forward( + Ort::Value x) const { + return impl_->Forward(std::move(x)); +} + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.h b/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.h new file mode 100644 index 000000000..b504c373f --- /dev/null +++ b/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.h @@ -0,0 +1,40 @@ +// sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.h +// +// Copyright (c) 2024 Xiaomi Corporation +#ifndef SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_SEGMENTATION_PYANNOTE_MODEL_H_ +#define SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_SEGMENTATION_PYANNOTE_MODEL_H_ + +#include + +#include "onnxruntime_cxx_api.h" // NOLINT +#include "sherpa-onnx/csrc/offline-speaker-segmentation-model-config.h" +#include "sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model-meta-data.h" + +namespace sherpa_onnx { + +class OfflineSpeakerSegmentationPyannoteModel { + public: + explicit OfflineSpeakerSegmentationPyannoteModel( + const OfflineSpeakerSegmentationModelConfig &config); + + ~OfflineSpeakerSegmentationPyannoteModel(); + + const OfflineSpeakerSegmentationPyannoteModelMetaData &GetModelMetaData() + const; + + /** + * @param x A 3-D float tensor of shape (batch_size, 1, num_samples) + * @return Return a float tensor of + * shape (batch_size, num_frames, num_speakers). Note that + * num_speakers here uses powerset encoding. + */ + Ort::Value Forward(Ort::Value x) const; + + private: + class Impl; + std::unique_ptr impl_; +}; + +} // namespace sherpa_onnx + +#endif // SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_SEGMENTATION_PYANNOTE_MODEL_H_ diff --git a/sherpa-onnx/csrc/provider-config.cc b/sherpa-onnx/csrc/provider-config.cc index 1db62aa6b..165e2d9a2 100644 --- a/sherpa-onnx/csrc/provider-config.cc +++ b/sherpa-onnx/csrc/provider-config.cc @@ -61,8 +61,10 @@ void TensorrtConfig::Register(ParseOptions *po) { bool TensorrtConfig::Validate() const { if (trt_max_workspace_size < 0) { - SHERPA_ONNX_LOGE("trt_max_workspace_size: %ld is not valid.", - trt_max_workspace_size); + std::ostringstream os; + os << "trt_max_workspace_size: " << trt_max_workspace_size + << " is not valid."; + SHERPA_ONNX_LOGE("%s", os.str().c_str()); return false; } if (trt_max_partition_iterations < 0) { diff --git a/sherpa-onnx/csrc/session.cc b/sherpa-onnx/csrc/session.cc index 7f6f685e0..9c5eb2b1a 100644 --- a/sherpa-onnx/csrc/session.cc +++ b/sherpa-onnx/csrc/session.cc @@ -35,9 +35,9 @@ static void OrtStatusFailure(OrtStatus *status, const char *s) { api.ReleaseStatus(status); } -static Ort::SessionOptions GetSessionOptionsImpl( +Ort::SessionOptions GetSessionOptionsImpl( int32_t num_threads, const std::string &provider_str, - const ProviderConfig *provider_config = nullptr) { + const ProviderConfig *provider_config /*= nullptr*/) { Provider p = StringToProvider(provider_str); Ort::SessionOptions sess_opts; @@ -259,10 +259,6 @@ Ort::SessionOptions GetSessionOptions(const OnlineModelConfig &config, &config.provider_config); } -Ort::SessionOptions GetSessionOptions(const OfflineModelConfig &config) { - return GetSessionOptionsImpl(config.num_threads, config.provider); -} - Ort::SessionOptions GetSessionOptions(const OfflineLMConfig &config) { return GetSessionOptionsImpl(config.lm_num_threads, config.lm_provider); } @@ -271,38 +267,4 @@ Ort::SessionOptions GetSessionOptions(const OnlineLMConfig &config) { return GetSessionOptionsImpl(config.lm_num_threads, config.lm_provider); } -Ort::SessionOptions GetSessionOptions(const VadModelConfig &config) { - return GetSessionOptionsImpl(config.num_threads, config.provider); -} - -#if SHERPA_ONNX_ENABLE_TTS -Ort::SessionOptions GetSessionOptions(const OfflineTtsModelConfig &config) { - return GetSessionOptionsImpl(config.num_threads, config.provider); -} -#endif - -Ort::SessionOptions GetSessionOptions( - const SpeakerEmbeddingExtractorConfig &config) { - return GetSessionOptionsImpl(config.num_threads, config.provider); -} - -Ort::SessionOptions GetSessionOptions( - const SpokenLanguageIdentificationConfig &config) { - return GetSessionOptionsImpl(config.num_threads, config.provider); -} - -Ort::SessionOptions GetSessionOptions(const AudioTaggingModelConfig &config) { - return GetSessionOptionsImpl(config.num_threads, config.provider); -} - -Ort::SessionOptions GetSessionOptions( - const OfflinePunctuationModelConfig &config) { - return GetSessionOptionsImpl(config.num_threads, config.provider); -} - -Ort::SessionOptions GetSessionOptions( - const OnlinePunctuationModelConfig &config) { - return GetSessionOptionsImpl(config.num_threads, config.provider); -} - } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/session.h b/sherpa-onnx/csrc/session.h index 1e8beb114..e19db6c20 100644 --- a/sherpa-onnx/csrc/session.h +++ b/sherpa-onnx/csrc/session.h @@ -8,53 +8,28 @@ #include #include "onnxruntime_cxx_api.h" // NOLINT -#include "sherpa-onnx/csrc/audio-tagging-model-config.h" #include "sherpa-onnx/csrc/offline-lm-config.h" -#include "sherpa-onnx/csrc/offline-model-config.h" -#include "sherpa-onnx/csrc/offline-punctuation-model-config.h" -#include "sherpa-onnx/csrc/online-punctuation-model-config.h" #include "sherpa-onnx/csrc/online-lm-config.h" #include "sherpa-onnx/csrc/online-model-config.h" -#include "sherpa-onnx/csrc/speaker-embedding-extractor.h" -#include "sherpa-onnx/csrc/spoken-language-identification.h" -#include "sherpa-onnx/csrc/vad-model-config.h" - -#if SHERPA_ONNX_ENABLE_TTS -#include "sherpa-onnx/csrc/offline-tts-model-config.h" -#endif namespace sherpa_onnx { -Ort::SessionOptions GetSessionOptions(const OnlineModelConfig &config); - -Ort::SessionOptions GetSessionOptions(const OnlineModelConfig &config, - const std::string &model_type); - -Ort::SessionOptions GetSessionOptions(const OfflineModelConfig &config); +Ort::SessionOptions GetSessionOptionsImpl( + int32_t num_threads, const std::string &provider_str, + const ProviderConfig *provider_config = nullptr); Ort::SessionOptions GetSessionOptions(const OfflineLMConfig &config); - Ort::SessionOptions GetSessionOptions(const OnlineLMConfig &config); -Ort::SessionOptions GetSessionOptions(const VadModelConfig &config); - -#if SHERPA_ONNX_ENABLE_TTS -Ort::SessionOptions GetSessionOptions(const OfflineTtsModelConfig &config); -#endif - -Ort::SessionOptions GetSessionOptions( - const SpeakerEmbeddingExtractorConfig &config); - -Ort::SessionOptions GetSessionOptions( - const SpokenLanguageIdentificationConfig &config); - -Ort::SessionOptions GetSessionOptions(const AudioTaggingModelConfig &config); +Ort::SessionOptions GetSessionOptions(const OnlineModelConfig &config); -Ort::SessionOptions GetSessionOptions( - const OfflinePunctuationModelConfig &config); +Ort::SessionOptions GetSessionOptions(const OnlineModelConfig &config, + const std::string &model_type); -Ort::SessionOptions GetSessionOptions( - const OnlinePunctuationModelConfig &config); +template +Ort::SessionOptions GetSessionOptions(const T &config) { + return GetSessionOptionsImpl(config.num_threads, config.provider); +} } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/sherpa-onnx-offline-speaker-diarization.cc b/sherpa-onnx/csrc/sherpa-onnx-offline-speaker-diarization.cc new file mode 100644 index 000000000..170973114 --- /dev/null +++ b/sherpa-onnx/csrc/sherpa-onnx-offline-speaker-diarization.cc @@ -0,0 +1,133 @@ +// sherpa-onnx/csrc/sherpa-onnx-offline-speaker-diarization.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include "sherpa-onnx/csrc/offline-speaker-diarization.h" +#include "sherpa-onnx/csrc/parse-options.h" +#include "sherpa-onnx/csrc/wave-reader.h" + +static int32_t ProgressCallback(int32_t processed_chunks, int32_t num_chunks, + void *arg) { + float progress = 100.0 * processed_chunks / num_chunks; + fprintf(stderr, "progress %.2f%%\n", progress); + + // the return value is currently ignored + return 0; +} + +int main(int32_t argc, char *argv[]) { + const char *kUsageMessage = R"usage( +Offline/Non-streaming speaker diarization with sherpa-onnx +Usage example: + +Step 1: Download a speaker segmentation model + +Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models +for a list of available models. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + +Step 2: Download a speaker embedding extractor model + +Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models +for a list of available models. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx + +Step 3. Download test wave files + +Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models +for a list of available test wave files. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/0-four-speakers-zh.wav + +Step 4. Build sherpa-onnx + +Step 5. Run it + + ./bin/sherpa-onnx-offline-speaker-diarization \ + --clustering.num-clusters=4 \ + --segmentation.pyannote-model=./sherpa-onnx-pyannote-segmentation-3-0/model.onnx \ + --embedding.model=./3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx \ + ./0-four-speakers-zh.wav + +Since we know that there are four speakers in the test wave file, we use +--clustering.num-clusters=4 in the above example. + +If we don't know number of speakers in the given wave file, we can use +the argument --clustering.cluster-threshold. The following is an example: + + ./bin/sherpa-onnx-offline-speaker-diarization \ + --clustering.cluster-threshold=0.90 \ + --segmentation.pyannote-model=./sherpa-onnx-pyannote-segmentation-3-0/model.onnx \ + --embedding.model=./3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx \ + ./0-four-speakers-zh.wav + +A larger threshold leads to few clusters, i.e., few speakers; +a smaller threshold leads to more clusters, i.e., more speakers + )usage"; + sherpa_onnx::OfflineSpeakerDiarizationConfig config; + sherpa_onnx::ParseOptions po(kUsageMessage); + config.Register(&po); + po.Read(argc, argv); + + std::cout << config.ToString() << "\n"; + + if (!config.Validate()) { + po.PrintUsage(); + std::cerr << "Errors in config!\n"; + return -1; + } + + if (po.NumArgs() != 1) { + std::cerr << "Error: Please provide exactly 1 wave file.\n\n"; + po.PrintUsage(); + return -1; + } + + sherpa_onnx::OfflineSpeakerDiarization sd(config); + + std::cout << "Started\n"; + const auto begin = std::chrono::steady_clock::now(); + const std::string wav_filename = po.GetArg(1); + int32_t sample_rate = -1; + bool is_ok = false; + const std::vector samples = + sherpa_onnx::ReadWave(wav_filename, &sample_rate, &is_ok); + if (!is_ok) { + std::cerr << "Failed to read " << wav_filename.c_str() << "\n"; + return -1; + } + + if (sample_rate != sd.SampleRate()) { + std::cerr << "Expect sample rate " << sd.SampleRate() + << ". Given: " << sample_rate << "\n"; + return -1; + } + + float duration = samples.size() / static_cast(sample_rate); + + auto result = + sd.Process(samples.data(), samples.size(), ProgressCallback, nullptr) + .SortByStartTime(); + + for (const auto &r : result) { + std::cout << r.ToString() << "\n"; + } + + const auto end = std::chrono::steady_clock::now(); + float elapsed_seconds = + std::chrono::duration_cast(end - begin) + .count() / + 1000.; + + fprintf(stderr, "Duration : %.3f s\n", duration); + fprintf(stderr, "Elapsed seconds: %.3f s\n", elapsed_seconds); + float rtf = elapsed_seconds / duration; + fprintf(stderr, "Real time factor (RTF): %.3f / %.3f = %.3f\n", + elapsed_seconds, duration, rtf); + + return 0; +} diff --git a/sherpa-onnx/csrc/sherpa-onnx-offline-tts.cc b/sherpa-onnx/csrc/sherpa-onnx-offline-tts.cc index 442ec1813..1ab8b68de 100644 --- a/sherpa-onnx/csrc/sherpa-onnx-offline-tts.cc +++ b/sherpa-onnx/csrc/sherpa-onnx-offline-tts.cc @@ -9,14 +9,15 @@ #include "sherpa-onnx/csrc/parse-options.h" #include "sherpa-onnx/csrc/wave-writer.h" -int32_t audioCallback(const float * /*samples*/, int32_t n, float progress) { +static int32_t AudioCallback(const float * /*samples*/, int32_t n, + float progress) { printf("sample=%d, progress=%f\n", n, progress); return 1; } int main(int32_t argc, char *argv[]) { const char *kUsageMessage = R"usage( -Offline text-to-speech with sherpa-onnx +Offline/Non-streaming text-to-speech with sherpa-onnx Usage example: @@ -79,7 +80,7 @@ or details. sherpa_onnx::OfflineTts tts(config); const auto begin = std::chrono::steady_clock::now(); - auto audio = tts.Generate(po.GetArg(1), sid, 1.0, audioCallback); + auto audio = tts.Generate(po.GetArg(1), sid, 1.0, AudioCallback); const auto end = std::chrono::steady_clock::now(); if (audio.samples.empty()) { diff --git a/sherpa-onnx/csrc/sherpa-onnx-online-punctuation.cc b/sherpa-onnx/csrc/sherpa-onnx-online-punctuation.cc index ea83cfaaf..faca83b98 100644 --- a/sherpa-onnx/csrc/sherpa-onnx-online-punctuation.cc +++ b/sherpa-onnx/csrc/sherpa-onnx-online-punctuation.cc @@ -19,7 +19,7 @@ The input text can contain English words. Usage: Please download the model from: -https://github.com/k2-fsa/sherpa-onnx/releases/download/punctuation-models/sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2 +https://github.com/k2-fsa/sherpa-onnx/releases/download/punctuation-models/sherpa-onnx-online-punct-en-2024-08-06.tar.bz2 ./bin/Release/sherpa-onnx-online-punctuation \ --cnn-bilstm=/path/to/model.onnx \ diff --git a/sherpa-onnx/csrc/speaker-embedding-extractor.cc b/sherpa-onnx/csrc/speaker-embedding-extractor.cc index 1c99de1a0..d90b0b1e0 100644 --- a/sherpa-onnx/csrc/speaker-embedding-extractor.cc +++ b/sherpa-onnx/csrc/speaker-embedding-extractor.cc @@ -26,12 +26,12 @@ void SpeakerEmbeddingExtractorConfig::Register(ParseOptions *po) { bool SpeakerEmbeddingExtractorConfig::Validate() const { if (model.empty()) { - SHERPA_ONNX_LOGE("Please provide --model"); + SHERPA_ONNX_LOGE("Please provide a speaker embedding extractor model"); return false; } if (!FileExists(model)) { - SHERPA_ONNX_LOGE("--speaker-embedding-model: '%s' does not exist", + SHERPA_ONNX_LOGE("speaker embedding extractor model: '%s' does not exist", model.c_str()); return false; } From 8535b1d3bbbf4bfedf3924abaf563e3e346ad042 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 9 Oct 2024 14:13:26 +0800 Subject: [PATCH 002/183] Python API for speaker diarization. (#1400) --- .github/scripts/test-python.sh | 15 +++ .github/workflows/windows-x64.yaml | 2 +- .github/workflows/windows-x86.yaml | 2 +- .../offline-speaker-diarization.py | 118 ++++++++++++++++++ ...ffline-speaker-diarization-pyannote-impl.h | 2 +- .../csrc/offline-speaker-diarization-result.h | 2 + .../csrc/offline-speaker-diarization.h | 7 +- sherpa-onnx/python/csrc/CMakeLists.txt | 2 + .../offline-speaker-diarization-result.cc | 32 +++++ .../csrc/offline-speaker-diarization-result.h | 16 +++ .../csrc/offline-speaker-diarization.cc | 92 ++++++++++++++ .../python/csrc/offline-speaker-diarization.h | 16 +++ sherpa-onnx/python/csrc/sherpa-onnx.cc | 12 +- sherpa-onnx/python/sherpa_onnx/__init__.py | 6 + 14 files changed, 315 insertions(+), 9 deletions(-) create mode 100755 python-api-examples/offline-speaker-diarization.py create mode 100644 sherpa-onnx/python/csrc/offline-speaker-diarization-result.cc create mode 100644 sherpa-onnx/python/csrc/offline-speaker-diarization-result.h create mode 100644 sherpa-onnx/python/csrc/offline-speaker-diarization.cc create mode 100644 sherpa-onnx/python/csrc/offline-speaker-diarization.h diff --git a/.github/scripts/test-python.sh b/.github/scripts/test-python.sh index de7297f2c..8c9d303b0 100755 --- a/.github/scripts/test-python.sh +++ b/.github/scripts/test-python.sh @@ -8,6 +8,21 @@ log() { echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" } +log "test offline speaker diarization" + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 +tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 +rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/0-four-speakers-zh.wav + +python3 ./python-api-examples/offline-speaker-diarization.py + +rm -rf *.wav *.onnx ./sherpa-onnx-pyannote-segmentation-3-0 + + log "test_clustering" pushd /tmp/ mkdir test-cluster diff --git a/.github/workflows/windows-x64.yaml b/.github/workflows/windows-x64.yaml index c67f3e0b5..758935918 100644 --- a/.github/workflows/windows-x64.yaml +++ b/.github/workflows/windows-x64.yaml @@ -93,7 +93,7 @@ jobs: shell: bash run: | du -h -d1 . - export PATH=$PWD/build/bin:$PATH + export PATH=$PWD/build/bin/Release:$PATH export EXE=sherpa-onnx-offline-speaker-diarization.exe .github/scripts/test-speaker-diarization.sh diff --git a/.github/workflows/windows-x86.yaml b/.github/workflows/windows-x86.yaml index 30394e90e..b9e473184 100644 --- a/.github/workflows/windows-x86.yaml +++ b/.github/workflows/windows-x86.yaml @@ -93,7 +93,7 @@ jobs: shell: bash run: | du -h -d1 . - export PATH=$PWD/build/bin:$PATH + export PATH=$PWD/build/bin/Release:$PATH export EXE=sherpa-onnx-offline-speaker-diarization.exe .github/scripts/test-speaker-diarization.sh diff --git a/python-api-examples/offline-speaker-diarization.py b/python-api-examples/offline-speaker-diarization.py new file mode 100755 index 000000000..3e3ff1618 --- /dev/null +++ b/python-api-examples/offline-speaker-diarization.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# Copyright (c) 2024 Xiaomi Corporation + +""" +This file shows how to use sherpa-onnx Python API for +offline/non-streaming speaker diarization. + +Usage: + +Step 1: Download a speaker segmentation model + +Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models +for a list of available models. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + +Step 2: Download a speaker embedding extractor model + +Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models +for a list of available models. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx + +Step 3. Download test wave files + +Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models +for a list of available test wave files. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/0-four-speakers-zh.wav + +Step 4. Run it + + python3 ./python-api-examples/offline-speaker-diarization.py + +""" +from pathlib import Path + +import sherpa_onnx +import soundfile as sf + + +def init_speaker_diarization(num_speakers: int = -1, cluster_threshold: float = 0.5): + """ + Args: + num_speakers: + If you know the actual number of speakers in the wave file, then please + specify it. Otherwise, leave it to -1 + cluster_threshold: + If num_speakers is -1, then this threshold is used for clustering. + A smaller cluster_threshold leads to more clusters, i.e., more speakers. + A larger cluster_threshold leads to fewer clusters, i.e., fewer speakers. + """ + segmentation_model = "./sherpa-onnx-pyannote-segmentation-3-0/model.onnx" + embedding_extractor_model = ( + "./3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx" + ) + + config = sherpa_onnx.OfflineSpeakerDiarizationConfig( + segmentation=sherpa_onnx.OfflineSpeakerSegmentationModelConfig( + pyannote=sherpa_onnx.OfflineSpeakerSegmentationPyannoteModelConfig( + model=segmentation_model + ), + ), + embedding=sherpa_onnx.SpeakerEmbeddingExtractorConfig( + model=embedding_extractor_model + ), + clustering=sherpa_onnx.FastClusteringConfig( + num_clusters=num_speakers, threshold=cluster_threshold + ), + min_duration_on=0.3, + min_duration_off=0.5, + ) + if not config.validate(): + raise RuntimeError( + "Please check your config and make sure all required files exist" + ) + + return sherpa_onnx.OfflineSpeakerDiarization(config) + + +def progress_callback(num_processed_chunk: int, num_total_chunks: int) -> int: + progress = num_processed_chunk / num_total_chunks * 100 + print(f"Progress: {progress:.3f}%") + return 0 + + +def main(): + wave_filename = "./0-four-speakers-zh.wav" + if not Path(wave_filename).is_file(): + raise RuntimeError(f"{wave_filename} does not exist") + + audio, sample_rate = sf.read(wave_filename, dtype="float32", always_2d=True) + audio = audio[:, 0] # only use the first channel + + # Since we know there are 4 speakers in the above test wave file, we use + # num_speakers 4 here + sd = init_speaker_diarization(num_speakers=4) + if sample_rate != sd.sample_rate: + raise RuntimeError( + f"Expected samples rate: {sd.sample_rate}, given: {sample_rate}" + ) + + show_porgress = True + + if show_porgress: + result = sd.process(audio, callback=progress_callback).sort_by_start_time() + else: + result = sd.process(audio).sort_by_start_time() + + for r in result: + print(f"{r.start:.3f} -- {r.end:.3f} speaker_{r.speaker:02}") + # print(r) # this one is simpler + + +if __name__ == "__main__": + main() diff --git a/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h b/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h index bcd0c93a4..64b087c00 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h +++ b/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h @@ -103,7 +103,7 @@ class OfflineSpeakerDiarizationPyannoteImpl auto chunk_speaker_samples_list_pair = GetChunkSpeakerSampleIndexes(labels); Matrix2D embeddings = ComputeEmbeddings(audio, n, chunk_speaker_samples_list_pair.second, - callback, callback_arg); + std::move(callback), callback_arg); std::vector cluster_labels = clustering_.Cluster( &embeddings(0, 0), embeddings.rows(), embeddings.cols()); diff --git a/sherpa-onnx/csrc/offline-speaker-diarization-result.h b/sherpa-onnx/csrc/offline-speaker-diarization-result.h index e71d054e5..5fb144f5c 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization-result.h +++ b/sherpa-onnx/csrc/offline-speaker-diarization-result.h @@ -28,6 +28,8 @@ class OfflineSpeakerDiarizationSegment { const std::string &Text() const { return text_; } float Duration() const { return end_ - start_; } + void SetText(const std::string &text) { text_ = text; } + std::string ToString() const; private: diff --git a/sherpa-onnx/csrc/offline-speaker-diarization.h b/sherpa-onnx/csrc/offline-speaker-diarization.h index ab9a440aa..e5d02c473 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization.h +++ b/sherpa-onnx/csrc/offline-speaker-diarization.h @@ -34,10 +34,13 @@ struct OfflineSpeakerDiarizationConfig { OfflineSpeakerDiarizationConfig( const OfflineSpeakerSegmentationModelConfig &segmentation, const SpeakerEmbeddingExtractorConfig &embedding, - const FastClusteringConfig &clustering) + const FastClusteringConfig &clustering, float min_duration_on, + float min_duration_off) : segmentation(segmentation), embedding(embedding), - clustering(clustering) {} + clustering(clustering), + min_duration_on(min_duration_on), + min_duration_off(min_duration_off) {} void Register(ParseOptions *po); bool Validate() const; diff --git a/sherpa-onnx/python/csrc/CMakeLists.txt b/sherpa-onnx/python/csrc/CMakeLists.txt index 7fd5efa33..2e971581a 100644 --- a/sherpa-onnx/python/csrc/CMakeLists.txt +++ b/sherpa-onnx/python/csrc/CMakeLists.txt @@ -62,6 +62,8 @@ endif() if(SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION) list(APPEND srcs fast-clustering.cc + offline-speaker-diarization-result.cc + offline-speaker-diarization.cc ) endif() diff --git a/sherpa-onnx/python/csrc/offline-speaker-diarization-result.cc b/sherpa-onnx/python/csrc/offline-speaker-diarization-result.cc new file mode 100644 index 000000000..d058c26a2 --- /dev/null +++ b/sherpa-onnx/python/csrc/offline-speaker-diarization-result.cc @@ -0,0 +1,32 @@ +// sherpa-onnx/python/csrc/offline-speaker-diarization-result.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include "sherpa-onnx/python/csrc/offline-speaker-diarization-result.h" + +#include "sherpa-onnx/csrc/offline-speaker-diarization-result.h" + +namespace sherpa_onnx { + +static void PybindOfflineSpeakerDiarizationSegment(py::module *m) { + using PyClass = OfflineSpeakerDiarizationSegment; + py::class_(*m, "OfflineSpeakerDiarizationSegment") + .def_property_readonly("start", &PyClass::Start) + .def_property_readonly("end", &PyClass::End) + .def_property_readonly("duration", &PyClass::Duration) + .def_property_readonly("speaker", &PyClass::Speaker) + .def_property("text", &PyClass::Text, &PyClass::SetText) + .def("__str__", &PyClass::ToString); +} + +void PybindOfflineSpeakerDiarizationResult(py::module *m) { + PybindOfflineSpeakerDiarizationSegment(m); + using PyClass = OfflineSpeakerDiarizationResult; + py::class_(*m, "OfflineSpeakerDiarizationResult") + .def_property_readonly("num_speakers", &PyClass::NumSpeakers) + .def_property_readonly("num_segments", &PyClass::NumSegments) + .def("sort_by_start_time", &PyClass::SortByStartTime) + .def("sort_by_speaker", &PyClass::SortBySpeaker); +} + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/python/csrc/offline-speaker-diarization-result.h b/sherpa-onnx/python/csrc/offline-speaker-diarization-result.h new file mode 100644 index 000000000..2c11e4073 --- /dev/null +++ b/sherpa-onnx/python/csrc/offline-speaker-diarization-result.h @@ -0,0 +1,16 @@ +// sherpa-onnx/python/csrc/offline-speaker-diarization-result.h +// +// Copyright (c) 2024 Xiaomi Corporation + +#ifndef SHERPA_ONNX_PYTHON_CSRC_OFFLINE_SPEAKER_DIARIZATION_RESULT_H_ +#define SHERPA_ONNX_PYTHON_CSRC_OFFLINE_SPEAKER_DIARIZATION_RESULT_H_ + +#include "sherpa-onnx/python/csrc/sherpa-onnx.h" + +namespace sherpa_onnx { + +void PybindOfflineSpeakerDiarizationResult(py::module *m); + +} + +#endif // SHERPA_ONNX_PYTHON_CSRC_OFFLINE_SPEAKER_DIARIZATION_RESULT_H_ diff --git a/sherpa-onnx/python/csrc/offline-speaker-diarization.cc b/sherpa-onnx/python/csrc/offline-speaker-diarization.cc new file mode 100644 index 000000000..c77979b3c --- /dev/null +++ b/sherpa-onnx/python/csrc/offline-speaker-diarization.cc @@ -0,0 +1,92 @@ +// sherpa-onnx/python/csrc/offline-speaker-diarization.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include "sherpa-onnx/python/csrc/offline-speaker-diarization.h" + +#include +#include + +#include "sherpa-onnx/csrc/offline-speaker-diarization.h" +#include "sherpa-onnx/csrc/offline-speaker-segmentation-model-config.h" +#include "sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model-config.h" + +namespace sherpa_onnx { + +static void PybindOfflineSpeakerSegmentationPyannoteModelConfig(py::module *m) { + using PyClass = OfflineSpeakerSegmentationPyannoteModelConfig; + py::class_(*m, "OfflineSpeakerSegmentationPyannoteModelConfig") + .def(py::init<>()) + .def(py::init(), py::arg("model")) + .def_readwrite("model", &PyClass::model) + .def("__str__", &PyClass::ToString) + .def("validate", &PyClass::Validate); +} + +static void PybindOfflineSpeakerSegmentationModelConfig(py::module *m) { + PybindOfflineSpeakerSegmentationPyannoteModelConfig(m); + + using PyClass = OfflineSpeakerSegmentationModelConfig; + py::class_(*m, "OfflineSpeakerSegmentationModelConfig") + .def(py::init<>()) + .def(py::init(), + py::arg("pyannote"), py::arg("num_threads") = 1, + py::arg("debug") = false, py::arg("provider") = "cpu") + .def_readwrite("pyannote", &PyClass::pyannote) + .def_readwrite("num_threads", &PyClass::num_threads) + .def_readwrite("debug", &PyClass::debug) + .def_readwrite("provider", &PyClass::provider) + .def("__str__", &PyClass::ToString) + .def("validate", &PyClass::Validate); +} + +static void PybindOfflineSpeakerDiarizationConfig(py::module *m) { + PybindOfflineSpeakerSegmentationModelConfig(m); + + using PyClass = OfflineSpeakerDiarizationConfig; + py::class_(*m, "OfflineSpeakerDiarizationConfig") + .def(py::init(), + py::arg("segmentation"), py::arg("embedding"), py::arg("clustering"), + py::arg("min_duration_on") = 0.3, py::arg("min_duration_off") = 0.5) + .def_readwrite("segmentation", &PyClass::segmentation) + .def_readwrite("embedding", &PyClass::embedding) + .def_readwrite("clustering", &PyClass::clustering) + .def_readwrite("min_duration_on", &PyClass::min_duration_on) + .def_readwrite("min_duration_off", &PyClass::min_duration_off) + .def("__str__", &PyClass::ToString) + .def("validate", &PyClass::Validate); +} + +void PybindOfflineSpeakerDiarization(py::module *m) { + PybindOfflineSpeakerDiarizationConfig(m); + + using PyClass = OfflineSpeakerDiarization; + py::class_(*m, "OfflineSpeakerDiarization") + .def(py::init(), + py::arg("config")) + .def_property_readonly("sample_rate", &PyClass::SampleRate) + .def( + "process", + [](const PyClass &self, const std::vector samples, + std::function callback) { + if (!callback) { + return self.Process(samples.data(), samples.size()); + } + + std::function callback_wrapper = + [callback](int32_t processed_chunks, int32_t num_chunks, + void *) -> int32_t { + callback(processed_chunks, num_chunks); + return 0; + }; + + return self.Process(samples.data(), samples.size(), + callback_wrapper); + }, + py::arg("samples"), py::arg("callback") = py::none()); +} + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/python/csrc/offline-speaker-diarization.h b/sherpa-onnx/python/csrc/offline-speaker-diarization.h new file mode 100644 index 000000000..523343062 --- /dev/null +++ b/sherpa-onnx/python/csrc/offline-speaker-diarization.h @@ -0,0 +1,16 @@ +// sherpa-onnx/python/csrc/offline-speaker-diarization.h +// +// Copyright (c) 2024 Xiaomi Corporation + +#ifndef SHERPA_ONNX_PYTHON_CSRC_OFFLINE_SPEAKER_DIARIZATION_H_ +#define SHERPA_ONNX_PYTHON_CSRC_OFFLINE_SPEAKER_DIARIZATION_H_ + +#include "sherpa-onnx/python/csrc/sherpa-onnx.h" + +namespace sherpa_onnx { + +void PybindOfflineSpeakerDiarization(py::module *m); + +} + +#endif // SHERPA_ONNX_PYTHON_CSRC_OFFLINE_SPEAKER_DIARIZATION_H_ diff --git a/sherpa-onnx/python/csrc/sherpa-onnx.cc b/sherpa-onnx/python/csrc/sherpa-onnx.cc index f668d626c..c73022f17 100644 --- a/sherpa-onnx/python/csrc/sherpa-onnx.cc +++ b/sherpa-onnx/python/csrc/sherpa-onnx.cc @@ -37,6 +37,8 @@ #if SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION == 1 #include "sherpa-onnx/python/csrc/fast-clustering.h" +#include "sherpa-onnx/python/csrc/offline-speaker-diarization-result.h" +#include "sherpa-onnx/python/csrc/offline-speaker-diarization.h" #endif namespace sherpa_onnx { @@ -74,14 +76,16 @@ PYBIND11_MODULE(_sherpa_onnx, m) { PybindOfflineTts(&m); #endif -#if SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION == 1 - PybindFastClustering(&m); -#endif - PybindSpeakerEmbeddingExtractor(&m); PybindSpeakerEmbeddingManager(&m); PybindSpokenLanguageIdentification(&m); +#if SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION == 1 + PybindFastClustering(&m); + PybindOfflineSpeakerDiarizationResult(&m); + PybindOfflineSpeakerDiarization(&m); +#endif + PybindAlsa(&m); } diff --git a/sherpa-onnx/python/sherpa_onnx/__init__.py b/sherpa-onnx/python/sherpa_onnx/__init__.py index 3568447b3..2d5e456dc 100644 --- a/sherpa-onnx/python/sherpa_onnx/__init__.py +++ b/sherpa-onnx/python/sherpa_onnx/__init__.py @@ -11,6 +11,12 @@ OfflinePunctuation, OfflinePunctuationConfig, OfflinePunctuationModelConfig, + OfflineSpeakerDiarization, + OfflineSpeakerDiarizationConfig, + OfflineSpeakerDiarizationResult, + OfflineSpeakerDiarizationSegment, + OfflineSpeakerSegmentationModelConfig, + OfflineSpeakerSegmentationPyannoteModelConfig, OfflineStream, OfflineTts, OfflineTtsConfig, From d468527f6202b61d72b8ff81532ba7c94f5b0f96 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 9 Oct 2024 17:10:03 +0800 Subject: [PATCH 003/183] C API for speaker diarization (#1402) --- .gitignore | 1 + README.md | 11 +- c-api-examples/CMakeLists.txt | 5 + .../offline-speaker-diarization-c-api.c | 131 ++++++++++++++++ sherpa-onnx/c-api/c-api.cc | 145 ++++++++++++++++++ sherpa-onnx/c-api/c-api.h | 116 +++++++++++++- sherpa-onnx/csrc/fast-clustering-config.h | 4 +- .../csrc/offline-speaker-diarization.cc | 10 ++ ...sherpa-onnx-offline-speaker-diarization.cc | 2 +- 9 files changed, 418 insertions(+), 7 deletions(-) create mode 100644 c-api-examples/offline-speaker-diarization-c-api.c diff --git a/.gitignore b/.gitignore index b0fbfae78..7e6708be4 100644 --- a/.gitignore +++ b/.gitignore @@ -120,3 +120,4 @@ vits-melo-tts-zh_en sherpa-onnx-online-punct-en-2024-08-06 *.mp4 *.mp3 +sherpa-onnx-pyannote-segmentation-3-0 diff --git a/README.md b/README.md index 890abe882..2f318d3ef 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,12 @@ ### Supported functions -|Speech recognition| Speech synthesis | Speaker verification | Speaker identification | -|------------------|------------------|----------------------|------------------------| -| ✔️ | ✔️ | ✔️ | ✔️ | +|Speech recognition| Speech synthesis | +|------------------|------------------| +| ✔️ | ✔️ | + +|Speaker identification| Speaker diarization | Speaker identification | +|----------------------|-------------------- |------------------------| +| ✔️ | ✔️ | ✔️ | | Spoken Language identification | Audio tagging | Voice activity detection | |--------------------------------|---------------|--------------------------| @@ -47,6 +51,7 @@ This repository supports running the following functions **locally** - Speech-to-text (i.e., ASR); both streaming and non-streaming are supported - Text-to-speech (i.e., TTS) + - Speaker diarization - Speaker identification - Speaker verification - Spoken language identification diff --git a/c-api-examples/CMakeLists.txt b/c-api-examples/CMakeLists.txt index 0bf526450..45ca9a156 100644 --- a/c-api-examples/CMakeLists.txt +++ b/c-api-examples/CMakeLists.txt @@ -9,6 +9,11 @@ if(SHERPA_ONNX_ENABLE_TTS) target_link_libraries(offline-tts-c-api sherpa-onnx-c-api cargs) endif() +if(SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION) + add_executable(offline-speaker-diarization-c-api offline-speaker-diarization-c-api.c) + target_link_libraries(offline-speaker-diarization-c-api sherpa-onnx-c-api) +endif() + add_executable(spoken-language-identification-c-api spoken-language-identification-c-api.c) target_link_libraries(spoken-language-identification-c-api sherpa-onnx-c-api) diff --git a/c-api-examples/offline-speaker-diarization-c-api.c b/c-api-examples/offline-speaker-diarization-c-api.c new file mode 100644 index 000000000..d5a17dd0b --- /dev/null +++ b/c-api-examples/offline-speaker-diarization-c-api.c @@ -0,0 +1,131 @@ +// c-api-examples/offline-sepaker-diarization-c-api.c +// +// Copyright (c) 2024 Xiaomi Corporation + +// +// This file demonstrates how to implement speaker diarization with +// sherpa-onnx's C API. + +// clang-format off +/* +Usage: + +Step 1: Download a speaker segmentation model + +Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models +for a list of available models. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + +Step 2: Download a speaker embedding extractor model + +Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models +for a list of available models. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx + +Step 3. Download test wave files + +Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models +for a list of available test wave files. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/0-four-speakers-zh.wav + +Step 4. Run it + + */ +// clang-format on + +#include +#include + +#include "sherpa-onnx/c-api/c-api.h" + +static int32_t ProgressCallback(int32_t num_processed_chunks, + int32_t num_total_chunks, void *arg) { + float progress = 100.0 * num_processed_chunks / num_total_chunks; + fprintf(stderr, "progress %.2f%%\n", progress); + + // the return value is currently ignored + return 0; +} + +int main() { + // Please see the comments at the start of this file for how to download + // the .onnx file and .wav files below + const char *segmentation_model = + "./sherpa-onnx-pyannote-segmentation-3-0/model.onnx"; + + const char *embedding_extractor_model = + "./3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx"; + + const char *wav_filename = "./0-four-speakers-zh.wav"; + + const SherpaOnnxWave *wave = SherpaOnnxReadWave(wav_filename); + if (wave == NULL) { + fprintf(stderr, "Failed to read %s\n", wav_filename); + return -1; + } + + SherpaOnnxOfflineSpeakerDiarizationConfig config; + memset(&config, 0, sizeof(config)); + + config.segmentation.pyannote.model = segmentation_model; + config.embedding.model = embedding_extractor_model; + + // the test wave ./0-four-speakers-zh.wav has 4 speakers, so + // we set num_clusters to 4 + // + config.clustering.num_clusters = 4; + // If you don't know the number of speakers in the test wave file, please + // use + // config.clustering.threshold = 0.5; // You need to tune this threshold + + const SherpaOnnxOfflineSpeakerDiarization *sd = + SherpaOnnxCreateOfflineSpeakerDiarization(&config); + + if (!sd) { + fprintf(stderr, "Failed to initialize offline speaker diarization\n"); + return -1; + } + + if (SherpaOnnxOfflineSpeakerDiarizationGetSampleRate(sd) != + wave->sample_rate) { + fprintf( + stderr, + "Expected sample rate: %d. Actual sample rate from the wave file: %d\n", + SherpaOnnxOfflineSpeakerDiarizationGetSampleRate(sd), + wave->sample_rate); + goto failed; + } + + const SherpaOnnxOfflineSpeakerDiarizationResult *result = + SherpaOnnxOfflineSpeakerDiarizationProcessWithCallback( + sd, wave->samples, wave->num_samples, ProgressCallback, NULL); + if (!result) { + fprintf(stderr, "Failed to do speaker diarization"); + goto failed; + } + + int32_t num_segments = + SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments(result); + + const SherpaOnnxOfflineSpeakerDiarizationSegment *segments = + SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime(result); + + for (int32_t i = 0; i != num_segments; ++i) { + fprintf(stderr, "%.3f -- %.3f speaker_%02d\n", segments[i].start, + segments[i].end, segments[i].speaker); + } + +failed: + + SherpaOnnxOfflineSpeakerDiarizationDestroySegment(segments); + SherpaOnnxOfflineSpeakerDiarizationDestroyResult(result); + SherpaOnnxDestroyOfflineSpeakerDiarization(sd); + SherpaOnnxFreeWave(wave); + + return 0; +} diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index 176557c75..322c4f79e 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -31,6 +31,10 @@ #include "sherpa-onnx/csrc/offline-tts.h" #endif +#if SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION == 1 +#include "sherpa-onnx/csrc/offline-speaker-diarization.h" +#endif + struct SherpaOnnxOnlineRecognizer { std::unique_ptr impl; }; @@ -1670,3 +1674,144 @@ void SherpaOnnxLinearResamplerReset(SherpaOnnxLinearResampler *p) { int32_t SherpaOnnxFileExists(const char *filename) { return sherpa_onnx::FileExists(filename); } + +#if SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION == 1 + +struct SherpaOnnxOfflineSpeakerDiarization { + std::unique_ptr impl; +}; + +struct SherpaOnnxOfflineSpeakerDiarizationResult { + sherpa_onnx::OfflineSpeakerDiarizationResult impl; +}; + +const SherpaOnnxOfflineSpeakerDiarization * +SherpaOnnxCreateOfflineSpeakerDiarization( + const SherpaOnnxOfflineSpeakerDiarizationConfig *config) { + sherpa_onnx::OfflineSpeakerDiarizationConfig sd_config; + + sd_config.segmentation.pyannote.model = + SHERPA_ONNX_OR(config->segmentation.pyannote.model, ""); + sd_config.segmentation.num_threads = + SHERPA_ONNX_OR(config->segmentation.num_threads, 1); + sd_config.segmentation.debug = config->segmentation.debug; + sd_config.segmentation.provider = + SHERPA_ONNX_OR(config->segmentation.provider, "cpu"); + if (sd_config.segmentation.provider.empty()) { + sd_config.segmentation.provider = "cpu"; + } + + sd_config.embedding.model = SHERPA_ONNX_OR(config->embedding.model, ""); + sd_config.embedding.num_threads = + SHERPA_ONNX_OR(config->embedding.num_threads, 1); + sd_config.embedding.debug = config->embedding.debug; + sd_config.embedding.provider = + SHERPA_ONNX_OR(config->embedding.provider, "cpu"); + if (sd_config.embedding.provider.empty()) { + sd_config.embedding.provider = "cpu"; + } + + sd_config.clustering.num_clusters = + SHERPA_ONNX_OR(config->clustering.num_clusters, -1); + + sd_config.clustering.threshold = + SHERPA_ONNX_OR(config->clustering.threshold, 0.5); + + sd_config.min_duration_on = SHERPA_ONNX_OR(config->min_duration_on, 0.3); + + sd_config.min_duration_off = SHERPA_ONNX_OR(config->min_duration_off, 0.5); + + if (!sd_config.Validate()) { + SHERPA_ONNX_LOGE("Errors in config"); + return nullptr; + } + + SherpaOnnxOfflineSpeakerDiarization *sd = + new SherpaOnnxOfflineSpeakerDiarization; + + sd->impl = + std::make_unique(sd_config); + + if (sd_config.segmentation.debug || sd_config.embedding.debug) { + SHERPA_ONNX_LOGE("%s\n", sd_config.ToString().c_str()); + } + + return sd; +} + +void SherpaOnnxDestroyOfflineSpeakerDiarization( + const SherpaOnnxOfflineSpeakerDiarization *sd) { + delete sd; +} + +int32_t SherpaOnnxOfflineSpeakerDiarizationGetSampleRate( + const SherpaOnnxOfflineSpeakerDiarization *sd) { + return sd->impl->SampleRate(); +} + +int32_t SherpaOnnxOfflineSpeakerDiarizationResultGetNumSpeakers( + const SherpaOnnxOfflineSpeakerDiarizationResult *r) { + return r->impl.NumSpeakers(); +} + +int32_t SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments( + const SherpaOnnxOfflineSpeakerDiarizationResult *r) { + return r->impl.NumSegments(); +} + +const SherpaOnnxOfflineSpeakerDiarizationSegment * +SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime( + const SherpaOnnxOfflineSpeakerDiarizationResult *r) { + if (r->impl.NumSegments() == 0) { + return nullptr; + } + + auto segments = r->impl.SortByStartTime(); + + int32_t n = segments.size(); + SherpaOnnxOfflineSpeakerDiarizationSegment *ans = + new SherpaOnnxOfflineSpeakerDiarizationSegment[n]; + + for (int32_t i = 0; i != n; ++i) { + const auto &s = segments[i]; + + ans[i].start = s.Start(); + ans[i].end = s.End(); + ans[i].speaker = s.Speaker(); + } + + return ans; +} + +void SherpaOnnxOfflineSpeakerDiarizationDestroySegment( + const SherpaOnnxOfflineSpeakerDiarizationSegment *s) { + delete[] s; +} + +const SherpaOnnxOfflineSpeakerDiarizationResult * +SherpaOnnxOfflineSpeakerDiarizationProcess( + const SherpaOnnxOfflineSpeakerDiarization *sd, const float *samples, + int32_t n) { + auto ans = new SherpaOnnxOfflineSpeakerDiarizationResult; + ans->impl = sd->impl->Process(samples, n); + + return ans; +} + +void SherpaOnnxOfflineSpeakerDiarizationDestroyResult( + const SherpaOnnxOfflineSpeakerDiarizationResult *r) { + delete r; +} + +const SherpaOnnxOfflineSpeakerDiarizationResult * +SherpaOnnxOfflineSpeakerDiarizationProcessWithCallback( + const SherpaOnnxOfflineSpeakerDiarization *sd, const float *samples, + int32_t n, SherpaOnnxOfflineSpeakerDiarizationProgressCallback callback, + void *arg) { + auto ans = new SherpaOnnxOfflineSpeakerDiarizationResult; + ans->impl = sd->impl->Process(samples, n, callback, arg); + + return ans; +} + +#endif diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index 58615fe48..d378dedec 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -927,7 +927,7 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOfflineTts SherpaOnnxOfflineTts; SHERPA_ONNX_API SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTts( const SherpaOnnxOfflineTtsConfig *config); -// Free the pointer returned by CreateOfflineTts() +// Free the pointer returned by SherpaOnnxCreateOfflineTts() SHERPA_ONNX_API void SherpaOnnxDestroyOfflineTts(SherpaOnnxOfflineTts *tts); // Return the sample rate of the current TTS object @@ -954,6 +954,11 @@ SherpaOnnxOfflineTtsGenerateWithCallback( const SherpaOnnxOfflineTts *tts, const char *text, int32_t sid, float speed, SherpaOnnxGeneratedAudioCallback callback); +const SherpaOnnxGeneratedAudio * +SherpaOnnxOfflineTtsGenerateWithProgressCallback( + const SherpaOnnxOfflineTts *tts, const char *text, int32_t sid, float speed, + SherpaOnnxGeneratedAudioProgressCallback callback); + // Same as SherpaOnnxGeneratedAudioCallback but you can pass an additional // `void* arg` to the callback. SHERPA_ONNX_API const SherpaOnnxGeneratedAudio * @@ -1384,6 +1389,115 @@ SHERPA_ONNX_API int32_t SherpaOnnxLinearResamplerResampleGetOutputSampleRate( // Return 1 if the file exists; return 0 if the file does not exist. SHERPA_ONNX_API int32_t SherpaOnnxFileExists(const char *filename); +// ========================================================================= +// For offline speaker diarization (i.e., non-streaming speaker diarization) +// ========================================================================= +SHERPA_ONNX_API typedef struct + SherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig { + const char *model; +} SherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig; + +SHERPA_ONNX_API typedef struct SherpaOnnxOfflineSpeakerSegmentationModelConfig { + SherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig pyannote; + int32_t num_threads; // 1 + int32_t debug; // false + const char *provider; // "cpu" +} SherpaOnnxOfflineSpeakerSegmentationModelConfig; + +SHERPA_ONNX_API typedef struct SherpaOnnxFastClusteringConfig { + // If greater than 0, then threshold is ignored. + // + // We strongly recommend that you set it if you know the number of clusters + // in advance + int32_t num_clusters; + + // distance threshold. + // + // The smaller, the more clusters it will generate. + // The larger, the fewer clusters it will generate. + float threshold; +} SherpaOnnxFastClusteringConfig; + +SHERPA_ONNX_API typedef struct SherpaOnnxOfflineSpeakerDiarizationConfig { + SherpaOnnxOfflineSpeakerSegmentationModelConfig segmentation; + SherpaOnnxSpeakerEmbeddingExtractorConfig embedding; + SherpaOnnxFastClusteringConfig clustering; + + // if a segment is less than this value, then it is discarded + float min_duration_on; // in seconds + + // if the gap between to segments of the same speaker is less than this value, + // then these two segments are merged into a single segment. + // We do this recursively. + float min_duration_off; // in seconds +} SherpaOnnxOfflineSpeakerDiarizationConfig; + +SHERPA_ONNX_API typedef struct SherpaOnnxOfflineSpeakerDiarization + SherpaOnnxOfflineSpeakerDiarization; + +// The users has to invoke SherpaOnnxDestroyOfflineSpeakerDiarization() +// to free the returned pointer to avoid memory leak +SHERPA_ONNX_API const SherpaOnnxOfflineSpeakerDiarization * +SherpaOnnxCreateOfflineSpeakerDiarization( + const SherpaOnnxOfflineSpeakerDiarizationConfig *config); + +// Free the pointer returned by SherpaOnnxCreateOfflineSpeakerDiarization() +SHERPA_ONNX_API void SherpaOnnxDestroyOfflineSpeakerDiarization( + const SherpaOnnxOfflineSpeakerDiarization *sd); + +// Expected sample rate of the input audio samples +SHERPA_ONNX_API int32_t SherpaOnnxOfflineSpeakerDiarizationGetSampleRate( + const SherpaOnnxOfflineSpeakerDiarization *sd); + +SHERPA_ONNX_API typedef struct SherpaOnnxOfflineSpeakerDiarizationResult + SherpaOnnxOfflineSpeakerDiarizationResult; + +SHERPA_ONNX_API typedef struct SherpaOnnxOfflineSpeakerDiarizationSegment { + float start; + float end; + int32_t speaker; +} SherpaOnnxOfflineSpeakerDiarizationSegment; + +SHERPA_ONNX_API int32_t SherpaOnnxOfflineSpeakerDiarizationResultGetNumSpeakers( + const SherpaOnnxOfflineSpeakerDiarizationResult *r); + +SHERPA_ONNX_API int32_t SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments( + const SherpaOnnxOfflineSpeakerDiarizationResult *r); + +// The user has to invoke SherpaOnnxOfflineSpeakerDiarizationDestroySegment() +// to free the returned pointer to avoid memory leak. +// +// The returned pointer is the start address of an array. +// Number of entries in the array equals to the value +// returned by SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments() +SHERPA_ONNX_API const SherpaOnnxOfflineSpeakerDiarizationSegment * +SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime( + const SherpaOnnxOfflineSpeakerDiarizationResult *r); + +SHERPA_ONNX_API void SherpaOnnxOfflineSpeakerDiarizationDestroySegment( + const SherpaOnnxOfflineSpeakerDiarizationSegment *s); + +typedef int32_t (*SherpaOnnxOfflineSpeakerDiarizationProgressCallback)( + int32_t num_processed_chunk, int32_t num_total_chunks, void *arg); + +// The user has to invoke SherpaOnnxOfflineSpeakerDiarizationDestroyResult() +// to free the returned pointer to avoid memory leak. +SHERPA_ONNX_API const SherpaOnnxOfflineSpeakerDiarizationResult * +SherpaOnnxOfflineSpeakerDiarizationProcess( + const SherpaOnnxOfflineSpeakerDiarization *sd, const float *samples, + int32_t n); + +// The user has to invoke SherpaOnnxOfflineSpeakerDiarizationDestroyResult() +// to free the returned pointer to avoid memory leak. +SHERPA_ONNX_API const SherpaOnnxOfflineSpeakerDiarizationResult * +SherpaOnnxOfflineSpeakerDiarizationProcessWithCallback( + const SherpaOnnxOfflineSpeakerDiarization *sd, const float *samples, + int32_t n, SherpaOnnxOfflineSpeakerDiarizationProgressCallback callback, + void *arg); + +SHERPA_ONNX_API void SherpaOnnxOfflineSpeakerDiarizationDestroyResult( + const SherpaOnnxOfflineSpeakerDiarizationResult *r); + #if defined(__GNUC__) #pragma GCC diagnostic pop #endif diff --git a/sherpa-onnx/csrc/fast-clustering-config.h b/sherpa-onnx/csrc/fast-clustering-config.h index 9b190d46b..4abf2b128 100644 --- a/sherpa-onnx/csrc/fast-clustering-config.h +++ b/sherpa-onnx/csrc/fast-clustering-config.h @@ -20,8 +20,8 @@ struct FastClusteringConfig { // distance threshold. // - // The lower, the more clusters it will generate. - // The higher, the fewer clusters it will generate. + // The smaller, the more clusters it will generate. + // The larger, the fewer clusters it will generate. float threshold = 0.5; FastClusteringConfig() = default; diff --git a/sherpa-onnx/csrc/offline-speaker-diarization.cc b/sherpa-onnx/csrc/offline-speaker-diarization.cc index aeff9b42d..4748b1cb4 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization.cc +++ b/sherpa-onnx/csrc/offline-speaker-diarization.cc @@ -43,6 +43,16 @@ bool OfflineSpeakerDiarizationConfig::Validate() const { return false; } + if (min_duration_on < 0) { + SHERPA_ONNX_LOGE("min_duration_on %.3f is negative", min_duration_on); + return false; + } + + if (min_duration_off < 0) { + SHERPA_ONNX_LOGE("min_duration_off %.3f is negative", min_duration_off); + return false; + } + return true; } diff --git a/sherpa-onnx/csrc/sherpa-onnx-offline-speaker-diarization.cc b/sherpa-onnx/csrc/sherpa-onnx-offline-speaker-diarization.cc index 170973114..31cda85fc 100644 --- a/sherpa-onnx/csrc/sherpa-onnx-offline-speaker-diarization.cc +++ b/sherpa-onnx/csrc/sherpa-onnx-offline-speaker-diarization.cc @@ -7,7 +7,7 @@ #include "sherpa-onnx/csrc/wave-reader.h" static int32_t ProgressCallback(int32_t processed_chunks, int32_t num_chunks, - void *arg) { + void *) { float progress = 100.0 * processed_chunks / num_chunks; fprintf(stderr, "progress %.2f%%\n", progress); From 97654122fa42ec7d029f7722327ee3381d45e5e3 Mon Sep 17 00:00:00 2001 From: Yongzeng Liu <18193094+YogiLiu@users.noreply.github.com> Date: Wed, 9 Oct 2024 18:12:41 +0800 Subject: [PATCH 004/183] docs(nodejs-addon-examples): add guide for pnpm user (#1401) --- nodejs-addon-examples/README.md | 14 +++++++++++++- scripts/node-addon-api/lib/addon.js | 15 +++++++++++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/nodejs-addon-examples/README.md b/nodejs-addon-examples/README.md index b979c5126..ef0c25395 100644 --- a/nodejs-addon-examples/README.md +++ b/nodejs-addon-examples/README.md @@ -12,19 +12,31 @@ Note: [../nodejs-examples](../nodejs-examples) uses WebAssembly to wrap Before you continue, please first run ```bash -npm install +npm install # or pnpm install # For macOS x64 +## With npm export DYLD_LIBRARY_PATH=$PWD/node_modules/sherpa-onnx-darwin-x64:$DYLD_LIBRARY_PATH +## With pnpm +export DYLD_LIBRARY_PATH=$PWD/node_modules/.pnpm/sherpa-onnx-node@/node_modules/sherpa-onnx-darwin-x64:$DYLD_LIBRARY_PATH # For macOS arm64 +## With npm export DYLD_LIBRARY_PATH=$PWD/node_modules/sherpa-onnx-darwin-arm64:$DYLD_LIBRARY_PATH +## With pnpm +export DYLD_LIBRARY_PATH=$PWD/node_modules/.pnpm/sherpa-onnx-node@/node_modules/sherpa-onnx-darwin-arm64:$DYLD_LIBRARY_PATH # For Linux x64 +## With npm export LD_LIBRARY_PATH=$PWD/node_modules/sherpa-onnx-linux-x64:$LD_LIBRARY_PATH +## With pnpm +export LD_LIBRARY_PATH=$PWD/node_modules/.pnpm/sherpa-onnx-node@/node_modules/sherpa-onnx-linux-x64:$LD_LIBRARY_PATH # For Linux arm64, e.g., Raspberry Pi 4 +## With npm export LD_LIBRARY_PATH=$PWD/node_modules/sherpa-onnx-linux-arm64:$LD_LIBRARY_PATH +## With pnpm +export LD_LIBRARY_PATH=$PWD/node_modules/.pnpm/sherpa-onnx-node@/node_modules/sherpa-onnx-linux-arm64:$LD_LIBRARY_PATH ``` # Examples diff --git a/scripts/node-addon-api/lib/addon.js b/scripts/node-addon-api/lib/addon.js index 9ba19351f..840ead710 100644 --- a/scripts/node-addon-api/lib/addon.js +++ b/scripts/node-addon-api/lib/addon.js @@ -1,4 +1,5 @@ const os = require('os'); +const path = require('path'); // Package name triggered spam for sherpa-onnx-win32-x64 // so we have renamed it to sherpa-onnx-win-x64 @@ -25,6 +26,14 @@ for (const p of possible_paths) { } if (!found) { + let addon_path = `${process.env.PWD}/node_modules/sherpa-onnx-${platform_arch}`; + const pnpmIndex = __dirname.indexOf(`node_modules${path.sep}.pnpm`); + if (pnpmIndex !== -1) { + const parts = __dirname.slice(pnpmIndex).split(path.sep); + parts.pop(); + addon_path = `${process.env.PWD}/${parts.join('/')}/sherpa-onnx-${platform_arch}`; + } + let msg = `Could not find sherpa-onnx-node. Tried\n\n ${ possible_paths.join('\n ')}\n` if (os.platform() == 'darwin' && @@ -34,8 +43,7 @@ if (!found) { msg += 'Please remeber to set the following environment variable and try again:\n'; - msg += `export DYLD_LIBRARY_PATH=${ - process.env.PWD}/node_modules/sherpa-onnx-${platform_arch}`; + msg += `export DYLD_LIBRARY_PATH=${addon_path}`; msg += ':$DYLD_LIBRARY_PATH\n'; } @@ -47,8 +55,7 @@ if (!found) { msg += 'Please remeber to set the following environment variable and try again:\n'; - msg += `export LD_LIBRARY_PATH=${ - process.env.PWD}/node_modules/sherpa-onnx-${platform_arch}`; + msg += `export LD_LIBRARY_PATH=${addon_path}`; msg += ':$LD_LIBRARY_PATH\n'; } From df681e980722b692a2c830224d9196debee3508a Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 9 Oct 2024 20:10:44 +0800 Subject: [PATCH 005/183] Go API for speaker diarization (#1403) --- .github/workflows/test-go-package.yaml | 44 +++++++ .github/workflows/test-go.yaml | 6 + .../non-streaming-speaker-diarization/go.mod | 3 + .../non-streaming-speaker-diarization/main.go | 87 +++++++++++++ .../non-streaming-speaker-diarization/run.sh | 20 +++ .../non-streaming-speaker-diarization/go.mod | 5 + .../non-streaming-speaker-diarization/main.go | 1 + .../non-streaming-speaker-diarization/run.sh | 1 + scripts/go/sherpa_onnx.go | 118 ++++++++++++++++++ 9 files changed, 285 insertions(+) create mode 100644 go-api-examples/non-streaming-speaker-diarization/go.mod create mode 100644 go-api-examples/non-streaming-speaker-diarization/main.go create mode 100755 go-api-examples/non-streaming-speaker-diarization/run.sh create mode 100644 scripts/go/_internal/non-streaming-speaker-diarization/go.mod create mode 120000 scripts/go/_internal/non-streaming-speaker-diarization/main.go create mode 120000 scripts/go/_internal/non-streaming-speaker-diarization/run.sh diff --git a/.github/workflows/test-go-package.yaml b/.github/workflows/test-go-package.yaml index 2634e5ca7..6839b0bab 100644 --- a/.github/workflows/test-go-package.yaml +++ b/.github/workflows/test-go-package.yaml @@ -68,6 +68,50 @@ jobs: run: | gcc --version + - name: Test non-streaming speaker diarization + if: matrix.os != 'windows-latest' + shell: bash + run: | + cd go-api-examples/non-streaming-speaker-diarization/ + ./run.sh + + - name: Test non-streaming speaker diarization + if: matrix.os == 'windows-latest' && matrix.arch == 'x64' + shell: bash + run: | + cd go-api-examples/non-streaming-speaker-diarization/ + go mod tidy + cat go.mod + go build + + echo $PWD + ls -lh /C/Users/runneradmin/go/pkg/mod/github.com/k2-fsa/ + ls -lh /C/Users/runneradmin/go/pkg/mod/github.com/k2-fsa/* + cp -v /C/Users/runneradmin/go/pkg/mod/github.com/k2-fsa/sherpa-onnx-go-windows*/lib/x86_64-pc-windows-gnu/*.dll . + + ./run.sh + + - name: Test non-streaming speaker diarization + if: matrix.os == 'windows-latest' && matrix.arch == 'x86' + shell: bash + run: | + cd go-api-examples/non-streaming-speaker-diarization/ + + go env GOARCH + go env -w GOARCH=386 + go env -w CGO_ENABLED=1 + + go mod tidy + cat go.mod + go build + + echo $PWD + ls -lh /C/Users/runneradmin/go/pkg/mod/github.com/k2-fsa/ + ls -lh /C/Users/runneradmin/go/pkg/mod/github.com/k2-fsa/* + cp -v /C/Users/runneradmin/go/pkg/mod/github.com/k2-fsa/sherpa-onnx-go-windows*/lib/i686-pc-windows-gnu/*.dll . + + ./run.sh + - name: Test streaming HLG decoding (Linux/macOS) if: matrix.os != 'windows-latest' shell: bash diff --git a/.github/workflows/test-go.yaml b/.github/workflows/test-go.yaml index 65c72e174..ccb3eb8d9 100644 --- a/.github/workflows/test-go.yaml +++ b/.github/workflows/test-go.yaml @@ -134,6 +134,12 @@ jobs: name: ${{ matrix.os }}-libs path: to-upload/ + - name: Test non-streaming speaker diarization + shell: bash + run: | + cd scripts/go/_internal/non-streaming-speaker-diarization/ + ./run.sh + - name: Test speaker identification shell: bash run: | diff --git a/go-api-examples/non-streaming-speaker-diarization/go.mod b/go-api-examples/non-streaming-speaker-diarization/go.mod new file mode 100644 index 000000000..39edcecf5 --- /dev/null +++ b/go-api-examples/non-streaming-speaker-diarization/go.mod @@ -0,0 +1,3 @@ +module non-streaming-speaker-diarization + +go 1.12 diff --git a/go-api-examples/non-streaming-speaker-diarization/main.go b/go-api-examples/non-streaming-speaker-diarization/main.go new file mode 100644 index 000000000..7b975bf61 --- /dev/null +++ b/go-api-examples/non-streaming-speaker-diarization/main.go @@ -0,0 +1,87 @@ +package main + +import ( + sherpa "github.com/k2-fsa/sherpa-onnx-go/sherpa_onnx" + "log" +) + +/* +Usage: + +Step 1: Download a speaker segmentation model + +Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models +for a list of available models. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + +Step 2: Download a speaker embedding extractor model + +Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models +for a list of available models. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx + +Step 3. Download test wave files + +Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models +for a list of available test wave files. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/0-four-speakers-zh.wav + +Step 4. Run it +*/ + +func initSpeakerDiarization() *sherpa.OfflineSpeakerDiarization { + config := sherpa.OfflineSpeakerDiarizationConfig{} + + config.Segmentation.Pyannote.Model = "./sherpa-onnx-pyannote-segmentation-3-0/model.onnx" + config.Embedding.Model = "./3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx" + + // The test wave file contains 4 speakers, so we use 4 here + config.Clustering.NumClusters = 4 + + // if you don't know the actual numbers in the wave file, + // then please don't set NumClusters; you need to use + // + // config.Clustering.Threshold = 0.5 + // + + // A larger Threshold leads to fewer clusters + // A smaller Threshold leads to more clusters + + sd := sherpa.NewOfflineSpeakerDiarization(&config) + return sd +} + +func main() { + wave_filename := "./0-four-speakers-zh.wav" + wave := sherpa.ReadWave(wave_filename) + if wave == nil { + log.Printf("Failed to read %v", wave_filename) + return + } + + sd := initSpeakerDiarization() + if sd == nil { + log.Printf("Please check your config") + return + } + + defer sherpa.DeleteOfflineSpeakerDiarization(sd) + + if wave.SampleRate != sd.SampleRate() { + log.Printf("Expected sample rate: %v, given: %d\n", sd.SampleRate(), wave.SampleRate) + return + } + + log.Println("Started") + segments := sd.Process(wave.Samples) + n := len(segments) + + for i := 0; i < n; i++ { + log.Printf("%.3f -- %.3f speaker_%02d\n", segments[i].Start, segments[i].End, segments[i].Speaker) + } +} diff --git a/go-api-examples/non-streaming-speaker-diarization/run.sh b/go-api-examples/non-streaming-speaker-diarization/run.sh new file mode 100755 index 000000000..1ebfd4aa1 --- /dev/null +++ b/go-api-examples/non-streaming-speaker-diarization/run.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + + +if [ ! -f ./sherpa-onnx-pyannote-segmentation-3-0/model.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 +fi + +if [ ! -f ./3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx +fi + +if [ ! -f ./0-four-speakers-zh.wav ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/0-four-speakers-zh.wav +fi + +go mod tidy +go build +./non-streaming-speaker-diarization diff --git a/scripts/go/_internal/non-streaming-speaker-diarization/go.mod b/scripts/go/_internal/non-streaming-speaker-diarization/go.mod new file mode 100644 index 000000000..38ae36d4d --- /dev/null +++ b/scripts/go/_internal/non-streaming-speaker-diarization/go.mod @@ -0,0 +1,5 @@ +module non-streaming-speaker-diarization + +go 1.12 + +replace github.com/k2-fsa/sherpa-onnx-go/sherpa_onnx => ../ diff --git a/scripts/go/_internal/non-streaming-speaker-diarization/main.go b/scripts/go/_internal/non-streaming-speaker-diarization/main.go new file mode 120000 index 000000000..2e4da65e0 --- /dev/null +++ b/scripts/go/_internal/non-streaming-speaker-diarization/main.go @@ -0,0 +1 @@ +../../../../go-api-examples/non-streaming-speaker-diarization/main.go \ No newline at end of file diff --git a/scripts/go/_internal/non-streaming-speaker-diarization/run.sh b/scripts/go/_internal/non-streaming-speaker-diarization/run.sh new file mode 120000 index 000000000..0746440f7 --- /dev/null +++ b/scripts/go/_internal/non-streaming-speaker-diarization/run.sh @@ -0,0 +1 @@ +../../../../go-api-examples/non-streaming-speaker-diarization/run.sh \ No newline at end of file diff --git a/scripts/go/sherpa_onnx.go b/scripts/go/sherpa_onnx.go index 85ab8e0b1..b8b9e6ee2 100644 --- a/scripts/go/sherpa_onnx.go +++ b/scripts/go/sherpa_onnx.go @@ -1175,7 +1175,14 @@ func ReadWave(filename string) *Wave { w := C.SherpaOnnxReadWave(s) defer C.SherpaOnnxFreeWave(w) + if w == nil { + return nil + } + n := int(w.num_samples) + if n == 0 { + return nil + } ans := &Wave{} ans.SampleRate = int(w.sample_rate) @@ -1189,3 +1196,114 @@ func ReadWave(filename string) *Wave { return ans } + +// ============================================================ +// For offline speaker diarization +// ============================================================ +type OfflineSpeakerSegmentationPyannoteModelConfig struct { + Model string +} + +type OfflineSpeakerSegmentationModelConfig struct { + Pyannote OfflineSpeakerSegmentationPyannoteModelConfig + NumThreads int + Debug int + Provider string +} + +type FastClusteringConfig struct { + NumClusters int + Threshold float32 +} + +type OfflineSpeakerDiarizationConfig struct { + Segmentation OfflineSpeakerSegmentationModelConfig + Embedding SpeakerEmbeddingExtractorConfig + Clustering FastClusteringConfig + MinDurationOn float32 + MinDurationOff float32 +} + +type OfflineSpeakerDiarization struct { + impl *C.struct_SherpaOnnxOfflineSpeakerDiarization +} + +func DeleteOfflineSpeakerDiarization(sd *OfflineSpeakerDiarization) { + C.SherpaOnnxDestroyOfflineSpeakerDiarization(sd.impl) + sd.impl = nil +} + +func NewOfflineSpeakerDiarization(config *OfflineSpeakerDiarizationConfig) *OfflineSpeakerDiarization { + c := C.struct_SherpaOnnxOfflineSpeakerDiarizationConfig{} + c.segmentation.pyannote.model = C.CString(config.Segmentation.Pyannote.Model) + defer C.free(unsafe.Pointer(c.segmentation.pyannote.model)) + + c.segmentation.num_threads = C.int(config.Segmentation.NumThreads) + + c.segmentation.debug = C.int(config.Segmentation.Debug) + + c.segmentation.provider = C.CString(config.Segmentation.Provider) + defer C.free(unsafe.Pointer(c.segmentation.provider)) + + c.embedding.model = C.CString(config.Embedding.Model) + defer C.free(unsafe.Pointer(c.embedding.model)) + + c.embedding.num_threads = C.int(config.Embedding.NumThreads) + + c.embedding.debug = C.int(config.Embedding.Debug) + + c.embedding.provider = C.CString(config.Embedding.Provider) + defer C.free(unsafe.Pointer(c.embedding.provider)) + + c.clustering.num_clusters = C.int(config.Clustering.NumClusters) + c.clustering.threshold = C.float(config.Clustering.Threshold) + c.min_duration_on = C.float(config.MinDurationOn) + c.min_duration_off = C.float(config.MinDurationOff) + + p := C.SherpaOnnxCreateOfflineSpeakerDiarization(&c) + + if p == nil { + return nil + } + + sd := &OfflineSpeakerDiarization{} + sd.impl = p + + return sd +} + +func (sd *OfflineSpeakerDiarization) SampleRate() int { + return int(C.SherpaOnnxOfflineSpeakerDiarizationGetSampleRate(sd.impl)) +} + +type OfflineSpeakerDiarizationSegment struct { + Start float32 + End float32 + Speaker int +} + +func (sd *OfflineSpeakerDiarization) Process(samples []float32) []OfflineSpeakerDiarizationSegment { + r := C.SherpaOnnxOfflineSpeakerDiarizationProcess(sd.impl, (*C.float)(&samples[0]), C.int(len(samples))) + defer C.SherpaOnnxOfflineSpeakerDiarizationDestroyResult(r) + + n := int(C.SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments(r)) + + if n == 0 { + return nil + } + + s := C.SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime(r) + defer C.SherpaOnnxOfflineSpeakerDiarizationDestroySegment(s) + + ans := make([]OfflineSpeakerDiarizationSegment, n) + + p := (*[1 << 28]C.struct_SherpaOnnxOfflineSpeakerDiarizationSegment)(unsafe.Pointer(s))[:n:n] + + for i := 0; i < n; i++ { + ans[i].Start = float32(p[i].start) + ans[i].End = float32(p[i].end) + ans[i].Speaker = int(p[i].speaker) + } + + return ans +} From 15713445095fc8bedef7087079021b1eb31ed08b Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 9 Oct 2024 23:25:39 +0800 Subject: [PATCH 006/183] Swift API for speaker diarization (#1404) --- .github/scripts/test-swift.sh | 5 + swift-api-examples/SherpaOnnx.swift | 113 ++++++++++++++++++ swift-api-examples/run-speaker-diarization.sh | 35 ++++++ swift-api-examples/speaker-diarization.swift | 56 +++++++++ 4 files changed, 209 insertions(+) create mode 100755 swift-api-examples/run-speaker-diarization.sh create mode 100644 swift-api-examples/speaker-diarization.swift diff --git a/.github/scripts/test-swift.sh b/.github/scripts/test-swift.sh index 18c9bed41..0da23eb24 100755 --- a/.github/scripts/test-swift.sh +++ b/.github/scripts/test-swift.sh @@ -7,6 +7,11 @@ echo "pwd: $PWD" cd swift-api-examples ls -lh +./run-speaker-diarization.sh +rm -rf *.onnx +rm -rf sherpa-onnx-pyannote-segmentation-3-0 +rm -fv *.wav + ./run-add-punctuations.sh rm ./add-punctuations rm -rf sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12 diff --git a/swift-api-examples/SherpaOnnx.swift b/swift-api-examples/SherpaOnnx.swift index 778bccb9b..881291fd6 100644 --- a/swift-api-examples/SherpaOnnx.swift +++ b/swift-api-examples/SherpaOnnx.swift @@ -1078,3 +1078,116 @@ class SherpaOnnxOfflinePunctuationWrapper { return ans } } + +func sherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig(model: String) + -> SherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig +{ + return SherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig(model: toCPointer(model)) +} + +func sherpaOnnxOfflineSpeakerSegmentationModelConfig( + pyannote: SherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig, + numThreads: Int = 1, + debug: Int = 0, + provider: String = "cpu" +) -> SherpaOnnxOfflineSpeakerSegmentationModelConfig { + return SherpaOnnxOfflineSpeakerSegmentationModelConfig( + pyannote: pyannote, + num_threads: Int32(numThreads), + debug: Int32(debug), + provider: toCPointer(provider) + ) +} + +func sherpaOnnxFastClusteringConfig(numClusters: Int = -1, threshold: Float = 0.5) + -> SherpaOnnxFastClusteringConfig +{ + return SherpaOnnxFastClusteringConfig(num_clusters: Int32(numClusters), threshold: threshold) +} + +func sherpaOnnxSpeakerEmbeddingExtractorConfig( + model: String, + numThreads: Int = 1, + debug: Int = 0, + provider: String = "cpu" +) -> SherpaOnnxSpeakerEmbeddingExtractorConfig { + return SherpaOnnxSpeakerEmbeddingExtractorConfig( + model: toCPointer(model), + num_threads: Int32(numThreads), + debug: Int32(debug), + provider: toCPointer(provider) + ) +} + +func sherpaOnnxOfflineSpeakerDiarizationConfig( + segmentation: SherpaOnnxOfflineSpeakerSegmentationModelConfig, + embedding: SherpaOnnxSpeakerEmbeddingExtractorConfig, + clustering: SherpaOnnxFastClusteringConfig, + minDurationOn: Float = 0.3, + minDurationOff: Float = 0.5 +) -> SherpaOnnxOfflineSpeakerDiarizationConfig { + return SherpaOnnxOfflineSpeakerDiarizationConfig( + segmentation: segmentation, + embedding: embedding, + clustering: clustering, + min_duration_on: minDurationOn, + min_duration_off: minDurationOff + ) +} + +struct SherpaOnnxOfflineSpeakerDiarizationSegmentWrapper { + var start: Float = 0 + var end: Float = 0 + var speaker: Int = 0 +} + +class SherpaOnnxOfflineSpeakerDiarizationWrapper { + /// A pointer to the underlying counterpart in C + let impl: OpaquePointer! + + init( + config: UnsafePointer! + ) { + impl = SherpaOnnxCreateOfflineSpeakerDiarization(config) + } + + deinit { + if let impl { + SherpaOnnxDestroyOfflineSpeakerDiarization(impl) + } + } + + var sampleRate: Int { + return Int(SherpaOnnxOfflineSpeakerDiarizationGetSampleRate(impl)) + } + + func process(samples: [Float]) -> [SherpaOnnxOfflineSpeakerDiarizationSegmentWrapper] { + let result = SherpaOnnxOfflineSpeakerDiarizationProcess( + impl, samples, Int32(samples.count)) + + if result == nil { + return [] + } + + let numSegments = Int(SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments(result)) + + let p: UnsafePointer? = + SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime(result) + + if p == nil { + return [] + } + + var ans: [SherpaOnnxOfflineSpeakerDiarizationSegmentWrapper] = [] + for i in 0.. [Float] { + return Array(UnsafeBufferPointer(self)) + } +} + +extension AVAudioPCMBuffer { + func array() -> [Float] { + return self.audioBufferList.pointee.mBuffers.array() + } +} + +func run() { + let segmentationModel = "./sherpa-onnx-pyannote-segmentation-3-0/model.onnx" + let embeddingExtractorModel = "./3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx" + let waveFilename = "./0-four-speakers-zh.wav" + + // There are 4 speakers in ./0-four-speakers-zh.wav, so we use 4 here + let numSpeakers = 4 + var config = sherpaOnnxOfflineSpeakerDiarizationConfig( + segmentation: sherpaOnnxOfflineSpeakerSegmentationModelConfig( + pyannote: sherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig(model: segmentationModel)), + embedding: sherpaOnnxSpeakerEmbeddingExtractorConfig(model: embeddingExtractorModel), + clustering: sherpaOnnxFastClusteringConfig(numClusters: numSpeakers) + ) + + let sd = SherpaOnnxOfflineSpeakerDiarizationWrapper(config: &config) + + let fileURL: NSURL = NSURL(fileURLWithPath: waveFilename) + let audioFile = try! AVAudioFile(forReading: fileURL as URL) + + let audioFormat = audioFile.processingFormat + assert(Int(audioFormat.sampleRate) == sd.sampleRate) + assert(audioFormat.channelCount == 1) + assert(audioFormat.commonFormat == AVAudioCommonFormat.pcmFormatFloat32) + + let audioFrameCount = UInt32(audioFile.length) + let audioFileBuffer = AVAudioPCMBuffer(pcmFormat: audioFormat, frameCapacity: audioFrameCount) + + try! audioFile.read(into: audioFileBuffer!) + let array: [Float]! = audioFileBuffer?.array() + print("Started!") + let segments = sd.process(samples: array) + for i in 0.. Date: Thu, 10 Oct 2024 10:27:14 +0800 Subject: [PATCH 007/183] Update readme to include more external projects using sherpa-onnx (#1405) --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2f318d3ef..00fe61fd0 100644 --- a/README.md +++ b/README.md @@ -196,10 +196,18 @@ for 新一代 Kaldi **微信交流群** and **QQ 交流群**. It shows how to use the ASR and TTS Python APIs with FastAPI. -### [TMSpeech](https://github.com/jxlpzqc/TMSpeech) +### [腾讯会议摸鱼工具 TMSpeech](https://github.com/jxlpzqc/TMSpeech) Uses streaming ASR in C# with graphical user interface. +Video demo in Chinese: [【开源】Windows实时字幕软件（网课/开会必备）](https://www.bilibili.com/video/BV1rX4y1p7Nx) + +### [lol互动助手](https://github.com/l1veIn/lol-wom-electron) + +It uses the JavaScript API of sherpa-onnx along with [Electron](https://electronjs.org/) + +Video demo in Chinese: [爆了！炫神教你开打字挂！真正影响胜率的英雄联盟工具！英雄联盟的最后一块拼图！和游戏中的每个人无障碍沟通！](https://www.bilibili.com/video/BV142tje9E74) + [sherpa-rs]: https://github.com/thewh1teagle/sherpa-rs [silero-vad]: https://github.com/snakers4/silero-vad From a45e5dba9986f07f13d4f64b13ff77589a9909e3 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 10 Oct 2024 14:29:05 +0800 Subject: [PATCH 008/183] C# API for speaker diarization (#1407) --- .github/scripts/test-dot-net.sh | 8 +- .github/workflows/test-dot-net.yaml | 71 +++------- .../offline-speaker-diarization/Program.cs | 83 ++++++++++++ .../offline-speaker-diarization.csproj | 15 +++ .../offline-speaker-diarization/run.sh | 18 +++ dotnet-examples/sherpa-onnx.sln | 6 + scripts/dotnet/FastClusteringConfig.cs | 20 +++ scripts/dotnet/OfflineSpeakerDiarization.cs | 122 ++++++++++++++++++ .../dotnet/OfflineSpeakerDiarizationConfig.cs | 31 +++++ .../OfflineSpeakerDiarizationSegment.cs | 33 +++++ .../OfflineSpeakerSegmentationModelConfig.cs | 32 +++++ ...eSpeakerSegmentationPyannoteModelConfig.cs | 20 +++ 12 files changed, 408 insertions(+), 51 deletions(-) create mode 100644 dotnet-examples/offline-speaker-diarization/Program.cs create mode 100644 dotnet-examples/offline-speaker-diarization/offline-speaker-diarization.csproj create mode 100755 dotnet-examples/offline-speaker-diarization/run.sh create mode 100644 scripts/dotnet/FastClusteringConfig.cs create mode 100644 scripts/dotnet/OfflineSpeakerDiarization.cs create mode 100644 scripts/dotnet/OfflineSpeakerDiarizationConfig.cs create mode 100644 scripts/dotnet/OfflineSpeakerDiarizationSegment.cs create mode 100644 scripts/dotnet/OfflineSpeakerSegmentationModelConfig.cs create mode 100644 scripts/dotnet/OfflineSpeakerSegmentationPyannoteModelConfig.cs diff --git a/.github/scripts/test-dot-net.sh b/.github/scripts/test-dot-net.sh index c397fc0cd..eec3b6bb4 100755 --- a/.github/scripts/test-dot-net.sh +++ b/.github/scripts/test-dot-net.sh @@ -2,7 +2,13 @@ cd dotnet-examples/ -cd ./offline-decode-files +cd ./offline-speaker-diarization +./run.sh +rm -rfv *.onnx +rm -fv *.wav +rm -rfv sherpa-onnx-pyannote-* + +cd ../offline-decode-files ./run-sense-voice-ctc.sh rm -rf sherpa-onnx-* diff --git a/.github/workflows/test-dot-net.yaml b/.github/workflows/test-dot-net.yaml index 6e32b155e..d046542ba 100644 --- a/.github/workflows/test-dot-net.yaml +++ b/.github/workflows/test-dot-net.yaml @@ -47,53 +47,10 @@ jobs: with: fetch-depth: 0 - - name: Free space - if: matrix.os == 'ubuntu-latest' - shell: bash - run: | - df -h - rm -rf /opt/hostedtoolcache - df -h - - - name: Free more space - if: matrix.os == 'ubuntu-latest' - shell: bash - run: | - # https://github.com/orgs/community/discussions/25678 - cd /opt - find . -maxdepth 1 -mindepth 1 '!' -path ./containerd '!' -path ./actionarchivecache '!' -path ./runner '!' -path ./runner-cache -exec rm -rf '{}' ';' - - sudo rm -rf /usr/share/dotnet - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - - - name: Free Disk Space (Ubuntu) - if: matrix.os == 'ubuntu-latest' - uses: jlumbroso/free-disk-space@main - with: - # this might remove tools that are actually needed, - # if set to "true" but frees about 6 GB - tool-cache: false - - # all of these default to true, but feel free to set to - # "false" if necessary for your workflow - android: true - dotnet: false - haskell: true - large-packages: true - docker-images: false - swap-storage: true - - - name: Check space - if: matrix.os == 'ubuntu-latest' - shell: bash - run: | - df -h - - name: ccache uses: hendrikmuhs/ccache-action@v1.2 with: - key: ${{ matrix.os }}-release-shared + key: ${{ matrix.os }}-dotnet-release-shared - name: Build sherpa-onnx shell: bash @@ -110,11 +67,16 @@ jobs: -DCMAKE_BUILD_TYPE=Release \ -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ -DBUILD_ESPEAK_NG_EXE=OFF \ - -DSHERPA_ONNX_ENABLE_BINARY=ON \ + -DSHERPA_ONNX_ENABLE_BINARY=OFF \ .. cmake --build . --target install --config Release + rm -rf install/share + rm -rf install/lib/pkg* + + ls -lh ./install/lib + - uses: actions/upload-artifact@v4 with: name: ${{ matrix.os }} @@ -148,7 +110,7 @@ jobs: uses: actions/download-artifact@v4 with: name: ubuntu-latest - path: /tmp/linux + path: /tmp/linux-x64 - name: Setup .NET uses: actions/setup-dotnet@v4 @@ -162,17 +124,21 @@ jobs: - name: Display files shell: bash run: | - echo "----------/tmp/----------" - ls -lh /tmp/ + echo "----------/tmp----------" + ls -lh /tmp - echo "----------/tmp/linux----------" - ls -lh /tmp/linux + echo "----------/tmp/linux-x64----------" + ls -lh /tmp/linux-x64 + df -h - name: Build shell: bash run: | cd scripts/dotnet ./run.sh + df -h + + ls -lh /tmp/packages - name: Copy files shell: bash @@ -181,9 +147,14 @@ jobs: ls -lh /tmp + df -h + - name: Run tests shell: bash run: | + dotnet nuget locals all --clear + df -h + .github/scripts/test-dot-net.sh - uses: actions/upload-artifact@v4 diff --git a/dotnet-examples/offline-speaker-diarization/Program.cs b/dotnet-examples/offline-speaker-diarization/Program.cs new file mode 100644 index 000000000..45316fe75 --- /dev/null +++ b/dotnet-examples/offline-speaker-diarization/Program.cs @@ -0,0 +1,83 @@ +// Copyright (c) 2024 Xiaomi Corporation +// + +// This file shows how to use sherpa-onnx C# API for speaker diarization +/* +Usage: + +Step 1: Download a speaker segmentation model + +Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models +for a list of available models. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + +Step 2: Download a speaker embedding extractor model + +Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models +for a list of available models. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx + +Step 3. Download test wave files + +Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models +for a list of available test wave files. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/0-four-speakers-zh.wav + +Step 4. Run it + + dotnet run +*/ + +using SherpaOnnx; +using System; + +class OfflineSpeakerDiarizationDemo +{ + static void Main(string[] args) + { + var config = new OfflineSpeakerDiarizationConfig(); + config.Segmentation.Pyannote.Model = "./sherpa-onnx-pyannote-segmentation-3-0/model.onnx"; + config.Embedding.Model = "./3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx"; + + // the test wave ./0-four-speakers-zh.wav has 4 speakers, so + // we set num_clusters to 4 + // + config.Clustering.NumClusters = 4; + // If you don't know the number of speakers in the test wave file, please + // use + // config.Clustering.Threshold = 0.5; // You need to tune this threshold + var sd = new OfflineSpeakerDiarization(config); + + var testWaveFile = "./0-four-speakers-zh.wav"; + WaveReader waveReader = new WaveReader(testWaveFile); + if (sd.SampleRate != waveReader.SampleRate) + { + Console.WriteLine($"Expected sample rate: {sd.SampleRate}. Given: {waveReader.SampleRate}"); + return; + } + + Console.WriteLine("Started"); + + // var segments = sd.Process(waveReader.Samples); // this one is also ok + + var MyProgressCallback = (int numProcessedChunks, int numTotalChunks, IntPtr arg) => + { + float progress = 100.0F * numProcessedChunks / numTotalChunks; + Console.WriteLine("Progress {0}%", String.Format("{0:0.00}", progress)); + return 0; + }; + + var callback = new OfflineSpeakerDiarizationProgressCallback(MyProgressCallback); + var segments = sd.ProcessWithCallback(waveReader.Samples, callback, IntPtr.Zero); + + foreach (var s in segments) + { + Console.WriteLine("{0} -- {1} speaker_{2}", String.Format("{0:0.00}", s.Start), String.Format("{0:0.00}", s.End), s.Speaker); + } + } +} diff --git a/dotnet-examples/offline-speaker-diarization/offline-speaker-diarization.csproj b/dotnet-examples/offline-speaker-diarization/offline-speaker-diarization.csproj new file mode 100644 index 000000000..3374dbca1 --- /dev/null +++ b/dotnet-examples/offline-speaker-diarization/offline-speaker-diarization.csproj @@ -0,0 +1,15 @@ + + + + Exe + net6.0 + offline_speaker_diarization + enable + enable + + + + + + + diff --git a/dotnet-examples/offline-speaker-diarization/run.sh b/dotnet-examples/offline-speaker-diarization/run.sh new file mode 100755 index 000000000..fe64412f9 --- /dev/null +++ b/dotnet-examples/offline-speaker-diarization/run.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + + +if [ ! -f ./sherpa-onnx-pyannote-segmentation-3-0/model.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 +fi + +if [ ! -f ./3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx +fi + +if [ ! -f ./0-four-speakers-zh.wav ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/0-four-speakers-zh.wav +fi + +dotnet run diff --git a/dotnet-examples/sherpa-onnx.sln b/dotnet-examples/sherpa-onnx.sln index 397fe99e5..0bff03f5c 100644 --- a/dotnet-examples/sherpa-onnx.sln +++ b/dotnet-examples/sherpa-onnx.sln @@ -31,6 +31,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "keyword-spotting-from-micro EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TTS", "TTS\TTS.csproj", "{DACE4A18-4FC8-4437-92BF-5A90BA81286C}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "offline-speaker-diarization", "offline-speaker-diarization\offline-speaker-diarization.csproj", "{D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -93,6 +95,10 @@ Global {DACE4A18-4FC8-4437-92BF-5A90BA81286C}.Debug|Any CPU.Build.0 = Debug|Any CPU {DACE4A18-4FC8-4437-92BF-5A90BA81286C}.Release|Any CPU.ActiveCfg = Release|Any CPU {DACE4A18-4FC8-4437-92BF-5A90BA81286C}.Release|Any CPU.Build.0 = Release|Any CPU + {D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/scripts/dotnet/FastClusteringConfig.cs b/scripts/dotnet/FastClusteringConfig.cs new file mode 100644 index 000000000..276ef9d86 --- /dev/null +++ b/scripts/dotnet/FastClusteringConfig.cs @@ -0,0 +1,20 @@ +/// Copyright (c) 2024 Xiaomi Corporation + +using System.Runtime.InteropServices; + +namespace SherpaOnnx +{ + + [StructLayout(LayoutKind.Sequential)] + public struct FastClusteringConfig + { + public FastClusteringConfig() + { + NumClusters = -1; + Threshold = 0.5F; + } + + public int NumClusters; + public float Threshold; + } +} diff --git a/scripts/dotnet/OfflineSpeakerDiarization.cs b/scripts/dotnet/OfflineSpeakerDiarization.cs new file mode 100644 index 000000000..b56cab9b6 --- /dev/null +++ b/scripts/dotnet/OfflineSpeakerDiarization.cs @@ -0,0 +1,122 @@ +/// Copyright (c) 2024 Xiaomi Corporation +using System; +using System.Runtime.InteropServices; +using System.Text; + +namespace SherpaOnnx +{ + // IntPtr is actually a `const float*` from C++ + public delegate int OfflineSpeakerDiarizationProgressCallback(int numProcessedChunks, int numTotalChunks, IntPtr arg); + + public class OfflineSpeakerDiarization : IDisposable + { + public OfflineSpeakerDiarization(OfflineSpeakerDiarizationConfig config) + { + IntPtr h = SherpaOnnxCreateOfflineSpeakerDiarization(ref config); + _handle = new HandleRef(this, h); + } + + public OfflineSpeakerDiarizationSegment[] Process(float[] samples) + { + IntPtr result = SherpaOnnxOfflineSpeakerDiarizationProcess(_handle.Handle, samples, samples.Length); + return ProcessImpl(result); + } + + public OfflineSpeakerDiarizationSegment[] ProcessWithCallback(float[] samples, OfflineSpeakerDiarizationProgressCallback callback, IntPtr arg) + { + IntPtr result = SherpaOnnxOfflineSpeakerDiarizationProcessWithCallback(_handle.Handle, samples, samples.Length, callback, arg); + return ProcessImpl(result); + } + + private OfflineSpeakerDiarizationSegment[] ProcessImpl(IntPtr result) + { + if (result == IntPtr.Zero) + { + return new OfflineSpeakerDiarizationSegment[] {}; + } + + int numSegments = SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments(result); + IntPtr p = SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime(result); + + OfflineSpeakerDiarizationSegment[] ans = new OfflineSpeakerDiarizationSegment[numSegments]; + unsafe + { + int size = sizeof(float) * 2 + sizeof(int); + for (int i = 0; i != numSegments; ++i) + { + IntPtr t = new IntPtr((byte*)p + i * size); + ans[i] = new OfflineSpeakerDiarizationSegment(t); + + // The following IntPtr.Add() does not support net20 + // ans[i] = new OfflineSpeakerDiarizationSegment(IntPtr.Add(p, i)); + } + } + + + SherpaOnnxOfflineSpeakerDiarizationDestroySegment(p); + SherpaOnnxOfflineSpeakerDiarizationDestroyResult(result); + + return ans; + + } + + public void Dispose() + { + Cleanup(); + // Prevent the object from being placed on the + // finalization queue + System.GC.SuppressFinalize(this); + } + + ~OfflineSpeakerDiarization() + { + Cleanup(); + } + + private void Cleanup() + { + SherpaOnnxDestroyOfflineSpeakerDiarization(_handle.Handle); + + // Don't permit the handle to be used again. + _handle = new HandleRef(this, IntPtr.Zero); + } + + private HandleRef _handle; + + public int SampleRate + { + get + { + return SherpaOnnxOfflineSpeakerDiarizationGetSampleRate(_handle.Handle); + } + } + + [DllImport(Dll.Filename)] + private static extern IntPtr SherpaOnnxCreateOfflineSpeakerDiarization(ref OfflineSpeakerDiarizationConfig config); + + [DllImport(Dll.Filename)] + private static extern void SherpaOnnxDestroyOfflineSpeakerDiarization(IntPtr handle); + + [DllImport(Dll.Filename)] + private static extern int SherpaOnnxOfflineSpeakerDiarizationGetSampleRate(IntPtr handle); + + [DllImport(Dll.Filename)] + private static extern int SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments(IntPtr handle); + + [DllImport(Dll.Filename)] + private static extern IntPtr SherpaOnnxOfflineSpeakerDiarizationProcess(IntPtr handle, float[] samples, int n); + + [DllImport(Dll.Filename, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr SherpaOnnxOfflineSpeakerDiarizationProcessWithCallback(IntPtr handle, float[] samples, int n, OfflineSpeakerDiarizationProgressCallback callback, IntPtr arg); + + [DllImport(Dll.Filename)] + private static extern void SherpaOnnxOfflineSpeakerDiarizationDestroyResult(IntPtr handle); + + [DllImport(Dll.Filename)] + private static extern IntPtr SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime(IntPtr handle); + + [DllImport(Dll.Filename)] + private static extern void SherpaOnnxOfflineSpeakerDiarizationDestroySegment(IntPtr handle); + } +} + diff --git a/scripts/dotnet/OfflineSpeakerDiarizationConfig.cs b/scripts/dotnet/OfflineSpeakerDiarizationConfig.cs new file mode 100644 index 000000000..94f57039f --- /dev/null +++ b/scripts/dotnet/OfflineSpeakerDiarizationConfig.cs @@ -0,0 +1,31 @@ +/// Copyright (c) 2024 Xiaomi Corporation + +using System.Runtime.InteropServices; + +namespace SherpaOnnx +{ + + [StructLayout(LayoutKind.Sequential)] + public struct OfflineSpeakerDiarizationConfig + { + public OfflineSpeakerDiarizationConfig() + { + Segmentation = new OfflineSpeakerSegmentationModelConfig(); + Embedding = new SpeakerEmbeddingExtractorConfig(); + Clustering = new FastClusteringConfig(); + + MinDurationOn = 0.3F; + MinDurationOff = 0.5F; + } + + public OfflineSpeakerSegmentationModelConfig Segmentation; + public SpeakerEmbeddingExtractorConfig Embedding; + public FastClusteringConfig Clustering; + + public float MinDurationOn; + public float MinDurationOff; + } +} + + + diff --git a/scripts/dotnet/OfflineSpeakerDiarizationSegment.cs b/scripts/dotnet/OfflineSpeakerDiarizationSegment.cs new file mode 100644 index 000000000..8985c977e --- /dev/null +++ b/scripts/dotnet/OfflineSpeakerDiarizationSegment.cs @@ -0,0 +1,33 @@ +/// Copyright (c) 2024 Xiaomi Corporation +using System; +using System.Runtime.InteropServices; +using System.Text; + +namespace SherpaOnnx +{ + + public class OfflineSpeakerDiarizationSegment + { + public OfflineSpeakerDiarizationSegment(IntPtr handle) + { + Impl impl = (Impl)Marshal.PtrToStructure(handle, typeof(Impl)); + + Start = impl.Start; + End = impl.End; + Speaker = impl.Speaker; + } + + [StructLayout(LayoutKind.Sequential)] + struct Impl + { + public float Start; + public float End; + public int Speaker; + } + + public float Start; + public float End; + public int Speaker; + } +} + diff --git a/scripts/dotnet/OfflineSpeakerSegmentationModelConfig.cs b/scripts/dotnet/OfflineSpeakerSegmentationModelConfig.cs new file mode 100644 index 000000000..1bd1f3842 --- /dev/null +++ b/scripts/dotnet/OfflineSpeakerSegmentationModelConfig.cs @@ -0,0 +1,32 @@ +/// Copyright (c) 2024 Xiaomi Corporation + +using System.Runtime.InteropServices; + +namespace SherpaOnnx +{ + + [StructLayout(LayoutKind.Sequential)] + public struct OfflineSpeakerSegmentationModelConfig + { + public OfflineSpeakerSegmentationModelConfig() + { + Pyannote = new OfflineSpeakerSegmentationPyannoteModelConfig(); + NumThreads = 1; + Debug = 0; + Provider = "cpu"; + } + + public OfflineSpeakerSegmentationPyannoteModelConfig Pyannote; + + /// Number of threads used to run the neural network model + public int NumThreads; + + /// true to print debug information of the model + public int Debug; + + [MarshalAs(UnmanagedType.LPStr)] + public string Provider; + } +} + + diff --git a/scripts/dotnet/OfflineSpeakerSegmentationPyannoteModelConfig.cs b/scripts/dotnet/OfflineSpeakerSegmentationPyannoteModelConfig.cs new file mode 100644 index 000000000..319762125 --- /dev/null +++ b/scripts/dotnet/OfflineSpeakerSegmentationPyannoteModelConfig.cs @@ -0,0 +1,20 @@ +/// Copyright (c) 2024 Xiaomi Corporation + +using System.Runtime.InteropServices; + +namespace SherpaOnnx +{ + + [StructLayout(LayoutKind.Sequential)] + public struct OfflineSpeakerSegmentationPyannoteModelConfig + { + public OfflineSpeakerSegmentationPyannoteModelConfig() + { + Model = ""; + } + + [MarshalAs(UnmanagedType.LPStr)] + public string Model; + } +} + From 67349b52f2c9c503013ca439ef6e854896ac9a9a Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 10 Oct 2024 15:51:31 +0800 Subject: [PATCH 009/183] JavaScript API (node-addon) for speaker diarization (#1408) --- .../scripts/node-addon/package-optional.json | 22 +- .github/scripts/node-addon/package.json | 22 +- .github/scripts/test-nodejs-addon-npm.sh | 14 + nodejs-addon-examples/README.md | 21 ++ .../test_offline_speaker_diarization.js | 62 ++++ scripts/node-addon-api/CMakeLists.txt | 1 + .../lib/non-streaming-speaker-diarization.js | 32 +++ scripts/node-addon-api/lib/sherpa-onnx.js | 2 + scripts/node-addon-api/package.json | 12 +- .../src/non-streaming-speaker-diarization.cc | 265 ++++++++++++++++++ .../src/sherpa-onnx-node-addon-api.cc | 3 + 11 files changed, 443 insertions(+), 13 deletions(-) create mode 100644 nodejs-addon-examples/test_offline_speaker_diarization.js create mode 100644 scripts/node-addon-api/lib/non-streaming-speaker-diarization.js create mode 100644 scripts/node-addon-api/src/non-streaming-speaker-diarization.cc diff --git a/.github/scripts/node-addon/package-optional.json b/.github/scripts/node-addon/package-optional.json index b3c71f9da..d2db2e192 100644 --- a/.github/scripts/node-addon/package-optional.json +++ b/.github/scripts/node-addon/package-optional.json @@ -1,7 +1,7 @@ { "name": "sherpa-onnx-PLATFORM2-ARCH", "version": "SHERPA_ONNX_VERSION", - "description": "Speech-to-text and text-to-speech using Next-gen Kaldi without internet connection", + "description": "Speech-to-text, text-to-speech, and speaker diarization using Next-gen Kaldi without internet connection", "main": "index.js", "scripts": { "test": "echo \"Error: no test specified\" && exit 1" @@ -16,8 +16,18 @@ "transcription", "real-time speech recognition", "without internet connection", + "locally", + "local", "embedded systems", "open source", + "diarization", + "speaker diarization", + "speaker recognition", + "speaker", + "speaker segmentation", + "speaker verification", + "spoken language identification", + "sherpa", "zipformer", "asr", "tts", @@ -30,13 +40,13 @@ "offline", "privacy", "open source", - "vad", - "speaker id", - "language id", - "node-addon-api", "streaming speech recognition", "speech", - "recognition" + "recognition", + "vad", + "node-addon-api", + "speaker id", + "language id" ], "author": "The next-gen Kaldi team", "license": "Apache-2.0", diff --git a/.github/scripts/node-addon/package.json b/.github/scripts/node-addon/package.json index 0444552fc..bc2d89e89 100644 --- a/.github/scripts/node-addon/package.json +++ b/.github/scripts/node-addon/package.json @@ -1,7 +1,7 @@ { "name": "sherpa-onnx-node", "version": "SHERPA_ONNX_VERSION", - "description": "Speech-to-text and text-to-speech using Next-gen Kaldi without internet connection", + "description": "Speech-to-text, text-to-speech, and speaker diarization using Next-gen Kaldi without internet connection", "main": "sherpa-onnx.js", "scripts": { "test": "echo \"Error: no test specified\" && exit 1" @@ -16,8 +16,18 @@ "transcription", "real-time speech recognition", "without internet connection", + "locally", + "local", "embedded systems", "open source", + "diarization", + "speaker diarization", + "speaker recognition", + "speaker", + "speaker segmentation", + "speaker verification", + "spoken language identification", + "sherpa", "zipformer", "asr", "tts", @@ -30,13 +40,13 @@ "offline", "privacy", "open source", - "vad", - "speaker id", - "language id", - "node-addon-api", "streaming speech recognition", "speech", - "recognition" + "recognition", + "vad", + "node-addon-api", + "speaker id", + "language id" ], "author": "The next-gen Kaldi team", "license": "Apache-2.0", diff --git a/.github/scripts/test-nodejs-addon-npm.sh b/.github/scripts/test-nodejs-addon-npm.sh index a46e2de8e..42c753ebf 100755 --- a/.github/scripts/test-nodejs-addon-npm.sh +++ b/.github/scripts/test-nodejs-addon-npm.sh @@ -10,6 +10,20 @@ arch=$(node -p "require('os').arch()") platform=$(node -p "require('os').platform()") node_version=$(node -p "process.versions.node.split('.')[0]") +echo "----------non-streaming speaker diarization----------" + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 +tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 +rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/0-four-speakers-zh.wav + +node ./test_offline_speaker_diarization.js + +rm -rfv *.onnx *.wav sherpa-onnx-pyannote-* + echo "----------non-streaming asr + vad----------" curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2 tar xvf sherpa-onnx-whisper-tiny.en.tar.bz2 diff --git a/nodejs-addon-examples/README.md b/nodejs-addon-examples/README.md index ef0c25395..8851c6265 100644 --- a/nodejs-addon-examples/README.md +++ b/nodejs-addon-examples/README.md @@ -43,6 +43,12 @@ export LD_LIBRARY_PATH=$PWD/node_modules/.pnpm/sherpa-onnx-node@ +#include + +#include "macros.h" // NOLINT +#include "napi.h" // NOLINT +#include "sherpa-onnx/c-api/c-api.h" + +static SherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig +GetOfflineSpeakerSegmentationPyannoteModelConfig(Napi::Object obj) { + SherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("pyannote") || !obj.Get("pyannote").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("pyannote").As(); + SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); + + return c; +} + +static SherpaOnnxOfflineSpeakerSegmentationModelConfig +GetOfflineSpeakerSegmentationModelConfig(Napi::Object obj) { + SherpaOnnxOfflineSpeakerSegmentationModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("segmentation") || !obj.Get("segmentation").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("segmentation").As(); + + c.pyannote = GetOfflineSpeakerSegmentationPyannoteModelConfig(o); + + SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); + + if (o.Has("debug") && + (o.Get("debug").IsNumber() || o.Get("debug").IsBoolean())) { + if (o.Get("debug").IsBoolean()) { + c.debug = o.Get("debug").As().Value(); + } else { + c.debug = o.Get("debug").As().Int32Value(); + } + } + + SHERPA_ONNX_ASSIGN_ATTR_STR(provider, provider); + + return c; +} + +static SherpaOnnxSpeakerEmbeddingExtractorConfig +GetSpeakerEmbeddingExtractorConfig(Napi::Object obj) { + SherpaOnnxSpeakerEmbeddingExtractorConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("embedding") || !obj.Get("embedding").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("embedding").As(); + + SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); + SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); + + if (o.Has("debug") && + (o.Get("debug").IsNumber() || o.Get("debug").IsBoolean())) { + if (o.Get("debug").IsBoolean()) { + c.debug = o.Get("debug").As().Value(); + } else { + c.debug = o.Get("debug").As().Int32Value(); + } + } + + SHERPA_ONNX_ASSIGN_ATTR_STR(provider, provider); + + return c; +} + +static SherpaOnnxFastClusteringConfig GetFastClusteringConfig( + Napi::Object obj) { + SherpaOnnxFastClusteringConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("clustering") || !obj.Get("clustering").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("clustering").As(); + + SHERPA_ONNX_ASSIGN_ATTR_INT32(num_clusters, numClusters); + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(threshold, threshold); + + return c; +} + +static Napi::External +CreateOfflineSpeakerDiarizationWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsObject()) { + Napi::TypeError::New(env, "Expect an object as the argument") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Object o = info[0].As(); + + SherpaOnnxOfflineSpeakerDiarizationConfig c; + memset(&c, 0, sizeof(c)); + + c.segmentation = GetOfflineSpeakerSegmentationModelConfig(o); + c.embedding = GetSpeakerEmbeddingExtractorConfig(o); + c.clustering = GetFastClusteringConfig(o); + + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(min_duration_on, minDurationOn); + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(min_duration_off, minDurationOff); + + const SherpaOnnxOfflineSpeakerDiarization *sd = + SherpaOnnxCreateOfflineSpeakerDiarization(&c); + + if (c.segmentation.pyannote.model) { + delete[] c.segmentation.pyannote.model; + } + + if (c.segmentation.provider) { + delete[] c.segmentation.provider; + } + + if (c.embedding.model) { + delete[] c.embedding.model; + } + + if (c.embedding.provider) { + delete[] c.embedding.provider; + } + + if (!sd) { + Napi::TypeError::New(env, "Please check your config!") + .ThrowAsJavaScriptException(); + + return {}; + } + + return Napi::External::New( + env, const_cast(sd), + [](Napi::Env env, SherpaOnnxOfflineSpeakerDiarization *sd) { + SherpaOnnxDestroyOfflineSpeakerDiarization(sd); + }); +} + +static Napi::Number OfflineSpeakerDiarizationGetSampleRateWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New( + env, "Argument 0 should be an offline speaker diarization pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + const SherpaOnnxOfflineSpeakerDiarization *sd = + info[0].As>().Data(); + + int32_t sample_rate = SherpaOnnxOfflineSpeakerDiarizationGetSampleRate(sd); + + return Napi::Number::New(env, sample_rate); +} + +static Napi::Array OfflineSpeakerDiarizationProcessWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New( + env, "Argument 0 should be an offline speaker diarization pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + const SherpaOnnxOfflineSpeakerDiarization *sd = + info[0].As>().Data(); + + if (!info[1].IsTypedArray()) { + Napi::TypeError::New(env, "Argument 1 should be a typed array") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Float32Array samples = info[1].As(); + + const SherpaOnnxOfflineSpeakerDiarizationResult *r = + SherpaOnnxOfflineSpeakerDiarizationProcess(sd, samples.Data(), + samples.ElementLength()); + + int32_t num_segments = + SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments(r); + + const SherpaOnnxOfflineSpeakerDiarizationSegment *segments = + SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime(r); + + Napi::Array ans = Napi::Array::New(env, num_segments); + + for (int32_t i = 0; i != num_segments; ++i) { + Napi::Object obj = Napi::Object::New(env); + obj.Set(Napi::String::New(env, "start"), segments[i].start); + obj.Set(Napi::String::New(env, "end"), segments[i].end); + obj.Set(Napi::String::New(env, "speaker"), segments[i].speaker); + + ans[i] = obj; + } + + SherpaOnnxOfflineSpeakerDiarizationDestroySegment(segments); + SherpaOnnxOfflineSpeakerDiarizationDestroyResult(r); + + return ans; +} + +void InitNonStreamingSpeakerDiarization(Napi::Env env, Napi::Object exports) { + exports.Set(Napi::String::New(env, "createOfflineSpeakerDiarization"), + Napi::Function::New(env, CreateOfflineSpeakerDiarizationWrapper)); + + exports.Set( + Napi::String::New(env, "getOfflineSpeakerDiarizationSampleRate"), + Napi::Function::New(env, OfflineSpeakerDiarizationGetSampleRateWrapper)); + + exports.Set( + Napi::String::New(env, "offlineSpeakerDiarizationProcess"), + Napi::Function::New(env, OfflineSpeakerDiarizationProcessWrapper)); +} diff --git a/scripts/node-addon-api/src/sherpa-onnx-node-addon-api.cc b/scripts/node-addon-api/src/sherpa-onnx-node-addon-api.cc index b86883d86..3f0affd79 100644 --- a/scripts/node-addon-api/src/sherpa-onnx-node-addon-api.cc +++ b/scripts/node-addon-api/src/sherpa-onnx-node-addon-api.cc @@ -25,6 +25,8 @@ void InitPunctuation(Napi::Env env, Napi::Object exports); void InitKeywordSpotting(Napi::Env env, Napi::Object exports); +void InitNonStreamingSpeakerDiarization(Napi::Env env, Napi::Object exports); + Napi::Object Init(Napi::Env env, Napi::Object exports) { InitStreamingAsr(env, exports); InitNonStreamingAsr(env, exports); @@ -37,6 +39,7 @@ Napi::Object Init(Napi::Env env, Napi::Object exports) { InitAudioTagging(env, exports); InitPunctuation(env, exports); InitKeywordSpotting(env, exports); + InitNonStreamingSpeakerDiarization(env, exports); return exports; } From 1d061df35522edb5f08cef36a9ada73f24bd83eb Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 10 Oct 2024 22:14:45 +0800 Subject: [PATCH 010/183] WebAssembly exmaple for speaker diarization (#1411) --- .../workflows/wasm-simd-hf-space-de-tts.yaml | 2 +- .../wasm-simd-hf-space-en-asr-zipformer.yaml | 2 +- .../workflows/wasm-simd-hf-space-en-tts.yaml | 2 +- .../wasm-simd-hf-space-silero-vad.yaml | 2 +- ...asm-simd-hf-space-speaker-diarization.yaml | 167 ++++++++++ .../workflows/wasm-simd-hf-space-vad-asr.yaml | 2 +- ...-space-zh-cantonese-en-asr-paraformer.yaml | 2 +- ...sm-simd-hf-space-zh-en-asr-paraformer.yaml | 2 +- ...asm-simd-hf-space-zh-en-asr-zipformer.yaml | 2 +- CMakeLists.txt | 14 +- README.md | 5 + build-wasm-simd-asr.sh | 4 +- build-wasm-simd-kws.sh | 4 +- build-wasm-simd-nodejs.sh | 4 +- build-wasm-simd-speaker-diarization.sh | 61 ++++ build-wasm-simd-tts.sh | 4 +- build-wasm-simd-vad-asr.sh | 4 +- build-wasm-simd-vad.sh | 4 +- scripts/dotnet/OfflineSpeakerDiarization.cs | 8 + scripts/go/sherpa_onnx.go | 10 + .../lib/non-streaming-speaker-diarization.js | 5 + .../src/non-streaming-speaker-diarization.cc | 44 +++ sherpa-onnx/c-api/c-api.cc | 14 + sherpa-onnx/c-api/c-api.h | 5 + .../csrc/offline-speaker-diarization-impl.h | 4 + ...ffline-speaker-diarization-pyannote-impl.h | 15 +- .../csrc/offline-speaker-diarization.cc | 5 + .../csrc/offline-speaker-diarization.h | 4 + .../csrc/offline-speaker-diarization.cc | 1 + swift-api-examples/SherpaOnnx.swift | 5 + wasm/CMakeLists.txt | 4 + wasm/speaker-diarization/CMakeLists.txt | 61 ++++ .../app-speaker-diarization.js | 124 ++++++++ wasm/speaker-diarization/assets/README.md | 30 ++ wasm/speaker-diarization/index.html | 48 +++ .../sherpa-onnx-speaker-diarization.js | 295 ++++++++++++++++++ ...erpa-onnx-wasm-main-speaker-diarization.cc | 63 ++++ 37 files changed, 1008 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/wasm-simd-hf-space-speaker-diarization.yaml create mode 100755 build-wasm-simd-speaker-diarization.sh create mode 100644 wasm/speaker-diarization/CMakeLists.txt create mode 100644 wasm/speaker-diarization/app-speaker-diarization.js create mode 100644 wasm/speaker-diarization/assets/README.md create mode 100644 wasm/speaker-diarization/index.html create mode 100644 wasm/speaker-diarization/sherpa-onnx-speaker-diarization.js create mode 100644 wasm/speaker-diarization/sherpa-onnx-wasm-main-speaker-diarization.cc diff --git a/.github/workflows/wasm-simd-hf-space-de-tts.yaml b/.github/workflows/wasm-simd-hf-space-de-tts.yaml index cbd3b1fce..76013291b 100644 --- a/.github/workflows/wasm-simd-hf-space-de-tts.yaml +++ b/.github/workflows/wasm-simd-hf-space-de-tts.yaml @@ -29,7 +29,7 @@ jobs: - name: Install emsdk uses: mymindstorm/setup-emsdk@v14 with: - version: 3.1.51 + version: 3.1.53 actions-cache-folder: 'emsdk-cache' - name: View emsdk version diff --git a/.github/workflows/wasm-simd-hf-space-en-asr-zipformer.yaml b/.github/workflows/wasm-simd-hf-space-en-asr-zipformer.yaml index 510a003c7..d34a182d4 100644 --- a/.github/workflows/wasm-simd-hf-space-en-asr-zipformer.yaml +++ b/.github/workflows/wasm-simd-hf-space-en-asr-zipformer.yaml @@ -28,7 +28,7 @@ jobs: - name: Install emsdk uses: mymindstorm/setup-emsdk@v14 with: - version: 3.1.51 + version: 3.1.53 actions-cache-folder: 'emsdk-cache' - name: View emsdk version diff --git a/.github/workflows/wasm-simd-hf-space-en-tts.yaml b/.github/workflows/wasm-simd-hf-space-en-tts.yaml index 9c5c1d446..d67ae8818 100644 --- a/.github/workflows/wasm-simd-hf-space-en-tts.yaml +++ b/.github/workflows/wasm-simd-hf-space-en-tts.yaml @@ -29,7 +29,7 @@ jobs: - name: Install emsdk uses: mymindstorm/setup-emsdk@v14 with: - version: 3.1.51 + version: 3.1.53 actions-cache-folder: 'emsdk-cache' - name: View emsdk version diff --git a/.github/workflows/wasm-simd-hf-space-silero-vad.yaml b/.github/workflows/wasm-simd-hf-space-silero-vad.yaml index dc8bada70..81052cac8 100644 --- a/.github/workflows/wasm-simd-hf-space-silero-vad.yaml +++ b/.github/workflows/wasm-simd-hf-space-silero-vad.yaml @@ -29,7 +29,7 @@ jobs: - name: Install emsdk uses: mymindstorm/setup-emsdk@v14 with: - version: 3.1.51 + version: 3.1.53 actions-cache-folder: 'emsdk-cache' - name: View emsdk version diff --git a/.github/workflows/wasm-simd-hf-space-speaker-diarization.yaml b/.github/workflows/wasm-simd-hf-space-speaker-diarization.yaml new file mode 100644 index 000000000..14301f9f0 --- /dev/null +++ b/.github/workflows/wasm-simd-hf-space-speaker-diarization.yaml @@ -0,0 +1,167 @@ +name: wasm-simd-hf-space-speaker-diarization + +on: + push: + branches: + - wasm + - wasm-speaker-diarization + tags: + - 'v[0-9]+.[0-9]+.[0-9]+*' + + workflow_dispatch: + +concurrency: + group: wasm-simd-hf-space-speaker-diarization-${{ github.ref }} + cancel-in-progress: true + +jobs: + wasm-simd-hf-space-speaker-diarization: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install emsdk + uses: mymindstorm/setup-emsdk@v14 + with: + version: 3.1.53 + actions-cache-folder: 'emsdk-cache' + + - name: View emsdk version + shell: bash + run: | + emcc -v + echo "--------------------" + emcc --check + + - name: Download model files + shell: bash + run: | + cd wasm/speaker-diarization/assets/ + ls -lh + echo "----------" + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + mv sherpa-onnx-pyannote-segmentation-3-0/model.onnx ./segmentation.onnx + rm -rf sherpa-onnx-pyannote-segmentation-3-0 + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx + mv 3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx ./embedding.onnx + + echo "----------" + + ls -lh + + - name: Build sherpa-onnx for WebAssembly + shell: bash + run: | + ./build-wasm-simd-speaker-diarization.sh + + - name: collect files + shell: bash + run: | + SHERPA_ONNX_VERSION=v$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + + dst=sherpa-onnx-wasm-simd-${SHERPA_ONNX_VERSION}-speaker-diarization + mv build-wasm-simd-speaker-diarization/install/bin/wasm/speaker-diarization $dst + ls -lh $dst + tar cjfv $dst.tar.bz2 ./$dst + + - name: Upload wasm files + uses: actions/upload-artifact@v4 + with: + name: sherpa-onnx-wasm-simd-speaker-diarization + path: ./sherpa-onnx-wasm-simd-*.tar.bz2 + + - name: Release + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + overwrite: true + file: ./*.tar.bz2 + + - name: Publish to ModelScope + # if: false + env: + MS_TOKEN: ${{ secrets.MODEL_SCOPE_GIT_TOKEN }} + uses: nick-fields/retry@v2 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + SHERPA_ONNX_VERSION=v$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + + rm -rf ms + export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false + + git clone https://www.modelscope.cn/studios/csukuangfj/web-assembly-speaker-diarization-sherpa-onnx.git ms + cd ms + rm -fv *.js + rm -fv *.data + git fetch + git pull + git merge -m "merge remote" --ff origin main + + cp -v ../sherpa-onnx-wasm-simd-${SHERPA_ONNX_VERSION}-*/* . + + git status + git lfs track "*.data" + git lfs track "*.wasm" + ls -lh + + git add . + git commit -m "update model" + git push https://oauth2:${MS_TOKEN}@www.modelscope.cn/studios/csukuangfj/web-assembly-speaker-diarization-sherpa-onnx.git + + - name: Publish to huggingface + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v2 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + SHERPA_ONNX_VERSION=v$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + + rm -rf huggingface + export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false + + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/spaces/k2-fsa/web-assembly-speaker-diarization-sherpa-onnx huggingface + ls -lh + + cd huggingface + rm -fv *.js + rm -fv *.data + git fetch + git pull + git merge -m "merge remote" --ff origin main + + cp -v ../sherpa-onnx-wasm-simd-${SHERPA_ONNX_VERSION}-*/* . + + git status + git lfs track "*.data" + git lfs track "*.wasm" + ls -lh + + git add . + git commit -m "update model" + git push https://csukuangfj:$HF_TOKEN@huggingface.co/spaces/k2-fsa/web-assembly-speaker-diarization-sherpa-onnx main diff --git a/.github/workflows/wasm-simd-hf-space-vad-asr.yaml b/.github/workflows/wasm-simd-hf-space-vad-asr.yaml index c093f0fe9..18c1c1d60 100644 --- a/.github/workflows/wasm-simd-hf-space-vad-asr.yaml +++ b/.github/workflows/wasm-simd-hf-space-vad-asr.yaml @@ -37,7 +37,7 @@ jobs: - name: Install emsdk uses: mymindstorm/setup-emsdk@v14 with: - version: 3.1.51 + version: 3.1.53 actions-cache-folder: 'emsdk-cache' - name: View emsdk version diff --git a/.github/workflows/wasm-simd-hf-space-zh-cantonese-en-asr-paraformer.yaml b/.github/workflows/wasm-simd-hf-space-zh-cantonese-en-asr-paraformer.yaml index c72e0cef2..02a328a9b 100644 --- a/.github/workflows/wasm-simd-hf-space-zh-cantonese-en-asr-paraformer.yaml +++ b/.github/workflows/wasm-simd-hf-space-zh-cantonese-en-asr-paraformer.yaml @@ -29,7 +29,7 @@ jobs: - name: Install emsdk uses: mymindstorm/setup-emsdk@v14 with: - version: 3.1.51 + version: 3.1.53 actions-cache-folder: 'emsdk-cache' - name: View emsdk version diff --git a/.github/workflows/wasm-simd-hf-space-zh-en-asr-paraformer.yaml b/.github/workflows/wasm-simd-hf-space-zh-en-asr-paraformer.yaml index b76f912b4..1a72be6ab 100644 --- a/.github/workflows/wasm-simd-hf-space-zh-en-asr-paraformer.yaml +++ b/.github/workflows/wasm-simd-hf-space-zh-en-asr-paraformer.yaml @@ -29,7 +29,7 @@ jobs: - name: Install emsdk uses: mymindstorm/setup-emsdk@v14 with: - version: 3.1.51 + version: 3.1.53 actions-cache-folder: 'emsdk-cache' - name: View emsdk version diff --git a/.github/workflows/wasm-simd-hf-space-zh-en-asr-zipformer.yaml b/.github/workflows/wasm-simd-hf-space-zh-en-asr-zipformer.yaml index 9bdd90ee2..8b7c2029f 100644 --- a/.github/workflows/wasm-simd-hf-space-zh-en-asr-zipformer.yaml +++ b/.github/workflows/wasm-simd-hf-space-zh-en-asr-zipformer.yaml @@ -29,7 +29,7 @@ jobs: - name: Install emsdk uses: mymindstorm/setup-emsdk@v14 with: - version: 3.1.51 + version: 3.1.53 actions-cache-folder: 'emsdk-cache' - name: View emsdk version diff --git a/CMakeLists.txt b/CMakeLists.txt index 9084a0216..d0b44be2a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,6 +32,7 @@ option(SHERPA_ONNX_ENABLE_WEBSOCKET "Whether to build webscoket server/client" O option(SHERPA_ONNX_ENABLE_GPU "Enable ONNX Runtime GPU support" OFF) option(SHERPA_ONNX_ENABLE_DIRECTML "Enable ONNX Runtime DirectML support" OFF) option(SHERPA_ONNX_ENABLE_WASM "Whether to enable WASM" OFF) +option(SHERPA_ONNX_ENABLE_WASM_SPEAKER_DIARIZATION "Whether to enable WASM for speaker diarization" OFF) option(SHERPA_ONNX_ENABLE_WASM_TTS "Whether to enable WASM for TTS" OFF) option(SHERPA_ONNX_ENABLE_WASM_ASR "Whether to enable WASM for ASR" OFF) option(SHERPA_ONNX_ENABLE_WASM_KWS "Whether to enable WASM for KWS" OFF) @@ -135,6 +136,7 @@ message(STATUS "SHERPA_ONNX_ENABLE_C_API ${SHERPA_ONNX_ENABLE_C_API}") message(STATUS "SHERPA_ONNX_ENABLE_WEBSOCKET ${SHERPA_ONNX_ENABLE_WEBSOCKET}") message(STATUS "SHERPA_ONNX_ENABLE_GPU ${SHERPA_ONNX_ENABLE_GPU}") message(STATUS "SHERPA_ONNX_ENABLE_WASM ${SHERPA_ONNX_ENABLE_WASM}") +message(STATUS "SHERPA_ONNX_ENABLE_WASM_SPEAKER_DIARIZATION ${SHERPA_ONNX_ENABLE_WASM_SPEAKER_DIARIZATION}") message(STATUS "SHERPA_ONNX_ENABLE_WASM_TTS ${SHERPA_ONNX_ENABLE_WASM_TTS}") message(STATUS "SHERPA_ONNX_ENABLE_WASM_ASR ${SHERPA_ONNX_ENABLE_WASM_ASR}") message(STATUS "SHERPA_ONNX_ENABLE_WASM_KWS ${SHERPA_ONNX_ENABLE_WASM_KWS}") @@ -196,9 +198,19 @@ else() add_definitions(-DSHERPA_ONNX_ENABLE_DIRECTML=0) endif() +if(SHERPA_ONNX_ENABLE_WASM_SPEAKER_DIARIZATION) + if(NOT SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION) + message(FATAL_ERROR "Please set SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION to ON if you want to build WASM for speaker diarization") + endif() + + if(NOT SHERPA_ONNX_ENABLE_WASM) + message(FATAL_ERROR "Please set SHERPA_ONNX_ENABLE_WASM to ON if you enable WASM for speaker diarization") + endif() +endif() + if(SHERPA_ONNX_ENABLE_WASM_TTS) if(NOT SHERPA_ONNX_ENABLE_TTS) - message(FATAL_ERROR "Please set SHERPA_ONNX_ENABLE_TTS to ON if you want to build wasm TTS") + message(FATAL_ERROR "Please set SHERPA_ONNX_ENABLE_TTS to ON if you want to build WASM for TTS") endif() if(NOT SHERPA_ONNX_ENABLE_WASM) diff --git a/README.md b/README.md index 00fe61fd0..1828847e5 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,7 @@ We also have spaces built using WebAssembly. They are listed below: |VAD + speech recognition (English + Chinese, 及多种中文方言) with Paraformer-small |[Click me][wasm-hf-vad-asr-zh-en-paraformer-small]| [地址][wasm-ms-vad-asr-zh-en-paraformer-small]| |Speech synthesis (English) |[Click me][wasm-hf-tts-piper-en]| [地址][wasm-ms-tts-piper-en]| |Speech synthesis (German) |[Click me][wasm-hf-tts-piper-de]| [地址][wasm-ms-tts-piper-de]| +|Speaker diarization |[Click me][wasm-hf-speaker-diarization]|[地址][wasm-ms-speaker-diarization]| ### Links for pre-built Android APKs @@ -173,6 +174,7 @@ We also have spaces built using WebAssembly. They are listed below: | Speaker identification (Speaker ID) | [Address][sid-models] | | Spoken language identification (Language ID)| See multi-lingual [Whisper][Whisper] ASR models from [Speech recognition][asr-models]| | Punctuation | [Address][punct-models] | +| Speaker segmentation | [Address][speaker-segmentation-models] | ### Useful links @@ -261,6 +263,8 @@ Video demo in Chinese: [爆了！炫神教你开打字挂！真正影响胜率 [wasm-ms-tts-piper-en]: https://modelscope.cn/studios/k2-fsa/web-assembly-tts-sherpa-onnx-en [wasm-hf-tts-piper-de]: https://huggingface.co/spaces/k2-fsa/web-assembly-tts-sherpa-onnx-de [wasm-ms-tts-piper-de]: https://modelscope.cn/studios/k2-fsa/web-assembly-tts-sherpa-onnx-de +[wasm-hf-speaker-diarization]: https://huggingface.co/spaces/k2-fsa/web-assembly-speaker-diarization-sherpa-onnx +[wasm-ms-speaker-diarization]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-speaker-diarization-sherpa-onnx [apk-streaming-asr]: https://k2-fsa.github.io/sherpa/onnx/android/apk.html [apk-streaming-asr-cn]: https://k2-fsa.github.io/sherpa/onnx/android/apk-cn.html [apk-tts]: https://k2-fsa.github.io/sherpa/onnx/tts/apk-engine.html @@ -303,5 +307,6 @@ Video demo in Chinese: [爆了！炫神教你开打字挂！真正影响胜率 [sid-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models [slid-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models [punct-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/punctuation-models +[speaker-segmentation-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models [GigaSpeech]: https://github.com/SpeechColab/GigaSpeech [WenetSpeech]: https://github.com/wenet-e2e/WenetSpeech diff --git a/build-wasm-simd-asr.sh b/build-wasm-simd-asr.sh index eda18f74d..f5e755047 100755 --- a/build-wasm-simd-asr.sh +++ b/build-wasm-simd-asr.sh @@ -14,8 +14,8 @@ if [ x"$EMSCRIPTEN" == x"" ]; then echo "git clone https://github.com/emscripten-core/emsdk.git" echo "cd emsdk" echo "git pull" - echo "./emsdk install latest" - echo "./emsdk activate latest" + echo "./emsdk install 3.1.53" + echo "./emsdk activate 3.1.53" echo "source ./emsdk_env.sh" exit 1 else diff --git a/build-wasm-simd-kws.sh b/build-wasm-simd-kws.sh index 6fdf8218f..301bd8711 100755 --- a/build-wasm-simd-kws.sh +++ b/build-wasm-simd-kws.sh @@ -9,8 +9,8 @@ if [ x"$EMSCRIPTEN" == x"" ]; then echo "git clone https://github.com/emscripten-core/emsdk.git" echo "cd emsdk" echo "git pull" - echo "./emsdk install latest" - echo "./emsdk activate latest" + echo "./emsdk install 3.1.53" + echo "./emsdk activate 3.1.53" echo "source ./emsdk_env.sh" exit 1 else diff --git a/build-wasm-simd-nodejs.sh b/build-wasm-simd-nodejs.sh index 3ad88d5d4..13bf3c854 100755 --- a/build-wasm-simd-nodejs.sh +++ b/build-wasm-simd-nodejs.sh @@ -16,8 +16,8 @@ if [ x"$EMSCRIPTEN" == x"" ]; then echo "git clone https://github.com/emscripten-core/emsdk.git" echo "cd emsdk" echo "git pull" - echo "./emsdk install latest" - echo "./emsdk activate latest" + echo "./emsdk install 3.1.53" + echo "./emsdk activate 3.1.53" echo "source ./emsdk_env.sh" exit 1 else diff --git a/build-wasm-simd-speaker-diarization.sh b/build-wasm-simd-speaker-diarization.sh new file mode 100755 index 000000000..da4be0381 --- /dev/null +++ b/build-wasm-simd-speaker-diarization.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Copyright (c) 2024 Xiaomi Corporation +# +# This script is to build sherpa-onnx for WebAssembly (speaker diarization) + +set -ex + +if [ x"$EMSCRIPTEN" == x"" ]; then + if ! command -v emcc &> /dev/null; then + echo "Please install emscripten first" + echo "" + echo "You can use the following commands to install it:" + echo "" + echo "git clone https://github.com/emscripten-core/emsdk.git" + echo "cd emsdk" + echo "git pull" + echo "./emsdk install 3.1.53" + echo "./emsdk activate 3.1.53" + echo "source ./emsdk_env.sh" + exit 1 + else + EMSCRIPTEN=$(dirname $(realpath $(which emcc))) + fi +fi + +export EMSCRIPTEN=$EMSCRIPTEN +echo "EMSCRIPTEN: $EMSCRIPTEN" +if [ ! -f $EMSCRIPTEN/cmake/Modules/Platform/Emscripten.cmake ]; then + echo "Cannot find $EMSCRIPTEN/cmake/Modules/Platform/Emscripten.cmake" + echo "Please make sure you have installed emsdk correctly" + exit 1 +fi + +mkdir -p build-wasm-simd-speaker-diarization +pushd build-wasm-simd-speaker-diarization + +export SHERPA_ONNX_IS_USING_BUILD_WASM_SH=ON + +cmake \ + -DCMAKE_INSTALL_PREFIX=./install \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_TOOLCHAIN_FILE=$EMSCRIPTEN/cmake/Modules/Platform/Emscripten.cmake \ + \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=OFF \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + -DSHERPA_ONNX_ENABLE_JNI=OFF \ + -DSHERPA_ONNX_ENABLE_C_API=ON \ + -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ + -DSHERPA_ONNX_ENABLE_GPU=OFF \ + -DSHERPA_ONNX_ENABLE_WASM=ON \ + -DSHERPA_ONNX_ENABLE_WASM_SPEAKER_DIARIZATION=ON \ + -DSHERPA_ONNX_ENABLE_BINARY=OFF \ + -DSHERPA_ONNX_LINK_LIBSTDCPP_STATICALLY=OFF \ + .. +make -j2 +make install + +ls -lh install/bin/wasm/speaker-diarization diff --git a/build-wasm-simd-tts.sh b/build-wasm-simd-tts.sh index 6835e4c43..4e37d2047 100755 --- a/build-wasm-simd-tts.sh +++ b/build-wasm-simd-tts.sh @@ -14,8 +14,8 @@ if [ x"$EMSCRIPTEN" == x"" ]; then echo "git clone https://github.com/emscripten-core/emsdk.git" echo "cd emsdk" echo "git pull" - echo "./emsdk install latest" - echo "./emsdk activate latest" + echo "./emsdk install 3.1.53" + echo "./emsdk activate 3.1.53" echo "source ./emsdk_env.sh" exit 1 else diff --git a/build-wasm-simd-vad-asr.sh b/build-wasm-simd-vad-asr.sh index 5d15cf651..4bf899da3 100755 --- a/build-wasm-simd-vad-asr.sh +++ b/build-wasm-simd-vad-asr.sh @@ -15,8 +15,8 @@ if [ x"$EMSCRIPTEN" == x"" ]; then echo "git clone https://github.com/emscripten-core/emsdk.git" echo "cd emsdk" echo "git pull" - echo "./emsdk install latest" - echo "./emsdk activate latest" + echo "./emsdk install 3.1.53" + echo "./emsdk activate 3.1.53" echo "source ./emsdk_env.sh" exit 1 else diff --git a/build-wasm-simd-vad.sh b/build-wasm-simd-vad.sh index c74f57d37..b73f7f156 100755 --- a/build-wasm-simd-vad.sh +++ b/build-wasm-simd-vad.sh @@ -14,8 +14,8 @@ if [ x"$EMSCRIPTEN" == x"" ]; then echo "git clone https://github.com/emscripten-core/emsdk.git" echo "cd emsdk" echo "git pull" - echo "./emsdk install latest" - echo "./emsdk activate latest" + echo "./emsdk install 3.1.53" + echo "./emsdk activate 3.1.53" echo "source ./emsdk_env.sh" exit 1 else diff --git a/scripts/dotnet/OfflineSpeakerDiarization.cs b/scripts/dotnet/OfflineSpeakerDiarization.cs index b56cab9b6..cfe28e941 100644 --- a/scripts/dotnet/OfflineSpeakerDiarization.cs +++ b/scripts/dotnet/OfflineSpeakerDiarization.cs @@ -16,6 +16,11 @@ public OfflineSpeakerDiarization(OfflineSpeakerDiarizationConfig config) _handle = new HandleRef(this, h); } + public void SetConfig(OfflineSpeakerDiarizationConfig config) + { + SherpaOnnxOfflineSpeakerDiarizationSetConfig(_handle.Handle, ref config); + } + public OfflineSpeakerDiarizationSegment[] Process(float[] samples) { IntPtr result = SherpaOnnxOfflineSpeakerDiarizationProcess(_handle.Handle, samples, samples.Length); @@ -117,6 +122,9 @@ public int SampleRate [DllImport(Dll.Filename)] private static extern void SherpaOnnxOfflineSpeakerDiarizationDestroySegment(IntPtr handle); + + [DllImport(Dll.Filename)] + private static extern void SherpaOnnxOfflineSpeakerDiarizationSetConfig(IntPtr handle, ref OfflineSpeakerDiarizationConfig config); } } diff --git a/scripts/go/sherpa_onnx.go b/scripts/go/sherpa_onnx.go index b8b9e6ee2..8055380c6 100644 --- a/scripts/go/sherpa_onnx.go +++ b/scripts/go/sherpa_onnx.go @@ -1276,6 +1276,16 @@ func (sd *OfflineSpeakerDiarization) SampleRate() int { return int(C.SherpaOnnxOfflineSpeakerDiarizationGetSampleRate(sd.impl)) } +// only config.Clustering is used. All other fields are ignored +func (sd *OfflineSpeakerDiarization) SetConfig(config *OfflineSpeakerDiarizationConfig) { + c := C.struct_SherpaOnnxOfflineSpeakerDiarizationConfig{} + + c.clustering.num_clusters = C.int(config.Clustering.NumClusters) + c.clustering.threshold = C.float(config.Clustering.Threshold) + + SherpaOnnxOfflineSpeakerDiarizationSetConfig(sd.impl, &c) +} + type OfflineSpeakerDiarizationSegment struct { Start float32 End float32 diff --git a/scripts/node-addon-api/lib/non-streaming-speaker-diarization.js b/scripts/node-addon-api/lib/non-streaming-speaker-diarization.js index ae5158517..8ec31ee10 100644 --- a/scripts/node-addon-api/lib/non-streaming-speaker-diarization.js +++ b/scripts/node-addon-api/lib/non-streaming-speaker-diarization.js @@ -25,6 +25,11 @@ class OfflineSpeakerDiarization { process(samples) { return addon.offlineSpeakerDiarizationProcess(this.handle, samples); } + + setConfig(config) { + addon.offlineSpeakerDiarizationSetConfig(config); + this.config.clustering = config.clustering; + } } module.exports = { diff --git a/scripts/node-addon-api/src/non-streaming-speaker-diarization.cc b/scripts/node-addon-api/src/non-streaming-speaker-diarization.cc index d8ac9033c..56767476a 100644 --- a/scripts/node-addon-api/src/non-streaming-speaker-diarization.cc +++ b/scripts/node-addon-api/src/non-streaming-speaker-diarization.cc @@ -251,6 +251,46 @@ static Napi::Array OfflineSpeakerDiarizationProcessWrapper( return ans; } +static void OfflineSpeakerDiarizationSetConfigWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New( + env, "Argument 0 should be an offline speaker diarization pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + const SherpaOnnxOfflineSpeakerDiarization *sd = + info[0].As>().Data(); + + if (!info[1].IsObject()) { + Napi::TypeError::New(env, "Expect an object as the argument") + .ThrowAsJavaScriptException(); + + return; + } + + Napi::Object o = info[0].As(); + + SherpaOnnxOfflineSpeakerDiarizationConfig c; + memset(&c, 0, sizeof(c)); + + c.clustering = GetFastClusteringConfig(o); + SherpaOnnxOfflineSpeakerDiarizationSetConfig(sd, &c); +} + void InitNonStreamingSpeakerDiarization(Napi::Env env, Napi::Object exports) { exports.Set(Napi::String::New(env, "createOfflineSpeakerDiarization"), Napi::Function::New(env, CreateOfflineSpeakerDiarizationWrapper)); @@ -262,4 +302,8 @@ void InitNonStreamingSpeakerDiarization(Napi::Env env, Napi::Object exports) { exports.Set( Napi::String::New(env, "offlineSpeakerDiarizationProcess"), Napi::Function::New(env, OfflineSpeakerDiarizationProcessWrapper)); + + exports.Set( + Napi::String::New(env, "offlineSpeakerDiarizationSetConfig"), + Napi::Function::New(env, OfflineSpeakerDiarizationSetConfigWrapper)); } diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index 322c4f79e..abcfc5b82 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -1749,6 +1749,20 @@ int32_t SherpaOnnxOfflineSpeakerDiarizationGetSampleRate( return sd->impl->SampleRate(); } +void SherpaOnnxOfflineSpeakerDiarizationSetConfig( + const SherpaOnnxOfflineSpeakerDiarization *sd, + const SherpaOnnxOfflineSpeakerDiarizationConfig *config) { + sherpa_onnx::OfflineSpeakerDiarizationConfig sd_config; + + sd_config.clustering.num_clusters = + SHERPA_ONNX_OR(config->clustering.num_clusters, -1); + + sd_config.clustering.threshold = + SHERPA_ONNX_OR(config->clustering.threshold, 0.5); + + sd->impl->SetConfig(sd_config); +} + int32_t SherpaOnnxOfflineSpeakerDiarizationResultGetNumSpeakers( const SherpaOnnxOfflineSpeakerDiarizationResult *r) { return r->impl.NumSpeakers(); diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index d378dedec..c9e7f9ee1 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -1449,6 +1449,11 @@ SHERPA_ONNX_API void SherpaOnnxDestroyOfflineSpeakerDiarization( SHERPA_ONNX_API int32_t SherpaOnnxOfflineSpeakerDiarizationGetSampleRate( const SherpaOnnxOfflineSpeakerDiarization *sd); +// Only config->clustering is used. All other fields are ignored +SHERPA_ONNX_API void SherpaOnnxOfflineSpeakerDiarizationSetConfig( + const SherpaOnnxOfflineSpeakerDiarization *sd, + const SherpaOnnxOfflineSpeakerDiarizationConfig *config); + SHERPA_ONNX_API typedef struct SherpaOnnxOfflineSpeakerDiarizationResult SherpaOnnxOfflineSpeakerDiarizationResult; diff --git a/sherpa-onnx/csrc/offline-speaker-diarization-impl.h b/sherpa-onnx/csrc/offline-speaker-diarization-impl.h index f7fe39499..3aed9d72f 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization-impl.h +++ b/sherpa-onnx/csrc/offline-speaker-diarization-impl.h @@ -20,6 +20,10 @@ class OfflineSpeakerDiarizationImpl { virtual int32_t SampleRate() const = 0; + // Note: Only config.clustering is used. All other fields in config are + // ignored + virtual void SetConfig(const OfflineSpeakerDiarizationConfig &config) = 0; + virtual OfflineSpeakerDiarizationResult Process( const float *audio, int32_t n, OfflineSpeakerDiarizationProgressCallback callback = nullptr, diff --git a/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h b/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h index 64b087c00..9667088d5 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h +++ b/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h @@ -60,7 +60,7 @@ class OfflineSpeakerDiarizationPyannoteImpl : config_(config), segmentation_model_(config_.segmentation), embedding_extractor_(config_.embedding), - clustering_(config_.clustering) { + clustering_(std::make_unique(config_.clustering)) { Init(); } @@ -70,6 +70,15 @@ class OfflineSpeakerDiarizationPyannoteImpl return meta_data.sample_rate; } + void SetConfig(const OfflineSpeakerDiarizationConfig &config) override { + if (!config.clustering.Validate()) { + SHERPA_ONNX_LOGE("Invalid clustering config. Skip it"); + return; + } + clustering_ = std::make_unique(config.clustering); + config_.clustering = config.clustering; + } + OfflineSpeakerDiarizationResult Process( const float *audio, int32_t n, OfflineSpeakerDiarizationProgressCallback callback = nullptr, @@ -105,7 +114,7 @@ class OfflineSpeakerDiarizationPyannoteImpl ComputeEmbeddings(audio, n, chunk_speaker_samples_list_pair.second, std::move(callback), callback_arg); - std::vector cluster_labels = clustering_.Cluster( + std::vector cluster_labels = clustering_->Cluster( &embeddings(0, 0), embeddings.rows(), embeddings.cols()); int32_t max_cluster_index = @@ -636,7 +645,7 @@ class OfflineSpeakerDiarizationPyannoteImpl OfflineSpeakerDiarizationConfig config_; OfflineSpeakerSegmentationPyannoteModel segmentation_model_; SpeakerEmbeddingExtractor embedding_extractor_; - FastClustering clustering_; + std::unique_ptr clustering_; Matrix2DInt32 powerset_mapping_; }; diff --git a/sherpa-onnx/csrc/offline-speaker-diarization.cc b/sherpa-onnx/csrc/offline-speaker-diarization.cc index 4748b1cb4..00733bfb2 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization.cc +++ b/sherpa-onnx/csrc/offline-speaker-diarization.cc @@ -79,6 +79,11 @@ int32_t OfflineSpeakerDiarization::SampleRate() const { return impl_->SampleRate(); } +void OfflineSpeakerDiarization::SetConfig( + const OfflineSpeakerDiarizationConfig &config) { + impl_->SetConfig(config); +} + OfflineSpeakerDiarizationResult OfflineSpeakerDiarization::Process( const float *audio, int32_t n, OfflineSpeakerDiarizationProgressCallback callback /*= nullptr*/, diff --git a/sherpa-onnx/csrc/offline-speaker-diarization.h b/sherpa-onnx/csrc/offline-speaker-diarization.h index e5d02c473..376e5f975 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization.h +++ b/sherpa-onnx/csrc/offline-speaker-diarization.h @@ -62,6 +62,10 @@ class OfflineSpeakerDiarization { // Expected sample rate of the input audio samples int32_t SampleRate() const; + // Note: Only config.clustering is used. All other fields in config are + // ignored + void SetConfig(const OfflineSpeakerDiarizationConfig &config); + OfflineSpeakerDiarizationResult Process( const float *audio, int32_t n, OfflineSpeakerDiarizationProgressCallback callback = nullptr, diff --git a/sherpa-onnx/python/csrc/offline-speaker-diarization.cc b/sherpa-onnx/python/csrc/offline-speaker-diarization.cc index c77979b3c..b3a332f70 100644 --- a/sherpa-onnx/python/csrc/offline-speaker-diarization.cc +++ b/sherpa-onnx/python/csrc/offline-speaker-diarization.cc @@ -68,6 +68,7 @@ void PybindOfflineSpeakerDiarization(py::module *m) { .def(py::init(), py::arg("config")) .def_property_readonly("sample_rate", &PyClass::SampleRate) + .def("set_config", &PyClass::SetConfig, py::arg("config")) .def( "process", [](const PyClass &self, const std::vector samples, diff --git a/swift-api-examples/SherpaOnnx.swift b/swift-api-examples/SherpaOnnx.swift index 881291fd6..783f59224 100644 --- a/swift-api-examples/SherpaOnnx.swift +++ b/swift-api-examples/SherpaOnnx.swift @@ -1161,6 +1161,11 @@ class SherpaOnnxOfflineSpeakerDiarizationWrapper { return Int(SherpaOnnxOfflineSpeakerDiarizationGetSampleRate(impl)) } + // only config.clustering is used. All other fields are ignored + func setConfig(config: UnsafePointer!) { + SherpaOnnxOfflineSpeakerDiarizationSetConfig(impl, config) + } + func process(samples: [Float]) -> [SherpaOnnxOfflineSpeakerDiarizationSegmentWrapper] { let result = SherpaOnnxOfflineSpeakerDiarizationProcess( impl, samples, Int32(samples.count)) diff --git a/wasm/CMakeLists.txt b/wasm/CMakeLists.txt index b143e57b8..7dd6ce7b5 100644 --- a/wasm/CMakeLists.txt +++ b/wasm/CMakeLists.txt @@ -18,6 +18,10 @@ if(SHERPA_ONNX_ENABLE_WASM_VAD_ASR) add_subdirectory(vad-asr) endif() +if(SHERPA_ONNX_ENABLE_WASM_SPEAKER_DIARIZATION) + add_subdirectory(speaker-diarization) +endif() + if(SHERPA_ONNX_ENABLE_WASM_NODEJS) add_subdirectory(nodejs) endif() diff --git a/wasm/speaker-diarization/CMakeLists.txt b/wasm/speaker-diarization/CMakeLists.txt new file mode 100644 index 000000000..71af018ac --- /dev/null +++ b/wasm/speaker-diarization/CMakeLists.txt @@ -0,0 +1,61 @@ +if(NOT $ENV{SHERPA_ONNX_IS_USING_BUILD_WASM_SH}) + message(FATAL_ERROR "Please use ./build-wasm-simd-speaker-diarization.sh to build for WASM for speaker diarization") +endif() + +if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/assets/segmentation.onnx" OR NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/assets/embedding.onnx") + message(FATAL_ERROR "Please read ${CMAKE_CURRENT_SOURCE_DIR}/assets/README.md before you continue") +endif() + +set(exported_functions + MyPrint + SherpaOnnxCreateOfflineSpeakerDiarization + SherpaOnnxDestroyOfflineSpeakerDiarization + SherpaOnnxOfflineSpeakerDiarizationDestroyResult + SherpaOnnxOfflineSpeakerDiarizationDestroySegment + SherpaOnnxOfflineSpeakerDiarizationGetSampleRate + SherpaOnnxOfflineSpeakerDiarizationProcess + SherpaOnnxOfflineSpeakerDiarizationProcessWithCallback + SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments + SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime + SherpaOnnxOfflineSpeakerDiarizationSetConfig +) +set(mangled_exported_functions) +foreach(x IN LISTS exported_functions) + list(APPEND mangled_exported_functions "_${x}") +endforeach() +list(JOIN mangled_exported_functions "," all_exported_functions) + + +include_directories(${CMAKE_SOURCE_DIR}) +set(MY_FLAGS " -s FORCE_FILESYSTEM=1 -s INITIAL_MEMORY=512MB -s ALLOW_MEMORY_GROWTH=1") +string(APPEND MY_FLAGS " -sSTACK_SIZE=10485760 ") # 10MB +string(APPEND MY_FLAGS " -sEXPORTED_FUNCTIONS=[_CopyHeap,_malloc,_free,${all_exported_functions}] ") +string(APPEND MY_FLAGS "--preload-file ${CMAKE_CURRENT_SOURCE_DIR}/assets@. ") +string(APPEND MY_FLAGS " -sEXPORTED_RUNTIME_METHODS=['ccall','stringToUTF8','setValue','getValue','lengthBytesUTF8','UTF8ToString'] ") + +message(STATUS "MY_FLAGS: ${MY_FLAGS}") + +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${MY_FLAGS}") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${MY_FLAGS}") +set(CMAKE_EXECUTBLE_LINKER_FLAGS "${CMAKE_EXECUTBLE_LINKER_FLAGS} ${MY_FLAGS}") + +if (NOT CMAKE_EXECUTABLE_SUFFIX STREQUAL ".js") + message(FATAL_ERROR "The default suffix for building executables should be .js!") +endif() +# set(CMAKE_EXECUTABLE_SUFFIX ".html") + +add_executable(sherpa-onnx-wasm-main-speaker-diarization sherpa-onnx-wasm-main-speaker-diarization.cc) +target_link_libraries(sherpa-onnx-wasm-main-speaker-diarization sherpa-onnx-c-api) +install(TARGETS sherpa-onnx-wasm-main-speaker-diarization DESTINATION bin/wasm/speaker-diarization) + +install( + FILES + "$/sherpa-onnx-wasm-main-speaker-diarization.js" + "index.html" + "sherpa-onnx-speaker-diarization.js" + "app-speaker-diarization.js" + "$/sherpa-onnx-wasm-main-speaker-diarization.wasm" + "$/sherpa-onnx-wasm-main-speaker-diarization.data" + DESTINATION + bin/wasm/speaker-diarization +) diff --git a/wasm/speaker-diarization/app-speaker-diarization.js b/wasm/speaker-diarization/app-speaker-diarization.js new file mode 100644 index 000000000..cb757fcfd --- /dev/null +++ b/wasm/speaker-diarization/app-speaker-diarization.js @@ -0,0 +1,124 @@ +const startBtn = document.getElementById('startBtn'); +const hint = document.getElementById('hint'); +const numClustersInput = document.getElementById('numClustersInputID'); +const thresholdInput = document.getElementById('thresholdInputID'); +const textArea = document.getElementById('text'); + +const fileSelectCtrl = document.getElementById('file'); + +let sd = null; +let float32Samples = null; + +Module = {}; +Module.onRuntimeInitialized = function() { + console.log('Model files downloaded!'); + + console.log('Initializing speaker diarization ......'); + sd = createOfflineSpeakerDiarization(Module) + console.log('sampleRate', sd.sampleRate); + + hint.innerText = + 'Initialized! Please select a wave file and click the Start button.'; + + fileSelectCtrl.disabled = false; +}; + +function onFileChange() { + var files = document.getElementById('file').files; + + if (files.length == 0) { + console.log('No file selected'); + float32Samples = null; + startBtn.disabled = true; + return; + } + textArea.value = ''; + + console.log('files: ' + files); + + const file = files[0]; + console.log(file); + console.log('file.name ' + file.name); + console.log('file.type ' + file.type); + console.log('file.size ' + file.size); + + let audioCtx = new AudioContext({sampleRate: sd.sampleRate}); + + let reader = new FileReader(); + reader.onload = function() { + console.log('reading file!'); + audioCtx.decodeAudioData(reader.result, decodedDone); + }; + + function decodedDone(decoded) { + let typedArray = new Float32Array(decoded.length); + float32Samples = decoded.getChannelData(0); + + startBtn.disabled = false; + } + + reader.readAsArrayBuffer(file); +} + +startBtn.onclick = function() { + textArea.value = ''; + if (float32Samples == null) { + alert('Empty audio samples!'); + + startBtn.disabled = true; + return; + } + + let numClusters = numClustersInput.value; + if (numClusters.trim().length == 0) { + alert( + 'Please provide numClusters. Use -1 if you are not sure how many speakers are there'); + return; + } + + if (!numClusters.match(/^\d+$/)) { + alert(`number of clusters ${ + numClusters} is not an integer .\nPlease enter an integer`); + return; + } + numClusters = parseInt(numClusters, 10); + if (numClusters < -1) { + alert(`Number of clusters should be >= -1`); + return; + } + + let threshold = 0.5; + if (numClusters <= 0) { + threshold = thresholdInput.value; + if (threshold.trim().length == 0) { + alert('Please provide a threshold.'); + return; + } + + threshold = parseFloat(threshold); + if (threshold < 0) { + alert(`Pleaser enter a positive threshold`); + return; + } + } + + let config = sd.config + config.clustering = {numClusters: numClusters, threshold: threshold}; + sd.setConfig(config); + let segments = sd.process(float32Samples); + if (segments == null) { + textArea.value = 'No speakers detected'; + return + } + + let s = ''; + let sep = ''; + + for (seg of segments) { + // clang-format off + s += sep + `${seg.start.toFixed(2)} -- ${seg.end.toFixed(2)} speaker_${seg.speaker}` + // clang-format on + sep = '\n'; + } + textArea.value = s; +} diff --git a/wasm/speaker-diarization/assets/README.md b/wasm/speaker-diarization/assets/README.md new file mode 100644 index 000000000..5c06139e2 --- /dev/null +++ b/wasm/speaker-diarization/assets/README.md @@ -0,0 +1,30 @@ +# Introduction + +Please refer to +https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models +to download a speaker segmentation model +and +refer to +https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models +to download a speaker embedding extraction model. + +Remember to rename the downloaded files. + +The following is an example. + + +```bash +cd wasm/speaker-diarization/assets/ + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 +tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 +rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 +cp sherpa-onnx-pyannote-segmentation-3-0/model.onnx ./segmentation.onnx +rm -rf sherpa-onnx-pyannote-segmentation-3-0 + + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx +mv 3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx ./embedding.onnx + + +``` diff --git a/wasm/speaker-diarization/index.html b/wasm/speaker-diarization/index.html new file mode 100644 index 000000000..55de8bd3b --- /dev/null +++ b/wasm/speaker-diarization/index.html @@ -0,0 +1,48 @@ + + + + + + Next-gen Kaldi WebAssembly with sherpa-onnx for Speaker Diarization + + + + +

+ Next-gen Kaldi + WebAssembly
+ Speaker Diarization
with sherpa-onnx +

+
+ Loading model ... ... +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+ + + + + diff --git a/wasm/speaker-diarization/sherpa-onnx-speaker-diarization.js b/wasm/speaker-diarization/sherpa-onnx-speaker-diarization.js new file mode 100644 index 000000000..ccfc8373c --- /dev/null +++ b/wasm/speaker-diarization/sherpa-onnx-speaker-diarization.js @@ -0,0 +1,295 @@ + +function freeConfig(config, Module) { + if ('buffer' in config) { + Module._free(config.buffer); + } + + if ('config' in config) { + freeConfig(config.config, Module) + } + + if ('segmentation' in config) { + freeConfig(config.segmentation, Module) + } + + if ('embedding' in config) { + freeConfig(config.embedding, Module) + } + + if ('clustering' in config) { + freeConfig(config.clustering, Module) + } + + Module._free(config.ptr); +} + +function initSherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig( + config, Module) { + const modelLen = Module.lengthBytesUTF8(config.model || '') + 1; + const n = modelLen; + const buffer = Module._malloc(n); + + const len = 1 * 4; + const ptr = Module._malloc(len); + + let offset = 0; + Module.stringToUTF8(config.model || '', buffer + offset, modelLen); + offset += modelLen; + + offset = 0; + Module.setValue(ptr, buffer + offset, 'i8*'); + + return { + buffer: buffer, ptr: ptr, len: len, + } +} + +function initSherpaOnnxOfflineSpeakerSegmentationModelConfig(config, Module) { + if (!('pyannote' in config)) { + config.pyannote = { + model: '', + }; + } + + const pyannote = initSherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig( + config.pyannote, Module); + + const len = pyannote.len + 3 * 4; + const ptr = Module._malloc(len); + + let offset = 0; + Module._CopyHeap(pyannote.ptr, pyannote.len, ptr + offset); + offset += pyannote.len; + + Module.setValue(ptr + offset, config.numThreads || 1, 'i32'); + offset += 4; + + Module.setValue(ptr + offset, config.debug || 1, 'i32'); + offset += 4; + + const providerLen = Module.lengthBytesUTF8(config.provider || 'cpu') + 1; + const buffer = Module._malloc(providerLen); + Module.stringToUTF8(config.provider || 'cpu', buffer, providerLen); + Module.setValue(ptr + offset, buffer, 'i8*'); + + return { + buffer: buffer, + ptr: ptr, + len: len, + config: pyannote, + }; +} + +function initSherpaOnnxSpeakerEmbeddingExtractorConfig(config, Module) { + const modelLen = Module.lengthBytesUTF8(config.model || '') + 1; + const providerLen = Module.lengthBytesUTF8(config.provider || 'cpu') + 1; + const n = modelLen + providerLen; + const buffer = Module._malloc(n); + + const len = 4 * 4; + const ptr = Module._malloc(len); + + let offset = 0; + Module.stringToUTF8(config.model || '', buffer + offset, modelLen); + offset += modelLen; + + Module.stringToUTF8(config.provider || 'cpu', buffer + offset, providerLen); + offset += providerLen; + + offset = 0 + Module.setValue(ptr + offset, buffer, 'i8*'); + offset += 4; + + Module.setValue(ptr + offset, config.numThreads || 1, 'i32'); + offset += 4; + + Module.setValue(ptr + offset, config.debug || 1, 'i32'); + offset += 4; + + Module.setValue(ptr + offset, buffer + modelLen, 'i8*'); + offset += 4; + + return { + buffer: buffer, + ptr: ptr, + len: len, + }; +} + +function initSherpaOnnxFastClusteringConfig(config, Module) { + const len = 2 * 4; + const ptr = Module._malloc(len); + + let offset = 0; + Module.setValue(ptr + offset, config.numClusters || -1, 'i32'); + offset += 4; + + Module.setValue(ptr + offset, config.threshold || 0.5, 'float'); + offset += 4; + + return { + ptr: ptr, + len: len, + }; +} + +function initSherpaOnnxOfflineSpeakerDiarizationConfig(config, Module) { + if (!('segmentation' in config)) { + config.segmentation = { + pyannote: {model: ''}, + numThreads: 1, + debug: 0, + provider: 'cpu', + }; + } + + if (!('embedding' in config)) { + config.embedding = { + model: '', + numThreads: 1, + debug: 0, + provider: 'cpu', + }; + } + + if (!('clustering' in config)) { + config.clustering = { + numClusters: -1, + threshold: 0.5, + }; + } + + const segmentation = initSherpaOnnxOfflineSpeakerSegmentationModelConfig( + config.segmentation, Module); + + const embedding = + initSherpaOnnxSpeakerEmbeddingExtractorConfig(config.embedding, Module); + + const clustering = + initSherpaOnnxFastClusteringConfig(config.clustering, Module); + + const len = segmentation.len + embedding.len + clustering.len + 2 * 4; + const ptr = Module._malloc(len); + + let offset = 0; + Module._CopyHeap(segmentation.ptr, segmentation.len, ptr + offset); + offset += segmentation.len; + + Module._CopyHeap(embedding.ptr, embedding.len, ptr + offset); + offset += embedding.len; + + Module._CopyHeap(clustering.ptr, clustering.len, ptr + offset); + offset += clustering.len; + + Module.setValue(ptr + offset, config.minDurationOn || 0.2, 'float'); + offset += 4; + + Module.setValue(ptr + offset, config.minDurationOff || 0.5, 'float'); + offset += 4; + + return { + ptr: ptr, len: len, segmentation: segmentation, embedding: embedding, + clustering: clustering, + } +} + +class OfflineSpeakerDiarization { + constructor(configObj, Module) { + const config = + initSherpaOnnxOfflineSpeakerDiarizationConfig(configObj, Module) + // Module._MyPrint(config.ptr); + + const handle = + Module._SherpaOnnxCreateOfflineSpeakerDiarization(config.ptr); + + freeConfig(config, Module); + + this.handle = handle; + this.sampleRate = + Module._SherpaOnnxOfflineSpeakerDiarizationGetSampleRate(this.handle); + this.Module = Module + + this.config = configObj; + } + + free() { + this.Module._SherpaOnnxDestroyOfflineSpeakerDiarization(this.handle); + this.handle = 0 + } + + setConfig(configObj) { + if (!('clustering' in configObj)) { + return; + } + + const config = + initSherpaOnnxOfflineSpeakerDiarizationConfig(configObj, this.Module); + + this.Module._SherpaOnnxOfflineSpeakerDiarizationSetConfig( + this.handle, config.ptr); + + freeConfig(config, Module); + + this.config.clustering = configObj.clustering; + } + + process(samples) { + const pointer = + this.Module._malloc(samples.length * samples.BYTES_PER_ELEMENT); + this.Module.HEAPF32.set(samples, pointer / samples.BYTES_PER_ELEMENT); + + let r = this.Module._SherpaOnnxOfflineSpeakerDiarizationProcess( + this.handle, pointer, samples.length); + this.Module._free(pointer); + + let numSegments = + this.Module._SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments(r); + + let segments = + this.Module._SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime( + r); + + let ans = []; + + let sizeOfSegment = 3 * 4; + for (let i = 0; i < numSegments; ++i) { + let p = segments + i * sizeOfSegment + + let start = this.Module.HEAPF32[p / 4 + 0]; + let end = this.Module.HEAPF32[p / 4 + 1]; + let speaker = this.Module.HEAP32[p / 4 + 2]; + + ans.push({start: start, end: end, speaker: speaker}); + } + + this.Module._SherpaOnnxOfflineSpeakerDiarizationDestroySegment(segments); + this.Module._SherpaOnnxOfflineSpeakerDiarizationDestroyResult(r); + + return ans; + } +} + +function createOfflineSpeakerDiarization(Module, myConfig) { + const config = { + segmentation: { + pyannote: {model: './segmentation.onnx'}, + }, + embedding: {model: './embedding.onnx'}, + clustering: {numClusters: -1, threshold: 0.5}, + minDurationOn: 0.3, + minDurationOff: 0.5, + }; + + if (myConfig) { + config = myConfig; + } + + return new OfflineSpeakerDiarization(config, Module); +} + +if (typeof process == 'object' && typeof process.versions == 'object' && + typeof process.versions.node == 'string') { + module.exports = { + createOfflineSpeakerDiarization, + }; +} diff --git a/wasm/speaker-diarization/sherpa-onnx-wasm-main-speaker-diarization.cc b/wasm/speaker-diarization/sherpa-onnx-wasm-main-speaker-diarization.cc new file mode 100644 index 000000000..6e83f61d8 --- /dev/null +++ b/wasm/speaker-diarization/sherpa-onnx-wasm-main-speaker-diarization.cc @@ -0,0 +1,63 @@ +// wasm/sherpa-onnx-wasm-main-speaker-diarization.cc +// +// Copyright (c) 2024 Xiaomi Corporation +#include + +#include +#include + +#include "sherpa-onnx/c-api/c-api.h" + +// see also +// https://emscripten.org/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.html + +extern "C" { + +static_assert(sizeof(SherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig) == + 1 * 4, + ""); + +static_assert( + sizeof(SherpaOnnxOfflineSpeakerSegmentationModelConfig) == + sizeof(SherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig) + 3 * 4, + ""); + +static_assert(sizeof(SherpaOnnxFastClusteringConfig) == 2 * 4, ""); + +static_assert(sizeof(SherpaOnnxSpeakerEmbeddingExtractorConfig) == 4 * 4, ""); + +static_assert(sizeof(SherpaOnnxOfflineSpeakerDiarizationConfig) == + sizeof(SherpaOnnxOfflineSpeakerSegmentationModelConfig) + + sizeof(SherpaOnnxSpeakerEmbeddingExtractorConfig) + + sizeof(SherpaOnnxFastClusteringConfig) + 2 * 4, + ""); + +void MyPrint(const SherpaOnnxOfflineSpeakerDiarizationConfig *sd_config) { + const auto &segmentation = sd_config->segmentation; + const auto &embedding = sd_config->embedding; + const auto &clustering = sd_config->clustering; + + fprintf(stdout, "----------segmentation config----------\n"); + fprintf(stdout, "pyannote model: %s\n", segmentation.pyannote.model); + fprintf(stdout, "num threads: %d\n", segmentation.num_threads); + fprintf(stdout, "debug: %d\n", segmentation.debug); + fprintf(stdout, "provider: %s\n", segmentation.provider); + + fprintf(stdout, "----------embedding config----------\n"); + fprintf(stdout, "model: %s\n", embedding.model); + fprintf(stdout, "num threads: %d\n", embedding.num_threads); + fprintf(stdout, "debug: %d\n", embedding.debug); + fprintf(stdout, "provider: %s\n", embedding.provider); + + fprintf(stdout, "----------clustering config----------\n"); + fprintf(stdout, "num_clusters: %d\n", clustering.num_clusters); + fprintf(stdout, "threshold: %.3f\n", clustering.threshold); + + fprintf(stdout, "min_duration_on: %.3f\n", sd_config->min_duration_on); + fprintf(stdout, "min_duration_off: %.3f\n", sd_config->min_duration_off); +} + +void CopyHeap(const char *src, int32_t num_bytes, char *dst) { + std::copy(src, src + num_bytes, dst); +} +} From f1b311ee4fe4d84468ed93d8479097b21d13c5d2 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 11 Oct 2024 10:27:16 +0800 Subject: [PATCH 011/183] Handle audio files less than 10s long for speaker diarization. (#1412) If the input audio file is less than 10 seconds long, there is only one chunk, and there is no need to compute embeddings or do clustering. We can use the segmentation result from the speaker segmentation model directly. --- ...ffline-speaker-diarization-pyannote-impl.h | 35 +++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h b/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h index 9667088d5..8f669e27c 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h +++ b/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h @@ -99,6 +99,14 @@ class OfflineSpeakerDiarizationPyannoteImpl segmentations.clear(); + if (labels.size() == 1) { + if (callback) { + callback(1, 1, callback_arg); + } + + return HandleOneChunkSpecialCase(labels[0], n); + } + // labels[i] is a 0-1 matrix of shape (num_frames, num_speakers) // speaker count per frame @@ -201,7 +209,7 @@ class OfflineSpeakerDiarizationPyannoteImpl } int32_t num_chunks = (n - window_size) / window_shift + 1; - bool has_last_chunk = (n - window_size) % window_shift > 0; + bool has_last_chunk = ((n - window_size) % window_shift) > 0; ans.reserve(num_chunks + has_last_chunk); @@ -524,9 +532,9 @@ class OfflineSpeakerDiarizationPyannoteImpl count(seq, Eigen::all).array() += labels[i].array(); } - bool has_last_chunk = (num_samples - window_size) % window_shift > 0; + bool has_last_chunk = ((num_samples - window_size) % window_shift) > 0; - if (has_last_chunk) { + if (!has_last_chunk) { return count; } @@ -622,6 +630,27 @@ class OfflineSpeakerDiarizationPyannoteImpl return ans; } + OfflineSpeakerDiarizationResult HandleOneChunkSpecialCase( + const Matrix2DInt32 &final_labels, int32_t num_samples) const { + const auto &meta_data = segmentation_model_.GetModelMetaData(); + int32_t window_size = meta_data.window_size; + int32_t window_shift = meta_data.window_shift; + int32_t receptive_field_shift = meta_data.receptive_field_shift; + + bool has_last_chunk = (num_samples - window_size) % window_shift > 0; + if (!has_last_chunk) { + return ComputeResult(final_labels); + } + + int32_t num_frames = final_labels.rows(); + + int32_t new_num_frames = num_samples / receptive_field_shift; + + num_frames = (new_num_frames <= num_frames) ? new_num_frames : num_frames; + + return ComputeResult(final_labels(Eigen::seq(0, num_frames), Eigen::all)); + } + void MergeSegments( std::vector *segments) const { float min_duration_off = config_.min_duration_off; From eefc17209589fe3b950f561bc9c8b8d1c9b8a742 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 11 Oct 2024 11:40:10 +0800 Subject: [PATCH 012/183] JavaScript API with WebAssembly for speaker diarization (#1414) #1408 uses [node-addon-api](https://github.com/nodejs/node-addon-api) to call C API from JavaScript, whereas this pull request uses WebAssembly to call C API from JavaScript. --- .github/scripts/test-nodejs-npm.sh | 12 ++++ .github/workflows/test-build-wheel.yaml | 2 +- .github/workflows/test-pip-install.yaml | 2 +- nodejs-examples/README.md | 16 +++++ .../test-offline-speaker-diarization.js | 64 +++++++++++++++++++ scripts/nodejs/index.js | 8 +++ wasm/nodejs/CMakeLists.txt | 12 ++++ wasm/speaker-diarization/assets/README.md | 4 -- .../sherpa-onnx-speaker-diarization.js | 12 ++-- 9 files changed, 122 insertions(+), 10 deletions(-) create mode 100644 nodejs-examples/test-offline-speaker-diarization.js diff --git a/.github/scripts/test-nodejs-npm.sh b/.github/scripts/test-nodejs-npm.sh index c41a0de65..03dec04aa 100755 --- a/.github/scripts/test-nodejs-npm.sh +++ b/.github/scripts/test-nodejs-npm.sh @@ -9,6 +9,18 @@ git status ls -lh ls -lh node_modules +echo '-----speaker diarization----------' +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 +tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 +rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/0-four-speakers-zh.wav + +node ./test-offline-speaker-diarization.js +rm -rfv *.wav *.onnx sherpa-onnx-pyannote-* + echo '-----vad+whisper----------' curl -LS -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2 diff --git a/.github/workflows/test-build-wheel.yaml b/.github/workflows/test-build-wheel.yaml index a9b2db589..8b7472b84 100644 --- a/.github/workflows/test-build-wheel.yaml +++ b/.github/workflows/test-build-wheel.yaml @@ -139,7 +139,7 @@ jobs: export PATH=/c/hostedtoolcache/windows/Python/3.9.13/x64/bin:$PATH export PATH=/c/hostedtoolcache/windows/Python/3.10.11/x64/bin:$PATH export PATH=/c/hostedtoolcache/windows/Python/3.11.9/x64/bin:$PATH - export PATH=/c/hostedtoolcache/windows/Python/3.12.6/x64/bin:$PATH + export PATH=/c/hostedtoolcache/windows/Python/3.12.7/x64/bin:$PATH which sherpa-onnx sherpa-onnx --help diff --git a/.github/workflows/test-pip-install.yaml b/.github/workflows/test-pip-install.yaml index 0f73e3643..b59b66b53 100644 --- a/.github/workflows/test-pip-install.yaml +++ b/.github/workflows/test-pip-install.yaml @@ -104,7 +104,7 @@ jobs: export PATH=/c/hostedtoolcache/windows/Python/3.9.13/x64/bin:$PATH export PATH=/c/hostedtoolcache/windows/Python/3.10.11/x64/bin:$PATH export PATH=/c/hostedtoolcache/windows/Python/3.11.9/x64/bin:$PATH - export PATH=/c/hostedtoolcache/windows/Python/3.12.6/x64/bin:$PATH + export PATH=/c/hostedtoolcache/windows/Python/3.12.7/x64/bin:$PATH sherpa-onnx --help sherpa-onnx-keyword-spotter --help diff --git a/nodejs-examples/README.md b/nodejs-examples/README.md index 73a85de77..496a0062b 100644 --- a/nodejs-examples/README.md +++ b/nodejs-examples/README.md @@ -22,6 +22,22 @@ In the following, we describe how to use [sherpa-onnx](https://github.com/k2-fsa for text-to-speech and speech-to-text. +# Speaker diarization + +In the following, we demonstrate how to run speaker diarization. + +```bash +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 +tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 +rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/0-four-speakers-zh.wav + +node ./test-offline-speaker-diarization.js +``` + # Text-to-speech In the following, we demonstrate how to run text-to-speech. diff --git a/nodejs-examples/test-offline-speaker-diarization.js b/nodejs-examples/test-offline-speaker-diarization.js new file mode 100644 index 000000000..de0f4a45b --- /dev/null +++ b/nodejs-examples/test-offline-speaker-diarization.js @@ -0,0 +1,64 @@ +// Copyright (c) 2024 Xiaomi Corporation +const sherpa_onnx = require('sherpa-onnx'); + +// clang-format off +/* Please use the following commands to download files + used in this script + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 +tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 +rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/0-four-speakers-zh.wav + + */ +// clang-format on + +const config = { + segmentation: { + pyannote: { + model: './sherpa-onnx-pyannote-segmentation-3-0/model.onnx', + debug: 1, + }, + }, + embedding: { + model: './3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx', + debug: 1, + }, + clustering: { + // since we know that the test wave file + // ./0-four-speakers-zh.wav contains 4 speakers, we use 4 for numClusters + // here. if you don't have such information, please set numClusters to -1 + numClusters: 4, + + // If numClusters is not -1, then threshold is ignored. + // + // A larger threshold leads to fewer clusters, i.e., fewer speakers + // A smaller threshold leads to more clusters, i.e., more speakers + // You need to tune it by yourself. + threshold: 0.5, + }, + + // If a segment is shorter than minDurationOn, we discard it + minDurationOn: 0.2, // in seconds + + // If the gap between two segments is less than minDurationOff, then we + // merge these two segments into a single one + minDurationOff: 0.5, // in seconds +}; + +const waveFilename = './0-four-speakers-zh.wav'; + +const sd = sherpa_onnx.createOfflineSpeakerDiarization(config); +console.log('Started') + +const wave = sherpa_onnx.readWave(waveFilename); +if (sd.sampleRate != wave.sampleRate) { + throw new Error( + `Expected sample rate: ${sd.sampleRate}, given: ${wave.sampleRate}`); +} + +const segments = sd.process(wave.samples); +console.log(segments); diff --git a/scripts/nodejs/index.js b/scripts/nodejs/index.js index 3f0789edb..b1b77841c 100644 --- a/scripts/nodejs/index.js +++ b/scripts/nodejs/index.js @@ -7,6 +7,8 @@ const sherpa_onnx_tts = require('./sherpa-onnx-tts.js'); const sherpa_onnx_kws = require('./sherpa-onnx-kws.js'); const sherpa_onnx_wave = require('./sherpa-onnx-wave.js'); const sherpa_onnx_vad = require('./sherpa-onnx-vad.js'); +const sherpa_onnx_speaker_diarization = + require('./sherpa-onnx-speaker-diarization.js'); function createOnlineRecognizer(config) { return sherpa_onnx_asr.createOnlineRecognizer(wasmModule, config); @@ -32,6 +34,11 @@ function createVad(config) { return sherpa_onnx_vad.createVad(wasmModule, config); } +function createOfflineSpeakerDiarization(config) { + return sherpa_onnx_speaker_diarization.createOfflineSpeakerDiarization( + wasmModule, config); +} + function readWave(filename) { return sherpa_onnx_wave.readWave(filename, wasmModule); } @@ -51,4 +58,5 @@ module.exports = { writeWave, createCircularBuffer, createVad, + createOfflineSpeakerDiarization, }; diff --git a/wasm/nodejs/CMakeLists.txt b/wasm/nodejs/CMakeLists.txt index 4efc879a1..dc8d8c854 100644 --- a/wasm/nodejs/CMakeLists.txt +++ b/wasm/nodejs/CMakeLists.txt @@ -70,6 +70,17 @@ set(exported_functions SherpaOnnxDestroySpeechSegment SherpaOnnxVoiceActivityDetectorReset SherpaOnnxVoiceActivityDetectorFlush + # Speaker diarization + SherpaOnnxCreateOfflineSpeakerDiarization + SherpaOnnxDestroyOfflineSpeakerDiarization + SherpaOnnxOfflineSpeakerDiarizationDestroyResult + SherpaOnnxOfflineSpeakerDiarizationDestroySegment + SherpaOnnxOfflineSpeakerDiarizationGetSampleRate + SherpaOnnxOfflineSpeakerDiarizationProcess + SherpaOnnxOfflineSpeakerDiarizationProcessWithCallback + SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments + SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime + SherpaOnnxOfflineSpeakerDiarizationSetConfig # SherpaOnnxFileExists SherpaOnnxReadWave @@ -109,6 +120,7 @@ install( ${CMAKE_SOURCE_DIR}/wasm/tts/sherpa-onnx-tts.js ${CMAKE_SOURCE_DIR}/wasm/kws/sherpa-onnx-kws.js ${CMAKE_SOURCE_DIR}/wasm/vad/sherpa-onnx-vad.js + ${CMAKE_SOURCE_DIR}/wasm/speaker-diarization/sherpa-onnx-speaker-diarization.js ${CMAKE_SOURCE_DIR}/wasm/nodejs/sherpa-onnx-wave.js "$/sherpa-onnx-wasm-nodejs.js" "$/sherpa-onnx-wasm-nodejs.wasm" diff --git a/wasm/speaker-diarization/assets/README.md b/wasm/speaker-diarization/assets/README.md index 5c06139e2..f09a5899d 100644 --- a/wasm/speaker-diarization/assets/README.md +++ b/wasm/speaker-diarization/assets/README.md @@ -12,7 +12,6 @@ Remember to rename the downloaded files. The following is an example. - ```bash cd wasm/speaker-diarization/assets/ @@ -22,9 +21,6 @@ rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 cp sherpa-onnx-pyannote-segmentation-3-0/model.onnx ./segmentation.onnx rm -rf sherpa-onnx-pyannote-segmentation-3-0 - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx mv 3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx ./embedding.onnx - - ``` diff --git a/wasm/speaker-diarization/sherpa-onnx-speaker-diarization.js b/wasm/speaker-diarization/sherpa-onnx-speaker-diarization.js index ccfc8373c..741013480 100644 --- a/wasm/speaker-diarization/sherpa-onnx-speaker-diarization.js +++ b/wasm/speaker-diarization/sherpa-onnx-speaker-diarization.js @@ -64,7 +64,7 @@ function initSherpaOnnxOfflineSpeakerSegmentationModelConfig(config, Module) { Module.setValue(ptr + offset, config.numThreads || 1, 'i32'); offset += 4; - Module.setValue(ptr + offset, config.debug || 1, 'i32'); + Module.setValue(ptr + offset, config.debug || 0, 'i32'); offset += 4; const providerLen = Module.lengthBytesUTF8(config.provider || 'cpu') + 1; @@ -103,7 +103,7 @@ function initSherpaOnnxSpeakerEmbeddingExtractorConfig(config, Module) { Module.setValue(ptr + offset, config.numThreads || 1, 'i32'); offset += 4; - Module.setValue(ptr + offset, config.debug || 1, 'i32'); + Module.setValue(ptr + offset, config.debug || 0, 'i32'); offset += 4; Module.setValue(ptr + offset, buffer + modelLen, 'i8*'); @@ -270,11 +270,15 @@ class OfflineSpeakerDiarization { } function createOfflineSpeakerDiarization(Module, myConfig) { - const config = { + let config = { segmentation: { pyannote: {model: './segmentation.onnx'}, + debug: 1, + }, + embedding: { + model: './embedding.onnx', + debug: 1, }, - embedding: {model: './embedding.onnx'}, clustering: {numClusters: -1, threshold: 0.5}, minDurationOn: 0.3, minDurationOff: 0.5, From 2d412b1190778bc35f337ef1feeb12292b5c9f92 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 11 Oct 2024 14:41:53 +0800 Subject: [PATCH 013/183] Kotlin API for speaker diarization (#1415) --- .../OfflineSpeakerDiarization.kt | 1 + kotlin-api-examples/run.sh | 31 +++ .../test_offline_speaker_diarization.kt | 53 +++++ .../csrc/offline-speaker-diarization-result.h | 2 +- sherpa-onnx/jni/CMakeLists.txt | 6 + .../jni/offline-speaker-diarization.cc | 219 ++++++++++++++++++ .../kotlin-api/OfflineSpeakerDiarization.kt | 101 ++++++++ 7 files changed, 412 insertions(+), 1 deletion(-) create mode 120000 kotlin-api-examples/OfflineSpeakerDiarization.kt create mode 100644 kotlin-api-examples/test_offline_speaker_diarization.kt create mode 100644 sherpa-onnx/jni/offline-speaker-diarization.cc create mode 100644 sherpa-onnx/kotlin-api/OfflineSpeakerDiarization.kt diff --git a/kotlin-api-examples/OfflineSpeakerDiarization.kt b/kotlin-api-examples/OfflineSpeakerDiarization.kt new file mode 120000 index 000000000..870612b4c --- /dev/null +++ b/kotlin-api-examples/OfflineSpeakerDiarization.kt @@ -0,0 +1 @@ +../sherpa-onnx/kotlin-api/OfflineSpeakerDiarization.kt \ No newline at end of file diff --git a/kotlin-api-examples/run.sh b/kotlin-api-examples/run.sh index 23e86886e..50e7816f1 100755 --- a/kotlin-api-examples/run.sh +++ b/kotlin-api-examples/run.sh @@ -285,6 +285,37 @@ function testPunctuation() { java -Djava.library.path=../build/lib -jar $out_filename } +function testOfflineSpeakerDiarization() { + if [ ! -f ./sherpa-onnx-pyannote-segmentation-3-0/model.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + fi + + if [ ! -f ./3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx + fi + + if [ ! -f ./0-four-speakers-zh.wav ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/0-four-speakers-zh.wav + fi + + out_filename=test_offline_speaker_diarization.jar + kotlinc-jvm -include-runtime -d $out_filename \ + test_offline_speaker_diarization.kt \ + OfflineSpeakerDiarization.kt \ + Speaker.kt \ + OnlineStream.kt \ + WaveReader.kt \ + faked-asset-manager.kt \ + faked-log.kt + + ls -lh $out_filename + + java -Djava.library.path=../build/lib -jar $out_filename +} + +testOfflineSpeakerDiarization testSpeakerEmbeddingExtractor testOnlineAsr testTts diff --git a/kotlin-api-examples/test_offline_speaker_diarization.kt b/kotlin-api-examples/test_offline_speaker_diarization.kt new file mode 100644 index 000000000..96c33f062 --- /dev/null +++ b/kotlin-api-examples/test_offline_speaker_diarization.kt @@ -0,0 +1,53 @@ +package com.k2fsa.sherpa.onnx + +fun main() { + testOfflineSpeakerDiarization() +} + +fun callback(numProcessedChunks: Int, numTotalChunks: Int, arg: Long): Int { + val progress = numProcessedChunks.toFloat() / numTotalChunks * 100 + val s = "%.2f".format(progress) + println("Progress: ${s}%"); + + return 0 +} + +fun testOfflineSpeakerDiarization() { + var config = OfflineSpeakerDiarizationConfig( + segmentation=OfflineSpeakerSegmentationModelConfig( + pyannote=OfflineSpeakerSegmentationPyannoteModelConfig("./sherpa-onnx-pyannote-segmentation-3-0/model.onnx"), + ), + embedding=SpeakerEmbeddingExtractorConfig( + model="./3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx", + ), + + // The test wave file ./0-four-speakers-zh.wav contains four speakers, so + // we use numClusters=4 here. If you don't know the number of speakers + // in the test wave file, please set the threshold like below. + // + // clustering=FastClusteringConfig(threshold=0.5), + // + // WARNING: You need to tune threshold by yourself. + // A larger threshold leads to fewer clusters, i.e., few speakers. + // A smaller threshold leads to more clusters, i.e., more speakers. + // + clustering=FastClusteringConfig(numClusters=4), + ) + + val sd = OfflineSpeakerDiarization(config=config) + + val waveData = WaveReader.readWave( + filename = "./0-four-speakers-zh.wav", + ) + + if (sd.sampleRate() != waveData.sampleRate) { + println("Expected sample rate: ${sd.sampleRate()}, given: ${waveData.sampleRate}") + return + } + + // val segments = sd.process(waveData.samples) // this one is also ok + val segments = sd.processWithCallback(waveData.samples, callback=::callback) + for (segment in segments) { + println("${segment.start} -- ${segment.end} speaker_${segment.speaker}") + } +} diff --git a/sherpa-onnx/csrc/offline-speaker-diarization-result.h b/sherpa-onnx/csrc/offline-speaker-diarization-result.h index 5fb144f5c..6298a87c7 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization-result.h +++ b/sherpa-onnx/csrc/offline-speaker-diarization-result.h @@ -58,7 +58,7 @@ class OfflineSpeakerDiarizationResult { std::vector> SortBySpeaker() const; - public: + private: std::vector segments_; }; diff --git a/sherpa-onnx/jni/CMakeLists.txt b/sherpa-onnx/jni/CMakeLists.txt index 998379084..23544c177 100644 --- a/sherpa-onnx/jni/CMakeLists.txt +++ b/sherpa-onnx/jni/CMakeLists.txt @@ -33,6 +33,12 @@ if(SHERPA_ONNX_ENABLE_TTS) ) endif() +if(SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION) + list(APPEND sources + offline-speaker-diarization.cc + ) +endif() + add_library(sherpa-onnx-jni ${sources}) target_compile_definitions(sherpa-onnx-jni PRIVATE SHERPA_ONNX_BUILD_SHARED_LIBS=1) diff --git a/sherpa-onnx/jni/offline-speaker-diarization.cc b/sherpa-onnx/jni/offline-speaker-diarization.cc new file mode 100644 index 000000000..a0eef8b9c --- /dev/null +++ b/sherpa-onnx/jni/offline-speaker-diarization.cc @@ -0,0 +1,219 @@ +// sherpa-onnx/jni/offline-speaker-diarization.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include "sherpa-onnx/csrc/offline-speaker-diarization.h" + +#include "sherpa-onnx/csrc/macros.h" +#include "sherpa-onnx/jni/common.h" + +namespace sherpa_onnx { + +static OfflineSpeakerDiarizationConfig GetOfflineSpeakerDiarizationConfig( + JNIEnv *env, jobject config) { + OfflineSpeakerDiarizationConfig ans; + + jclass cls = env->GetObjectClass(config); + jfieldID fid; + + //---------- segmentation ---------- + fid = env->GetFieldID( + cls, "segmentation", + "Lcom/k2fsa/sherpa/onnx/OfflineSpeakerSegmentationModelConfig;"); + jobject segmentation_config = env->GetObjectField(config, fid); + jclass segmentation_config_cls = env->GetObjectClass(segmentation_config); + + fid = env->GetFieldID( + segmentation_config_cls, "pyannote", + "Lcom/k2fsa/sherpa/onnx/OfflineSpeakerSegmentationPyannoteModelConfig;"); + jobject pyannote_config = env->GetObjectField(segmentation_config, fid); + jclass pyannote_config_cls = env->GetObjectClass(pyannote_config); + + fid = env->GetFieldID(pyannote_config_cls, "model", "Ljava/lang/String;"); + jstring s = (jstring)env->GetObjectField(pyannote_config, fid); + const char *p = env->GetStringUTFChars(s, nullptr); + ans.segmentation.pyannote.model = p; + env->ReleaseStringUTFChars(s, p); + + fid = env->GetFieldID(segmentation_config_cls, "numThreads", "I"); + ans.segmentation.num_threads = env->GetIntField(segmentation_config, fid); + + fid = env->GetFieldID(segmentation_config_cls, "debug", "Z"); + ans.segmentation.debug = env->GetBooleanField(segmentation_config, fid); + + fid = env->GetFieldID(segmentation_config_cls, "provider", + "Ljava/lang/String;"); + s = (jstring)env->GetObjectField(segmentation_config, fid); + p = env->GetStringUTFChars(s, nullptr); + ans.segmentation.provider = p; + env->ReleaseStringUTFChars(s, p); + + //---------- embedding ---------- + fid = env->GetFieldID( + cls, "embedding", + "Lcom/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractorConfig;"); + jobject embedding_config = env->GetObjectField(config, fid); + jclass embedding_config_cls = env->GetObjectClass(embedding_config); + + fid = env->GetFieldID(embedding_config_cls, "model", "Ljava/lang/String;"); + s = (jstring)env->GetObjectField(embedding_config, fid); + p = env->GetStringUTFChars(s, nullptr); + ans.embedding.model = p; + env->ReleaseStringUTFChars(s, p); + + fid = env->GetFieldID(embedding_config_cls, "numThreads", "I"); + ans.embedding.num_threads = env->GetIntField(embedding_config, fid); + + fid = env->GetFieldID(embedding_config_cls, "debug", "Z"); + ans.embedding.debug = env->GetBooleanField(embedding_config, fid); + + fid = env->GetFieldID(embedding_config_cls, "provider", "Ljava/lang/String;"); + s = (jstring)env->GetObjectField(embedding_config, fid); + p = env->GetStringUTFChars(s, nullptr); + ans.embedding.provider = p; + env->ReleaseStringUTFChars(s, p); + + //---------- clustering ---------- + fid = env->GetFieldID(cls, "clustering", + "Lcom/k2fsa/sherpa/onnx/FastClusteringConfig;"); + jobject clustering_config = env->GetObjectField(config, fid); + jclass clustering_config_cls = env->GetObjectClass(clustering_config); + + fid = env->GetFieldID(clustering_config_cls, "numClusters", "I"); + ans.clustering.num_clusters = env->GetIntField(clustering_config, fid); + + fid = env->GetFieldID(clustering_config_cls, "threshold", "F"); + ans.clustering.threshold = env->GetFloatField(clustering_config, fid); + + // its own fields + fid = env->GetFieldID(cls, "minDurationOn", "F"); + ans.min_duration_on = env->GetFloatField(config, fid); + + fid = env->GetFieldID(cls, "minDurationOff", "F"); + ans.min_duration_off = env->GetFloatField(config, fid); + + return ans; +} + +} // namespace sherpa_onnx + +SHERPA_ONNX_EXTERN_C +JNIEXPORT jlong JNICALL +Java_com_k2fsa_sherpa_onnx_OfflineSpeakerDiarization_newFromAsset( + JNIEnv *env, jobject /*obj*/, jobject asset_manager, jobject _config) { + return 0; +} + +SHERPA_ONNX_EXTERN_C +JNIEXPORT jlong JNICALL +Java_com_k2fsa_sherpa_onnx_OfflineSpeakerDiarization_newFromFile( + JNIEnv *env, jobject /*obj*/, jobject _config) { + auto config = sherpa_onnx::GetOfflineSpeakerDiarizationConfig(env, _config); + SHERPA_ONNX_LOGE("config:\n%s", config.ToString().c_str()); + + if (!config.Validate()) { + SHERPA_ONNX_LOGE("Errors found in config!"); + return 0; + } + + auto sd = new sherpa_onnx::OfflineSpeakerDiarization(config); + + return (jlong)sd; +} + +SHERPA_ONNX_EXTERN_C +JNIEXPORT void JNICALL +Java_com_k2fsa_sherpa_onnx_OfflineSpeakerDiarization_setConfig( + JNIEnv *env, jobject /*obj*/, jlong ptr, jobject _config) { + auto config = sherpa_onnx::GetOfflineSpeakerDiarizationConfig(env, _config); + SHERPA_ONNX_LOGE("config:\n%s", config.ToString().c_str()); + + auto sd = reinterpret_cast(ptr); + sd->SetConfig(config); +} + +SHERPA_ONNX_EXTERN_C +JNIEXPORT void JNICALL +Java_com_k2fsa_sherpa_onnx_OfflineSpeakerDiarization_delete(JNIEnv * /*env*/, + jobject /*obj*/, + jlong ptr) { + delete reinterpret_cast(ptr); +} + +static jobjectArray ProcessImpl( + JNIEnv *env, + const std::vector + &segments) { + jclass cls = + env->FindClass("com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationSegment"); + + jobjectArray obj_arr = + (jobjectArray)env->NewObjectArray(segments.size(), cls, nullptr); + + jmethodID constructor = env->GetMethodID(cls, "", "(FFI)V"); + + for (int32_t i = 0; i != segments.size(); ++i) { + const auto &s = segments[i]; + jobject segment = + env->NewObject(cls, constructor, s.Start(), s.End(), s.Speaker()); + env->SetObjectArrayElement(obj_arr, i, segment); + } + + return obj_arr; +} + +SHERPA_ONNX_EXTERN_C +JNIEXPORT jobjectArray JNICALL +Java_com_k2fsa_sherpa_onnx_OfflineSpeakerDiarization_process( + JNIEnv *env, jobject /*obj*/, jlong ptr, jfloatArray samples) { + auto sd = reinterpret_cast(ptr); + + jfloat *p = env->GetFloatArrayElements(samples, nullptr); + jsize n = env->GetArrayLength(samples); + auto segments = sd->Process(p, n).SortByStartTime(); + env->ReleaseFloatArrayElements(samples, p, JNI_ABORT); + + return ProcessImpl(env, segments); +} + +SHERPA_ONNX_EXTERN_C +JNIEXPORT jobjectArray JNICALL +Java_com_k2fsa_sherpa_onnx_OfflineSpeakerDiarization_processWithCallback( + JNIEnv *env, jobject /*obj*/, jlong ptr, jfloatArray samples, + jobject callback, jlong arg) { + std::function callback_wrapper = + [env, callback](int32_t num_processed_chunks, int32_t num_total_chunks, + void *data) -> int { + jclass cls = env->GetObjectClass(callback); + + jmethodID mid = env->GetMethodID(cls, "invoke", "(IIJ)Ljava/lang/Integer;"); + if (mid == nullptr) { + SHERPA_ONNX_LOGE("Failed to get the callback. Ignore it."); + return 0; + } + + jobject ret = env->CallObjectMethod(callback, mid, num_processed_chunks, + num_total_chunks, (jlong)data); + jclass jklass = env->GetObjectClass(ret); + jmethodID int_value_mid = env->GetMethodID(jklass, "intValue", "()I"); + return env->CallIntMethod(ret, int_value_mid); + }; + + auto sd = reinterpret_cast(ptr); + + jfloat *p = env->GetFloatArrayElements(samples, nullptr); + jsize n = env->GetArrayLength(samples); + auto segments = + sd->Process(p, n, callback_wrapper, (void *)arg).SortByStartTime(); + env->ReleaseFloatArrayElements(samples, p, JNI_ABORT); + + return ProcessImpl(env, segments); +} + +SHERPA_ONNX_EXTERN_C +JNIEXPORT jint JNICALL +Java_com_k2fsa_sherpa_onnx_OfflineSpeakerDiarization_getSampleRate( + JNIEnv * /*env*/, jobject /*obj*/, jlong ptr) { + return reinterpret_cast(ptr) + ->SampleRate(); +} diff --git a/sherpa-onnx/kotlin-api/OfflineSpeakerDiarization.kt b/sherpa-onnx/kotlin-api/OfflineSpeakerDiarization.kt new file mode 100644 index 000000000..de0a9dffd --- /dev/null +++ b/sherpa-onnx/kotlin-api/OfflineSpeakerDiarization.kt @@ -0,0 +1,101 @@ +package com.k2fsa.sherpa.onnx + +import android.content.res.AssetManager + +data class OfflineSpeakerSegmentationPyannoteModelConfig( + var model: String, +) + +data class OfflineSpeakerSegmentationModelConfig( + var pyannote: OfflineSpeakerSegmentationPyannoteModelConfig, + var numThreads: Int = 1, + var debug: Boolean = false, + var provider: String = "cpu", +) + +data class FastClusteringConfig( + var numClusters: Int = -1, + var threshold: Float = 0.5f, +) + +data class OfflineSpeakerDiarizationConfig( + var segmentation: OfflineSpeakerSegmentationModelConfig, + var embedding: SpeakerEmbeddingExtractorConfig, + var clustering: FastClusteringConfig, + var minDurationOn: Float = 0.2f, + var minDurationOff: Float = 0.5f, +) + +data class OfflineSpeakerDiarizationSegment( + val start: Float, // in seconds + val end: Float, // in seconds + val speaker: Int, // ID of the speaker; count from 0 +) + +class OfflineSpeakerDiarization( + assetManager: AssetManager? = null, + config: OfflineSpeakerDiarizationConfig, +) { + private var ptr: Long + + init { + ptr = if (assetManager != null) { + newFromAsset(assetManager, config) + } else { + newFromFile(config) + } + } + + protected fun finalize() { + if (ptr != 0L) { + delete(ptr) + ptr = 0 + } + } + + fun release() = finalize() + + // Only config.clustering is used. All other fields in config + // are ignored + fun setConfig(config: OfflineSpeakerDiarizationConfig) = setConfig(ptr, config) + + fun sampleRate() = getSampleRate(ptr) + + fun process(samples: FloatArray) = process(ptr, samples) + + fun processWithCallback( + samples: FloatArray, + callback: (numProcessedChunks: Int, numTotalChunks: Int, arg: Long) -> Int, + arg: Long = 0, + ) = processWithCallback(ptr, samples, callback, arg) + + private external fun delete(ptr: Long) + + private external fun newFromAsset( + assetManager: AssetManager, + config: OfflineSpeakerDiarizationConfig, + ): Long + + private external fun newFromFile( + config: OfflineSpeakerDiarizationConfig, + ): Long + + private external fun setConfig(ptr: Long, config: OfflineSpeakerDiarizationConfig) + + private external fun getSampleRate(ptr: Long): Int + + private external fun process(ptr: Long, samples: FloatArray): Array + + private external fun processWithCallback( + ptr: Long, + samples: FloatArray, + callback: (numProcessedChunks: Int, numTotalChunks: Int, arg: Long) -> Int, + arg: Long, + ): Array + + companion object { + init { + System.loadLibrary("sherpa-onnx-jni") + } + } +} From 1851ff63373ed1d3ef614b431a153bcc6528e4e2 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 11 Oct 2024 16:51:40 +0800 Subject: [PATCH 014/183] Java API for speaker diarization (#1416) --- .github/workflows/run-java-test.yaml | 7 ++ .../OfflineSpeakerDiarizationDemo.java | 99 +++++++++++++++++++ java-api-examples/README.md | 6 ++ .../run-offline-speaker-diarization.sh | 45 +++++++++ sherpa-onnx/java-api/Makefile | 9 ++ .../sherpa/onnx/FastClusteringConfig.java | 44 +++++++++ .../onnx/OfflineSpeakerDiarization.java | 61 ++++++++++++ .../OfflineSpeakerDiarizationCallback.java | 8 ++ .../onnx/OfflineSpeakerDiarizationConfig.java | 79 +++++++++++++++ .../OfflineSpeakerDiarizationSegment.java | 27 +++++ ...OfflineSpeakerSegmentationModelConfig.java | 52 ++++++++++ ...peakerSegmentationPyannoteModelConfig.java | 32 ++++++ .../k2fsa/sherpa/onnx/OfflineTtsCallback.java | 2 + .../onnx/SpeakerEmbeddingExtractorConfig.java | 1 - 14 files changed, 471 insertions(+), 1 deletion(-) create mode 100644 java-api-examples/OfflineSpeakerDiarizationDemo.java create mode 100755 java-api-examples/run-offline-speaker-diarization.sh create mode 100644 sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/FastClusteringConfig.java create mode 100644 sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarization.java create mode 100644 sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationCallback.java create mode 100644 sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationConfig.java create mode 100644 sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationSegment.java create mode 100644 sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerSegmentationModelConfig.java create mode 100644 sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerSegmentationPyannoteModelConfig.java diff --git a/.github/workflows/run-java-test.yaml b/.github/workflows/run-java-test.yaml index 3e932707c..5759ea5d8 100644 --- a/.github/workflows/run-java-test.yaml +++ b/.github/workflows/run-java-test.yaml @@ -107,6 +107,13 @@ jobs: make -j4 ls -lh lib + - name: Run java test (speaker diarization) + shell: bash + run: | + cd ./java-api-examples + ./run-offline-speaker-diarization.sh + rm -rfv *.onnx *.wav sherpa-onnx-pyannote-* + - name: Run java test (kws) shell: bash run: | diff --git a/java-api-examples/OfflineSpeakerDiarizationDemo.java b/java-api-examples/OfflineSpeakerDiarizationDemo.java new file mode 100644 index 000000000..a5ef8d1f4 --- /dev/null +++ b/java-api-examples/OfflineSpeakerDiarizationDemo.java @@ -0,0 +1,99 @@ +// Copyright 2024 Xiaomi Corporation + +// This file shows how to use sherpa-onnx Java API for speaker diarization, +import com.k2fsa.sherpa.onnx.*; + +public class OfflineSpeakerDiarizationDemo { + public static void main(String[] args) { + /* Please use the following commands to download files used in this file + Step 1: Download a speaker segmentation model + + Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models + for a list of available models. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + + Step 2: Download a speaker embedding extractor model + + Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models + for a list of available models. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx + + Step 3. Download test wave files + + Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models + for a list of available test wave files. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/0-four-speakers-zh.wav + + Step 4. Run it + */ + + String segmentationModel = "./sherpa-onnx-pyannote-segmentation-3-0/model.onnx"; + String embeddingModel = "./3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx"; + String waveFilename = "./0-four-speakers-zh.wav"; + + WaveReader reader = new WaveReader(waveFilename); + + OfflineSpeakerSegmentationPyannoteModelConfig pyannote = + OfflineSpeakerSegmentationPyannoteModelConfig.builder().setModel(segmentationModel).build(); + + OfflineSpeakerSegmentationModelConfig segmentation = + OfflineSpeakerSegmentationModelConfig.builder() + .setPyannote(pyannote) + .setDebug(true) + .build(); + + SpeakerEmbeddingExtractorConfig embedding = + SpeakerEmbeddingExtractorConfig.builder().setModel(embeddingModel).setDebug(true).build(); + + // The test wave file ./0-four-speakers-zh.wav contains four speakers, so + // we use numClusters=4 here. If you don't know the number of speakers + // in the test wave file, please set the numClusters to -1 and provide + // threshold for clustering + FastClusteringConfig clustering = + FastClusteringConfig.builder() + .setNumClusters(4) // set it to -1 if you don't know the actual number + .setThreshold(0.5f) + .build(); + + OfflineSpeakerDiarizationConfig config = + OfflineSpeakerDiarizationConfig.builder() + .setSegmentation(segmentation) + .setEmbedding(embedding) + .setClustering(clustering) + .setMinDurationOn(0.2f) + .setMinDurationOff(0.5f) + .build(); + + OfflineSpeakerDiarization sd = new OfflineSpeakerDiarization(config); + if (sd.getSampleRate() != reader.getSampleRate()) { + System.out.printf( + "Expected sample rate: %d, given: %d\n", sd.getSampleRate(), reader.getSampleRate()); + return; + } + + // OfflineSpeakerDiarizationSegment[] segments = sd.process(reader.getSamples()); + // without callback is also ok + + // or you can use a callback to show the progress + OfflineSpeakerDiarizationSegment[] segments = + sd.processWithCallback( + reader.getSamples(), + (int numProcessedChunks, int numTotalChunks, long arg) -> { + float progress = 100.0f * numProcessedChunks / numTotalChunks; + System.out.printf("Progress: %.2f%%\n", progress); + + return 0; + }); + + for (OfflineSpeakerDiarizationSegment s : segments) { + System.out.printf("%.3f -- %.3f speaker_%02d\n", s.getStart(), s.getEnd(), s.getSpeaker()); + } + + sd.release(); + } +} diff --git a/java-api-examples/README.md b/java-api-examples/README.md index 697f0c876..779c1b254 100755 --- a/java-api-examples/README.md +++ b/java-api-examples/README.md @@ -4,6 +4,12 @@ This directory contains examples for the JAVA API of sherpa-onnx. # Usage +## Non-streaming speaker diarization + +```bash +./run-offline-speaker-diarization.sh +``` + ## Streaming Speech recognition ``` diff --git a/java-api-examples/run-offline-speaker-diarization.sh b/java-api-examples/run-offline-speaker-diarization.sh new file mode 100755 index 000000000..d5cd63b5f --- /dev/null +++ b/java-api-examples/run-offline-speaker-diarization.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -ex + +if [[ ! -f ../build/lib/libsherpa-onnx-jni.dylib && ! -f ../build/lib/libsherpa-onnx-jni.so ]]; then + mkdir -p ../build + pushd ../build + cmake \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + -DSHERPA_ONNX_ENABLE_JNI=ON \ + .. + + make -j4 + ls -lh lib + popd +fi + +if [ ! -f ../sherpa-onnx/java-api/build/sherpa-onnx.jar ]; then + pushd ../sherpa-onnx/java-api + make + popd +fi + +if [ ! -f ./sherpa-onnx-pyannote-segmentation-3-0/model.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 +fi + +if [ ! -f ./3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx +fi + +if [ ! -f ./0-four-speakers-zh.wav ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/0-four-speakers-zh.wav +fi + +java \ + -Djava.library.path=$PWD/../build/lib \ + -cp ../sherpa-onnx/java-api/build/sherpa-onnx.jar \ + ./OfflineSpeakerDiarizationDemo.java diff --git a/sherpa-onnx/java-api/Makefile b/sherpa-onnx/java-api/Makefile index 69c3631b4..6e4778ae7 100644 --- a/sherpa-onnx/java-api/Makefile +++ b/sherpa-onnx/java-api/Makefile @@ -68,6 +68,15 @@ java_files += KeywordSpotterConfig.java java_files += KeywordSpotterResult.java java_files += KeywordSpotter.java +java_files += OfflineSpeakerSegmentationPyannoteModelConfig.java +java_files += OfflineSpeakerSegmentationModelConfig.java +java_files += FastClusteringConfig.java +java_files += OfflineSpeakerDiarizationConfig.java +java_files += OfflineSpeakerDiarizationSegment.java +java_files += OfflineSpeakerDiarizationCallback.java +java_files += OfflineSpeakerDiarization.java + + class_files := $(java_files:%.java=%.class) java_files := $(addprefix src/$(package_dir)/,$(java_files)) diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/FastClusteringConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/FastClusteringConfig.java new file mode 100644 index 000000000..f2e957259 --- /dev/null +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/FastClusteringConfig.java @@ -0,0 +1,44 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class FastClusteringConfig { + private final int numClusters; + private final float threshold; + + private FastClusteringConfig(Builder builder) { + this.numClusters = builder.numClusters; + this.threshold = builder.threshold; + } + + public static Builder builder() { + return new Builder(); + } + + public int getNumClusters() { + return numClusters; + } + + public float getThreshold() { + return threshold; + } + + public static class Builder { + private int numClusters = -1; + private float threshold = 0.5f; + + public FastClusteringConfig build() { + return new FastClusteringConfig(this); + } + + public Builder setNumClusters(int numClusters) { + this.numClusters = numClusters; + return this; + } + + public Builder setThreshold(float threshold) { + this.threshold = threshold; + return this; + } + } +} diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarization.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarization.java new file mode 100644 index 000000000..b75cd09ea --- /dev/null +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarization.java @@ -0,0 +1,61 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class OfflineSpeakerDiarization { + static { + System.loadLibrary("sherpa-onnx-jni"); + } + + private long ptr = 0; + + public OfflineSpeakerDiarization(OfflineSpeakerDiarizationConfig config) { + ptr = newFromFile(config); + } + + public int getSampleRate() { + return getSampleRate(ptr); + } + + // Only config.clustering is used. All other fields are ignored + public void setConfig(OfflineSpeakerDiarizationConfig config) { + setConfig(ptr, config); + } + + public OfflineSpeakerDiarizationSegment[] process(float[] samples) { + return process(ptr, samples); + } + + public OfflineSpeakerDiarizationSegment[] processWithCallback(float[] samples, OfflineSpeakerDiarizationCallback callback) { + return processWithCallback(ptr, samples, callback, 0); + } + + public OfflineSpeakerDiarizationSegment[] processWithCallback(float[] samples, OfflineSpeakerDiarizationCallback callback, long arg) { + return processWithCallback(ptr, samples, callback, arg); + } + + protected void finalize() throws Throwable { + release(); + } + + // You'd better call it manually if it is not used anymore + public void release() { + if (this.ptr == 0) { + return; + } + delete(this.ptr); + this.ptr = 0; + } + + private native int getSampleRate(long ptr); + + private native void delete(long ptr); + + private native long newFromFile(OfflineSpeakerDiarizationConfig config); + + private native void setConfig(long ptr, OfflineSpeakerDiarizationConfig config); + + private native OfflineSpeakerDiarizationSegment[] process(long ptr, float[] samples); + + private native OfflineSpeakerDiarizationSegment[] processWithCallback(long ptr, float[] samples, OfflineSpeakerDiarizationCallback callback, long arg); +} \ No newline at end of file diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationCallback.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationCallback.java new file mode 100644 index 000000000..7787386d3 --- /dev/null +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationCallback.java @@ -0,0 +1,8 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +@FunctionalInterface +public interface OfflineSpeakerDiarizationCallback { + Integer invoke(int numProcessedChunks, int numTotalCunks, long arg); +} diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationConfig.java new file mode 100644 index 000000000..9965c5742 --- /dev/null +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationConfig.java @@ -0,0 +1,79 @@ +package com.k2fsa.sherpa.onnx; + +public class OfflineSpeakerDiarizationConfig { + private final OfflineSpeakerSegmentationModelConfig segmentation; + private final SpeakerEmbeddingExtractorConfig embedding; + private final FastClusteringConfig clustering; + private final float minDurationOn; + private final float minDurationOff; + + private OfflineSpeakerDiarizationConfig(Builder builder) { + this.segmentation = builder.segmentation; + this.embedding = builder.embedding; + this.clustering = builder.clustering; + this.minDurationOff = builder.minDurationOff; + this.minDurationOn = builder.minDurationOn; + } + + public static Builder builder() { + return new Builder(); + } + + public OfflineSpeakerSegmentationModelConfig getSegmentation() { + return segmentation; + } + + public SpeakerEmbeddingExtractorConfig getEmbedding() { + return embedding; + } + + public FastClusteringConfig getClustering() { + return clustering; + } + + public float getMinDurationOff() { + return minDurationOff; + } + + public float getMinDurationOn() { + return minDurationOn; + } + + public static class Builder { + private OfflineSpeakerSegmentationModelConfig segmentation = OfflineSpeakerSegmentationModelConfig.builder().build(); + private SpeakerEmbeddingExtractorConfig embedding = SpeakerEmbeddingExtractorConfig.builder().build(); + private FastClusteringConfig clustering = FastClusteringConfig.builder().build(); + private float minDurationOn = 0.2f; + private float minDurationOff = 0.5f; + + public OfflineSpeakerDiarizationConfig build() { + return new OfflineSpeakerDiarizationConfig(this); + } + + public Builder setSegmentation(OfflineSpeakerSegmentationModelConfig segmentation) { + this.segmentation = segmentation; + return this; + } + + public Builder setEmbedding(SpeakerEmbeddingExtractorConfig embedding) { + this.embedding = embedding; + return this; + } + + public Builder setClustering(FastClusteringConfig clustering) { + this.clustering = clustering; + return this; + } + + public Builder setMinDurationOff(float minDurationOff) { + this.minDurationOff = minDurationOff; + return this; + } + + public Builder setMinDurationOn(float minDurationOn) { + this.minDurationOn = minDurationOn; + return this; + } + } + +} diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationSegment.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationSegment.java new file mode 100644 index 000000000..1bb1a7635 --- /dev/null +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationSegment.java @@ -0,0 +1,27 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class OfflineSpeakerDiarizationSegment { + private final float start; + private final float end; + private final int speaker; + + public OfflineSpeakerDiarizationSegment(float start, float end, int speaker) { + this.start = start; + this.end = end; + this.speaker = speaker; + } + + public float getStart() { + return start; + } + + public float getEnd() { + return end; + } + + public int getSpeaker() { + return speaker; + } +} diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerSegmentationModelConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerSegmentationModelConfig.java new file mode 100644 index 000000000..55df6c295 --- /dev/null +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerSegmentationModelConfig.java @@ -0,0 +1,52 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class OfflineSpeakerSegmentationModelConfig { + private final OfflineSpeakerSegmentationPyannoteModelConfig pyannote; + private final int numThreads; + private final boolean debug; + private final String provider; + + private OfflineSpeakerSegmentationModelConfig(Builder builder) { + this.pyannote = builder.pyannote; + this.numThreads = builder.numThreads; + this.debug = builder.debug; + this.provider = builder.provider; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private OfflineSpeakerSegmentationPyannoteModelConfig pyannote = OfflineSpeakerSegmentationPyannoteModelConfig.builder().build(); + private int numThreads = 1; + private boolean debug = true; + private String provider = "cpu"; + + public OfflineSpeakerSegmentationModelConfig build() { + return new OfflineSpeakerSegmentationModelConfig(this); + } + + public Builder setPyannote(OfflineSpeakerSegmentationPyannoteModelConfig pyannote) { + this.pyannote = pyannote; + return this; + } + + public Builder setNumThreads(int numThreads) { + this.numThreads = numThreads; + return this; + } + + public Builder setDebug(boolean debug) { + this.debug = debug; + return this; + } + + public Builder setProvider(String provider) { + this.provider = provider; + return this; + } + } +} \ No newline at end of file diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerSegmentationPyannoteModelConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerSegmentationPyannoteModelConfig.java new file mode 100644 index 000000000..51fd99874 --- /dev/null +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineSpeakerSegmentationPyannoteModelConfig.java @@ -0,0 +1,32 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class OfflineSpeakerSegmentationPyannoteModelConfig { + private final String model; + + private OfflineSpeakerSegmentationPyannoteModelConfig(Builder builder) { + this.model = builder.model; + } + + public static Builder builder() { + return new Builder(); + } + + public String getModel() { + return model; + } + + public static class Builder { + private String model = ""; + + public OfflineSpeakerSegmentationPyannoteModelConfig build() { + return new OfflineSpeakerSegmentationPyannoteModelConfig(this); + } + + public Builder setModel(String model) { + this.model = model; + return this; + } + } +} diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsCallback.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsCallback.java index 396594a96..2fc1d45dd 100644 --- a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsCallback.java +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsCallback.java @@ -1,3 +1,5 @@ +// Copyright 2024 Xiaomi Corporation + package com.k2fsa.sherpa.onnx; @FunctionalInterface diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractorConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractorConfig.java index ffc688f34..80f800cdc 100644 --- a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractorConfig.java +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractorConfig.java @@ -50,5 +50,4 @@ public Builder setProvider(String provider) { return this; } } - } From 1ed803adc13a3b060a6b972253e3adfa81be8126 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 11 Oct 2024 21:17:41 +0800 Subject: [PATCH 015/183] Dart API for speaker diarization (#1418) --- .github/scripts/test-dart.sh | 5 + .github/workflows/test-dart.yaml | 1 + dart-api-examples/README.md | 1 + .../speaker-diarization/.gitignore | 3 + .../speaker-diarization/CHANGELOG.md | 3 + .../speaker-diarization/README.md | 7 + .../speaker-diarization/analysis_options.yaml | 30 ++ .../speaker-diarization/bin/init.dart | 1 + .../bin/speaker-diarization.dart | 100 +++++++ .../speaker-diarization/pubspec.yaml | 17 ++ dart-api-examples/speaker-diarization/run.sh | 21 ++ flutter/sherpa_onnx/example/example.md | 1 + flutter/sherpa_onnx/lib/sherpa_onnx.dart | 1 + .../lib/src/offline_speaker_diarization.dart | 243 ++++++++++++++++ .../lib/src/sherpa_onnx_bindings.dart | 263 +++++++++++++++++- flutter/sherpa_onnx/pubspec.yaml | 8 +- scripts/dart/speaker-diarization-pubspec.yaml | 16 ++ sherpa-onnx/c-api/c-api.cc | 16 ++ sherpa-onnx/c-api/c-api.h | 9 + ...ffline-speaker-diarization-pyannote-impl.h | 1 + .../jni/offline-speaker-diarization.cc | 3 +- 21 files changed, 733 insertions(+), 17 deletions(-) create mode 100644 dart-api-examples/speaker-diarization/.gitignore create mode 100644 dart-api-examples/speaker-diarization/CHANGELOG.md create mode 100644 dart-api-examples/speaker-diarization/README.md create mode 100644 dart-api-examples/speaker-diarization/analysis_options.yaml create mode 120000 dart-api-examples/speaker-diarization/bin/init.dart create mode 100644 dart-api-examples/speaker-diarization/bin/speaker-diarization.dart create mode 100644 dart-api-examples/speaker-diarization/pubspec.yaml create mode 100755 dart-api-examples/speaker-diarization/run.sh create mode 100644 flutter/sherpa_onnx/lib/src/offline_speaker_diarization.dart create mode 100644 scripts/dart/speaker-diarization-pubspec.yaml diff --git a/.github/scripts/test-dart.sh b/.github/scripts/test-dart.sh index 0aff2085e..27c21573a 100755 --- a/.github/scripts/test-dart.sh +++ b/.github/scripts/test-dart.sh @@ -4,6 +4,11 @@ set -ex cd dart-api-examples +pushd speaker-diarization +echo '----------speaker diarization----------' +./run.sh +popd + pushd speaker-identification echo '----------3d speaker----------' ./run-3d-speaker.sh diff --git a/.github/workflows/test-dart.yaml b/.github/workflows/test-dart.yaml index 58d505490..d9e27e86f 100644 --- a/.github/workflows/test-dart.yaml +++ b/.github/workflows/test-dart.yaml @@ -114,6 +114,7 @@ jobs: cp scripts/dart/audio-tagging-pubspec.yaml dart-api-examples/audio-tagging/pubspec.yaml cp scripts/dart/add-punctuations-pubspec.yaml dart-api-examples/add-punctuations/pubspec.yaml cp scripts/dart/speaker-id-pubspec.yaml dart-api-examples/speaker-identification/pubspec.yaml + cp scripts/dart/speaker-diarization-pubspec.yaml dart-api-examples/speaker-diarization/pubspec.yaml cp scripts/dart/sherpa-onnx-pubspec.yaml flutter/sherpa_onnx/pubspec.yaml diff --git a/dart-api-examples/README.md b/dart-api-examples/README.md index 9370372e7..3d66cb04e 100644 --- a/dart-api-examples/README.md +++ b/dart-api-examples/README.md @@ -9,6 +9,7 @@ https://pub.dev/packages/sherpa_onnx | Directory | Description | |-----------|-------------| +| [./speaker-diarization](./speaker-diarization)| Example for speaker diarization.| | [./add-punctuations](./add-punctuations)| Example for adding punctuations to text.| | [./audio-tagging](./audio-tagging)| Example for audio tagging.| | [./keyword-spotter](./keyword-spotter)| Example for keyword spotting| diff --git a/dart-api-examples/speaker-diarization/.gitignore b/dart-api-examples/speaker-diarization/.gitignore new file mode 100644 index 000000000..3a8579040 --- /dev/null +++ b/dart-api-examples/speaker-diarization/.gitignore @@ -0,0 +1,3 @@ +# https://dart.dev/guides/libraries/private-files +# Created by `dart pub` +.dart_tool/ diff --git a/dart-api-examples/speaker-diarization/CHANGELOG.md b/dart-api-examples/speaker-diarization/CHANGELOG.md new file mode 100644 index 000000000..effe43c82 --- /dev/null +++ b/dart-api-examples/speaker-diarization/CHANGELOG.md @@ -0,0 +1,3 @@ +## 1.0.0 + +- Initial version. diff --git a/dart-api-examples/speaker-diarization/README.md b/dart-api-examples/speaker-diarization/README.md new file mode 100644 index 000000000..d4d8c4fd2 --- /dev/null +++ b/dart-api-examples/speaker-diarization/README.md @@ -0,0 +1,7 @@ +# Introduction + +This example shows how to use the Dart API from sherpa-onnx for speaker diarization. + +# Usage + +Please see [./run.sh](./run.sh) diff --git a/dart-api-examples/speaker-diarization/analysis_options.yaml b/dart-api-examples/speaker-diarization/analysis_options.yaml new file mode 100644 index 000000000..dee8927aa --- /dev/null +++ b/dart-api-examples/speaker-diarization/analysis_options.yaml @@ -0,0 +1,30 @@ +# This file configures the static analysis results for your project (errors, +# warnings, and lints). +# +# This enables the 'recommended' set of lints from `package:lints`. +# This set helps identify many issues that may lead to problems when running +# or consuming Dart code, and enforces writing Dart using a single, idiomatic +# style and format. +# +# If you want a smaller set of lints you can change this to specify +# 'package:lints/core.yaml'. These are just the most critical lints +# (the recommended set includes the core lints). +# The core lints are also what is used by pub.dev for scoring packages. + +include: package:lints/recommended.yaml + +# Uncomment the following section to specify additional rules. + +# linter: +# rules: +# - camel_case_types + +# analyzer: +# exclude: +# - path/to/excluded/files/** + +# For more information about the core and recommended set of lints, see +# https://dart.dev/go/core-lints + +# For additional information about configuring this file, see +# https://dart.dev/guides/language/analysis-options diff --git a/dart-api-examples/speaker-diarization/bin/init.dart b/dart-api-examples/speaker-diarization/bin/init.dart new file mode 120000 index 000000000..48508cfd3 --- /dev/null +++ b/dart-api-examples/speaker-diarization/bin/init.dart @@ -0,0 +1 @@ +../../vad/bin/init.dart \ No newline at end of file diff --git a/dart-api-examples/speaker-diarization/bin/speaker-diarization.dart b/dart-api-examples/speaker-diarization/bin/speaker-diarization.dart new file mode 100644 index 000000000..760adc868 --- /dev/null +++ b/dart-api-examples/speaker-diarization/bin/speaker-diarization.dart @@ -0,0 +1,100 @@ +// Copyright (c) 2024 Xiaomi Corporation +import 'dart:io'; +import 'dart:typed_data'; +import 'dart:ffi'; + +import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; +import './init.dart'; + +void main(List arguments) async { + await initSherpaOnnx(); + + /* Please use the following commands to download files used in this file + Step 1: Download a speaker segmentation model + + Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models + for a list of available models. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + + Step 2: Download a speaker embedding extractor model + + Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models + for a list of available models. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx + + Step 3. Download test wave files + + Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models + for a list of available test wave files. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/0-four-speakers-zh.wav + + Step 4. Run it + */ + + final segmentationModel = + "./sherpa-onnx-pyannote-segmentation-3-0/model.onnx"; + + final embeddingModel = + "./3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx"; + + final waveFilename = "./0-four-speakers-zh.wav"; + + final segmentationConfig = sherpa_onnx.OfflineSpeakerSegmentationModelConfig( + pyannote: sherpa_onnx.OfflineSpeakerSegmentationPyannoteModelConfig( + model: segmentationModel), + ); + + final embeddingConfig = + sherpa_onnx.SpeakerEmbeddingExtractorConfig(model: embeddingModel); + + // since we know there are 4 speakers in ./0-four-speakers-zh.wav, we set + // numClusters to 4. If you don't know the exact number, please set it to -1. + // in that case, you have to set threshold. A larger threshold leads to + // fewer clusters, i.e., fewer speakers. + final clusteringConfig = + sherpa_onnx.FastClusteringConfig(numClusters: 4, threshold: 0.5); + + var config = sherpa_onnx.OfflineSpeakerDiarizationConfig( + segmentation: segmentationConfig, + embedding: embeddingConfig, + clustering: clusteringConfig, + minDurationOn: 0.2, + minDurationOff: 0.5); + + final sd = sherpa_onnx.OfflineSpeakerDiarization(config); + if (sd.ptr == nullptr) { + return; + } + + final waveData = sherpa_onnx.readWave(waveFilename); + if (sd.sampleRate != waveData.sampleRate) { + print( + 'Expected sample rate: ${sd.sampleRate}, given: ${waveData.sampleRate}'); + return; + } + + print('started'); + + // Use the following statement if you don't want to use a callback + // final segments = sd.process(samples: waveData.samples); + + final segments = sd.processWithCallback( + samples: waveData.samples, + callback: (int numProcessedChunk, int numTotalChunks) { + final progress = 100.0 * numProcessedChunk / numTotalChunks; + + print('Progress ${progress.toStringAsFixed(2)}%'); + + return 0; + }); + + for (int i = 0; i < segments.length; ++i) { + print( + '${segments[i].start.toStringAsFixed(3)} -- ${segments[i].end.toStringAsFixed(3)} speaker_${segments[i].speaker}'); + } +} diff --git a/dart-api-examples/speaker-diarization/pubspec.yaml b/dart-api-examples/speaker-diarization/pubspec.yaml new file mode 100644 index 000000000..28154a49c --- /dev/null +++ b/dart-api-examples/speaker-diarization/pubspec.yaml @@ -0,0 +1,17 @@ +name: speaker_diarization +description: > + This example demonstrates how to use the Dart API for speaker diarization. + +version: 1.0.0 + +environment: + sdk: ">=3.0.0 <4.0.0" + +dependencies: + sherpa_onnx: ^1.10.27 + # sherpa_onnx: + # path: ../../flutter/sherpa_onnx + path: ^1.9.0 + +dev_dependencies: + lints: ^3.0.0 diff --git a/dart-api-examples/speaker-diarization/run.sh b/dart-api-examples/speaker-diarization/run.sh new file mode 100755 index 000000000..7717870dc --- /dev/null +++ b/dart-api-examples/speaker-diarization/run.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -ex + +dart pub get + +if [ ! -f ./sherpa-onnx-pyannote-segmentation-3-0/model.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 +fi + +if [ ! -f ./3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx +fi + +if [ ! -f ./0-four-speakers-zh.wav ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/0-four-speakers-zh.wav +fi + +dart run ./bin/speaker-diarization.dart diff --git a/flutter/sherpa_onnx/example/example.md b/flutter/sherpa_onnx/example/example.md index 7e7e8031d..0c24a79b2 100644 --- a/flutter/sherpa_onnx/example/example.md +++ b/flutter/sherpa_onnx/example/example.md @@ -11,6 +11,7 @@ | Functions | URL | Supported Platforms| |---|---|---| +|Speaker diarization| [Address](https://github.com/k2-fsa/sherpa-onnx/tree/master/dart-api-examples/speaker-diarization)| macOS, Windows, Linux| |Streaming speech recognition| [Address](https://github.com/k2-fsa/sherpa-onnx/tree/master/dart-api-examples/streaming-asr)| macOS, Windows, Linux| |Non-Streaming speech recognition| [Address](https://github.com/k2-fsa/sherpa-onnx/tree/master/dart-api-examples/non-streaming-asr)| macOS, Windows, Linux| |Text to speech| [Address](https://github.com/k2-fsa/sherpa-onnx/tree/master/dart-api-examples/tts)| macOS, Windows, Linux| diff --git a/flutter/sherpa_onnx/lib/sherpa_onnx.dart b/flutter/sherpa_onnx/lib/sherpa_onnx.dart index b15e67532..9fcd2872f 100644 --- a/flutter/sherpa_onnx/lib/sherpa_onnx.dart +++ b/flutter/sherpa_onnx/lib/sherpa_onnx.dart @@ -6,6 +6,7 @@ export 'src/audio_tagging.dart'; export 'src/feature_config.dart'; export 'src/keyword_spotter.dart'; export 'src/offline_recognizer.dart'; +export 'src/offline_speaker_diarization.dart'; export 'src/offline_stream.dart'; export 'src/online_recognizer.dart'; export 'src/online_stream.dart'; diff --git a/flutter/sherpa_onnx/lib/src/offline_speaker_diarization.dart b/flutter/sherpa_onnx/lib/src/offline_speaker_diarization.dart new file mode 100644 index 000000000..5981e3c04 --- /dev/null +++ b/flutter/sherpa_onnx/lib/src/offline_speaker_diarization.dart @@ -0,0 +1,243 @@ +// Copyright (c) 2024 Xiaomi Corporation +import 'dart:ffi'; +import 'dart:typed_data'; + +import 'package:ffi/ffi.dart'; + +import './sherpa_onnx_bindings.dart'; +import './speaker_identification.dart'; + +class OfflineSpeakerDiarizationSegment { + const OfflineSpeakerDiarizationSegment({ + required this.start, + required this.end, + required this.speaker, + }); + + @override + String toString() { + return 'OfflineSpeakerDiarizationSegment(start: $start, end: $end, speaker: $speaker)'; + } + + final double start; + final double end; + final int speaker; +} + +class OfflineSpeakerSegmentationPyannoteModelConfig { + const OfflineSpeakerSegmentationPyannoteModelConfig({ + this.model = '', + }); + + @override + String toString() { + return 'OfflineSpeakerSegmentationPyannoteModelConfig(model: $model)'; + } + + final String model; +} + +class OfflineSpeakerSegmentationModelConfig { + const OfflineSpeakerSegmentationModelConfig({ + this.pyannote = const OfflineSpeakerSegmentationPyannoteModelConfig(), + this.numThreads = 1, + this.debug = true, + this.provider = 'cpu', + }); + + @override + String toString() { + return 'OfflineSpeakerSegmentationModelConfig(pyannote: $pyannote, numThreads: $numThreads, debug: $debug, provider: $provider)'; + } + + final OfflineSpeakerSegmentationPyannoteModelConfig pyannote; + + final int numThreads; + final bool debug; + final String provider; +} + +class FastClusteringConfig { + const FastClusteringConfig({ + this.numClusters = -1, + this.threshold = 0.5, + }); + + @override + String toString() { + return 'FastClusteringConfig(numClusters: $numClusters, threshold: $threshold)'; + } + + final int numClusters; + final double threshold; +} + +class OfflineSpeakerDiarizationConfig { + const OfflineSpeakerDiarizationConfig({ + this.segmentation = const OfflineSpeakerSegmentationModelConfig(), + this.embedding = const SpeakerEmbeddingExtractorConfig(model: ''), + this.clustering = const FastClusteringConfig(), + this.minDurationOn = 0.2, + this.minDurationOff = 0.5, + }); + + @override + String toString() { + return 'OfflineSpeakerDiarizationConfig(segmentation: $segmentation, embedding: $embedding, clustering: $clustering, minDurationOn: $minDurationOn, minDurationOff: $minDurationOff)'; + } + + final OfflineSpeakerSegmentationModelConfig segmentation; + final SpeakerEmbeddingExtractorConfig embedding; + final FastClusteringConfig clustering; + final double minDurationOff; // in seconds + final double minDurationOn; // in seconds +} + +class OfflineSpeakerDiarization { + OfflineSpeakerDiarization._( + {required this.ptr, required this.config, required this.sampleRate}); + + void free() { + SherpaOnnxBindings.sherpaOnnxDestroyOfflineSpeakerDiarization?.call(ptr); + ptr = nullptr; + } + + /// The user is responsible to call the OfflineSpeakerDiarization.free() + /// method of the returned instance to avoid memory leak. + factory OfflineSpeakerDiarization(OfflineSpeakerDiarizationConfig config) { + final c = calloc(); + + c.ref.segmentation.pyannote.model = + config.segmentation.pyannote.model.toNativeUtf8(); + c.ref.segmentation.numThreads = config.segmentation.numThreads; + c.ref.segmentation.debug = config.segmentation.debug ? 1 : 0; + c.ref.segmentation.provider = config.segmentation.provider.toNativeUtf8(); + + c.ref.embedding.model = config.embedding.model.toNativeUtf8(); + c.ref.embedding.numThreads = config.embedding.numThreads; + c.ref.embedding.debug = config.embedding.debug ? 1 : 0; + c.ref.embedding.provider = config.embedding.provider.toNativeUtf8(); + + c.ref.clustering.numClusters = config.clustering.numClusters; + c.ref.clustering.threshold = config.clustering.threshold; + + c.ref.minDurationOn = config.minDurationOn; + c.ref.minDurationOff = config.minDurationOff; + + final ptr = + SherpaOnnxBindings.sherpaOnnxCreateOfflineSpeakerDiarization?.call(c) ?? + nullptr; + + calloc.free(c.ref.embedding.provider); + calloc.free(c.ref.embedding.model); + calloc.free(c.ref.segmentation.provider); + calloc.free(c.ref.segmentation.pyannote.model); + + int sampleRate = 0; + if (ptr != nullptr) { + sampleRate = SherpaOnnxBindings + .sherpaOnnxOfflineSpeakerDiarizationGetSampleRate + ?.call(ptr) ?? + 0; + } + return OfflineSpeakerDiarization._( + ptr: ptr, config: config, sampleRate: sampleRate); + } + + List process( + {required Float32List samples}) { + if (ptr == nullptr) { + return []; + } + + final n = samples.length; + final Pointer p = calloc(n); + + final pList = p.asTypedList(n); + pList.setAll(0, samples); + + final r = SherpaOnnxBindings.sherpaOnnxOfflineSpeakerDiarizationProcess + ?.call(ptr, p, n) ?? + nullptr; + + final ans = _processImpl(r); + + SherpaOnnxBindings.sherpaOnnxOfflineSpeakerDiarizationDestroyResult + ?.call(r); + + return ans; + } + + List processWithCallback({ + required Float32List samples, + required int Function(int numProcessedChunks, int numTotalChunks) callback, + }) { + if (ptr == nullptr) { + return []; + } + + final n = samples.length; + final Pointer p = calloc(n); + + final pList = p.asTypedList(n); + pList.setAll(0, samples); + + final wrapper = NativeCallable< + SherpaOnnxOfflineSpeakerDiarizationProgressCallbackNoArgNative>.isolateLocal( + (int numProcessedChunks, int numTotalChunks) { + return callback(numProcessedChunks, numTotalChunks); + }, exceptionalReturn: 0); + + final r = SherpaOnnxBindings + .sherpaOnnxOfflineSpeakerDiarizationProcessWithCallbackNoArg + ?.call(ptr, p, n, wrapper.nativeFunction) ?? + nullptr; + + wrapper.close(); + + final ans = _processImpl(r); + + SherpaOnnxBindings.sherpaOnnxOfflineSpeakerDiarizationDestroyResult + ?.call(r); + + return ans; + } + + List _processImpl( + Pointer r) { + if (r == nullptr) { + return []; + } + + final numSegments = SherpaOnnxBindings + .sherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments + ?.call(r) ?? + 0; + final segments = SherpaOnnxBindings + .sherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime + ?.call(r) ?? + nullptr; + + if (segments == nullptr) { + return []; + } + + final ans = []; + for (int i = 0; i != numSegments; ++i) { + final s = segments + i; + + final tmp = OfflineSpeakerDiarizationSegment( + start: s.ref.start, end: s.ref.end, speaker: s.ref.speaker); + ans.add(tmp); + } + + SherpaOnnxBindings.sherpaOnnxOfflineSpeakerDiarizationDestroySegment + ?.call(segments); + + return ans; + } + + Pointer ptr; + OfflineSpeakerDiarizationConfig config; + final int sampleRate; +} diff --git a/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart b/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart index 42294c2d4..8a8817d63 100644 --- a/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart +++ b/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart @@ -2,6 +2,66 @@ import 'dart:ffi'; import 'package:ffi/ffi.dart'; +final class SherpaOnnxSpeakerEmbeddingExtractorConfig extends Struct { + external Pointer model; + + @Int32() + external int numThreads; + + @Int32() + external int debug; + + external Pointer provider; +} + +final class SherpaOnnxOfflineSpeakerDiarizationSegment extends Struct { + @Float() + external double start; + + @Float() + external double end; + + @Int32() + external int speaker; +} + +final class SherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig + extends Struct { + external Pointer model; +} + +final class SherpaOnnxOfflineSpeakerSegmentationModelConfig extends Struct { + external SherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig pyannote; + + @Int32() + external int numThreads; + + @Int32() + external int debug; + + external Pointer provider; +} + +final class SherpaOnnxFastClusteringConfig extends Struct { + @Int32() + external int numClusters; + + @Float() + external double threshold; +} + +final class SherpaOnnxOfflineSpeakerDiarizationConfig extends Struct { + external SherpaOnnxOfflineSpeakerSegmentationModelConfig segmentation; + external SherpaOnnxSpeakerEmbeddingExtractorConfig embedding; + external SherpaOnnxFastClusteringConfig clustering; + + @Float() + external double minDurationOn; + + @Float() + external double minDurationOff; +} + final class SherpaOnnxOfflinePunctuationModelConfig extends Struct { external Pointer ctTransformer; @@ -341,18 +401,6 @@ final class SherpaOnnxWave extends Struct { external int numSamples; } -final class SherpaOnnxSpeakerEmbeddingExtractorConfig extends Struct { - external Pointer model; - - @Int32() - external int numThreads; - - @Int32() - external int debug; - - external Pointer provider; -} - final class SherpaOnnxKeywordSpotterConfig extends Struct { external SherpaOnnxFeatureConfig feat; @@ -402,10 +450,101 @@ final class SherpaOnnxSpeakerEmbeddingExtractor extends Opaque {} final class SherpaOnnxSpeakerEmbeddingManager extends Opaque {} +final class SherpaOnnxOfflineSpeakerDiarization extends Opaque {} + +final class SherpaOnnxOfflineSpeakerDiarizationResult extends Opaque {} + +typedef SherpaOnnxCreateOfflineSpeakerDiarizationNative + = Pointer Function( + Pointer); + +typedef SherpaOnnxCreateOfflineSpeakerDiarization + = SherpaOnnxCreateOfflineSpeakerDiarizationNative; + +typedef SherpaOnnxDestroyOfflineSpeakerDiarizationNative = Void Function( + Pointer); + +typedef SherpaOnnxDestroyOfflineSpeakerDiarization = void Function( + Pointer); + typedef SherpaOnnxCreateOfflinePunctuationNative = Pointer Function( Pointer); +typedef SherpaOnnxOfflineSpeakerDiarizationGetSampleRateNative = Int32 Function( + Pointer); + +typedef SherpaOnnxOfflineSpeakerDiarizationGetSampleRate = int Function( + Pointer); + +typedef SherpaOnnxOfflineSpeakerDiarizationSetConfigNative = Void Function( + Pointer, + Pointer); + +typedef SherpaOnnxOfflineSpeakerDiarizationResultGetNumSpeakersNative = Int32 + Function(Pointer); + +typedef SherpaOnnxOfflineSpeakerDiarizationResultGetNumSpeakers = int Function( + Pointer); + +typedef SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegmentsNative = Int32 + Function(Pointer); + +typedef SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments = int Function( + Pointer); + +typedef SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTimeNative + = Pointer Function( + Pointer); + +typedef SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime + = SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTimeNative; + +typedef SherpaOnnxOfflineSpeakerDiarizationDestroySegmentNative = Void Function( + Pointer); + +typedef SherpaOnnxOfflineSpeakerDiarizationDestroySegment = void Function( + Pointer); + +typedef SherpaOnnxOfflineSpeakerDiarizationProcessNative + = Pointer Function( + Pointer, Pointer, Int32); + +typedef SherpaOnnxOfflineSpeakerDiarizationProcess + = Pointer Function( + Pointer, Pointer, int); + +typedef SherpaOnnxOfflineSpeakerDiarizationProgressCallbackNoArgNative = Int32 + Function(Int32, Int32); + +typedef SherpaOnnxOfflineSpeakerDiarizationProcessWithCallbackNoArgNative + = Pointer Function( + Pointer, + Pointer, + Int32, + Pointer< + NativeFunction< + SherpaOnnxOfflineSpeakerDiarizationProgressCallbackNoArgNative>>); + +typedef SherpaOnnxOfflineSpeakerDiarizationProcessWithCallbackNoArg + = Pointer Function( + Pointer, + Pointer, + int, + Pointer< + NativeFunction< + SherpaOnnxOfflineSpeakerDiarizationProgressCallbackNoArgNative>>); + +typedef SherpaOnnxOfflineSpeakerDiarizationDestroyResultNative = Void Function( + Pointer); + +typedef SherpaOnnxOfflineSpeakerDiarizationDestroyResult = void Function( + Pointer); + +typedef SherpaOnnxOfflineSpeakerDiarizationSetConfig = void Function( + Pointer, + Pointer); + typedef SherpaOnnxCreateOfflinePunctuation = SherpaOnnxCreateOfflinePunctuationNative; @@ -940,6 +1079,29 @@ typedef SherpaOnnxFreeWaveNative = Void Function(Pointer); typedef SherpaOnnxFreeWave = void Function(Pointer); class SherpaOnnxBindings { + static SherpaOnnxCreateOfflineSpeakerDiarization? + sherpaOnnxCreateOfflineSpeakerDiarization; + static SherpaOnnxDestroyOfflineSpeakerDiarization? + sherpaOnnxDestroyOfflineSpeakerDiarization; + static SherpaOnnxOfflineSpeakerDiarizationGetSampleRate? + sherpaOnnxOfflineSpeakerDiarizationGetSampleRate; + static SherpaOnnxOfflineSpeakerDiarizationSetConfig? + sherpaOnnxOfflineSpeakerDiarizationSetConfig; + static SherpaOnnxOfflineSpeakerDiarizationResultGetNumSpeakers? + sherpaOnnxOfflineSpeakerDiarizationResultGetNumSpeakers; + static SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments? + sherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments; + static SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime? + sherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime; + static SherpaOnnxOfflineSpeakerDiarizationDestroySegment? + sherpaOnnxOfflineSpeakerDiarizationDestroySegment; + static SherpaOnnxOfflineSpeakerDiarizationProcess? + sherpaOnnxOfflineSpeakerDiarizationProcess; + static SherpaOnnxOfflineSpeakerDiarizationDestroyResult? + sherpaOnnxOfflineSpeakerDiarizationDestroyResult; + static SherpaOnnxOfflineSpeakerDiarizationProcessWithCallbackNoArg? + sherpaOnnxOfflineSpeakerDiarizationProcessWithCallbackNoArg; + static SherpaOnnxCreateOfflinePunctuation? sherpaOnnxCreateOfflinePunctuation; static SherpaOnnxDestroyOfflinePunctuation? sherpaOnnxDestroyOfflinePunctuation; @@ -1107,6 +1269,83 @@ class SherpaOnnxBindings { static SherpaOnnxFreeWave? freeWave; static void init(DynamicLibrary dynamicLibrary) { + sherpaOnnxCreateOfflineSpeakerDiarization ??= dynamicLibrary + .lookup< + NativeFunction< + SherpaOnnxCreateOfflineSpeakerDiarizationNative>>( + 'SherpaOnnxCreateOfflineSpeakerDiarization') + .asFunction(); + + sherpaOnnxDestroyOfflineSpeakerDiarization ??= dynamicLibrary + .lookup< + NativeFunction< + SherpaOnnxDestroyOfflineSpeakerDiarizationNative>>( + 'SherpaOnnxDestroyOfflineSpeakerDiarization') + .asFunction(); + + sherpaOnnxOfflineSpeakerDiarizationGetSampleRate ??= dynamicLibrary + .lookup< + NativeFunction< + SherpaOnnxOfflineSpeakerDiarizationGetSampleRateNative>>( + 'SherpaOnnxOfflineSpeakerDiarizationGetSampleRate') + .asFunction(); + + sherpaOnnxOfflineSpeakerDiarizationSetConfig ??= dynamicLibrary + .lookup< + NativeFunction< + SherpaOnnxOfflineSpeakerDiarizationSetConfigNative>>( + 'SherpaOnnxOfflineSpeakerDiarizationSetConfig') + .asFunction(); + + sherpaOnnxOfflineSpeakerDiarizationResultGetNumSpeakers ??= dynamicLibrary + .lookup< + NativeFunction< + SherpaOnnxOfflineSpeakerDiarizationResultGetNumSpeakersNative>>( + 'SherpaOnnxOfflineSpeakerDiarizationResultGetNumSpeakers') + .asFunction(); + + sherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments ??= dynamicLibrary + .lookup< + NativeFunction< + SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegmentsNative>>( + 'SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments') + .asFunction(); + + sherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime ??= dynamicLibrary + .lookup< + NativeFunction< + SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTimeNative>>( + 'SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime') + .asFunction(); + + sherpaOnnxOfflineSpeakerDiarizationDestroySegment ??= dynamicLibrary + .lookup< + NativeFunction< + SherpaOnnxOfflineSpeakerDiarizationDestroySegmentNative>>( + 'SherpaOnnxOfflineSpeakerDiarizationDestroySegment') + .asFunction(); + + sherpaOnnxOfflineSpeakerDiarizationProcess ??= dynamicLibrary + .lookup< + NativeFunction< + SherpaOnnxOfflineSpeakerDiarizationProcessNative>>( + 'SherpaOnnxOfflineSpeakerDiarizationProcess') + .asFunction(); + + sherpaOnnxOfflineSpeakerDiarizationProcessWithCallbackNoArg ??= dynamicLibrary + .lookup< + NativeFunction< + SherpaOnnxOfflineSpeakerDiarizationProcessWithCallbackNoArgNative>>( + 'SherpaOnnxOfflineSpeakerDiarizationProcessWithCallbackNoArg') + .asFunction(); + + sherpaOnnxOfflineSpeakerDiarizationDestroyResult ??= dynamicLibrary + .lookup< + NativeFunction< + SherpaOnnxOfflineSpeakerDiarizationDestroyResultNative>>( + 'SherpaOnnxOfflineSpeakerDiarizationDestroyResult') + .asFunction(); + sherpaOnnxCreateOfflinePunctuation ??= dynamicLibrary .lookup>( 'SherpaOnnxCreateOfflinePunctuation') diff --git a/flutter/sherpa_onnx/pubspec.yaml b/flutter/sherpa_onnx/pubspec.yaml index 5b693ef0b..e92071833 100644 --- a/flutter/sherpa_onnx/pubspec.yaml +++ b/flutter/sherpa_onnx/pubspec.yaml @@ -1,8 +1,8 @@ name: sherpa_onnx description: > - Speech recognition, speech synthesis, and speaker recognition using next-gen Kaldi - with onnxruntime without Internet connection. + Speech recognition, speech synthesis, speaker diarization, and speaker recognition + using next-gen Kaldi with onnxruntime without Internet connection. repository: https://github.com/k2-fsa/sherpa-onnx/tree/master/flutter @@ -12,7 +12,7 @@ documentation: https://k2-fsa.github.io/sherpa/onnx/ topics: - speech-recognition - speech-synthesis - - speaker-identification + - speaker-diarization - audio-tagging - voice-activity-detection @@ -41,7 +41,7 @@ dependencies: sherpa_onnx_linux: ^1.10.27 # sherpa_onnx_linux: # path: ../sherpa_onnx_linux - # + sherpa_onnx_windows: ^1.10.27 # sherpa_onnx_windows: # path: ../sherpa_onnx_windows diff --git a/scripts/dart/speaker-diarization-pubspec.yaml b/scripts/dart/speaker-diarization-pubspec.yaml new file mode 100644 index 000000000..fec147e75 --- /dev/null +++ b/scripts/dart/speaker-diarization-pubspec.yaml @@ -0,0 +1,16 @@ +name: speaker_diarization +description: > + This example demonstrates how to use the Dart API for speaker diarization. + +version: 1.0.0 + +environment: + sdk: ">=3.0.0 <4.0.0" + +dependencies: + sherpa_onnx: + path: ../../flutter/sherpa_onnx + path: ^1.9.0 + +dev_dependencies: + lints: ^3.0.0 diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index abcfc5b82..4ba0a4a60 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -1828,4 +1828,20 @@ SherpaOnnxOfflineSpeakerDiarizationProcessWithCallback( return ans; } +const SherpaOnnxOfflineSpeakerDiarizationResult * +SherpaOnnxOfflineSpeakerDiarizationProcessWithCallbackNoArg( + const SherpaOnnxOfflineSpeakerDiarization *sd, const float *samples, + int32_t n, + SherpaOnnxOfflineSpeakerDiarizationProgressCallbackNoArg callback) { + auto wrapper = [callback](int32_t num_processed_chunks, + int32_t num_total_chunks, void *) { + return callback(num_processed_chunks, num_total_chunks); + }; + + auto ans = new SherpaOnnxOfflineSpeakerDiarizationResult; + ans->impl = sd->impl->Process(samples, n, wrapper); + + return ans; +} + #endif diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index c9e7f9ee1..4b41a81a9 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -1485,6 +1485,9 @@ SHERPA_ONNX_API void SherpaOnnxOfflineSpeakerDiarizationDestroySegment( typedef int32_t (*SherpaOnnxOfflineSpeakerDiarizationProgressCallback)( int32_t num_processed_chunk, int32_t num_total_chunks, void *arg); +typedef int32_t (*SherpaOnnxOfflineSpeakerDiarizationProgressCallbackNoArg)( + int32_t num_processed_chunk, int32_t num_total_chunks); + // The user has to invoke SherpaOnnxOfflineSpeakerDiarizationDestroyResult() // to free the returned pointer to avoid memory leak. SHERPA_ONNX_API const SherpaOnnxOfflineSpeakerDiarizationResult * @@ -1500,6 +1503,12 @@ SherpaOnnxOfflineSpeakerDiarizationProcessWithCallback( int32_t n, SherpaOnnxOfflineSpeakerDiarizationProgressCallback callback, void *arg); +SHERPA_ONNX_API const SherpaOnnxOfflineSpeakerDiarizationResult * +SherpaOnnxOfflineSpeakerDiarizationProcessWithCallbackNoArg( + const SherpaOnnxOfflineSpeakerDiarization *sd, const float *samples, + int32_t n, + SherpaOnnxOfflineSpeakerDiarizationProgressCallbackNoArg callback); + SHERPA_ONNX_API void SherpaOnnxOfflineSpeakerDiarizationDestroyResult( const SherpaOnnxOfflineSpeakerDiarizationResult *r); diff --git a/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h b/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h index 8f669e27c..0c70f0bc6 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h +++ b/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h @@ -5,6 +5,7 @@ #define SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_DIARIZATION_PYANNOTE_IMPL_H_ #include +#include #include #include #include diff --git a/sherpa-onnx/jni/offline-speaker-diarization.cc b/sherpa-onnx/jni/offline-speaker-diarization.cc index a0eef8b9c..e82962c80 100644 --- a/sherpa-onnx/jni/offline-speaker-diarization.cc +++ b/sherpa-onnx/jni/offline-speaker-diarization.cc @@ -204,7 +204,8 @@ Java_com_k2fsa_sherpa_onnx_OfflineSpeakerDiarization_processWithCallback( jfloat *p = env->GetFloatArrayElements(samples, nullptr); jsize n = env->GetArrayLength(samples); auto segments = - sd->Process(p, n, callback_wrapper, (void *)arg).SortByStartTime(); + sd->Process(p, n, callback_wrapper, reinterpret_cast(arg)) + .SortByStartTime(); env->ReleaseFloatArrayElements(samples, p, JNI_ABORT); return ProcessImpl(env, segments); From 5e273c5be44e349b8e65cb649bc6e7e05f4f5ba7 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sat, 12 Oct 2024 12:28:38 +0800 Subject: [PATCH 016/183] Pascal API for speaker diarization (#1420) --- .github/workflows/pascal.yaml | 15 + pascal-api-examples/README.md | 1 + .../speaker-diarization/main.pas | 104 ++++++ .../speaker-diarization/run.sh | 49 +++ sherpa-onnx/pascal-api/sherpa_onnx.pas | 339 +++++++++++++++++- 5 files changed, 506 insertions(+), 2 deletions(-) create mode 100644 pascal-api-examples/speaker-diarization/main.pas create mode 100755 pascal-api-examples/speaker-diarization/run.sh diff --git a/.github/workflows/pascal.yaml b/.github/workflows/pascal.yaml index 2ed213184..ba9a73163 100644 --- a/.github/workflows/pascal.yaml +++ b/.github/workflows/pascal.yaml @@ -127,6 +127,21 @@ jobs: cp -v ../sherpa-onnx/pascal-api/*.pas ../pascal-api-examples/tts fi + - name: Run Pascal test (Speaker diarization) + shell: bash + run: | + export PATH=/c/lazarus/fpc/3.2.2/bin/x86_64-win64:$PATH + + cd ./pascal-api-examples + pushd speaker-diarization + + ./run.sh + rm -rfv *.onnx *.wav sherpa-onnx-* + ls -lh + echo "---" + + popd + - name: Run Pascal test (TTS) shell: bash run: | diff --git a/pascal-api-examples/README.md b/pascal-api-examples/README.md index 5475d825b..5e709cd7e 100644 --- a/pascal-api-examples/README.md +++ b/pascal-api-examples/README.md @@ -9,6 +9,7 @@ https://k2-fsa.github.io/sherpa/onnx/pascal-api/index.html |Directory| Description| |---------|------------| |[read-wav](./read-wav)|It shows how to read a wave file.| +|[speaker-diarization](./speaker-diarization)|It shows how to use Pascal API for speaker diarization.| |[streaming-asr](./streaming-asr)| It shows how to use streaming models for speech recognition.| |[non-streaming-asr](./non-streaming-asr)| It shows how to use non-streaming models for speech recognition.| |[vad](./vad)| It shows how to use the voice activity detection API.| diff --git a/pascal-api-examples/speaker-diarization/main.pas b/pascal-api-examples/speaker-diarization/main.pas new file mode 100644 index 000000000..35d915d0b --- /dev/null +++ b/pascal-api-examples/speaker-diarization/main.pas @@ -0,0 +1,104 @@ +{ Copyright (c) 2024 Xiaomi Corporation } +{ +This file shows how to use the Pascal API from sherpa-onnx +for speaker diarization. + +Usage: + +Step 1: Download a speaker segmentation model + +Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models +for a list of available models. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + +Step 2: Download a speaker embedding extractor model + +Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models +for a list of available models. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx + +Step 3. Download test wave files + +Please visit https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models +for a list of available test wave files. The following is an example + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/0-four-speakers-zh.wav + +Step 4. Run it +} + +program main; + +{$mode delphi} + +uses + sherpa_onnx, + ctypes, + SysUtils; + +function ProgressCallback( + NumProcessedChunks: cint32; + NumTotalChunks: cint32): cint32; cdecl; +var + Progress: Single; +begin + Progress := 100.0 * NumProcessedChunks / NumTotalChunks; + WriteLn(Format('Progress: %.3f%%', [Progress])); + + Result := 0; +end; + +var + Wave: TSherpaOnnxWave; + Config: TSherpaOnnxOfflineSpeakerDiarizationConfig; + Sd: TSherpaOnnxOfflineSpeakerDiarization; + Segments: TSherpaOnnxOfflineSpeakerDiarizationSegmentArray; + I: Integer; +begin + Wave := SherpaOnnxReadWave('./0-four-speakers-zh.wav'); + + Config.Segmentation.Pyannote.Model := './sherpa-onnx-pyannote-segmentation-3-0/model.onnx'; + Config.Embedding.Model := './3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx'; + + { + Since we know that there are 4 speakers in ./0-four-speakers-zh.wav, we + set NumClusters to 4 here. + If you don't have such information, please set NumClusters to -1. + In that case, you have to set Config.Clustering.Threshold. + A larger threshold leads to fewer clusters, i.e., fewer speakers. + } + Config.Clustering.NumClusters := 4; + Config.Segmentation.Debug := True; + Config.Embedding.Debug := True; + + Sd := TSherpaOnnxOfflineSpeakerDiarization.Create(Config); + if Sd.GetHandle = nil then + begin + WriteLn('Please check you config'); + Exit; + end; + + if Sd.GetSampleRate <> Wave.SampleRate then + begin + WriteLn(Format('Expected sample rate: %d, given: %d', [Sd.GetSampleRate, Wave.SampleRate])); + Exit; + end; + + { + // If you don't want to use a callback + Segments := Sd.Process(Wave.Samples); + } + Segments := Sd.Process(Wave.Samples, @ProgressCallback); + + for I := Low(Segments) to High(Segments) do + begin + WriteLn(Format('%.3f -- %.3f speaker_%d', + [Segments[I].Start, Segments[I].Stop, Segments[I].Speaker])); + end; + + FreeAndNil(Sd); +end. diff --git a/pascal-api-examples/speaker-diarization/run.sh b/pascal-api-examples/speaker-diarization/run.sh new file mode 100755 index 000000000..866dc63c9 --- /dev/null +++ b/pascal-api-examples/speaker-diarization/run.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -ex + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +SHERPA_ONNX_DIR=$(cd $SCRIPT_DIR/../.. && pwd) + +echo "SHERPA_ONNX_DIR: $SHERPA_ONNX_DIR" + +if [[ ! -f ../../build/install/lib/libsherpa-onnx-c-api.dylib && ! -f ../../build/install/lib/libsherpa-onnx-c-api.so && ! -f ../../build/install/lib/sherpa-onnx-c-api.dll ]]; then + mkdir -p ../../build + pushd ../../build + cmake \ + -DCMAKE_INSTALL_PREFIX=./install \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + .. + + cmake --build . --target install --config Release + popd +fi + +fpc \ + -dSHERPA_ONNX_USE_SHARED_LIBS \ + -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \ + -Fl$SHERPA_ONNX_DIR/build/install/lib \ + ./main.pas + +export LD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$LD_LIBRARY_PATH +export DYLD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$DYLD_LIBRARY_PATH + +if [ ! -f ./sherpa-onnx-pyannote-segmentation-3-0/model.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 +fi + +if [ ! -f ./3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx +fi + +if [ ! -f ./0-four-speakers-zh.wav ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/0-four-speakers-zh.wav +fi + +./main diff --git a/sherpa-onnx/pascal-api/sherpa_onnx.pas b/sherpa-onnx/pascal-api/sherpa_onnx.pas index 7f05793e1..1b24dec80 100644 --- a/sherpa-onnx/pascal-api/sherpa_onnx.pas +++ b/sherpa-onnx/pascal-api/sherpa_onnx.pas @@ -102,7 +102,7 @@ TSherpaOnnxOfflineTts = class function Generate(Text: AnsiString; SpeakerId: Integer; Speed: Single; - Callback:PSherpaOnnxGeneratedAudioCallbackWithArg; + Callback: PSherpaOnnxGeneratedAudioCallbackWithArg; Arg: Pointer ): TSherpaOnnxGeneratedAudio; overload; @@ -398,6 +398,78 @@ TSherpaOnnxVoiceActivityDetector = class property GetHandle: Pointer Read Handle; end; + + TSherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig = record + Model: AnsiString; + function ToString: AnsiString; + end; + + TSherpaOnnxOfflineSpeakerSegmentationModelConfig = record + Pyannote: TSherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig; + NumThreads: Integer; + Debug: Boolean; + Provider: AnsiString; + function ToString: AnsiString; + class operator Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineSpeakerSegmentationModelConfig); + end; + + TSherpaOnnxFastClusteringConfig = record + NumClusters: Integer; + Threshold: Single; + function ToString: AnsiString; + class operator Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxFastClusteringConfig); + end; + + TSherpaOnnxSpeakerEmbeddingExtractorConfig = record + Model: AnsiString; + NumThreads: Integer; + Debug: Boolean; + Provider: AnsiString; + function ToString: AnsiString; + class operator Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxSpeakerEmbeddingExtractorConfig); + end; + + TSherpaOnnxOfflineSpeakerDiarizationConfig = record + Segmentation: TSherpaOnnxOfflineSpeakerSegmentationModelConfig; + Embedding: TSherpaOnnxSpeakerEmbeddingExtractorConfig; + Clustering: TSherpaOnnxFastClusteringConfig; + MinDurationOn: Single; + MinDurationOff: Single; + function ToString: AnsiString; + class operator Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineSpeakerDiarizationConfig); + end; + + TSherpaOnnxOfflineSpeakerDiarizationSegment = record + Start: Single; + Stop: Single; + Speaker: Integer; + function ToString: AnsiString; + end; + + TSherpaOnnxOfflineSpeakerDiarizationSegmentArray = array of TSherpaOnnxOfflineSpeakerDiarizationSegment; + + PSherpaOnnxOfflineSpeakerDiarizationProgressCallbackNoArg = ^TSherpaOnnxOfflineSpeakerDiarizationProgressCallbackNoArg; + + TSherpaOnnxOfflineSpeakerDiarizationProgressCallbackNoArg = function( + NumProcessChunks: cint32; + NumTotalChunks: cint32): cint32; cdecl; + + TSherpaOnnxOfflineSpeakerDiarization = class + private + Handle: Pointer; + SampleRate: Integer; + _Config: TSherpaOnnxOfflineSpeakerDiarizationConfig; + public + constructor Create(Config: TSherpaOnnxOfflineSpeakerDiarizationConfig); + destructor Destroy; override; + procedure SetConfig(Config: TSherpaOnnxOfflineSpeakerDiarizationConfig); + function Process(Samples: array of Single): TSherpaOnnxOfflineSpeakerDiarizationSegmentArray; overload; + function Process(Samples: array of Single; Callback: PSherpaOnnxOfflineSpeakerDiarizationProgressCallbackNoArg): TSherpaOnnxOfflineSpeakerDiarizationSegmentArray; overload; + property GetHandle: Pointer Read Handle; + property GetSampleRate: Integer Read SampleRate; + end; + + { It supports reading a single channel wave with 16-bit encoded samples. Samples are normalized to the range [-1, 1]. } @@ -656,6 +728,47 @@ SherpaOnnxResampleOut = record PSherpaOnnxResampleOut = ^SherpaOnnxResampleOut; + SherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig = record + Model: PAnsiChar; + end; + + SherpaOnnxOfflineSpeakerSegmentationModelConfig = record + Pyannote: SherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig; + NumThreads: cint32; + Debug: cint32; + Provider: PAnsiChar; + end; + + SherpaOnnxFastClusteringConfig = record + NumClusters: cint32; + Threshold: cfloat; + end; + + SherpaOnnxSpeakerEmbeddingExtractorConfig = record + Model: PAnsiChar; + NumThreads: cint32; + Debug: cint32; + Provider: PAnsiChar; + end; + + SherpaOnnxOfflineSpeakerDiarizationConfig = record + Segmentation: SherpaOnnxOfflineSpeakerSegmentationModelConfig; + Embedding: SherpaOnnxSpeakerEmbeddingExtractorConfig; + Clustering: SherpaOnnxFastClusteringConfig; + MinDurationOn: cfloat; + MinDurationOff: cfloat; + end; + + SherpaOnnxOfflineSpeakerDiarizationSegment = record + Start: cfloat; + Stop: cfloat; + Speaker: cint32; + end; + + PSherpaOnnxOfflineSpeakerDiarizationSegment = ^SherpaOnnxOfflineSpeakerDiarizationSegment; + + PSherpaOnnxOfflineSpeakerDiarizationConfig = ^SherpaOnnxOfflineSpeakerDiarizationConfig; + function SherpaOnnxCreateLinearResampler(SampleRateInHz: cint32; SampleRateOutHz: cint32; FilterCutoffHz: cfloat; @@ -677,6 +790,37 @@ procedure SherpaOnnxLinearResamplerResampleFree(P: PSherpaOnnxResampleOut); cdec procedure SherpaOnnxLinearResamplerReset(P: Pointer); cdecl; external SherpaOnnxLibName; +function SherpaOnnxCreateOfflineSpeakerDiarization(Config: PSherpaOnnxOfflineSpeakerDiarizationConfig): Pointer; cdecl; + external SherpaOnnxLibName; + +procedure SherpaOnnxDestroyOfflineSpeakerDiarization(P: Pointer); cdecl; + external SherpaOnnxLibName; + +function SherpaOnnxOfflineSpeakerDiarizationGetSampleRate(P: Pointer): cint32; cdecl; + external SherpaOnnxLibName; + +procedure SherpaOnnxOfflineSpeakerDiarizationSetConfig(P: Pointer; Config: PSherpaOnnxOfflineSpeakerDiarizationConfig); cdecl; + external SherpaOnnxLibName; + +function SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments(P: Pointer): cint32; cdecl; + external SherpaOnnxLibName; + +function SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime(P: Pointer): PSherpaOnnxOfflineSpeakerDiarizationSegment; cdecl; + external SherpaOnnxLibName; + +procedure SherpaOnnxOfflineSpeakerDiarizationDestroySegment(P: Pointer); cdecl; + external SherpaOnnxLibName; + +function SherpaOnnxOfflineSpeakerDiarizationProcess(P: Pointer; Samples: pcfloat; N: cint32): Pointer; cdecl; + external SherpaOnnxLibName; + +function SherpaOnnxOfflineSpeakerDiarizationProcessWithCallbackNoArg(P: Pointer; + Samples: pcfloat; N: cint32; Callback: PSherpaOnnxOfflineSpeakerDiarizationProgressCallbackNoArg): Pointer; cdecl; + external SherpaOnnxLibName; + +procedure SherpaOnnxOfflineSpeakerDiarizationDestroyResult(P: Pointer); cdecl; + external SherpaOnnxLibName; + function SherpaOnnxCreateOfflineTts(Config: PSherpaOnnxOfflineTtsConfig): Pointer; cdecl; external SherpaOnnxLibName; @@ -1773,7 +1917,7 @@ function TSherpaOnnxOfflineTts.Generate(Text: AnsiString; SpeakerId: Integer; function TSherpaOnnxOfflineTts.Generate(Text: AnsiString; SpeakerId: Integer; Speed: Single; - Callback:PSherpaOnnxGeneratedAudioCallbackWithArg; + Callback: PSherpaOnnxGeneratedAudioCallbackWithArg; Arg: Pointer ): TSherpaOnnxGeneratedAudio; var @@ -1847,4 +1991,195 @@ procedure TSherpaOnnxLinearResampler.Reset; SherpaOnnxLinearResamplerReset(Self.Handle); end; +function TSherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig.ToString: AnsiString; +begin + Result := Format('TSherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig(' + + 'Model := %s)',[Self.Model]); +end; + +function TSherpaOnnxOfflineSpeakerSegmentationModelConfig.ToString: AnsiString; +begin + Result := Format('TSherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig(' + + 'Pyannote := %s, ' + + 'NumThreads := %d, ' + + 'Debug := %s, ' + + 'Provider := %s)', + [Self.Pyannote.ToString, Self.NumThreads, + Self.Debug.ToString, Self.Provider]); +end; + +class operator TSherpaOnnxOfflineSpeakerSegmentationModelConfig.Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineSpeakerSegmentationModelConfig); +begin + Dest.NumThreads := 1; + Dest.Debug := False; + Dest.Provider := 'cpu'; +end; + +function TSherpaOnnxFastClusteringConfig.ToString: AnsiString; +begin + Result := Format('TSherpaOnnxFastClusteringConfig(' + + 'NumClusters := %d, Threshold := %.3f)', + [Self.NumClusters, Self.Threshold]); +end; + +class operator TSherpaOnnxFastClusteringConfig.Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxFastClusteringConfig); +begin + Dest.NumClusters := -1; + Dest.Threshold := 0.5; +end; + +function TSherpaOnnxSpeakerEmbeddingExtractorConfig.ToString: AnsiString; +begin + Result := Format('TSherpaOnnxSpeakerEmbeddingExtractorConfig(' + + 'Model := %s, '+ + 'NumThreads := %d, '+ + 'Debug := %s, '+ + 'Provider := %s)', + [Self.Model, Self.NumThreads, Self.Debug.ToString, Self.Provider]); +end; + +class operator TSherpaOnnxSpeakerEmbeddingExtractorConfig.Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxSpeakerEmbeddingExtractorConfig); +begin + Dest.NumThreads := 1; + Dest.Debug := False; + Dest.Provider := 'cpu'; +end; + +function TSherpaOnnxOfflineSpeakerDiarizationConfig.ToString: AnsiString; +begin + Result := Format('TSherpaOnnxOfflineSpeakerDiarizationConfig(' + + 'Segmentation := %s, '+ + 'Embedding := %s, '+ + 'Clustering := %s, '+ + 'MinDurationOn := %.3f, '+ + 'MinDurationOff := %.3f)', + [Self.Segmentation.ToString, Self.Embedding.ToString, + Self.Clustering.ToString, Self.MinDurationOn, Self.MinDurationOff]); +end; + +class operator TSherpaOnnxOfflineSpeakerDiarizationConfig.Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineSpeakerDiarizationConfig); +begin + Dest.MinDurationOn := 0.2; + Dest.MinDurationOff := 0.5; +end; + +function TSherpaOnnxOfflineSpeakerDiarizationSegment.ToString: AnsiString; +begin + Result := Format('TSherpaOnnxOfflineSpeakerDiarizationSegment(' + + 'Start := %.3f, '+ + 'Stop := %.3f, '+ + 'Speaker := %d)', + [Self.Start, Self.Stop, Self.Speaker]); +end; + +constructor TSherpaOnnxOfflineSpeakerDiarization.Create(Config: TSherpaOnnxOfflineSpeakerDiarizationConfig); +var + C: SherpaOnnxOfflineSpeakerDiarizationConfig; +begin + C := Default(SherpaOnnxOfflineSpeakerDiarizationConfig); + C.Segmentation.Pyannote.Model := PAnsiChar(Config.Segmentation.Pyannote.Model); + C.Segmentation.NumThreads := Config.Segmentation.NumThreads; + C.Segmentation.Debug := Ord(Config.Segmentation.Debug); + C.Segmentation.Provider := PAnsiChar(Config.Segmentation.Provider); + + C.Embedding.Model := PAnsiChar(Config.Embedding.Model); + C.Embedding.NumThreads := Config.Embedding.NumThreads; + C.Embedding.Debug := Ord(Config.Embedding.Debug); + C.Embedding.Provider := PAnsiChar(Config.Embedding.Provider); + + C.Clustering.NumClusters := Config.Clustering.NumClusters; + C.Clustering.Threshold := Config.Clustering.Threshold; + + C.MinDurationOn := Config.MinDurationOn; + C.MinDurationOff := Config.MinDurationOff; + + Self.Handle := SherpaOnnxCreateOfflineSpeakerDiarization(@C); + Self._Config := Config; + Self.SampleRate := 0; + + if Self.Handle <> nil then + begin + Self.SampleRate := SherpaOnnxOfflineSpeakerDiarizationGetSampleRate(Self.Handle); + end; +end; + +destructor TSherpaOnnxOfflineSpeakerDiarization.Destroy; +begin + SherpaOnnxDestroyOfflineSpeakerDiarization(Self.Handle); + Self.Handle := nil; +end; + +procedure TSherpaOnnxOfflineSpeakerDiarization.SetConfig(Config: TSherpaOnnxOfflineSpeakerDiarizationConfig); +var + C: SherpaOnnxOfflineSpeakerDiarizationConfig; +begin + C := Default(SherpaOnnxOfflineSpeakerDiarizationConfig); + + C.Clustering.NumClusters := Config.Clustering.NumClusters; + C.Clustering.Threshold := Config.Clustering.Threshold; + + SherpaOnnxOfflineSpeakerDiarizationSetConfig(Self.Handle, @C); +end; + +function TSherpaOnnxOfflineSpeakerDiarization.Process(Samples: array of Single): TSherpaOnnxOfflineSpeakerDiarizationSegmentArray; +var + R: Pointer; + NumSegments: Integer; + I: Integer; + Segments: PSherpaOnnxOfflineSpeakerDiarizationSegment; +begin + Result := nil; + + R := SherpaOnnxOfflineSpeakerDiarizationProcess(Self.Handle, pcfloat(Samples), Length(Samples)); + if R = nil then + begin + Exit + end; + NumSegments := SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments(R); + + Segments := SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime(R); + + SetLength(Result, NumSegments); + for I := Low(Result) to High(Result) do + begin + Result[I].Start := Segments[I].Start; + Result[I].Stop := Segments[I].Stop; + Result[I].Speaker := Segments[I].Speaker; + end; + + SherpaOnnxOfflineSpeakerDiarizationDestroySegment(Segments); + SherpaOnnxOfflineSpeakerDiarizationDestroyResult(R); +end; + +function TSherpaOnnxOfflineSpeakerDiarization.Process(Samples: array of Single; + callback: PSherpaOnnxOfflineSpeakerDiarizationProgressCallbackNoArg): TSherpaOnnxOfflineSpeakerDiarizationSegmentArray; +var + R: Pointer; + NumSegments: Integer; + I: Integer; + Segments: PSherpaOnnxOfflineSpeakerDiarizationSegment; +begin + Result := nil; + + R := SherpaOnnxOfflineSpeakerDiarizationProcessWithCallbackNoArg(Self.Handle, pcfloat(Samples), Length(Samples), callback); + if R = nil then + begin + Exit + end; + NumSegments := SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments(R); + + Segments := SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime(R); + + SetLength(Result, NumSegments); + for I := Low(Result) to High(Result) do + begin + Result[I].Start := Segments[I].Start; + Result[I].Stop := Segments[I].Stop; + Result[I].Speaker := Segments[I].Speaker; + end; + + SherpaOnnxOfflineSpeakerDiarizationDestroySegment(Segments); + SherpaOnnxOfflineSpeakerDiarizationDestroyResult(R); +end; + end. From 94b26ff07c1b6275d1830cd2987081a0bdbedacb Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sat, 12 Oct 2024 13:03:48 +0800 Subject: [PATCH 017/183] Android JNI support for speaker diarization (#1421) --- .../csrc/offline-speaker-diarization-impl.cc | 14 ++++++++++++++ .../csrc/offline-speaker-diarization-impl.h | 10 ++++++++++ ...ffline-speaker-diarization-pyannote-impl.h | 16 ++++++++++++++++ .../csrc/offline-speaker-diarization.cc | 6 ++++++ .../csrc/offline-speaker-diarization.h | 10 ++++++++++ ...ine-speaker-segmentation-pyannote-model.cc | 18 ++++++++++++++++++ ...line-speaker-segmentation-pyannote-model.h | 10 ++++++++++ .../sherpa-onnx-vad-microphone-offline-asr.cc | 2 +- sherpa-onnx/jni/audio-tagging.cc | 1 + sherpa-onnx/jni/keyword-spotter.cc | 2 ++ sherpa-onnx/jni/offline-punctuation.cc | 2 ++ sherpa-onnx/jni/offline-recognizer.cc | 2 ++ .../jni/offline-speaker-diarization.cc | 19 ++++++++++++++++++- sherpa-onnx/jni/offline-tts.cc | 1 + sherpa-onnx/jni/online-recognizer.cc | 1 + .../jni/speaker-embedding-extractor.cc | 1 + .../jni/spoken-language-identification.cc | 1 + sherpa-onnx/jni/voice-activity-detector.cc | 2 ++ 18 files changed, 116 insertions(+), 2 deletions(-) diff --git a/sherpa-onnx/csrc/offline-speaker-diarization-impl.cc b/sherpa-onnx/csrc/offline-speaker-diarization-impl.cc index e41a7767a..15c3a2eb4 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization-impl.cc +++ b/sherpa-onnx/csrc/offline-speaker-diarization-impl.cc @@ -23,4 +23,18 @@ OfflineSpeakerDiarizationImpl::Create( return nullptr; } +#if __ANDROID_API__ >= 9 +std::unique_ptr +OfflineSpeakerDiarizationImpl::Create( + AAssetManager *mgr, const OfflineSpeakerDiarizationConfig &config) { + if (!config.segmentation.pyannote.model.empty()) { + return std::make_unique(mgr, config); + } + + SHERPA_ONNX_LOGE("Please specify a speaker segmentation model."); + + return nullptr; +} +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-speaker-diarization-impl.h b/sherpa-onnx/csrc/offline-speaker-diarization-impl.h index 3aed9d72f..41f0e1e2f 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization-impl.h +++ b/sherpa-onnx/csrc/offline-speaker-diarization-impl.h @@ -8,6 +8,11 @@ #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + #include "sherpa-onnx/csrc/offline-speaker-diarization.h" namespace sherpa_onnx { @@ -16,6 +21,11 @@ class OfflineSpeakerDiarizationImpl { static std::unique_ptr Create( const OfflineSpeakerDiarizationConfig &config); +#if __ANDROID_API__ >= 9 + static std::unique_ptr Create( + AAssetManager *mgr, const OfflineSpeakerDiarizationConfig &config); +#endif + virtual ~OfflineSpeakerDiarizationImpl() = default; virtual int32_t SampleRate() const = 0; diff --git a/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h b/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h index 0c70f0bc6..aaedc3be0 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h +++ b/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h @@ -10,6 +10,11 @@ #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + #include "Eigen/Dense" #include "sherpa-onnx/csrc/fast-clustering.h" #include "sherpa-onnx/csrc/math.h" @@ -65,6 +70,17 @@ class OfflineSpeakerDiarizationPyannoteImpl Init(); } +#if __ANDROID_API__ >= 9 + OfflineSpeakerDiarizationPyannoteImpl( + AAssetManager *mgr, const OfflineSpeakerDiarizationConfig &config) + : config_(config), + segmentation_model_(mgr, config_.segmentation), + embedding_extractor_(mgr, config_.embedding), + clustering_(std::make_unique(config_.clustering)) { + Init(); + } +#endif + int32_t SampleRate() const override { const auto &meta_data = segmentation_model_.GetModelMetaData(); diff --git a/sherpa-onnx/csrc/offline-speaker-diarization.cc b/sherpa-onnx/csrc/offline-speaker-diarization.cc index 00733bfb2..f34ea4e0e 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization.cc +++ b/sherpa-onnx/csrc/offline-speaker-diarization.cc @@ -73,6 +73,12 @@ OfflineSpeakerDiarization::OfflineSpeakerDiarization( const OfflineSpeakerDiarizationConfig &config) : impl_(OfflineSpeakerDiarizationImpl::Create(config)) {} +#if __ANDROID_API__ >= 9 +OfflineSpeakerDiarization::OfflineSpeakerDiarization( + AAssetManager *mgr, const OfflineSpeakerDiarizationConfig &config) + : impl_(OfflineSpeakerDiarizationImpl::Create(mgr, config)) {} +#endif + OfflineSpeakerDiarization::~OfflineSpeakerDiarization() = default; int32_t OfflineSpeakerDiarization::SampleRate() const { diff --git a/sherpa-onnx/csrc/offline-speaker-diarization.h b/sherpa-onnx/csrc/offline-speaker-diarization.h index 376e5f975..4a517fbb2 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization.h +++ b/sherpa-onnx/csrc/offline-speaker-diarization.h @@ -9,6 +9,11 @@ #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + #include "sherpa-onnx/csrc/fast-clustering-config.h" #include "sherpa-onnx/csrc/offline-speaker-diarization-result.h" #include "sherpa-onnx/csrc/offline-speaker-segmentation-model-config.h" @@ -57,6 +62,11 @@ class OfflineSpeakerDiarization { explicit OfflineSpeakerDiarization( const OfflineSpeakerDiarizationConfig &config); +#if __ANDROID_API__ >= 9 + OfflineSpeakerDiarization(AAssetManager *mgr, + const OfflineSpeakerDiarizationConfig &config); +#endif + ~OfflineSpeakerDiarization(); // Expected sample rate of the input audio samples diff --git a/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.cc b/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.cc index 3f3323698..e3768dcf4 100644 --- a/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.cc +++ b/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.cc @@ -24,6 +24,17 @@ class OfflineSpeakerSegmentationPyannoteModel::Impl { Init(buf.data(), buf.size()); } +#if __ANDROID_API__ >= 9 + Impl(AAssetManager *mgr, const OfflineSpeakerSegmentationModelConfig &config) + : config_(config), + env_(ORT_LOGGING_LEVEL_ERROR), + sess_opts_(GetSessionOptions(config)), + allocator_{} { + auto buf = ReadFile(mgr, config_.pyannote.model); + Init(buf.data(), buf.size()); + } +#endif + const OfflineSpeakerSegmentationPyannoteModelMetaData &GetModelMetaData() const { return meta_data_; @@ -92,6 +103,13 @@ OfflineSpeakerSegmentationPyannoteModel:: const OfflineSpeakerSegmentationModelConfig &config) : impl_(std::make_unique(config)) {} +#if __ANDROID_API__ >= 9 +OfflineSpeakerSegmentationPyannoteModel:: + OfflineSpeakerSegmentationPyannoteModel( + AAssetManager *mgr, const OfflineSpeakerSegmentationModelConfig &config) + : impl_(std::make_unique(mgr, config)) {} +#endif + OfflineSpeakerSegmentationPyannoteModel:: ~OfflineSpeakerSegmentationPyannoteModel() = default; diff --git a/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.h b/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.h index b504c373f..6b835763b 100644 --- a/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.h +++ b/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.h @@ -6,6 +6,11 @@ #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/offline-speaker-segmentation-model-config.h" #include "sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model-meta-data.h" @@ -17,6 +22,11 @@ class OfflineSpeakerSegmentationPyannoteModel { explicit OfflineSpeakerSegmentationPyannoteModel( const OfflineSpeakerSegmentationModelConfig &config); +#if __ANDROID_API__ >= 9 + OfflineSpeakerSegmentationPyannoteModel( + AAssetManager *mgr, const OfflineSpeakerSegmentationModelConfig &config); +#endif + ~OfflineSpeakerSegmentationPyannoteModel(); const OfflineSpeakerSegmentationPyannoteModelMetaData &GetModelMetaData() diff --git a/sherpa-onnx/csrc/sherpa-onnx-vad-microphone-offline-asr.cc b/sherpa-onnx/csrc/sherpa-onnx-vad-microphone-offline-asr.cc index c90c29c52..df3e250a5 100644 --- a/sherpa-onnx/csrc/sherpa-onnx-vad-microphone-offline-asr.cc +++ b/sherpa-onnx/csrc/sherpa-onnx-vad-microphone-offline-asr.cc @@ -211,7 +211,7 @@ to download models for offline ASR. } while (!vad->Empty()) { - auto &segment = vad->Front(); + const auto &segment = vad->Front(); auto s = recognizer.CreateStream(); s->AcceptWaveform(sample_rate, segment.samples.data(), segment.samples.size()); diff --git a/sherpa-onnx/jni/audio-tagging.cc b/sherpa-onnx/jni/audio-tagging.cc index ff8db0089..7ad6e7d53 100644 --- a/sherpa-onnx/jni/audio-tagging.cc +++ b/sherpa-onnx/jni/audio-tagging.cc @@ -70,6 +70,7 @@ JNIEXPORT jlong JNICALL Java_com_k2fsa_sherpa_onnx_AudioTagging_newFromAsset( AAssetManager *mgr = AAssetManager_fromJava(env, asset_manager); if (!mgr) { SHERPA_ONNX_LOGE("Failed to get asset manager: %p", mgr); + return 0; } #endif diff --git a/sherpa-onnx/jni/keyword-spotter.cc b/sherpa-onnx/jni/keyword-spotter.cc index ca0c229c2..4ac80a294 100644 --- a/sherpa-onnx/jni/keyword-spotter.cc +++ b/sherpa-onnx/jni/keyword-spotter.cc @@ -115,10 +115,12 @@ JNIEXPORT jlong JNICALL Java_com_k2fsa_sherpa_onnx_KeywordSpotter_newFromAsset( AAssetManager *mgr = AAssetManager_fromJava(env, asset_manager); if (!mgr) { SHERPA_ONNX_LOGE("Failed to get asset manager: %p", mgr); + return 0; } #endif auto config = sherpa_onnx::GetKwsConfig(env, _config); SHERPA_ONNX_LOGE("config:\n%s", config.ToString().c_str()); + auto kws = new sherpa_onnx::KeywordSpotter( #if __ANDROID_API__ >= 9 mgr, diff --git a/sherpa-onnx/jni/offline-punctuation.cc b/sherpa-onnx/jni/offline-punctuation.cc index 5056a3ac4..efe03cac0 100644 --- a/sherpa-onnx/jni/offline-punctuation.cc +++ b/sherpa-onnx/jni/offline-punctuation.cc @@ -53,10 +53,12 @@ Java_com_k2fsa_sherpa_onnx_OfflinePunctuation_newFromAsset( AAssetManager *mgr = AAssetManager_fromJava(env, asset_manager); if (!mgr) { SHERPA_ONNX_LOGE("Failed to get asset manager: %p", mgr); + return 0; } #endif auto config = sherpa_onnx::GetOfflinePunctuationConfig(env, _config); SHERPA_ONNX_LOGE("config:\n%s", config.ToString().c_str()); + auto model = new sherpa_onnx::OfflinePunctuation( #if __ANDROID_API__ >= 9 mgr, diff --git a/sherpa-onnx/jni/offline-recognizer.cc b/sherpa-onnx/jni/offline-recognizer.cc index 8c1265bba..5e4b359b6 100644 --- a/sherpa-onnx/jni/offline-recognizer.cc +++ b/sherpa-onnx/jni/offline-recognizer.cc @@ -233,10 +233,12 @@ Java_com_k2fsa_sherpa_onnx_OfflineRecognizer_newFromAsset(JNIEnv *env, AAssetManager *mgr = AAssetManager_fromJava(env, asset_manager); if (!mgr) { SHERPA_ONNX_LOGE("Failed to get asset manager: %p", mgr); + return 0; } #endif auto config = sherpa_onnx::GetOfflineConfig(env, _config); SHERPA_ONNX_LOGE("config:\n%s", config.ToString().c_str()); + auto model = new sherpa_onnx::OfflineRecognizer( #if __ANDROID_API__ >= 9 mgr, diff --git a/sherpa-onnx/jni/offline-speaker-diarization.cc b/sherpa-onnx/jni/offline-speaker-diarization.cc index e82962c80..ba4e14bc3 100644 --- a/sherpa-onnx/jni/offline-speaker-diarization.cc +++ b/sherpa-onnx/jni/offline-speaker-diarization.cc @@ -101,7 +101,24 @@ SHERPA_ONNX_EXTERN_C JNIEXPORT jlong JNICALL Java_com_k2fsa_sherpa_onnx_OfflineSpeakerDiarization_newFromAsset( JNIEnv *env, jobject /*obj*/, jobject asset_manager, jobject _config) { - return 0; +#if __ANDROID_API__ >= 9 + AAssetManager *mgr = AAssetManager_fromJava(env, asset_manager); + if (!mgr) { + SHERPA_ONNX_LOGE("Failed to get asset manager: %p", mgr); + return 0; + } +#endif + + auto config = sherpa_onnx::GetOfflineSpeakerDiarizationConfig(env, _config); + SHERPA_ONNX_LOGE("config:\n%s", config.ToString().c_str()); + + auto sd = new sherpa_onnx::OfflineSpeakerDiarization( +#if __ANDROID_API__ >= 9 + mgr, +#endif + config); + + return (jlong)sd; } SHERPA_ONNX_EXTERN_C diff --git a/sherpa-onnx/jni/offline-tts.cc b/sherpa-onnx/jni/offline-tts.cc index 43a93e0e0..4d67afc27 100644 --- a/sherpa-onnx/jni/offline-tts.cc +++ b/sherpa-onnx/jni/offline-tts.cc @@ -105,6 +105,7 @@ JNIEXPORT jlong JNICALL Java_com_k2fsa_sherpa_onnx_OfflineTts_newFromAsset( AAssetManager *mgr = AAssetManager_fromJava(env, asset_manager); if (!mgr) { SHERPA_ONNX_LOGE("Failed to get asset manager: %p", mgr); + return 0; } #endif auto config = sherpa_onnx::GetOfflineTtsConfig(env, _config); diff --git a/sherpa-onnx/jni/online-recognizer.cc b/sherpa-onnx/jni/online-recognizer.cc index 1793cf73b..dbe205c4e 100644 --- a/sherpa-onnx/jni/online-recognizer.cc +++ b/sherpa-onnx/jni/online-recognizer.cc @@ -267,6 +267,7 @@ Java_com_k2fsa_sherpa_onnx_OnlineRecognizer_newFromAsset(JNIEnv *env, AAssetManager *mgr = AAssetManager_fromJava(env, asset_manager); if (!mgr) { SHERPA_ONNX_LOGE("Failed to get asset manager: %p", mgr); + return 0; } #endif auto config = sherpa_onnx::GetConfig(env, _config); diff --git a/sherpa-onnx/jni/speaker-embedding-extractor.cc b/sherpa-onnx/jni/speaker-embedding-extractor.cc index b1190bffc..33d630ee6 100644 --- a/sherpa-onnx/jni/speaker-embedding-extractor.cc +++ b/sherpa-onnx/jni/speaker-embedding-extractor.cc @@ -45,6 +45,7 @@ Java_com_k2fsa_sherpa_onnx_SpeakerEmbeddingExtractor_newFromAsset( AAssetManager *mgr = AAssetManager_fromJava(env, asset_manager); if (!mgr) { SHERPA_ONNX_LOGE("Failed to get asset manager: %p", mgr); + return 0; } #endif auto config = sherpa_onnx::GetSpeakerEmbeddingExtractorConfig(env, _config); diff --git a/sherpa-onnx/jni/spoken-language-identification.cc b/sherpa-onnx/jni/spoken-language-identification.cc index 278c6adbf..fcb6f228a 100644 --- a/sherpa-onnx/jni/spoken-language-identification.cc +++ b/sherpa-onnx/jni/spoken-language-identification.cc @@ -62,6 +62,7 @@ Java_com_k2fsa_sherpa_onnx_SpokenLanguageIdentification_newFromAsset( AAssetManager *mgr = AAssetManager_fromJava(env, asset_manager); if (!mgr) { SHERPA_ONNX_LOGE("Failed to get asset manager: %p", mgr); + return 0; } #endif diff --git a/sherpa-onnx/jni/voice-activity-detector.cc b/sherpa-onnx/jni/voice-activity-detector.cc index 319edd09b..a30423f70 100644 --- a/sherpa-onnx/jni/voice-activity-detector.cc +++ b/sherpa-onnx/jni/voice-activity-detector.cc @@ -71,10 +71,12 @@ JNIEXPORT jlong JNICALL Java_com_k2fsa_sherpa_onnx_Vad_newFromAsset( AAssetManager *mgr = AAssetManager_fromJava(env, asset_manager); if (!mgr) { SHERPA_ONNX_LOGE("Failed to get asset manager: %p", mgr); + return 0; } #endif auto config = sherpa_onnx::GetVadModelConfig(env, _config); SHERPA_ONNX_LOGE("config:\n%s", config.ToString().c_str()); + auto model = new sherpa_onnx::VoiceActivityDetector( #if __ANDROID_API__ >= 9 mgr, From 5a22f74b2b0700b9f986bc9f01ae93b58b2117c9 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sun, 13 Oct 2024 14:02:57 +0800 Subject: [PATCH 018/183] Android demo for speaker diarization (#1423) --- .../workflows/apk-speaker-diarization.yaml | 175 +++++++++++++++ .../workflows/apk-speaker-identification.yaml | 62 ++++++ .github/workflows/apk-vad.yaml | 2 +- README.md | 51 +++-- android/README.md | 2 + .../SherpaOnnxSpeakerDiarization/.gitignore | 15 ++ .../app/.gitignore | 1 + .../app/build.gradle.kts | 71 ++++++ .../app/proguard-rules.pro | 21 ++ .../diarization/ExampleInstrumentedTest.kt | 24 ++ .../app/src/main/AndroidManifest.xml | 32 +++ .../app/src/main/assets/.gitkeep | 0 .../onnx/speaker/diarization/BarItem.kt | 13 ++ .../onnx/speaker/diarization/MainActivity.kt | 132 +++++++++++ .../onnx/speaker/diarization/NavBarItems.kt | 20 ++ .../onnx/speaker/diarization/NavRoutes.kt | 6 + .../diarization/OfflineSpeakerDiarization.kt | 1 + .../onnx/speaker/diarization/ReadWaveFile.kt | 137 ++++++++++++ .../diarization/SpeakerDiarizationObject.kt | 66 ++++++ .../SpeakerEmbeddingExtractorConfig.kt | 1 + .../onnx/speaker/diarization/screens/Help.kt | 38 ++++ .../onnx/speaker/diarization/screens/Home.kt | 210 ++++++++++++++++++ .../speaker/diarization/ui/theme/Color.kt | 11 + .../speaker/diarization/ui/theme/Theme.kt | 58 +++++ .../onnx/speaker/diarization/ui/theme/Type.kt | 34 +++ .../app/src/main/jniLibs/arm64-v8a/.gitkeep | 0 .../app/src/main/jniLibs/armeabi-v7a/.gitkeep | 0 .../app/src/main/jniLibs/x86/.gitkeep | 0 .../app/src/main/jniLibs/x86_64/.gitkeep | 0 .../drawable-v24/ic_launcher_foreground.xml | 30 +++ .../res/drawable/ic_launcher_background.xml | 170 ++++++++++++++ .../res/mipmap-anydpi-v26/ic_launcher.xml | 6 + .../mipmap-anydpi-v26/ic_launcher_round.xml | 6 + .../src/main/res/mipmap-hdpi/ic_launcher.webp | Bin 0 -> 1404 bytes .../res/mipmap-hdpi/ic_launcher_round.webp | Bin 0 -> 2898 bytes .../src/main/res/mipmap-mdpi/ic_launcher.webp | Bin 0 -> 982 bytes .../res/mipmap-mdpi/ic_launcher_round.webp | Bin 0 -> 1772 bytes .../main/res/mipmap-xhdpi/ic_launcher.webp | Bin 0 -> 1900 bytes .../res/mipmap-xhdpi/ic_launcher_round.webp | Bin 0 -> 3918 bytes .../main/res/mipmap-xxhdpi/ic_launcher.webp | Bin 0 -> 2884 bytes .../res/mipmap-xxhdpi/ic_launcher_round.webp | Bin 0 -> 5914 bytes .../main/res/mipmap-xxxhdpi/ic_launcher.webp | Bin 0 -> 3844 bytes .../res/mipmap-xxxhdpi/ic_launcher_round.webp | Bin 0 -> 7778 bytes .../app/src/main/res/values/colors.xml | 10 + .../app/src/main/res/values/strings.xml | 3 + .../app/src/main/res/values/themes.xml | 5 + .../app/src/main/res/xml/backup_rules.xml | 13 ++ .../main/res/xml/data_extraction_rules.xml | 19 ++ .../speaker/diarization/ExampleUnitTest.kt | 17 ++ .../build.gradle.kts | 5 + .../gradle.properties | 23 ++ .../gradle/libs.versions.toml | 35 +++ .../gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 59203 bytes .../gradle/wrapper/gradle-wrapper.properties | 6 + android/SherpaOnnxSpeakerDiarization/gradlew | 185 +++++++++++++++ .../SherpaOnnxSpeakerDiarization/gradlew.bat | 89 ++++++++ .../settings.gradle.kts | 23 ++ .../SpeakerEmbeddingExtractorConfig.kt | 1 + .../SpeakerEmbeddingExtractorConfig.kt | 1 + kotlin-api-examples/run.sh | 2 + scripts/apk/build-apk-speaker-diarization.sh | 73 ++++++ .../kotlin-api/OfflineSpeakerDiarization.kt | 11 +- sherpa-onnx/kotlin-api/Speaker.kt | 7 - .../SpeakerEmbeddingExtractorConfig.kt | 8 + 64 files changed, 1905 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/apk-speaker-diarization.yaml create mode 100644 android/SherpaOnnxSpeakerDiarization/.gitignore create mode 100644 android/SherpaOnnxSpeakerDiarization/app/.gitignore create mode 100644 android/SherpaOnnxSpeakerDiarization/app/build.gradle.kts create mode 100644 android/SherpaOnnxSpeakerDiarization/app/proguard-rules.pro create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/androidTest/java/com/k2fsa/sherpa/onnx/speaker/diarization/ExampleInstrumentedTest.kt create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/AndroidManifest.xml create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/assets/.gitkeep create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/BarItem.kt create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/MainActivity.kt create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/NavBarItems.kt create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/NavRoutes.kt create mode 120000 android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/OfflineSpeakerDiarization.kt create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/ReadWaveFile.kt create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/SpeakerDiarizationObject.kt create mode 120000 android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/SpeakerEmbeddingExtractorConfig.kt create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/screens/Help.kt create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/screens/Home.kt create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/ui/theme/Color.kt create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/ui/theme/Theme.kt create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/ui/theme/Type.kt create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/jniLibs/arm64-v8a/.gitkeep create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/jniLibs/armeabi-v7a/.gitkeep create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/jniLibs/x86/.gitkeep create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/jniLibs/x86_64/.gitkeep create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/res/drawable-v24/ic_launcher_foreground.xml create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/res/drawable/ic_launcher_background.xml create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-hdpi/ic_launcher.webp create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-mdpi/ic_launcher.webp create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-xhdpi/ic_launcher.webp create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/res/values/colors.xml create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/res/values/strings.xml create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/res/values/themes.xml create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/res/xml/backup_rules.xml create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/main/res/xml/data_extraction_rules.xml create mode 100644 android/SherpaOnnxSpeakerDiarization/app/src/test/java/com/k2fsa/sherpa/onnx/speaker/diarization/ExampleUnitTest.kt create mode 100644 android/SherpaOnnxSpeakerDiarization/build.gradle.kts create mode 100644 android/SherpaOnnxSpeakerDiarization/gradle.properties create mode 100644 android/SherpaOnnxSpeakerDiarization/gradle/libs.versions.toml create mode 100644 android/SherpaOnnxSpeakerDiarization/gradle/wrapper/gradle-wrapper.jar create mode 100644 android/SherpaOnnxSpeakerDiarization/gradle/wrapper/gradle-wrapper.properties create mode 100755 android/SherpaOnnxSpeakerDiarization/gradlew create mode 100644 android/SherpaOnnxSpeakerDiarization/gradlew.bat create mode 100644 android/SherpaOnnxSpeakerDiarization/settings.gradle.kts create mode 120000 android/SherpaOnnxSpeakerIdentification/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/identification/SpeakerEmbeddingExtractorConfig.kt create mode 120000 kotlin-api-examples/SpeakerEmbeddingExtractorConfig.kt create mode 100755 scripts/apk/build-apk-speaker-diarization.sh create mode 100644 sherpa-onnx/kotlin-api/SpeakerEmbeddingExtractorConfig.kt diff --git a/.github/workflows/apk-speaker-diarization.yaml b/.github/workflows/apk-speaker-diarization.yaml new file mode 100644 index 000000000..19f0b99bc --- /dev/null +++ b/.github/workflows/apk-speaker-diarization.yaml @@ -0,0 +1,175 @@ +name: apk-speaker-diarization + +on: + push: + branches: + - apk + - android-demo-speaker-diarization-2 + + workflow_dispatch: + +concurrency: + group: apk-speaker-diarization-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: write + +jobs: + apk_speaker_identification: + if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' + runs-on: ${{ matrix.os }} + name: apk for speaker diarization ${{ matrix.index }}/${{ matrix.total }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + total: ["1"] + index: ["0"] + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # https://github.com/actions/setup-java + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' # See 'Supported distributions' for available options + java-version: '21' + + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ matrix.os }}-android + + - name: Display NDK HOME + shell: bash + run: | + echo "ANDROID_NDK_LATEST_HOME: ${ANDROID_NDK_LATEST_HOME}" + ls -lh ${ANDROID_NDK_LATEST_HOME} + + - name: Install Python dependencies + shell: bash + run: | + python3 -m pip install --upgrade pip jinja2 + + - name: Setup build tool version variable + shell: bash + run: | + echo "---" + ls -lh /usr/local/lib/android/ + echo "---" + + ls -lh /usr/local/lib/android/sdk + echo "---" + + ls -lh /usr/local/lib/android/sdk/build-tools + echo "---" + + BUILD_TOOL_VERSION=$(ls /usr/local/lib/android/sdk/build-tools/ | tail -n 1) + echo "BUILD_TOOL_VERSION=$BUILD_TOOL_VERSION" >> $GITHUB_ENV + echo "Last build tool version is: $BUILD_TOOL_VERSION" + + - name: Generate build script + shell: bash + run: | + cd scripts/apk + + chmod +x build-apk-speaker-diarization.sh + mv -v ./build-apk-speaker-diarization.sh ../.. + + - name: build APK + shell: bash + run: | + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + cmake --version + + export ANDROID_NDK=$ANDROID_NDK_LATEST_HOME + ./build-apk-speaker-diarization.sh + + - name: Display APK + shell: bash + run: | + ls -lh ./apks/ + du -h -d1 . + + # https://github.com/marketplace/actions/sign-android-release + - uses: r0adkll/sign-android-release@v1 + name: Sign app APK + with: + releaseDirectory: ./apks + signingKeyBase64: ${{ secrets.ANDROID_SIGNING_KEY }} + alias: ${{ secrets.ANDROID_SIGNING_KEY_ALIAS }} + keyStorePassword: ${{ secrets.ANDROID_SIGNING_KEY_STORE_PASSWORD }} + env: + BUILD_TOOLS_VERSION: ${{ env.BUILD_TOOL_VERSION }} + + - name: Display APK after signing + shell: bash + run: | + ls -lh ./apks/ + du -h -d1 . + + - name: Rename APK after signing + shell: bash + run: | + cd apks + rm -fv signingKey.jks + rm -fv *.apk.idsig + rm -fv *-aligned.apk + + all_apks=$(ls -1 *-signed.apk) + echo "----" + echo $all_apks + echo "----" + for apk in ${all_apks[@]}; do + n=$(echo $apk | sed -e s/-signed//) + mv -v $apk $n + done + + cd .. + + ls -lh ./apks/ + du -h -d1 . + + - name: Display APK after rename + shell: bash + run: | + ls -lh ./apks/ + du -h -d1 . + + - name: Publish to huggingface + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v3 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + + rm -rf huggingface + export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false + + SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" + + git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface + cd huggingface + git fetch + git pull + git merge -m "merge remote" --ff origin main + + d=speaker-diarization/$SHERPA_ONNX_VERSION + mkdir -p $d/ + cp -v ../apks/*.apk $d/ + git status + git lfs track "*.apk" + git add . + git commit -m "add more apks" + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-apk main diff --git a/.github/workflows/apk-speaker-identification.yaml b/.github/workflows/apk-speaker-identification.yaml index ca89ec49f..e32ad3bc9 100644 --- a/.github/workflows/apk-speaker-identification.yaml +++ b/.github/workflows/apk-speaker-identification.yaml @@ -53,6 +53,23 @@ jobs: run: | python3 -m pip install --upgrade pip jinja2 + - name: Setup build tool version variable + shell: bash + run: | + echo "---" + ls -lh /usr/local/lib/android/ + echo "---" + + ls -lh /usr/local/lib/android/sdk + echo "---" + + ls -lh /usr/local/lib/android/sdk/build-tools + echo "---" + + BUILD_TOOL_VERSION=$(ls /usr/local/lib/android/sdk/build-tools/ | tail -n 1) + echo "BUILD_TOOL_VERSION=$BUILD_TOOL_VERSION" >> $GITHUB_ENV + echo "Last build tool version is: $BUILD_TOOL_VERSION" + - name: Generate build script shell: bash run: | @@ -82,6 +99,51 @@ jobs: ls -lh ./apks/ du -h -d1 . + # https://github.com/marketplace/actions/sign-android-release + - uses: r0adkll/sign-android-release@v1 + name: Sign app APK + with: + releaseDirectory: ./apks + signingKeyBase64: ${{ secrets.ANDROID_SIGNING_KEY }} + alias: ${{ secrets.ANDROID_SIGNING_KEY_ALIAS }} + keyStorePassword: ${{ secrets.ANDROID_SIGNING_KEY_STORE_PASSWORD }} + env: + BUILD_TOOLS_VERSION: ${{ env.BUILD_TOOL_VERSION }} + + - name: Display APK after signing + shell: bash + run: | + ls -lh ./apks/ + du -h -d1 . + + - name: Rename APK after signing + shell: bash + run: | + cd apks + rm -fv signingKey.jks + rm -fv *.apk.idsig + rm -fv *-aligned.apk + + all_apks=$(ls -1 *-signed.apk) + echo "----" + echo $all_apks + echo "----" + for apk in ${all_apks[@]}; do + n=$(echo $apk | sed -e s/-signed//) + mv -v $apk $n + done + + cd .. + + ls -lh ./apks/ + du -h -d1 . + + - name: Display APK after rename + shell: bash + run: | + ls -lh ./apks/ + du -h -d1 . + - name: Publish to huggingface env: HF_TOKEN: ${{ secrets.HF_TOKEN }} diff --git a/.github/workflows/apk-vad.yaml b/.github/workflows/apk-vad.yaml index 8253145b6..d9af75477 100644 --- a/.github/workflows/apk-vad.yaml +++ b/.github/workflows/apk-vad.yaml @@ -166,7 +166,7 @@ jobs: git pull git merge -m "merge remote" --ff origin main - d=vad/SHERPA_ONNX_VERSION + d=vad/$SHERPA_ONNX_VERSION mkdir -p $d cp -v ../apks/*.apk $d/ git status diff --git a/README.md b/README.md index 1828847e5..32d141f90 100644 --- a/README.md +++ b/README.md @@ -84,8 +84,9 @@ with the following APIs ### Links for Huggingface Spaces -You can visit the following Huggingface spaces to try `sherpa-onnx` without -installing anything. All you need is a browser. +
+You can visit the following Huggingface spaces to try sherpa-onnx without +installing anything. All you need is a browser. | Description | URL | |-------------------------------------------------------|------------------------------------| @@ -118,23 +119,34 @@ We also have spaces built using WebAssembly. They are listed below: |Speech synthesis (German) |[Click me][wasm-hf-tts-piper-de]| [地址][wasm-ms-tts-piper-de]| |Speaker diarization |[Click me][wasm-hf-speaker-diarization]|[地址][wasm-ms-speaker-diarization]| +
+ ### Links for pre-built Android APKs -| Description | URL | 中国用户 | -|----------------------------------------|------------------------------|-----------------------------| -| Streaming speech recognition | [Address][apk-streaming-asr] | [点此][apk-streaming-asr-cn]| -| Text-to-speech | [Address][apk-tts] | [点此][apk-tts-cn] | -| Voice activity detection (VAD) | [Address][apk-vad] | [点此][apk-vad-cn] | -| VAD + non-streaming speech recognition | [Address][apk-vad-asr] | [点此][apk-vad-asr-cn] | -| Two-pass speech recognition | [Address][apk-2pass] | [点此][apk-2pass-cn] | -| Audio tagging | [Address][apk-at] | [点此][apk-at-cn] | -| Audio tagging (WearOS) | [Address][apk-at-wearos] | [点此][apk-at-wearos-cn] | -| Speaker identification | [Address][apk-sid] | [点此][apk-sid-cn] | -| Spoken language identification | [Address][apk-slid] | [点此][apk-slid-cn] | -| Keyword spotting | [Address][apk-kws] | [点此][apk-kws-cn] | +
+ +You can find pre-built Android APKs for this repository in the following table + +| Description | URL | 中国用户 | +|----------------------------------------|------------------------------------|-----------------------------------| +| Speaker diarization | [Address][apk-speaker-diarization] | [点此][apk-speaker-diarization-cn]| +| Streaming speech recognition | [Address][apk-streaming-asr] | [点此][apk-streaming-asr-cn] | +| Text-to-speech | [Address][apk-tts] | [点此][apk-tts-cn] | +| Voice activity detection (VAD) | [Address][apk-vad] | [点此][apk-vad-cn] | +| VAD + non-streaming speech recognition | [Address][apk-vad-asr] | [点此][apk-vad-asr-cn] | +| Two-pass speech recognition | [Address][apk-2pass] | [点此][apk-2pass-cn] | +| Audio tagging | [Address][apk-at] | [点此][apk-at-cn] | +| Audio tagging (WearOS) | [Address][apk-at-wearos] | [点此][apk-at-wearos-cn] | +| Speaker identification | [Address][apk-sid] | [点此][apk-sid-cn] | +| Spoken language identification | [Address][apk-slid] | [点此][apk-slid-cn] | +| Keyword spotting | [Address][apk-kws] | [点此][apk-kws-cn] | + +
### Links for pre-built Flutter APPs +
+ #### Real-time speech recognition | Description | URL | 中国用户 | @@ -153,17 +165,24 @@ We also have spaces built using WebAssembly. They are listed below: > Note: You need to build from source for iOS. +
+ ### Links for pre-built Lazarus APPs +
+ #### Generating subtitles | Description | URL | 中国用户 | |--------------------------------|----------------------------|----------------------------| | Generate subtitles (生成字幕) | [Address][lazarus-subtitle]| [点此][lazarus-subtitle-cn]| +
### Links for pre-trained models +
+ | Description | URL | |---------------------------------------------|---------------------------------------------------------------------------------------| | Speech recognition (speech to text, ASR) | [Address][asr-models] | @@ -176,6 +195,8 @@ We also have spaces built using WebAssembly. They are listed below: | Punctuation | [Address][punct-models] | | Speaker segmentation | [Address][speaker-segmentation-models] | +
+ ### Useful links - Documentation: https://k2-fsa.github.io/sherpa/onnx/ @@ -265,6 +286,8 @@ Video demo in Chinese: [爆了！炫神教你开打字挂！真正影响胜率 [wasm-ms-tts-piper-de]: https://modelscope.cn/studios/k2-fsa/web-assembly-tts-sherpa-onnx-de [wasm-hf-speaker-diarization]: https://huggingface.co/spaces/k2-fsa/web-assembly-speaker-diarization-sherpa-onnx [wasm-ms-speaker-diarization]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-speaker-diarization-sherpa-onnx +[apk-speaker-diarization]: https://k2-fsa.github.io/sherpa/onnx/speaker-diarization/apk.html +[apk-speaker-diarization-cn]: https://k2-fsa.github.io/sherpa/onnx/speaker-diarization/apk-cn.html [apk-streaming-asr]: https://k2-fsa.github.io/sherpa/onnx/android/apk.html [apk-streaming-asr-cn]: https://k2-fsa.github.io/sherpa/onnx/android/apk-cn.html [apk-tts]: https://k2-fsa.github.io/sherpa/onnx/tts/apk-engine.html diff --git a/android/README.md b/android/README.md index 42b29e08f..bae335598 100644 --- a/android/README.md +++ b/android/README.md @@ -4,6 +4,8 @@ Please refer to https://k2-fsa.github.io/sherpa/onnx/android/index.html for usage. +- [SherpaOnnxSpeakerDiarization](./SherpaOnnxSpeakerDiarization) It is for speaker diarization. + - [SherpaOnnx](./SherpaOnnx) It uses a streaming ASR model. - [SherpaOnnx2Pass](./SherpaOnnx2Pass) It uses a streaming ASR model diff --git a/android/SherpaOnnxSpeakerDiarization/.gitignore b/android/SherpaOnnxSpeakerDiarization/.gitignore new file mode 100644 index 000000000..aa724b770 --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/.gitignore @@ -0,0 +1,15 @@ +*.iml +.gradle +/local.properties +/.idea/caches +/.idea/libraries +/.idea/modules.xml +/.idea/workspace.xml +/.idea/navEditor.xml +/.idea/assetWizardSettings.xml +.DS_Store +/build +/captures +.externalNativeBuild +.cxx +local.properties diff --git a/android/SherpaOnnxSpeakerDiarization/app/.gitignore b/android/SherpaOnnxSpeakerDiarization/app/.gitignore new file mode 100644 index 000000000..42afabfd2 --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/android/SherpaOnnxSpeakerDiarization/app/build.gradle.kts b/android/SherpaOnnxSpeakerDiarization/app/build.gradle.kts new file mode 100644 index 000000000..7a390ba42 --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/build.gradle.kts @@ -0,0 +1,71 @@ +plugins { + alias(libs.plugins.android.application) + alias(libs.plugins.jetbrains.kotlin.android) +} + +android { + namespace = "com.k2fsa.sherpa.onnx.speaker.diarization" + compileSdk = 34 + + defaultConfig { + applicationId = "com.k2fsa.sherpa.onnx.speaker.diarization" + minSdk = 21 + targetSdk = 34 + versionCode = 1 + versionName = "1.0" + + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + vectorDrawables { + useSupportLibrary = true + } + } + + buildTypes { + release { + isMinifyEnabled = false + proguardFiles( + getDefaultProguardFile("proguard-android-optimize.txt"), + "proguard-rules.pro" + ) + } + } + compileOptions { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + } + kotlinOptions { + jvmTarget = "1.8" + } + buildFeatures { + compose = true + } + composeOptions { + kotlinCompilerExtensionVersion = "1.5.1" + } + packaging { + resources { + excludes += "/META-INF/{AL2.0,LGPL2.1}" + } + } +} + +dependencies { + + implementation(libs.androidx.core.ktx) + implementation(libs.androidx.lifecycle.runtime.ktx) + implementation(libs.androidx.activity.compose) + implementation(platform(libs.androidx.compose.bom)) + implementation(libs.androidx.ui) + implementation(libs.androidx.ui.graphics) + implementation(libs.androidx.ui.tooling.preview) + implementation(libs.androidx.material3) + implementation(libs.androidx.navigation.compose) + implementation(libs.androidx.documentfile) + testImplementation(libs.junit) + androidTestImplementation(libs.androidx.junit) + androidTestImplementation(libs.androidx.espresso.core) + androidTestImplementation(platform(libs.androidx.compose.bom)) + androidTestImplementation(libs.androidx.ui.test.junit4) + debugImplementation(libs.androidx.ui.tooling) + debugImplementation(libs.androidx.ui.test.manifest) +} \ No newline at end of file diff --git a/android/SherpaOnnxSpeakerDiarization/app/proguard-rules.pro b/android/SherpaOnnxSpeakerDiarization/app/proguard-rules.pro new file mode 100644 index 000000000..481bb4348 --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile \ No newline at end of file diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/androidTest/java/com/k2fsa/sherpa/onnx/speaker/diarization/ExampleInstrumentedTest.kt b/android/SherpaOnnxSpeakerDiarization/app/src/androidTest/java/com/k2fsa/sherpa/onnx/speaker/diarization/ExampleInstrumentedTest.kt new file mode 100644 index 000000000..53d7af15f --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/src/androidTest/java/com/k2fsa/sherpa/onnx/speaker/diarization/ExampleInstrumentedTest.kt @@ -0,0 +1,24 @@ +package com.k2fsa.sherpa.onnx.speaker.diarization + +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.ext.junit.runners.AndroidJUnit4 + +import org.junit.Test +import org.junit.runner.RunWith + +import org.junit.Assert.* + +/** + * Instrumented test, which will execute on an Android device. + * + * See [testing documentation](http://d.android.com/tools/testing). + */ +@RunWith(AndroidJUnit4::class) +class ExampleInstrumentedTest { + @Test + fun useAppContext() { + // Context of the app under test. + val appContext = InstrumentationRegistry.getInstrumentation().targetContext + assertEquals("com.k2fsa.sherpa.onnx.speaker.diarization", appContext.packageName) + } +} \ No newline at end of file diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/AndroidManifest.xml b/android/SherpaOnnxSpeakerDiarization/app/src/main/AndroidManifest.xml new file mode 100644 index 000000000..d58f7e8d7 --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/src/main/AndroidManifest.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/assets/.gitkeep b/android/SherpaOnnxSpeakerDiarization/app/src/main/assets/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/BarItem.kt b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/BarItem.kt new file mode 100644 index 000000000..0895cf52c --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/BarItem.kt @@ -0,0 +1,13 @@ +package com.k2fsa.sherpa.onnx.speaker.diarization + +import androidx.compose.ui.graphics.vector.ImageVector + +data class BarItem( + val title: String, + + // see https://www.composables.com/icons + // and + // https://developer.android.com/reference/kotlin/androidx/compose/material/icons/filled/package-summary + val image: ImageVector, + val route: String, +) \ No newline at end of file diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/MainActivity.kt b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/MainActivity.kt new file mode 100644 index 000000000..7a25d49b9 --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/MainActivity.kt @@ -0,0 +1,132 @@ +package com.k2fsa.sherpa.onnx.speaker.diarization + +import android.os.Bundle +import androidx.activity.ComponentActivity +import androidx.activity.compose.setContent +import androidx.activity.enableEdgeToEdge +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.padding +import androidx.compose.material3.CenterAlignedTopAppBar +import androidx.compose.material3.ExperimentalMaterial3Api +import androidx.compose.material3.Icon +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.NavigationBar +import androidx.compose.material3.NavigationBarItem +import androidx.compose.material3.Scaffold +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.material3.TopAppBarDefaults +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.tooling.preview.Preview +import androidx.navigation.NavGraph.Companion.findStartDestination +import androidx.navigation.NavHostController +import androidx.navigation.compose.NavHost +import androidx.navigation.compose.composable +import androidx.navigation.compose.currentBackStackEntryAsState +import androidx.navigation.compose.rememberNavController +import com.k2fsa.sherpa.onnx.speaker.diarization.screens.HelpScreen +import com.k2fsa.sherpa.onnx.speaker.diarization.screens.HomeScreen +import com.k2fsa.sherpa.onnx.speaker.diarization.ui.theme.SherpaOnnxSpeakerDiarizationTheme + +const val TAG = "sherpa-onnx-sd" + +class MainActivity : ComponentActivity() { + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + enableEdgeToEdge() + setContent { + SherpaOnnxSpeakerDiarizationTheme { + // A surface container using the 'background' color from the theme + Surface( + modifier = Modifier.fillMaxSize(), + color = MaterialTheme.colorScheme.background + ) { + MainScreen() + } + } + } + SpeakerDiarizationObject.initSpeakerDiarization(this.assets) + } +} + +@OptIn(ExperimentalMaterial3Api::class) +@Composable +fun MainScreen(modifier: Modifier = Modifier) { + val navController = rememberNavController() + Scaffold( + topBar = { + CenterAlignedTopAppBar( + colors = TopAppBarDefaults.topAppBarColors( + containerColor = MaterialTheme.colorScheme.primaryContainer, + titleContentColor = MaterialTheme.colorScheme.primary, + ), + title = { + Text( + "Next-gen Kaldi: Speaker Diarization", + fontWeight = FontWeight.Bold, + ) + }, + ) + }, + content = { padding -> + Column(Modifier.padding(padding)) { + NavigationHost(navController = navController) + + } + }, + bottomBar = { + BottomNavigationBar(navController = navController) + } + ) +} + +@Composable +fun NavigationHost(navController: NavHostController) { + NavHost(navController = navController, startDestination = NavRoutes.Home.route) { + composable(NavRoutes.Home.route) { + HomeScreen() + } + + composable(NavRoutes.Help.route) { + HelpScreen() + } + } +} + +@Composable +fun BottomNavigationBar(navController: NavHostController) { + NavigationBar { + val backStackEntry by navController.currentBackStackEntryAsState() + val currentRoute = backStackEntry?.destination?.route + + NavBarItems.BarItems.forEach { navItem -> + NavigationBarItem(selected = currentRoute == navItem.route, + onClick = { + navController.navigate(navItem.route) { + popUpTo(navController.graph.findStartDestination().id) { + saveState = true + } + launchSingleTop = true + restoreState = true + } + }, + icon = { + Icon(imageVector = navItem.image, contentDescription = navItem.title) + }, label = { + Text(text = navItem.title) + }) + } + } +} + +@Preview(showBackground = true) +@Composable +fun MainScreenPreview() { + SherpaOnnxSpeakerDiarizationTheme { + MainScreen() + } +} \ No newline at end of file diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/NavBarItems.kt b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/NavBarItems.kt new file mode 100644 index 000000000..65c737f97 --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/NavBarItems.kt @@ -0,0 +1,20 @@ +package com.k2fsa.sherpa.onnx.speaker.diarization + +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Home +import androidx.compose.material.icons.filled.Info + +object NavBarItems { + val BarItems = listOf( + BarItem( + title = "Home", + image = Icons.Filled.Home, + route = "home", + ), + BarItem( + title = "Help", + image = Icons.Filled.Info, + route = "help", + ), + ) +} \ No newline at end of file diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/NavRoutes.kt b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/NavRoutes.kt new file mode 100644 index 000000000..2e1ae90b5 --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/NavRoutes.kt @@ -0,0 +1,6 @@ +package com.k2fsa.sherpa.onnx.speaker.diarization + +sealed class NavRoutes(val route: String) { + object Home : NavRoutes("home") + object Help : NavRoutes("help") +} \ No newline at end of file diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/OfflineSpeakerDiarization.kt b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/OfflineSpeakerDiarization.kt new file mode 120000 index 000000000..459cc22cc --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/OfflineSpeakerDiarization.kt @@ -0,0 +1 @@ +../../../../../../../../../../../../sherpa-onnx/kotlin-api/OfflineSpeakerDiarization.kt \ No newline at end of file diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/ReadWaveFile.kt b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/ReadWaveFile.kt new file mode 100644 index 000000000..940a2b643 --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/ReadWaveFile.kt @@ -0,0 +1,137 @@ +package com.k2fsa.sherpa.onnx.speaker.diarization.screens + +import android.content.Context +import android.media.AudioFormat +import android.media.MediaCodec +import android.media.MediaExtractor +import android.media.MediaFormat +import android.net.Uri + +data class WaveData( + val sampleRate: Int? = null, + val samples: FloatArray? = null, + val msg: String? = null +) + +// It supports only 16-bit encoded wave files +// +// References +// - https://gist.github.com/a-m-s/1991ab18fbcb0fcc2cf9 +// - https://github.com/taehwandev/MediaCodecExample/blob/master/app/src/main/java/tech/thdev/mediacodecexample/audio/AACAudioDecoderThread.kt +fun readUri(context: Context, uri: Uri): WaveData { + val extractor = MediaExtractor() + extractor.setDataSource(context, uri, null) + + val samplesList: MutableList = ArrayList() + + for (i in 0 until extractor.trackCount) { + val format = extractor.getTrackFormat(i) + val mime = format.getString(MediaFormat.KEY_MIME) + if (mime?.startsWith("audio/") == true) { + extractor.selectTrack(i) + + var encoding: Int = -1 + try { + encoding = format.getInteger(MediaFormat.KEY_PCM_ENCODING) + } catch (_: Exception) { + } + + if (encoding != AudioFormat.ENCODING_PCM_16BIT) { + return WaveData(msg = "We support only 16-bit encoded wave files") + } + + val sampleRate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE) + val decoder = MediaCodec.createDecoderByType(mime) + decoder.configure(format, null, null, 0) + decoder.start() + + val inputBuffers = decoder.inputBuffers + var outputBuffers = decoder.outputBuffers + + val info = MediaCodec.BufferInfo() + var eof = false + + var outputBufferIndex = -1 + + while (true) { + if (!eof) { + val inputBufferIndex = decoder.dequeueInputBuffer(10000) + if (inputBufferIndex > 0) { + val size = extractor.readSampleData(inputBuffers[inputBufferIndex], 0) + if (size < 0) { + decoder.queueInputBuffer( + inputBufferIndex, + 0, + 0, + 0, + MediaCodec.BUFFER_FLAG_END_OF_STREAM + ) + eof = true + } else { + decoder.queueInputBuffer( + inputBufferIndex, + 0, + size, + extractor.sampleTime, + 0 + ) + extractor.advance() + } + } + } // if (!eof) + + if (outputBufferIndex >= 0) { + outputBuffers[outputBufferIndex].position(0) + } + + outputBufferIndex = decoder.dequeueOutputBuffer(info, 10000) + if (outputBufferIndex >= 0) { + if (info.flags != 0) { + decoder.stop() + decoder.release() + + var k = 0 + for (s in samplesList) { + k += s.size + } + if (k == 0) { + return WaveData(msg = "Failed to read selected file") + } + + val ans = FloatArray(k) + k = 0 + for (s in samplesList) { + s.copyInto(ans, k) + k += s.size + } + + return WaveData(sampleRate = sampleRate, samples = ans) + } + + val buffer = outputBuffers[outputBufferIndex] + val chunk = ByteArray(info.size) + buffer[chunk] + buffer.clear() + + val numSamples = info.size / 2 + + val samples = FloatArray(numSamples) + for (k in 0 until numSamples) { + // assume little endian + val s = chunk[2 * k] + (chunk[2 * k + 1] * 256.0f) + + samples[k] = s / 32768.0f + } + samplesList.add(samples) + + decoder.releaseOutputBuffer(outputBufferIndex, false) + } else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { + outputBuffers = decoder.outputBuffers + } + } + } + } + + extractor.release() + return WaveData(msg = "not an audio file") +} \ No newline at end of file diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/SpeakerDiarizationObject.kt b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/SpeakerDiarizationObject.kt new file mode 100644 index 000000000..f4bc24554 --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/SpeakerDiarizationObject.kt @@ -0,0 +1,66 @@ +package com.k2fsa.sherpa.onnx.speaker.diarization + +import android.content.res.AssetManager +import android.util.Log +import com.k2fsa.sherpa.onnx.FastClusteringConfig +import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarization +import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationConfig +import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationModelConfig +import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationPyannoteModelConfig +import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig + +// Please download +// https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 +// then unzip it, rename model.onnx to segmentation.onnx, and mv +// segmentation.onnx to the assets folder +val segmentationModel = "segmentation.onnx" + +// please download it from +// https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx +// and move it to the assets folder +val embeddingModel = "3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx" + +// in the end, your assets folder should look like below +/* +(py38) fangjuns-MacBook-Pro:assets fangjun$ pwd +/Users/fangjun/open-source/sherpa-onnx/android/SherpaOnnxSpeakerDiarization/app/src/main/assets +(py38) fangjuns-MacBook-Pro:assets fangjun$ ls -lh +total 89048 +-rw-r--r-- 1 fangjun staff 38M Oct 12 20:28 3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx +-rw-r--r-- 1 fangjun staff 5.7M Oct 12 20:28 segmentation.onnx + */ + +object SpeakerDiarizationObject { + var _sd: OfflineSpeakerDiarization? = null + val sd: OfflineSpeakerDiarization + get() { + return _sd!! + } + + fun initSpeakerDiarization(assetManager: AssetManager? = null) { + synchronized(this) { + if (_sd != null) { + return + } + Log.i(TAG, "Initializing sherpa-onnx speaker diarization") + + val config = OfflineSpeakerDiarizationConfig( + segmentation = OfflineSpeakerSegmentationModelConfig( + pyannote = OfflineSpeakerSegmentationPyannoteModelConfig( + segmentationModel + ), + debug = true, + ), + embedding = SpeakerEmbeddingExtractorConfig( + model = embeddingModel, + debug = true, + numThreads = 2, + ), + clustering = FastClusteringConfig(numClusters = -1, threshold = 0.5f), + minDurationOn = 0.2f, + minDurationOff = 0.5f, + ) + _sd = OfflineSpeakerDiarization(assetManager = assetManager, config = config) + } + } +} \ No newline at end of file diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/SpeakerEmbeddingExtractorConfig.kt b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/SpeakerEmbeddingExtractorConfig.kt new file mode 120000 index 000000000..9bab8fe88 --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/SpeakerEmbeddingExtractorConfig.kt @@ -0,0 +1 @@ +../../../../../../../../../../../../sherpa-onnx/kotlin-api/SpeakerEmbeddingExtractorConfig.kt \ No newline at end of file diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/screens/Help.kt b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/screens/Help.kt new file mode 100644 index 000000000..b3640b9e9 --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/screens/Help.kt @@ -0,0 +1,38 @@ +package com.k2fsa.sherpa.onnx.speaker.diarization.screens + +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.ui.Modifier +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp + +@Composable +fun HelpScreen() { + Box(modifier = Modifier.fillMaxSize()) { + Column( + modifier = Modifier.padding(8.dp) + ) { + Text( + "This app accepts only 16kHz 16-bit 1-channel *.wav files. " + + "It has two arguments: Number of speakers and clustering threshold. " + + "If you know the actual number of speakers in the file, please set it. " + + "Otherwise, please set it to 0. In that case, you have to set the threshold. " + + "A larger threshold leads to fewer segmented speakers." + ) + Spacer(modifier = Modifier.height(5.dp)) + Text("The speaker segmentation model is from " + + "pyannote-audio (https://huggingface.co/pyannote/segmentation-3.0), "+ + "whereas the embedding extractor model is from 3D-Speaker (https://github.com/modelscope/3D-Speaker)") + Spacer(modifier = Modifier.height(5.dp)) + Text("Please see http://github.com/k2-fsa/sherpa-onnx ") + Spacer(modifier = Modifier.height(5.dp)) + Text("Everything is open-sourced!", fontSize = 20.sp) + } + } +} diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/screens/Home.kt b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/screens/Home.kt new file mode 100644 index 000000000..a5a9cd31c --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/screens/Home.kt @@ -0,0 +1,210 @@ +package com.k2fsa.sherpa.onnx.speaker.diarization.screens + +import android.util.Log +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.verticalScroll +import androidx.compose.material3.Button +import androidx.compose.material3.OutlinedTextField +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalClipboardManager +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.text.AnnotatedString +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import androidx.documentfile.provider.DocumentFile +import com.k2fsa.sherpa.onnx.speaker.diarization.SpeakerDiarizationObject +import com.k2fsa.sherpa.onnx.speaker.diarization.TAG +import kotlin.concurrent.thread + + +private var samples: FloatArray? = null + +@Composable +fun HomeScreen() { + val context = LocalContext.current + + var sampleRate: Int + var filename by remember { mutableStateOf("") } + var status by remember { mutableStateOf("") } + var progress by remember { mutableStateOf("") } + val clipboardManager = LocalClipboardManager.current + var done by remember { mutableStateOf(false) } + var fileIsOk by remember { mutableStateOf(false) } + var started by remember { mutableStateOf(false) } + var numSpeakers by remember { mutableStateOf(0) } + var threshold by remember { mutableStateOf(0.5f) } + + + val callback = here@{ numProcessedChunks: Int, numTotalChunks: Int, arg: Long -> + Int + val percent = 100.0 * numProcessedChunks / numTotalChunks + progress = "%.2f%%".format(percent) + Log.i(TAG, progress) + return@here 0 + } + + val launcher = rememberLauncherForActivityResult(ActivityResultContracts.OpenDocument()) { + it?.let { + val documentFile = DocumentFile.fromSingleUri(context, it) + filename = documentFile?.name ?: "" + + progress = "" + done = false + fileIsOk = false + + if (filename.isNotEmpty()) { + val data = readUri(context, it) + Log.i(TAG, "sample rate: ${data.sampleRate}") + Log.i(TAG, "numSamples: ${data.samples?.size ?: 0}") + if (data.msg != null) { + Log.i(TAG, "failed to read $filename") + status = data.msg + } else if (data.sampleRate != SpeakerDiarizationObject.sd.sampleRate()) { + status = + "Expected sample rate: ${SpeakerDiarizationObject.sd.sampleRate()}. Given wave file with sample rate: ${data.sampleRate}" + } else { + samples = data.samples!! + fileIsOk = true + } + } + } + } + + Column( + modifier = Modifier.padding(10.dp), + verticalArrangement = Arrangement.Top, + ) { + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.SpaceEvenly, + verticalAlignment = Alignment.CenterVertically + ) { + + Button(onClick = { + launcher.launch(arrayOf("audio/*")) + }) { + Text("Select a .wav file") + } + + Button(enabled = fileIsOk && !started, + onClick = { + Log.i(TAG, "started") + Log.i(TAG, "num samples: ${samples?.size}") + started = true + progress = "" + + val config = SpeakerDiarizationObject.sd.config + config.clustering.numClusters = numSpeakers + config.clustering.threshold = threshold + + SpeakerDiarizationObject.sd.setConfig(config) + + thread(true) { + done = false + status = "Started! Please wait" + val segments = SpeakerDiarizationObject.sd.processWithCallback( + samples!!, + callback = callback, + ) + done = true + started = false + status = "" + for (s in segments) { + val start = "%.2f".format(s.start) + val end = "%.2f".format(s.end) + val speaker = "speaker_%02d".format(s.speaker) + status += "$start -- $end $speaker\n" + Log.i(TAG, "$start -- $end $speaker") + } + + Log.i(TAG, status) + } + }) { + Text("Start") + } + if (progress.isNotEmpty()) { + Text(progress, fontSize = 25.sp) + } + } + + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.SpaceEvenly, + verticalAlignment = Alignment.CenterVertically + ) { + OutlinedTextField( + value = numSpeakers.toString(), + onValueChange = { + if (it.isEmpty() || it.isBlank()) { + numSpeakers = 0 + } else { + numSpeakers = it.toIntOrNull() ?: 0 + } + }, + label = { + Text("Number of Speakers") + }, + ) + } + + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.SpaceEvenly, + verticalAlignment = Alignment.CenterVertically + ) { + OutlinedTextField( + value = threshold.toString(), + onValueChange = { + if (it.isEmpty() || it.isBlank()) { + threshold = 0.5f + } else { + threshold = it.toFloatOrNull() ?: 0.5f + } + }, + label = { + Text("Clustering threshold") + }, + ) + } + + if (filename.isNotEmpty()) { + Text(text = "Selected $filename") + Spacer(Modifier.size(20.dp)) + } + + if (done) { + Button(onClick = { + clipboardManager.setText(AnnotatedString(status)) + progress = "Copied!" + }) { + Text("Copy result") + } + Spacer(Modifier.size(20.dp)) + } + + if (status.isNotEmpty()) { + Text( + status, + modifier = Modifier.verticalScroll(rememberScrollState()), + ) + } + + + } +} \ No newline at end of file diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/ui/theme/Color.kt b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/ui/theme/Color.kt new file mode 100644 index 000000000..a96515d3d --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/ui/theme/Color.kt @@ -0,0 +1,11 @@ +package com.k2fsa.sherpa.onnx.speaker.diarization.ui.theme + +import androidx.compose.ui.graphics.Color + +val Purple80 = Color(0xFFD0BCFF) +val PurpleGrey80 = Color(0xFFCCC2DC) +val Pink80 = Color(0xFFEFB8C8) + +val Purple40 = Color(0xFF6650a4) +val PurpleGrey40 = Color(0xFF625b71) +val Pink40 = Color(0xFF7D5260) \ No newline at end of file diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/ui/theme/Theme.kt b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/ui/theme/Theme.kt new file mode 100644 index 000000000..5dbbe7e59 --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/ui/theme/Theme.kt @@ -0,0 +1,58 @@ +package com.k2fsa.sherpa.onnx.speaker.diarization.ui.theme + +import android.app.Activity +import android.os.Build +import androidx.compose.foundation.isSystemInDarkTheme +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.darkColorScheme +import androidx.compose.material3.dynamicDarkColorScheme +import androidx.compose.material3.dynamicLightColorScheme +import androidx.compose.material3.lightColorScheme +import androidx.compose.runtime.Composable +import androidx.compose.ui.platform.LocalContext + +private val DarkColorScheme = darkColorScheme( + primary = Purple80, + secondary = PurpleGrey80, + tertiary = Pink80 +) + +private val LightColorScheme = lightColorScheme( + primary = Purple40, + secondary = PurpleGrey40, + tertiary = Pink40 + + /* Other default colors to override + background = Color(0xFFFFFBFE), + surface = Color(0xFFFFFBFE), + onPrimary = Color.White, + onSecondary = Color.White, + onTertiary = Color.White, + onBackground = Color(0xFF1C1B1F), + onSurface = Color(0xFF1C1B1F), + */ +) + +@Composable +fun SherpaOnnxSpeakerDiarizationTheme( + darkTheme: Boolean = isSystemInDarkTheme(), + // Dynamic color is available on Android 12+ + dynamicColor: Boolean = true, + content: @Composable () -> Unit +) { + val colorScheme = when { + dynamicColor && Build.VERSION.SDK_INT >= Build.VERSION_CODES.S -> { + val context = LocalContext.current + if (darkTheme) dynamicDarkColorScheme(context) else dynamicLightColorScheme(context) + } + + darkTheme -> DarkColorScheme + else -> LightColorScheme + } + + MaterialTheme( + colorScheme = colorScheme, + typography = Typography, + content = content + ) +} \ No newline at end of file diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/ui/theme/Type.kt b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/ui/theme/Type.kt new file mode 100644 index 000000000..39a81b941 --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/src/main/java/com/k2fsa/sherpa/onnx/speaker/diarization/ui/theme/Type.kt @@ -0,0 +1,34 @@ +package com.k2fsa.sherpa.onnx.speaker.diarization.ui.theme + +import androidx.compose.material3.Typography +import androidx.compose.ui.text.TextStyle +import androidx.compose.ui.text.font.FontFamily +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.unit.sp + +// Set of Material typography styles to start with +val Typography = Typography( + bodyLarge = TextStyle( + fontFamily = FontFamily.Default, + fontWeight = FontWeight.Normal, + fontSize = 16.sp, + lineHeight = 24.sp, + letterSpacing = 0.5.sp + ) + /* Other default text styles to override + titleLarge = TextStyle( + fontFamily = FontFamily.Default, + fontWeight = FontWeight.Normal, + fontSize = 22.sp, + lineHeight = 28.sp, + letterSpacing = 0.sp + ), + labelSmall = TextStyle( + fontFamily = FontFamily.Default, + fontWeight = FontWeight.Medium, + fontSize = 11.sp, + lineHeight = 16.sp, + letterSpacing = 0.5.sp + ) + */ +) \ No newline at end of file diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/jniLibs/arm64-v8a/.gitkeep b/android/SherpaOnnxSpeakerDiarization/app/src/main/jniLibs/arm64-v8a/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/jniLibs/armeabi-v7a/.gitkeep b/android/SherpaOnnxSpeakerDiarization/app/src/main/jniLibs/armeabi-v7a/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/jniLibs/x86/.gitkeep b/android/SherpaOnnxSpeakerDiarization/app/src/main/jniLibs/x86/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/jniLibs/x86_64/.gitkeep b/android/SherpaOnnxSpeakerDiarization/app/src/main/jniLibs/x86_64/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/res/drawable-v24/ic_launcher_foreground.xml b/android/SherpaOnnxSpeakerDiarization/app/src/main/res/drawable-v24/ic_launcher_foreground.xml new file mode 100644 index 000000000..2b068d114 --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/src/main/res/drawable-v24/ic_launcher_foreground.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/res/drawable/ic_launcher_background.xml b/android/SherpaOnnxSpeakerDiarization/app/src/main/res/drawable/ic_launcher_background.xml new file mode 100644 index 000000000..07d5da9cb --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/src/main/res/drawable/ic_launcher_background.xml @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml b/android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml new file mode 100644 index 000000000..6f3b755bf --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml b/android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml new file mode 100644 index 000000000..6f3b755bf --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-hdpi/ic_launcher.webp b/android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-hdpi/ic_launcher.webp new file mode 100644 index 0000000000000000000000000000000000000000..c209e78ecd372343283f4157dcfd918ec5165bb3 GIT binary patch literal 1404 zcmV-?1%vuhNk&F=1pok7MM6+kP&il$0000G0000-002h-06|PpNX!5L00Dqw+t%{r zzW2vH!KF=w&cMnnN@{whkTw+#mAh0SV?YL=)3MimFYCWp#fpdtz~8$hD5VPuQgtcN zXl<@<#Cme5f5yr2h%@8TWh?)bSK`O z^Z@d={gn7J{iyxL_y_%J|L>ep{dUxUP8a{byupH&!UNR*OutO~0{*T4q5R6@ApLF! z5{w?Z150gC7#>(VHFJZ-^6O@PYp{t!jH(_Z*nzTK4 zkc{fLE4Q3|mA2`CWQ3{8;gxGizgM!zccbdQoOLZc8hThi-IhN90RFT|zlxh3Ty&VG z?Fe{#9RrRnxzsu|Lg2ddugg7k%>0JeD+{XZ7>Z~{=|M+sh1MF7~ zz>To~`~LVQe1nNoR-gEzkpe{Ak^7{{ZBk2i_<+`Bq<^GB!RYG+z)h;Y3+<{zlMUYd zrd*W4w&jZ0%kBuDZ1EW&KLpyR7r2=}fF2%0VwHM4pUs}ZI2egi#DRMYZPek*^H9YK zay4Iy3WXFG(F14xYsoDA|KXgGc5%2DhmQ1gFCkrgHBm!lXG8I5h*uf{rn48Z!_@ z4Bk6TJAB2CKYqPjiX&mWoW>OPFGd$wqroa($ne7EUK;#3VYkXaew%Kh^3OrMhtjYN?XEoY`tRPQsAkH-DSL^QqyN0>^ zmC>{#F14jz4GeW{pJoRpLFa_*GI{?T93^rX7SPQgT@LbLqpNA}<@2wH;q493)G=1Y z#-sCiRNX~qf3KgiFzB3I>4Z%AfS(3$`-aMIBU+6?gbgDb!)L~A)je+;fR0jWLL-Fu z4)P{c7{B4Hp91&%??2$v9iRSFnuckHUm}or9seH6 z>%NbT+5*@L5(I9j@06@(!{ZI?U0=pKn8uwIg&L{JV14+8s2hnvbRrU|hZCd}IJu7*;;ECgO%8_*W Kmw_-CKmY()leWbG literal 0 HcmV?d00001 diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp b/android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp new file mode 100644 index 0000000000000000000000000000000000000000..b2dfe3d1ba5cf3ee31b3ecc1ced89044a1f3b7a9 GIT binary patch literal 2898 zcmV-Y3$650Nk&FW3jhFDMM6+kP&il$0000G0000-002h-06|PpNWB9900E$G+qN-D z+81ABX7q?;bwx%xBg?kcwr$(C-Tex-ZCkHUw(Y9#+`E5-zuONG5fgw~E2WDng@Bc@ z24xy+R1n%~6xI#u9vJ8zREI)sb<&Il(016}Z~V1n^PU3-_H17A*Bf^o)&{_uBv}Py zulRfeE8g(g6HFhk_?o_;0@tz?1I+l+Y#Q*;RVC?(ud`_cU-~n|AX-b`JHrOIqn(-t&rOg-o`#C zh0LPxmbOAEb;zHTu!R3LDh1QO zZTf-|lJNUxi-PpcbRjw3n~n-pG;$+dIF6eqM5+L();B2O2tQ~|p{PlpNcvDbd1l%c zLtXn%lu(3!aNK!V#+HNn_D3lp z2%l+hK-nsj|Bi9;V*WIcQRTt5j90A<=am+cc`J zTYIN|PsYAhJ|=&h*4wI4ebv-C=Be#u>}%m;a{IGmJDU`0snWS&$9zdrT(z8#{OZ_Y zxwJx!ZClUi%YJjD6Xz@OP8{ieyJB=tn?>zaI-4JN;rr`JQbb%y5h2O-?_V@7pG_+y z(lqAsqYr!NyVb0C^|uclHaeecG)Sz;WV?rtoqOdAAN{j%?Uo%owya(F&qps@Id|Of zo@~Y-(YmfB+chv^%*3g4k3R0WqvuYUIA+8^SGJ{2Bl$X&X&v02>+0$4?di(34{pt* zG=f#yMs@Y|b&=HyH3k4yP&goF2LJ#tBLJNNDo6lG06r}ghC-pC4Q*=x3;|+W04zte zAl>l4kzUBQFYF(E`KJy?ZXd1tnfbH+Z~SMmA21KokJNs#eqcXWKUIC>{TuoKe^vhF z);H)o`t9j~`$h1D`#bxe@E`oE`cM9w(@)5Bp8BNukIwM>wZHfd0S;5bcXA*5KT3bj zc&_~`&{z7u{Et!Z_k78H75gXf4g8<_ul!H$eVspPeU3j&&Au=2R*Zp#M9$9s;fqwgzfiX=E_?BwVcfx3tG9Q-+<5fw z%Hs64z)@Q*%s3_Xd5>S4dg$s>@rN^ixeVj*tqu3ZV)biDcFf&l?lGwsa zWj3rvK}?43c{IruV2L`hUU0t^MemAn3U~x3$4mFDxj=Byowu^Q+#wKRPrWywLjIAp z9*n}eQ9-gZmnd9Y0WHtwi2sn6n~?i#n9VN1B*074_VbZZ=WrpkMYr{RsI ztM_8X1)J*DZejxkjOTRJ&a*lrvMKBQURNP#K)a5wIitfu(CFYV4FT?LUB$jVwJSZz zNBFTWg->Yk0j&h3e*a5>B=-xM7dE`IuOQna!u$OoxLlE;WdrNlN)1 z7**de7-hZ!(%_ZllHBLg`Ir#|t>2$*xVOZ-ADZKTN?{(NUeLU9GbuG-+Axf*AZ-P1 z0ZZ*fx+ck4{XtFsbcc%GRStht@q!m*ImssGwuK+P@%gEK!f5dHymg<9nSCXsB6 zQ*{<`%^bxB($Z@5286^-A(tR;r+p7B%^%$N5h%lb*Vlz-?DL9x;!j<5>~kmXP$E}m zQV|7uv4SwFs0jUervsxVUm>&9Y3DBIzc1XW|CUZrUdb<&{@D5yuLe%Xniw^x&{A2s z0q1+owDSfc3Gs?ht;3jw49c#mmrViUfX-yvc_B*wY|Lo7; zGh!t2R#BHx{1wFXReX*~`NS-LpSX z#TV*miO^~B9PF%O0huw!1Zv>^d0G3$^8dsC6VI!$oKDKiXdJt{mGkyA`+Gwd4D-^1qtNTUK)`N*=NTG-6}=5k6suNfdLt*dt8D| z%H#$k)z#ZRcf|zDWB|pn<3+7Nz>?WW9WdkO5(a^m+D4WRJ9{wc>Y}IN)2Kbgn;_O? zGqdr&9~|$Y0tP=N(k7^Eu;iO*w+f%W`20BNo)=Xa@M_)+o$4LXJyiw{F?a633SC{B zl~9FH%?^Rm*LVz`lkULs)%idDX^O)SxQol(3jDRyBVR!7d`;ar+D7do)jQ}m`g$TevUD5@?*P8)voa?kEe@_hl{_h8j&5eB-5FrYW&*FHVt$ z$kRF9Nstj%KRzpjdd_9wO=4zO8ritN*NPk_9avYrsF(!4))tm{Ga#OY z(r{0buexOzu7+rw8E08Gxd`LTOID{*AC1m*6Nw@osfB%0oBF5sf<~wH1kL;sd zo)k6^VyRFU`)dt*iX^9&QtWbo6yE8XXH?`ztvpiOLgI3R+=MOBQ9=rMVgi<*CU%+d1PQQ0a1U=&b0vkF207%xU0ssI2 literal 0 HcmV?d00001 diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-mdpi/ic_launcher.webp b/android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-mdpi/ic_launcher.webp new file mode 100644 index 0000000000000000000000000000000000000000..4f0f1d64e58ba64d180ce43ee13bf9a17835fbca GIT binary patch literal 982 zcmV;{11bDcNk&G_0{{S5MM6+kP&il$0000G0000l001ul06|PpNU8t;00Dqo+t#w^ z^1csucXz7-Qrhzl9HuHB%l>&>1tG2^vb*E&k^T3$FG1eQZ51g$uv4V+kI`0<^1Z@N zk?Jjh$olyC%l>)Xq;7!>{iBj&BjJ`P&$fsCfpve_epJOBkTF?nu-B7D!hO=2ZR}

C%4 zc_9eOXvPbC4kzU8YowIA8cW~Uv|eB&yYwAObSwL2vY~UYI7NXPvf3b+c^?wcs~_t{ ze_m66-0)^{JdOMKPwjpQ@Sna!*?$wTZ~su*tNv7o!gXT!GRgivP}ec?5>l1!7<(rT zds|8x(qGc673zrvYIz;J23FG{9nHMnAuP}NpAED^laz3mAN1sy+NXK)!6v1FxQ;lh zOBLA>$~P3r4b*NcqR;y6pwyhZ3_PiDb|%n1gGjl3ZU}ujInlP{eks-#oA6>rh&g+!f`hv#_%JrgYPu z(U^&XLW^QX7F9Z*SRPpQl{B%x)_AMp^}_v~?j7 zapvHMKxSf*Mtyx8I}-<*UGn3)oHd(nn=)BZ`d$lDBwq_GL($_TPaS{UeevT(AJ`p0 z9%+hQb6z)U9qjbuXjg|dExCLjpS8$VKQ55VsIC%@{N5t{NsW)=hNGI`J=x97_kbz@ E0Of=7!TQj4N+cqN`nQhxvX7dAV-`K|Ub$-q+H-5I?Tx0g9jWxd@A|?POE8`3b8fO$T))xP* z(X?&brZw({`)WU&rdAs1iTa0x6F@PIxJ&&L|dpySV!ID|iUhjCcKz(@mE z!x@~W#3H<)4Ae(4eQJRk`Iz3<1)6^m)0b_4_TRZ+cz#eD3f8V;2r-1fE!F}W zEi0MEkTTx}8i1{`l_6vo0(Vuh0HD$I4SjZ=?^?k82R51bC)2D_{y8mi_?X^=U?2|F{Vr7s!k(AZC$O#ZMyavHhlQ7 zUR~QXuH~#o#>(b$u4?s~HLF*3IcF7023AlwAYudn0FV~|odGH^05AYPEfR)8p`i{n zwg3zPVp{+wOsxKc>)(pMupKF!Y2HoUqQ3|Yu|8lwR=?5zZuhG6J?H`bSNk_wPoM{u zSL{c@pY7+c2kck>`^q1^^gR0QB7Y?KUD{vz-uVX~;V-rW)PDcI)$_UjgVV?S?=oLR zf4}zz{#*R_{LkiJ#0RdQLNC^2Vp%JPEUvG9ra2BVZ92(p9h7Ka@!yf9(lj#}>+|u* z;^_?KWdzkM`6gqPo9;;r6&JEa)}R3X{(CWv?NvgLeOTq$cZXqf7|sPImi-7cS8DCN zGf;DVt3Am`>hH3{4-WzH43Ftx)SofNe^-#|0HdCo<+8Qs!}TZP{HH8~z5n`ExcHuT zDL1m&|DVpIy=xsLO>8k92HcmfSKhflQ0H~9=^-{#!I1g(;+44xw~=* zxvNz35vfsQE)@)Zsp*6_GjYD};Squ83<_?^SbALb{a`j<0Gn%6JY!zhp=Fg}Ga2|8 z52e1WU%^L1}15Ex0fF$e@eCT(()_P zvV?CA%#Sy08_U6VPt4EtmVQraWJX` zh=N|WQ>LgrvF~R&qOfB$!%D3cGv?;Xh_z$z7k&s4N)$WYf*k=|*jCEkO19{h_(%W4 zPuOqbCw`SeAX*R}UUsbVsgtuG?xs(#Ikx9`JZoQFz0n*7ZG@Fv@kZk`gzO$HoA9kN z8U5{-yY zvV{`&WKU2$mZeoBmiJrEdzUZAv1sRxpePdg1)F*X^Y)zp^Y*R;;z~vOv-z&)&G)JQ{m!C9cmziu1^nHA z`#`0c>@PnQ9CJKgC5NjJD8HM3|KC(g5nnCq$n0Gsu_DXk36@ql%npEye|?%RmG)

FJ$wK}0tWNB{uH;AM~i literal 0 HcmV?d00001 diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-xhdpi/ic_launcher.webp b/android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-xhdpi/ic_launcher.webp new file mode 100644 index 0000000000000000000000000000000000000000..948a3070fe34c611c42c0d3ad3013a0dce358be0 GIT binary patch literal 1900 zcmV-y2b1_xNk&Fw2LJ$9MM6+kP&il$0000G0001A003VA06|PpNH75a00DqwTbm-~ zullQTcXxO9ki!OCRx^i?oR|n!<8G0=kI^!JSjFi-LL*`V;ET0H2IXfU0*i>o6o6Gy zRq6Ap5(_{XLdXcL-MzlN`ugSdZY_`jXhcENAu)N_0?GhF))9R;E`!bo9p?g?SRgw_ zEXHhFG$0{qYOqhdX<(wE4N@es3VIo$%il%6xP9gjiBri+2pI6aY4 zJbgh-Ud|V%3O!IcHKQx1FQH(_*TK;1>FQWbt^$K1zNn^cczkBs=QHCYZ8b&l!UV{K z{L0$KCf_&KR^}&2Fe|L&?1I7~pBENnCtCuH3sjcx6$c zwqkNkru);ie``q+_QI;IYLD9OV0ZxkuyBz|5<$1BH|vtey$> z5oto4=l-R-Aaq`Dk0}o9N0VrkqW_#;!u{!bJLDq%0092{Ghe=F;(kn} z+sQ@1=UlX30+2nWjkL$B^b!H2^QYO@iFc0{(-~yXj2TWz?VG{v`Jg zg}WyYnwGgn>{HFaG7E~pt=)sOO}*yd(UU-D(E&x{xKEl6OcU?pl)K%#U$dn1mDF19 zSw@l8G!GNFB3c3VVK0?uyqN&utT-D5%NM4g-3@Sii9tSXKtwce~uF zS&Jn746EW^wV~8zdQ1XC28~kXu8+Yo9p!<8h&(Q({J*4DBglPdpe4M_mD8AguZFn~ ztiuO~{6Bx?SfO~_ZV(GIboeR9~hAym{{fV|VM=77MxDrbW6`ujX z<3HF(>Zr;#*uCvC*bpoSr~C$h?_%nXps@A)=l_;({Fo#6Y1+Zv`!T5HB+)#^-Ud_; zBwftPN=d8Vx)*O1Mj+0oO=mZ+NVH*ptNDC-&zZ7Hwho6UQ#l-yNvc0Cm+2$$6YUk2D2t#vdZX-u3>-Be1u9gtTBiMB^xwWQ_rgvGpZ6(C@e23c!^K=>ai-Rqu zhqT`ZQof;9Bu!AD(i^PCbYV%yha9zuoKMp`U^z;3!+&d@Hud&_iy!O-$b9ZLcSRh? z)R|826w}TU!J#X6P%@Zh=La$I6zXa#h!B;{qfug}O%z@K{EZECu6zl)7CiNi%xti0 zB{OKfAj83~iJvmpTU|&q1^?^cIMn2RQ?jeSB95l}{DrEPTW{_gmU_pqTc)h@4T>~& zluq3)GM=xa(#^VU5}@FNqpc$?#SbVsX!~RH*5p0p@w z;~v{QMX0^bFT1!cXGM8K9FP+=9~-d~#TK#ZE{4umGT=;dfvWi?rYj;^l_Zxywze`W z^Cr{55U@*BalS}K%Czii_80e0#0#Zkhlij4-~I@}`-JFJ7$5{>LnoJSs??J8kWVl6|8A}RCGAu9^rAsfCE=2}tHwl93t0C?#+jMpvr7O3`2=tr{Hg$=HlnjVG^ewm|Js0J*kfPa6*GhtB>`fN!m#9J(sU!?(OSfzY*zS(FJ<-Vb zfAIg+`U)YaXv#sY(c--|X zEB+TVyZ%Ie4L$gi#Fc++`h6%vzsS$pjz9aLt+ZL(g;n$Dzy5=m=_TV(3H8^C{r0xd zp#a%}ht55dOq?yhwYPrtp-m1xXp;4X;)NhxxUpgP%XTLmO zcjaFva^}dP3$&sfFTIR_jC=2pHh9kpI@2(6V*GQo7Ws)`j)hd+tr@P~gR*2gO@+1? zG<`_tB+LJuF|SZ9tIec;h%}}6WClT`L>HSW?E{Hp1h^+mlbf_$9zA>!ug>NALJsO{ mU%z=YwVD?}XMya)Bp;vlyE5&E_6!fzx9pwrdz474!~g(M6R?N? literal 0 HcmV?d00001 diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp b/android/SherpaOnnxSpeakerDiarization/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp new file mode 100644 index 0000000000000000000000000000000000000000..1b9a6956b3acdc11f40ce2bb3f6efbd845cc243f GIT binary patch literal 3918 zcmV-U53%r4Nk&FS4*&pHMM6+kP&il$0000G0001A003VA06|PpNSy@$00HoY|G(*G z+qV7x14$dSO^Re!iqt-AAIE9iwr$(CZQJL$blA4B`>;C3fBY6Q8_YSjb2%a=fc}4E zrSzssacq<^nmW|Rs93PJni30R<8w<(bK_$LO4L?!_OxLl$}K$MUEllnMK|rg=f3;y z*?;3j|Nh>)p0JQ3A~rf(MibH2r+)3cyV1qF&;8m{w-S*y+0mM){KTK^M5}ksc`qX3 zy>rf^b>~l>SSHds8(I@hz3&PD@LmEs4&prkT=BjsBCXTMhN$_)+kvnl0bLKW5rEsj z*d#KXGDB4P&>etx0X+`R19yC=LS)j!mgs5M0L~+o-T~Jl!p!AJxnGAhV%~rhYUL4hlWhgES3Kb5oA&X z{}?3OBSS-{!v$nCIGj->(-TAG)8LR{htr41^gxsT8yqt2@DEG6Yl`Uma3Nd4;YUoW zTbkYl3CMU5ypMF3EIkYmWL|*BknM`0+Kq6CpvO(y$#j94e+q{vI{Zp8cV_6RK!`&C zob$*5Q|$IZ09dW=L!V zw@#2wviu|<#3lgGE8GEhcx+zBt`} zOwP8j9X%^f7i_bth4PiJ$LYtFJSCN$3xwDN;8mr*B;CJwBP2G0TMq0uNt7S^DO_wE zepk!Wrn#Z#03j{`c*Rf~y3o7?J}w?tEELRUR2cgxB*Y{LzA#pxHgf}q?u5idu>077 zd^=p)`nA}6e`|@`p?u}YU66PP_MA}Zqqe!c{nK&z%Jwq1N4e_q<#4g^xaz=ao;u|6 zwpRcW2Lax=ZGbx=Q*HhlJ`Ns#Y*r0*%!T?P*TTiX;rb)$CGLz=rSUum$)3Qyv{BL2 zO*=OI2|%(Yz~`pNEOnLp>+?T@glq-DujlIp?hdJeZ7ctP4_OKx|5@EOps3rr(pWzg zK4d3&oN-X2qN(d_MkfwB4I)_)!I_6nj2iA9u^pQ{;GckGLxBGrJUM2Wdda!k)Y>lq zmjws>dVQ*vW9lvEMkiN3wE-__6OWD0txS&Qn0n22cyj4Q*8(nG4!G{6OOwNvsrPIL zCl-$W9UwkEUVuLwyD%|inbOF*xMODZ4VMEVAq_zUxZ+K#Gdqf!DW$5f)?7UNOFMz! zrB~tuu=6X2FE(p^iqgxr+?ZK;=yz`e;C$#_@D9Lj-+TDVOrva>(#*PVbaHO>A)mhl z07OJWCqYC60518$!&c`eNBcBW%GnfaQ*$eazV^2_AW?j)h;J1nUjN(I9=0+!RVx~% z3@Tf!P0TE+98jA?WceK-}A1% zW!K)lyKcGqy#M~})315-A#2NXQ`?6NR#Apo=S!oF=JfpX>iR*49ec{7AN$xxpK{D$ z2d%Fz&rdfSqourN$~Y^NFIMV1CZ?J*bMx~H3k&meGtH@q9ra2vZxmA$S(#jaaj-g4 ztJmxG+DLV<*q<|sDXPp$X>E)#S}Vm&sRaO5P&goh2><}FEdZSXDqsL$06sAkh(e+v zAsBhKSRexgwg6tIy~GFJzaTxXD(}|+0eOwFDA%rn`X;MVwDHT9=4=g%OaJ9s%3b9>9EUTnnp0t;2Zpa{*>mk~hZqItE_!dQ zOtC>8`$l|mV43Jbudf0N6&&X;{=z}Zi}d1`2qmJ}i|0*GsulD3>GgQXHN)pkR6sf1 z?5ZU%&xtL}oH;YiAA)d*^Ndw2T$+Mjuzyzz@-SM`9df7LqTxLuIwC~S0092~+=qYv z@*ja;?Wt!T!{U?c*Z0YtGe)XbI&y-?B&G2$`JDM)(dIV9G`Sc#6?sI60de6kv+)Qb zUW~2|WjvJq3TA8`0+sWA3zRhY9a~ow)O~&StBkG2{*{TGiY~S8ep{V&Vo2l<6LWsu z^#p0-v*t2?3&aA1)ozu|%efSR=XnpX$lvTeRdKlvM!@|pM5p2w3u-6 zU>}t2xiYLS+{|%C65AzX+23Mtlq?BS&YdYcYsVjoiE&rT>;Necn6l^K)T^lmE`5u{ zm1i+-a-gc;Z&v-{;8r)z6NYfBUv+=_L}ef}qa9FX01)+Aaf+;xj(mL6|JUzGJR1|fnanb%?BPPIp>SCjP|8qE5qJ{=n5ZGw?81z3(k;pzH%1CtlX50{E7h)$h{qGKfzC`e2o`*IqA#tjA z`Fz&^%$b9F*N`)U-#6>a)Z`55`$Dd0cfcs0$d13^ONrdCu9xcv_=n#WQo8stcz3jP9|2EvdI-RhJM3%Q%oM&!OlShM|0 z?gz?wHZSnm45njLtsz8PVT1S&jAlbKg5kVam$p16=EK@Sj4EP0OtH zmJDmdc^v)x>56Qg_wmYHz6h)>kl_h$>0@J!ypv%APmjZTAQVLy6Fu50RGY&JAVNhx zrF_qG6`x9MkT;1SFWo$)l{M$;3qUDn9JwE}z zRl#E_bDRJFii61kPgBybIgp8dNW!Cc1b*^YYk-#oWLJvtM_v^hQx~9?8LD4VFFxBF z3MlrsSC%f9Oupn*ctPL0U1fwfX?`tRhPD{PSLFPQOmIt$mDy0SgpNVvHS+f#Do>h1Gn?LZU9(KaN>Q_=Y*_T zvtD7%_u^^+{g`0VGzg(VZrpVQ6Ub5M=tI_p7T93R8@3Zulu3|#{iNcu!oiHxZ4Rf*( zfmiN$$ru(*_Zqn=`Gq#OuHRTSwp7uH_SokR&|)RuW5yo=Z|_4?qU-JU+tpt>!B&Is z@N(=SG;bpVc;AO@zbmMM zScqq1)b-ZQIrs={oD}|?6y{$HNB1U0^LsBh8JI&3!GBZxOXI<}&5-$lgkAaYqhOTb z?2vEnZ$-kk;*M_17(upJF3%+iH*s0-r{vttXVB2OUwI1s^+G(Ft(U8gYFXC}#P&E^ z>T@C^tS`Z7{6HT4_nF~n>JlZtk5&qDBl6r|^kzQYe`wq!C)n@$c>WOPA61NDFj<<6 zGW71NMMhwAl!U-yqrq2xrSFqRCI8acw7?}3j;ynxo*-b7Co;g5r%^j=H@9({PXXBf z@r>U>>N;E)81wx`B4f%{PB~MHka_);%kBCb(d|Jy5!MqJ%2p`t&@L)4$T2j&-WHvG zv3(uyA_gwqNu(k?jQTtv3dgPKRZoH8prxe7>pQBW5L&dpumS&5Ld2?(sCpJjvc4L5 zEnh&?91WVm)ZdTj=fjJ$pPDdgAttLXuke+?KdKxu*;kTC(r!tQk6;gxj4h%FdHAt(^M3YvYj(!tOeN)+Hvj6+< zzyJRG?^lZfWuR#t!tUKP&(?%3v&Zd$R2YN>lB(Lq`OInY48%4%yTv2 zYe1{G`3)(PDEio5Y@-I5tUf`c%%OCJMtSW56g3iEg%3`$7XSJJHyA z<|7&N)5Xrlgv~%BO24eFd;Hd;uiK%D`EdK|quUeRZDqbh9l)%j%J#0lfrZumvA<_w zu&=AVvdChf6}eqh(bUz`(`Ue*p01{fBAcTgKyDYLs_I+YyJEk+rM@avU~>fB$n)HS zM7pfJydu`i%gfS<{PF94kZDv$t>06sAkheDzu40NJ$5CMW%n^Lls?8^p^QGWURbKu3ZduZQZ((s2? zzE`}<{;Zt7<$C|9R8A~DJ~@%x>TfP zF>TX8)@v|t)q4GjRt<}5s6hLHwRel7>V@&r-O|Av(yh;Q1A{E>Ir>p+%dHD|=l+lT zpr(Dg&>#Nu=!)6bCLr-ZS%|;h)Ij$+e@r8_{qO19QvDe=&1tmpY*0lcA^Cc-#{9fQ z<~$*<&P$Q<_jy#<$40PMofM7aQ}C=jphI`4kLg}Z7CIN#26D{-4v-_CA-LiE@(%{y!BzsU%gG`Q?sjLUf%qFSl0y)2#ae*+EI>s|i`d^V$Dn)qmzqRq6VJRY|{4ujsIU%#bnqU6MR&-1I_43=|5(6Jr;Jvert) zE?S|Tmn}Tv<-??sxV5@9t}3D=>YZ0JrQe$CO~|EY=Lj9RM&4svQHPQL6%pV5fPFiH zfXDx;l@~et{*{U*#c#Dvzu)|znDO7$#CRx)Z&yp-}SrD{&|(MQtfUz~n35@RLfUy=aqrhCX0M}J_r5QsK~NmRCR|Nm&L z41UdsLjWxSUlL41r^0K&nCCK>fdR-!MYjFg(z9_mF^C|#ZQw?`)f6uVzF^`bRnVY& zo}@M06J&_+>w9@jpaO4snmU;0t-(zYW1qVBHtuD!d?%?AtN7Plp><-1Y8Rqb20ZaP zTCgn*-Sri4Q8Xn>=gNaWQ57%!D35UkA@ksOlPB*Dvw}t02ENAqw|kFhn%ZyyW%+t{ zNdM!uqEM^;2}f+tECHbwLmH*!nZVrb$-az%t50Y2pg(HqhvY-^-lb}>^6l{$jOI6} zo_kBzj%8aX|6H5M0Y<)7pzz_wLkIpRm!;PzY)9+24wk2&TT{w--phDGDCOz{cN_ca zpnm7`$oDy=HX%0i-`769*0M6(e5j-?(?24%)<)&46y0e&6@HCDZAm9W6Ib#Y#BF6- z=30crHGg+RRTe%VBC>T00OV6F+gQDAK38Ne3N9bm|62tPccBJi)5{B z4zc^Db72XiBd}v$CF|yU{Z=M|DZ%-(XarYNclODlb1Kz1_EKLy(NSLCN`eUl(rBCL zT*jx@wNvze0|TSqgE(QArOZU)_?qH(sj#TwzElLs9q)(0u!_P|R%Cy_0JFQxgGV>1 zz4?_uq<8_gM0`c*Hh|;UMz~vrg1gQXp{ufg`hM_qU;U>+zmvc5blCLSq@PrEBSGR# z&8=2Z4uXN`F3p73ueD1l{s{k$WipAvSh5W7ABe?4)t;r@V?y`bNB5FvBuE|0VRTb< zM1Hn^?DSsJY+sX@T5xW=#>T9VEV|?<(=6|ge$X6Sb05!LFdjDcoq*gM(Zq=t;_)Le&jyt(&9jzR73noru`a# zN*<`KwGa^gZU3-)MSLF0aFag#f0<>E(bYTeHmtdbns#|I)-$)mJ`q9ctQ8g0=ET?| zdO}eZ*b_p>ygRTtR^5Ggdam=Zb5wmd{}np+Jn1d_=M`~P=M67jj})fH4ztb5yQqQW z^C|C&^LHAK-u+ooIK)yM)QM?t;|<{P;;{`p=BclzAN#JzL4jCwXkQB1Dy{=^KR`=~ zTrr)y7eiYBzSNs_DvO=4A6#EgGS-zY%Vi)N*Yb`U;6o}KR}dq{r9pT5wqZ@3NOE8- z9-(}D|Nc5732CSYQbL)!gPQ#RbD8BhK3dl{sUuPvei0tkvnJBxDEAYTesU8H$)g(Plra{VH(v3u^CO1~(+ zU0O7#)jaS4{NcwA+LuSm&VBcX2#Im3xg)W}ySNw%->orn1taZ&+d)}8gJTqA!u|5P z{yv?zol_3|(1(%M(EVU=cp?L`{Pi|ixk{U)*guFML3P!OSlz;zGA#T+E@8@cgQ_mv1o7RSU=Zo_82F?&&2r;WE z@wk}JHYEZ9nYUc(Vv~iTCa3u8e4q(yq<29VoNbKk|`mq%I6u)My=gPIDuUb&lzf4`MEA9^g8u z)vp8|$$HE9m_BTV?lOosIGa4jud=jIbw)O2eCMfyw2*S8?hjWw^nqws$O*M$3I1)x zR0PWFb3$ySOcGTe1dz%N0l;RPc`x%05FtT^f^j{YCP}*Q=lvp4$ZXrTZQHhO+w%wJn3c8j%+5C3UAFD&%8dBl_qi9D5g8fry}6Ev z2_Q~)5^N$!IU`BPh1O|=BxQ#*C5*}`lluC515$lxc-vNC)IgW=K|=z7o%cWFpndn= zX}f{`!VK02_kU+Q5a3m37J;c} zTzbxteE{GNf?yLt5X=Bzc-mio^Up0nunMCgp*ZJ;%MJvPM3QK)BryP(_v@ei4UvHr z6+sbCifQaOkL6-;5fL8$W($zZ_;CZp305C;~$hhRquZr-r)jjd1z z31%ZK{-(`P#|Um_Sivn@p$-vz46uqT>QG0B1w9znfS9A8PB2LaHdzA|_)yjXVR*l{ zkcu3@vEf7bxH0nkh`q?8FmoO_Ucui*>_a~P?qQrlZ9@+D7%MTpSnztpylXrt5!-k8_QPB?YL8Kx_On8WD zgT+111d(Op$^$&KLAN5+@?>f7F4~wFi(8TL8+szgVmcMDTp5l&k6~=rA{Dt}!gb^r zSWY<)M7D|Z2P0cEodj6E42PV>&>DFmQpgt)E-|#sSUU@uKed+F680H@<;-x{p|nuH4!_mn85rx>wz;0mPi2ZkL#k6;sznu?cXh!T0S>{w6 zL^gvR05NY64l*<+_L>On$rjx9!US;l;LX6@z}yi#2XHh)F@Oo+l)h%fq$v}DNmF2> zfs^_t0)3N-W<9-N?uedVv{)-J0W5mh#29QM5R5h&KuiRM=0Zvnf#lF=K#WlCgc#9c zS;qvh(P$!_a8JwyhI^ZJV2k+B6Z^64?w|1?5gyo6y{}923CRZfYVe1#?F% z7h2SUiNO3;T#JUOyovSs@@C1GtwipycA=*x5{BpIZ_#GCMuV8XK=x;qCNy{d7?wA~ zC+=vjls;ci&zW=6$H~4^K%v{p}Ab?U%C6Z4p%eC<3ExqU$XR<}LLF67A$Sr20DR_pJ3yeBa~ z^sw{V0FI5;UpwXsScYuhbqGQ`YQ25;6p6W^+tgL&;Ml;>S3CGpSZ>VrTn0m1$y$HU z&65)I!c?oREz};c=nLCliriqQX->4uivHTgd${GqeAlf*!P^B|jkU|*IdNP(&6C>4 zqOW$)Nw9nvjy^&`?E|gotDV{JmJ9Q~vuhy<`^C4XIUDt|j4o6rK^e8_(=YqC zuaR6TRVf@tUFHB079o4MBIh{M~4>WwnGgesQH*3?w(RA%hCZ*7)b!aNV=yOQ%o_Y=Lt0Sl*(9^jfRnC210Om$=y>*o|3z} zAR&vAdrB#mWoaB0fJSw9xw|Am$fzK>rx-~R#7IFSAwdu_EI|SRfB*yl0w8oX09H^q zAjl2?0I)v*odGJ40FVGaF&2qJq9Gv`>V>2r0|c`GX8h>CX8eHcOy>S0@<;M3<_6UM z7yCEpug5NZL!H_0>Hg_HasQGxR`rY&Z{geOy?N92Z z{lER^um|$*?*G63*njwc(R?NT)Bei*3jVzR>FWUDb^gKhtL4A=kE_1p-%Fo2`!8M} z(0AjuCiS;G{?*^1tB-uY%=)SRx&D)pK4u@>f6@KPe3}2j_har$>HqzH;UCR^ssFD0 z7h+VLO4o@_Yt>>AeaZKUxqyvxWCAjKB>qjQ30UA)#w z&=RmdwlT`7a8J8Yae=7*c8XL|{@%wA8uvCqfsNX^?UZsS>wX}QD{K}ad4y~iO*p%4 z_cS{u7Ek%?WV6em2(U9#d8(&JDirb^u~7wK4+xP$iiI6IlD|a&S)6o=kG;59N|>K1 zn(0mUqbG3YIY7dQd+*4~)`!S9m7H6HP6YcKHhBc#b%1L}VIisp%;TckEkcu0>lo@u995$<*Em;XNodjTiCdC%R+TX|_ZR#|1`RR|`^@Teh zl#w@8fI1FTx2Dy+{blUT{`^kY*V-AZUd?ZZqCS4gW(kY5?retkLbF=>p=59Nl|=sf zo1Pc|{{N4>5nt#627ylGF`3n>X%`w%bw-Y~zWM_{Si$dc82|=YhISal{N7OY?O`C4 zD|qb}6nLWJ`hUyL+E>-;ricg9J@ZNYP(x(Sct&OI$Y!QWr*=^VN;G3#i>^1n4e#Je zOVhbFbLpXVu*16enDM+ic;97@R~u&kh__kgP#!R`*rQEnA+_dLkNP~L`0alC|J;c; zeiK=s8;BsLE)KbG3BD&Br@(Ha@SBT&$?xX`=$;eeel=|R_dIr6-Ro?=HEjnsJ_b`1 zK6Yg^-6;^2aW!xeTK)A~3Rm|L^FCHB_I>jIju7ZGo&N_1*QHkxH2!!%@o4iZ?vntS;&zJdPe1dH#04YD93A44o-MpfD zP{rn_aq>U%RDvC2+bp;xPlsOzauIi3*Lf42`jVKKZCRuKdYhi>FDuL2l=v{$BCN#Q6796s%r-AG$Q^t(3c@ zD?w0UhYr11@feiyl9kY_@H8~|xlmO<8PfQmj1!$@WieW@VxR@Psxfe-v9WCi1+f>F4VL?0O~K7T?m4-u|pSkBpUJZZe*16_wAp zSYZ@;k`3;W3UHKUWc8QeI}0jH5Ly=cGWQPw(Kr2fm=-5L(d`lcXofy8tJY3@Tuadz zYWXR{mW7XT!RF#RVCe%}=tM*O6!AD3^(!8un~opNI%Uko7$5t@<8+?; zTxDys(MyyGsUjtSu9$+|_-t!U3fVb1dkK?l`17<+jfl=hrBHnDSV>^R1=TnQeyqbW z>ov#l%!1|S!1>8UUxIdhQq`_klcHVx0{?#>K3#$4GlXncwldt!g17TcvKq-jo_996 z>oA=tH9CqRl6Yw?Uc`am!V?lHJbizOJaVaScf1UP5e7Dbgabq=b!B~T&_F6?ooU>w%x0A zH~&MHJ=q`fCH{U<7MDXE4SD32cDZA)WJeWkllJ`UspWaS#eDe^kg^oU_A14UE9zG-a^g{xaXf$})Wik>gT zl#dkzGr(;h0JZDuFn(+k8wNq?PZ5grQ<+sM?wBGt@JnH6v0#or-5wBQWKU~(S_> zkE!tc*ZJ1Y&*p(xX84POb3cClRMd!^qJ#CAZfIepEj-<`VURS_yCz0(?*Ixcj4 z-!zV1_QZhpm=0<;*(nm+F>T=)o?ep@CK5I%g^VAA+RB25ab?7)A~z~egru=I1S|@v zH7tXV!0wmGS^qj#e+MY;C5eUjEAp$Y?LDkS^QPZ}8WN85?r$u<-Epi;yZ1|J2J`se z$D6DpH~2F=eI0B&=UFAUnJvZAmClJlK)sutJ?M>xpZiWV&0=G4MZP+x+p>EX=HbCz zxls%Mw?*u^;LbHWIWCyq+yi)`GmFn9J112CZda_u@YIP%i;srFg_paU02Ifij*7}l z&CF-(3|>*a|+vbNR`^RP=9G?ymEJ0Z~)d&c*UE$UMepZ zcITr{0WqhxkjUnM15js_gW=e3Uh|y6ZReaXHIz-=p`x5VvB&rH9y>Amv@^WmXFEw) zQXYrk3feir=a{jMQ+wDIkkFnZ$k{sJakHn*?u za%4b!00ev8NVLM1TY=cl?KB&55BY_MU-sg?c>=Dbz_W{(Z~c?HJi*XpYL)C6Bd8WH zt+v-#0&o~@t4qESi*)+eW%@VD0|o^yF)n0hME$UtXF$*Lvh}7sso{`|pn*JDIy5^Fm3s$5*zEE=?u5<=l8FJc3r%+H} zdfoNl2J0^~!-*mOL5o-x32|e0Im*E!yY7F7E5N)W3>+v_LBydlEx?4$RL5f2oYRD# zaR0wv(-p~wO0eLDl3K=%`{5+0Gd$ktO=W)gWlGZJ0`K z$_RNA=ckrfa;H0KA~dR^p�(p-{x$&=IACIfoAR!za)F-^da-t3#0Dycnp zwO~NVXwXCl;jE<}>%@xz|=8fIJAB?>+E{7)|4l${4ngA3G|=r z2Dyv;VVWSgZx9Wj>qUjleGl3Ei9K4>h!(lPS%8VOG>Xu0%6VDz^O=bjJmuP7>DeUv zrbI}MlHB^^d?{zv6d=@_ZD2lg1&G7UjnVN{1}9WkaM3H~btX0GtSzB+tZ^qRgWo4m z!GmimlG$=wgXCnr6j@m<1gAL46#T~5Bnm=2{^@>|t&`9mkEPddj zAvG~@Tv~TAm2i%VW}R-g(Z0)z-Y|szHr@rk>4MAyG*Ma*7Yh#H7(!-5>DZ@8r;_dx z{prSe<>~099F8vsYd2xff7uAS%7{S)f(|@me3t2$iy&NEc7OUEchp@9A|X;;IA>8!oX+y(BKJ$EzV* znR$z;!L$s7uy@{OT~nG#B!NRraT8(X##Ho!0r_o@gg0CA-9H^;-uE&?$2$nHv_00o z%cbuUc-tCx$Uh&EZ4Nf4Zgqv)Y6>usG3>GeQnxx_Z6+PcbX-+ysbt1hQ`K1LDpOE? zrAhIZhSN9yVIAOa22gn577tbc&i3|3V8NWy&!tw##`}9*x}gtI^h1DzZRA>UuaJG) zaZ7j)dq!O}{?#8Y7~7i6fHh4{`pL?>-18|p!S75Y#^DM>-S3)vuZG+Q7l@ek zQP~#cBpWgg#mApc_sPYjpw8odQuRokmTkzcNl`^CcKB7e&;zViV;{Y{o^Y$%7i0m# z62%#1Lq!RC?}lK>%mp}T!3Xv;L*0v*>USLm``N%>w>@fwC+#T&Tx2bN4w(20JB}oU zuSa6v^kXi0xPs?pbaOHnyiqq6By1EZY9OZ^^QA>{q-Hsd&m`pbQ%8121aWG-F5xf zlZ%;B{;C>X19|`^_?dVyCq>n+41w7|!tUS!{9rHlbhX=SZO5CQ^;!Du_E7*`GiR^Q w)2!4MKjfSAeNo!9>IaV6aUZ*?W>} zs4%E?srLW`CJh0GCIK@hTkrW7A15Iu%N&?Q^$0+!{Tv&|t^Y@u%!L zglTg&?Q5q#ijZ;&HBQ?FNPp;k3J5!&{^+SGq?AX~SiOM9jJMRpyP?RCr@z38AQyy&WRMaC;n4una$~nJKSp?q|s8F00c9?Q! zY_ovvjTFm+DeQM^LXJ#v0}6HRt3R1%5PT*}W!k8BEM;Jrj8dIceFo2fhzTqaB3KKk zGlCLI)gU25(#u6ch6GeB1k@eHq7l{EHXv0n6xE#ws#ri}08kkCf8hUt{|Ejb`2YW* zvg}0nSSX1m=76s?sZhRY$K=3dpJ+y*eDULGnL2}4>4nvW^7_<~wIM_5fjvwt4h1|g z)g0Z6ZFq9j<~9~b8((~TN{Z?ZQfw|is&Xp~AC61sj;xItKyCHdI|tCMC_LbXF>~vR z=w6V3^H=W4CbAgR4#xw}ETTwu2guW~=Crl@SMXv85jQ=%y!s^?m4PI0My7MWICO;- z175jm%&PcPWh8QdOU(#8bp4!N7ET-+)N}N2zk2)8ch|4Q&lPFNQgT-thu053`r*h3 z_8dI@G;`zn;lH$zX3RzIk`E8~`J=BBdR}qD%n@vVG1834)!pS1Y?zVkJGtsa(sB~y zNfMYKsOJb%5J(0ivK8d+l2D2y&5X!cg3BG!AJ}910|_${nF}sC1QF^nLIhzXk-Y#x z0)&1iK!O;Og0Ky!;`b~v%b$`S4E&fB)1NB4v@8wr( z&+NX4e^&o)ecb=)dd~C!{(1e6t?&9j{l8%U*k4)?`(L3;Qjw z#w7FS+U(94MaJKS!J9O8^$)36_J8;thW#2$y9i{bB{?M{QS_inZIJ!jwqAbfXYVd$ zQ5fC$6Nc9hFi8m^;oI-%C#BS|c8vy+@{jx6hFcf^_;2VRgkoN(0h!_VSGmgNPRsxI z8$rTo0LaYq-H5i&gtj81=&xU?H-Y2==G@uQV7E`@+2E9XQW@{&j`?EOktk|Ho{HU>ZqDzvgjwBmdex z&uZNd2C1h{{}2k6Ys9$*nFP3;K%u!MhW`uZy7Sn`1M1zs@Es&;z*Z>Gsh@-3Fe6pE zQD2@cqF((NrRevgvLsvM_8;;iNyJ5nyPyy?e!kvKjGj`6diRFBEe49Oa7wwkJFV7Z z$YT&DWloYu-H?3<0BKn9L&JYDT-SK~*6c5pi18P26$JESKRYj{T7Zk6KiRJcbvOO*{P56Q6s8msbeI3>|j>K9}Q9UBeq*inXKemCm`-<5|-$ZyN4u$(3 z&HcvqehFD%5Yrmykg-^d`=BSa8(i=>ZoC77^mWY{evp(km@aHqhUECBz76YiR+VYK zY_avFC~V3$=`6C4JhfHAQ@DZtUOwH`L;oYX6zK0-uI^?hS$ALfq}A7evR;ohJHij} zHSZdW?EKv9U1s4oD*<(0oQ*;MaQ6@cvGL zuHCPgm_NhVsgp^sfr*ia^Db}swo1?O(_Q2)y+S$CBm+g=9wCOUPbz(x)_GbaKa@A7 zuI&!ynLiZRT#V%_y_-D`0Z5lT*auoe{(U5NylTzFSJW()W-#F6*&A`LNO1bV#Y;QJ zSbLBnp|B^dtK|KIWC|No>JjWBWE@n7O)x{&^E(WMeMvp57#qA8m* zeTow*U@_86B#Fm*rxyYu5PRWaWHx8y> z*qmHEp(AMDl0v)ij(AY8fnH=~ZwwjVAbu*m5;xPfidh@ov6d8g zfJsi&!QyK53Es%sC39ts;54V68koALD4b|%tNHW0bIkZAJKa=W&FomJSEDT>W1xIX z1x%Z>AvNIsSPLcn3RTcHXb@KB?cuM)=x6fcIx>&(GxqZ8w3p#jJ(GVgc*`c0HG}dv zIop&Qim!K1NFwic%07KcjWgHBPUkq7f~lj;TPqVGTiT#cUeim>;nY`>h@a*S{qQex zQ`z62WK|Mj)Y{tfF{;T4P;c8$Q|KU?Joh zIkA^z%X7z|r>4aTh@|StTi!-r1D!g=zb#3d#{{&K3CqE$Iz-UH<%37c zRfkO`&uM%#AD3PHv`g5t0e^O%nVL0d{Xlx^EjEC3#skF@`zl-7PF^0oxW)1!C!JxR zWvuAHH?)61FKA1QeT*_sY7;_Id#!GmV4n`MO{~sv}VLSK` zXRw=Y=Clz*00B(5y^K;gCZMAzjT5+c3IC=)l(9VIDdatpxj3y89WwI|bH&$!ZEvp` zPR!T@#!(|KfI-w?!&+7$N3F6>tD{YO4Qg$d_`nNEdfVCha9vaPn0jI0`)`@*72hq! zpU5ND^P*RoEkbD5o#az(-g=Y)L>HH>Oc%}$ zT3Rs_ih0;4+Lv4Y;@Iv(;fUbQ=i-G(#>vghec~*j(I#r|5mqFiJBpzi&hzEcD{u$< zRsm0BVYn=pT;0>R(itW|*D&;O%bOc7et9ACaH#J>z3A1A~6fdP>pmbM%xzm4>|;c_?B+%sl;Qs2{t!60$^u zH1t@9^6>;?!FuusnISi$f5CL&;z?EqJN$FBuWDA#D5`cy_UvCFIVvf{c?4N0teh;d zET$7aVbj08KTQS!x?Nd1Is8q8qFzs}a=!@nJ;7FSfCY^T@D-gpw`w<6e#X3+;O}1h z$%I!M)0bg|EKUA04Qjn@+x{Rj8vt6Wn!R|3A92z}^$KfF5(#CWr4y#~re1CN4i4w0 z#GsypBR{xA3Er7sgAi(|}1-W?s~n$7?K|9WL8kpVfw-;#b9 z+mn;=ep!162U5R>_t}fOt~tE?s#m( zO-S$7>Ay6*hHdZ)7_oU915WYYCIX;hFI-U2EWYX!pllONr@Q--2o~`!isi6vTPLJ4@(|o=%NHYjo0_S&q*UQIROw@*N-By@PaQ&;YxFZ0aR zX&}LeOEz);#m~Hwm^VAY8DK}b$F4bo{jMN?d!lxKPhNklzr^Cd`0f4oJr^z=I|l`* zm8AHm*fPV`0=lF3Pnnp}&J0N1X@}-D94YvmUabFrLGSnTz7Mu^21F#O5tN#CuY9Vh zUZBH=ez%h*wkf0hBtXJh1SN3d+IF{gzT7lp)j}n?03lt;XSQRAh7qd&v;RwTYDuQ# zbI2*r<>?x-G0@hM{;%{VBD7nLKt~D`T~-HAt5;h%i0_=Ifs=yHma5dhJ+QMG?Ux(a z|E?1CMy1!~oA`FP!k~iG=t&5#>bVdz=peT8HMB6Y)#7PpETtNryT^+Rv3vpJaF^zP z{H}0-LyV9Fu21ID%wO9f1IKlFr1p4c{o-?03vyB-tr5duk^&L$;m_|f$vs`^Sl{j2 z95}oY{LlY+=ZS%J+tZoXCd0*sSU7w^gjovXn+g7uyra5{cU49@yHf#Z^Jl-$9cIfo z+AJuxH$VLb=#+uBbVmUjnx zxb1pZ@-O9=AIk4@S)m6fJ2?{HrNYwwnL3a45muuNjr;6$O`bGEM0T4A2_S$t=86*- zcO+0mywg*j#A4mU}enR_!cGmIYQ;qwfchWtFEXL)AK%*;=j znYne+hS4EMy3S)C*mZ1KI>!+)0V@9!N6H$Y}~MJ{rYuf zz^KljIWvFi-?#?V@LPR&c6Nn{!=XM z>}-h$S76;$H{E{Y%@^zlmOl^efBwa%UU+jJD9UVukQ3ti_kH-?H*RC0?M1W%FCvMB zM_+v6fk$6X2sx)-p~B3&Kl{nscK}pNLM*qjtpaf9>AU{-iPKQZR8yCg!TY}Qg*(;) z)gdvCcB%kppZc$VdvsK@)3l1{&DG!d_6OHOS`y=ITLEVu`unSKA2E%JD*DVX{LJ}K z9l>hMRDqxQh0lnpGHpVYneX}eA3Pt|2v%=q;rt)``R|#bDyB)OXY&vI_@|*}h}G?^ z@aZ4_!7cQPX`!fW_?{oT1NTwHs#l5L-0`E|y@48<3Q^HFf8=Idi zpJYD%1MkII!~|7I^WGo)IF=?{>ACnjJ_WUi39C}!Q{QnheVJqeKKqq5^o5CBde(g9 zvw$X6^jz_^E2$wSw4!q5*RG(C2_^XO$HBn_55vbl44OnTTRwRaePP0vo{K)U1#99& z<>rq7V&V(<&@I%MFoN5zrY}sz=(*-L&}1QQ*a%`u25h{cFj===17eB_uGuzG&byQ< zrm8BJZl4r_E$3k|Wo6FW0-6M7>qac5uFQsQcmkLWGfeH74S3Z_rJ!jgN++!@i=HW8 zkyjI(oPH-+-N#Qc^-mpNO`bc6r=2-<%&Wy5K1vfFJB(L_IkpS6fY^NmuL8qsgj>MD zn~BHH9WM~32_3vd=W&B)k7F9q%stJx+b_L_X-4zr^LVUMCmyCTA3sWtkvsmME?Xiy z?xOSfB=_$oY06~J-HcCq&)qcW{j;uP;?Dm}=hkq?zh&n!;m((-G-u_t|6x399Q;>A zgNpxoJNj{u|MFDH7Rhq@FCAl0dE|ddnl!oh9{Lq?@JDoR6L;C941IK`ISfdE$4S zE0AUQ8+2|Ncl_q5QkSp#AODp~(^mfP&%Au@@|TBQwoP`UU+V{6u8|)6ZA{~uKmQ*M zmrMTDU8S~8Eqi{^v0Ug&5Upcm#y7Z1(RbgZAG8jB$eRwCspQ)>5;U)oGZ&E5aeR*K z8Yt`Y0$G))Yd(Y3KH}tA4`-_QmNke5hU_|nq=xtyjwW(_o?itz>B>WM&^63bNdQ)k@-IgDHW*RW$Xo9#RzrTrCn7L2H{9Amq|qNg@#eZY=|P zCoI?2s+L)zsM%WX(NbVEY^`C>lFjIBYmJ6@DKJ0ZT4&F&WHW!dwa%QzOG!?jY_2(S zDcEzZbz*2Q!43|z))9yOP9X1Xt%DXzwY(3tl-TR=Qb_MbZYRrooh;dYYmS!U_as1(=YVB?Q_A|tNu5Ut&_q3jbfDM zoFxT^uEuH`nX3*sB%K?GuHUkweYReBwnHqh3P)~`+s3+Tj!rDA1e)8vuBv5J*IsxC zkd^~b(aGzArj08{>cnzOuy04C+C`}gb|Yz-1avxeWzev3NzcHbz_&4W@QCr$z3~w=8Ua- z`;vfG1~BP8CyLb=F7t1am~ph_#|O%$khSJ9%Vtcn)YmpgQxF?xM^_Vb+5fnpB^W0I`f%X8gb9#X{Q-yJG0{Z56aWeI&zPxnf5pdJA38bM`cYnS#x)% z`n1tFf$i)W-hGm(f9mde^=X@NcV_lFb=P`4&CI&H=IArijGwdCk&X@uQ$5xmj!~^? z#$ROCI)V-~t%L%GS#wo@U27ddR`4`3)WoB{R-4snfNrfee|kI8^bu#yDgYqOwas9# zmcb`3!kRJ`Cr=_tq)8aMt{aGtUZsqwVlj6DgCGre>AEt&x8H_in!x@uwgExIh|-mA zjdaC(29~CTVSaaF7HPbql&*9Uo8P@f)>LqCXclr}peS7_1BQ28u9PO8Eq1@`l3q9o zkfKCaO2?T?ZyA6loW<#9_c^O=m<&h}CA!ineAD@=(gbq`vyT|tiJ6#^B1$P;;qax` z55k&Q?wEh#87niLo*+n4L@65J(Nz~=Ya%7^(miLb(E>A3B@|Jjl;FU&D>o|9#7PJH z?|ago!o;WC^h=|T7PVBg(DAB}72cyUS zb(f>Bwbr!F1eTCO5fpj<{PqhY5>143p?~5ZA5H40);=@M#MYvrB6gqHbU_!GSY??i z%s=>-ciA4*zOOZHds0a(kWewZ4h(k8h(ua7HX)Au&mY~H8KY6(_cb$_&fA@QjIW-*heP3%$d!m5^AdnT}`12qA^c@!g3DOwZ5WwE2?)-yU z!)Vx#Mtxt?FzFTwK!77sy7)sMzUd->w4^bxtpM2j!b1pjgyk zGKwWGeb4)^zjy{9Es&PU1}gwg?|J#L$KJB7ett9@4M%-nGtIQr0>Fl@8-yh`-+1ed zS6r}(MeSvgSoFmH*_WPu@i?}!AB~2?;i&IxrkNg~cQ9Som98tcq)k^|eeER|Zl77t za-TVUc;DNvzVXJ%w52+#weN?+;i#{f#!Oc&z?81*N>^e~ltRS%ZI@lR{rs()HmqG! zx*}ZrI-EZ}ckJMiy>A^oofwDfC~IH)z8{VHKGT@#E5I(Ll&+MnMCl>~AV7+>Gi%mF zkU1QlKASdR0B80!YhP<$Ywi0?W2Ux45oPfxv9QolWzJPD^weBfvo4SONxP35106sAmh(e+vAs0GboFD@PvNs)jNPvarhW}0YliZEg{Gazv z+JDIpoojRVPr<*C|BTq<`6ga{5q^8^!|0cxe=rZ!zxH3%f5ZO0cQ*Z<^$Yt2{|Ek0 zyT|*F+CO@K;(owBKtGg!S^xj-Z~rga2m6nxKl9J=fBSuNKW_dLKWhJKeg^-Xe`^1? z`TyJj)8E!#>_3Y?uKrwqq3LJ#SGU>AzUO|6`nR^u&3FNN_jGOc zw)Nw`wr3yIKhgcee6IaN=ws>M{6677%)hPwx&HzC(f&u~&)6@b2kNRzBDQAP0*H73 zq%McOmRk{B3i47qRe=DA*$&odrbEJZ*pV9XXa&p@wlW~@Yfs>V{yiTtplMhgM*-Bz zsSnlq&pG;z0OUN%$~$3=g1UF+G*>+17eRbBf3=y79J}KR8owon@$1Z7MIrvvWWH)34nK2SD)GsrJ{l z1Cl#oVo3A8qY3e=aF)qzms~FG#2$LzT=gs&aVMOj>(%{y<&O0cG!nCiESl~x=^dF{ zKvj8F1K8Ng171wwM5Fh4KoQw`_c6#y$(5cAm7e}~nJ#A*fx+c9;y#&W!#VukR)ugk zKp3=+;Ut+IYn%m+r4d*<`L2h%aDnX5}^!5R|H;(34AoVWjRx(msBZvk;rCI*|~ zdOijqI@9Z{Vu!~jvHW{lBa$rnl4+!s_5sfK3bCGk-B%iDe&@-}+%fOKU|(9?V1 zHE8&@4z)Kx!RAvAs z!Wic9=o#(bg?kc-G68-m(jZ`^=XGUXb)}t(%&~sjFnV^sEX%hSy6UKC4iOhgV=BHV z2w`4g7Y=s#Vu2B_?#VQ|hP39@eArgfX>-0S+dd&^mx0*wp}>)x;c4RUgxz%;oNe?& z-7-lJ@Y^2^C;=qJsxx5|xF)*pTGhch2B&kxtn;f!7=gznk}I3}Dh}(CoMXgA5-p&kS202!l?!fT3t|HG*rIP~mS* z$Wjo}jq3}z$Qq!9yrtd3fM0N629ZM?LU$nv@Tv9b7I;D|;0H2dsA~g7Z7zp1| zB)XmrkMgF6OQr|R)HHD^TE{Y#j!~SR?b`Xt3Qs`B+x<hxexYeAjMUWdZ-*n9%(1)Wb(n2U<><7&9dwGJmrob)4%H? zlQ%z+L-^$dFhhH|@u$%97Qz?*Ynh2VG@q|?8vY&L74&fs&_b&3$x&Oyjl~LQDRRap zJU4U*R+(2Dd!G+lh8!V{pT_UJn+^1Qg6$` zqkNm(a#hWyc6SP+p5=C4HL8-m`pO`5o~`-LI?_h5CsH?F_%?nDodmz&pWR20WTpJE z?N|wSzLjMUK8E)a2tI}Lf;+;*M|h3Y(U#>)g1>zk9|Hd}oZAa2 zLYBWBoSW!Ts!RwXr^8h+U*@{9{zqS^iH)Op<;r`Uw~nc}<^$V~_i%$GFjaG?X1@E|M`h)nekvFKt`Dh-f>@|0-`Xoq)o` zx;JmzDfOV9qCx|EVpogEe0LK~tGS?5$$L_i6P$P6wIsCQaP_;d{{N=iV@+8LI}o#( zvo*Ejy=IIn{rdIQh1&q-{EuohpVOjJ^Q3lD*YTp37$^RRgn8ihpdu5{Ct%5-KO!VL zcNB6dUajXI9jkm-P|i3~GB-A(X`P1Oqqb$tcku)UJw0w3GeUijb__#QT4j%64z%EeB7S?jlWwx_7&+EEvB|6N=kV}DwnyAlX=?j`) zmU#!$*^@NIu#n_d7;WoJV@*Fbv9|yJO4;n|BNF2xy(54RyB>t~8lUOUW$&2%Nwi1y zx6JxW88>U2$#qhl^6KUbtmg9}D0o5vYDT7kWJthLGkpGnN4T>{St^_EU>4;DmLF9o zr|LqsA8_MoNLQ=}w?8u!ziSZ@PC#Y<#9uJFo-ozVo6D;<8j^1$c|qAE3ZTE5i~zmE z$BU5lw6l=EWsg^y^;8>r9qH{xfL|~PZYK#md$zZ0?o11gV<*WSW~cgy2GYGQir%wf zt4iW8D+;s*;RGrmd(-T<@2&j(Cb9xhV*l-x`TpK`xq|7p?5R%5*s!69?2c!cC*VY* z2DE^9pvOPLU!1e}wA8S8opcTJ3`NB>hY=JQnL~QFXR4K8A$BqJnoEB$wn-%u@E6Mh zCfMF4kusv3N!(aHC}4)Xs^xoOwXd%e^6pi5|DZo=Q25j+6HlJ^7FodH6y1bMROR^q zGu6)fopS`h%Sw<;ZH%TEPf+#81-#_v+@8nlR0jLcIDKQtLleOC)6yLZgC!D9X3GgS zohwU{v$jl=quD#Go^hB{`@Qw*a%`(^jyT~=q^bWgGzRj;|12J55HWdCWV}EB|K=%N z3Nq-qxJJ`>^|1MNN+q}zTB&ooE3j==AgK@^UW<^oSbeALa2peF)Th6{@sj0KyMNHZ zksk1+MXN2tv+22A%cQOGpS9)77(uP9mh+!5T5ERLvF@b}$+WvXM45Z?-kCa)fb~f1 znVbTD$Gx-0Zxc`0D@YgHakge6SL0H`-vN_x?AP0>iGH0_EE&=v83hMJgaKAI0jJXm zVxVz;X<$v6WW7}fxROO7vr#YLP;;lij5VrX{;>7kK6TtOH&6|Ar^xo>00%+u$C4@# z>!jOt6*3><171+WxoZnKDTzJtDRw+T030;yI}~uV@9fCnei^I*j>Bp&mzP2d=FPb_ zCM*l_+$LDR3B*a!A$g#>xsrZvw0lckxmMg>0aQd7tPyN=t{dgXb;Ie+T8{fZH=gdu zM7Rg9c(kg(Jg0?ARRRl=AONFKrvFj)lTY$KfT%6^6s`mk*ABGhsce*LsoD>K{z_M2 ziPpnu+lw22PfF!CoId^6n*G4H(Ix+#+N{C(da7t1BYMGEaE#PdpOLxsVD5riQXHp@OX;`S`8VnpM~)I920w~<3|mo0 zf8~Az`*?2?H&gZ&*K&bRkV@qzvMlRHXys8*Ze2+1c?5o!^+$&MHxB@4Ee5cke52R! zmn7AZtY6ST%ixgU5)%$%QcwHj7Es-Qu^kLAPwy%7pGBw_4Q9#da^W2$}axNHr03)_nw z5?yuNmXrI5HgS46)c5&}B)Tts49oU92>3xBLLy}FMUW=84DQbVq^;7_e7|(Sdz|&J z73N+M`rc2rt*oSWu#7S{*s~nH6HRHJS1SmzeXk|;CA)FI4bat3<%}nkB%;;?=F>B7ms9QSxv#@+69;@>QaR?REYX4&)=itG>rM{<{A79Rmk)`5ON#GL`*KX%}Ihk3w(RtM-WLt z?f&FLF}4N^yE!(pZ&Yj&Bc`~K0@4_}*0Om?wN|}4WJ>WL;G^H2*QpgEkGA~OET-Km zkwz|5{6dnz1U<2Pe9DNL>3g5FEIvp1jzP&2K#z~j%g6!7B;^zF+o95?fV{3mnB8*RMhCDNp>Am-3e@jNfMj?jHV$MWjk!DDKP zkAz$Y?Sr)!GUOX}qTQ5aMh|wq1uq}~joWyKl=b_LboM#wi{CMuz5x6BKlA-qy++cM01D3b7`uD z#l6M4pI;JCypO8JZ6?U&wNxR!{4oB_ zlV!x9+-&Qy6{%MQ{~yoZGkKiTSC`YS_j22~G;xUV855g2&C(zm^V!(wpcm@zn{%!g z4}JGo(sGZ1O~to-}le

UmY2RIYtNPVDpE$%vda+HD#3m z&VuXJ{BK&Qe+rBa7eq}Q(bq|tn(RrJAk|ztj2(i{d>nmQnM?;HF2k&9sA6up5tmjl z7lySlzMbifH17-m-Lwa_F&e7nOH?ESi3#ckR3tsM+jsck3`oG!uMS}|eAwVXv>}qxwq?QY%QJ0}r@^;fhuUA9W z*BVl>TGo&N004@xSiwDUXUvp51sVmqO3m)=B55aPwf@0=e}cN+$-BdKxY`YrT_4)0 z_d10#i44Q*rFr8MC>*)v$EJvz``(pb{e&*6k+b zsMz%($|1+8hn8c2?P(l@;Rb&CsZeYoCI3?2!LqjbwPXW3z4G$Qfj=cT5Yb%vY0(AX oeb?AaKtwrnc|$|zzw9vfvn^aJJ!zd)XFXqqy0000001=f@-~a#s literal 0 HcmV?d00001 diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/res/values/colors.xml b/android/SherpaOnnxSpeakerDiarization/app/src/main/res/values/colors.xml new file mode 100644 index 000000000..f8c6127d3 --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/src/main/res/values/colors.xml @@ -0,0 +1,10 @@ + + + #FFBB86FC + #FF6200EE + #FF3700B3 + #FF03DAC5 + #FF018786 + #FF000000 + #FFFFFFFF + \ No newline at end of file diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/res/values/strings.xml b/android/SherpaOnnxSpeakerDiarization/app/src/main/res/values/strings.xml new file mode 100644 index 000000000..05f2df090 --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/src/main/res/values/strings.xml @@ -0,0 +1,3 @@ + + SherpaOnnxSpeakerDiarization + \ No newline at end of file diff --git a/android/SherpaOnnxSpeakerDiarization/app/src/main/res/values/themes.xml b/android/SherpaOnnxSpeakerDiarization/app/src/main/res/values/themes.xml new file mode 100644 index 000000000..34d1d96ed --- /dev/null +++ b/android/SherpaOnnxSpeakerDiarization/app/src/main/res/values/themes.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/android/SherpaOnnxJavaDemo/app/src/main/res/values/colors.xml b/android/SherpaOnnxJavaDemo/app/src/main/res/values/colors.xml new file mode 100644 index 000000000..f8c6127d3 --- /dev/null +++ b/android/SherpaOnnxJavaDemo/app/src/main/res/values/colors.xml @@ -0,0 +1,10 @@ + + + #FFBB86FC + #FF6200EE + #FF3700B3 + #FF03DAC5 + #FF018786 + #FF000000 + #FFFFFFFF + \ No newline at end of file diff --git a/android/SherpaOnnxJavaDemo/app/src/main/res/values/strings.xml b/android/SherpaOnnxJavaDemo/app/src/main/res/values/strings.xml new file mode 100644 index 000000000..31aa7267d --- /dev/null +++ b/android/SherpaOnnxJavaDemo/app/src/main/res/values/strings.xml @@ -0,0 +1,3 @@ + + SherpaOnnxJavaDemo + \ No newline at end of file diff --git a/android/SherpaOnnxJavaDemo/app/src/main/res/values/themes.xml b/android/SherpaOnnxJavaDemo/app/src/main/res/values/themes.xml new file mode 100644 index 000000000..d9f132e85 --- /dev/null +++ b/android/SherpaOnnxJavaDemo/app/src/main/res/values/themes.xml @@ -0,0 +1,16 @@ + + + + \ No newline at end of file diff --git a/android/SherpaOnnxJavaDemo/app/src/main/res/xml/backup_rules.xml b/android/SherpaOnnxJavaDemo/app/src/main/res/xml/backup_rules.xml new file mode 100644 index 000000000..fa0f996d2 --- /dev/null +++ b/android/SherpaOnnxJavaDemo/app/src/main/res/xml/backup_rules.xml @@ -0,0 +1,13 @@ + + + + \ No newline at end of file diff --git a/android/SherpaOnnxJavaDemo/app/src/main/res/xml/data_extraction_rules.xml b/android/SherpaOnnxJavaDemo/app/src/main/res/xml/data_extraction_rules.xml new file mode 100644 index 000000000..9ee9997b0 --- /dev/null +++ b/android/SherpaOnnxJavaDemo/app/src/main/res/xml/data_extraction_rules.xml @@ -0,0 +1,19 @@ + + + + + + + \ No newline at end of file diff --git a/android/SherpaOnnxJavaDemo/build.gradle b/android/SherpaOnnxJavaDemo/build.gradle new file mode 100644 index 000000000..5ae9a7b01 --- /dev/null +++ b/android/SherpaOnnxJavaDemo/build.gradle @@ -0,0 +1,9 @@ +// Top-level build file where you can add configuration options common to all sub-projects/modules. +plugins { + id 'com.android.application' version '7.2.2' apply false + id 'com.android.library' version '7.2.2' apply false +} + +task clean(type: Delete) { + delete rootProject.buildDir +} \ No newline at end of file diff --git a/android/SherpaOnnxJavaDemo/gradle.properties b/android/SherpaOnnxJavaDemo/gradle.properties new file mode 100644 index 000000000..dab7c28bf --- /dev/null +++ b/android/SherpaOnnxJavaDemo/gradle.properties @@ -0,0 +1,21 @@ +# Project-wide Gradle settings. +# IDE (e.g. Android Studio) users: +# Gradle settings configured through the IDE *will override* +# any settings specified in this file. +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8 +# When configured, Gradle will run in incubating parallel mode. +# This option should only be used with decoupled projects. More details, visit +# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects +# org.gradle.parallel=true +# AndroidX package structure to make it clearer which packages are bundled with the +# Android operating system, and which are packaged with your app"s APK +# https://developer.android.com/topic/libraries/support-library/androidx-rn +android.useAndroidX=true +# Enables namespacing of each library's R class so that its R class includes only the +# resources declared in the library itself and none from the library's dependencies, +# thereby reducing the size of the R class for that library +android.nonTransitiveRClass=true \ No newline at end of file diff --git a/android/SherpaOnnxJavaDemo/gradle/wrapper/gradle-wrapper.jar b/android/SherpaOnnxJavaDemo/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..e708b1c023ec8b20f512888fe07c5bd3ff77bb8f GIT binary patch literal 59203 zcma&O1CT9Y(k9%tZQHhO+qUh#ZQHhO+qmuS+qP|E@9xZO?0h@l{(r>DQ>P;GjjD{w zH}lENr;dU&FbEU?00aa80D$0M0RRB{U*7-#kbjS|qAG&4l5%47zyJ#WrfA#1$1Ctx zf&Z_d{GW=lf^w2#qRJ|CvSJUi(^E3iv~=^Z(zH}F)3Z%V3`@+rNB7gTVU{Bb~90p|f+0(v;nz01EG7yDMX9@S~__vVgv%rS$+?IH+oZ03D5zYrv|^ zC1J)SruYHmCki$jLBlTaE5&dFG9-kq3!^i>^UQL`%gn6)jz54$WDmeYdsBE9;PqZ_ zoGd=P4+|(-u4U1dbAVQrFWoNgNd;0nrghPFbQrJctO>nwDdI`Q^i0XJDUYm|T|RWc zZ3^Qgo_Qk$%Fvjj-G}1NB#ZJqIkh;kX%V{THPqOyiq)d)0+(r9o(qKlSp*hmK#iIY zA^)Vr$-Hz<#SF=0@tL@;dCQsm`V9s1vYNq}K1B)!XSK?=I1)tX+bUV52$YQu*0%fnWEukW>mxkz+%3-S!oguE8u#MGzST8_Dy^#U?fA@S#K$S@9msUiX!gd_ow>08w5)nX{-KxqMOo7d?k2&?Vf z&diGDtZr(0cwPe9z9FAUSD9KC)7(n^lMWuayCfxzy8EZsns%OEblHFSzP=cL6}?J| z0U$H!4S_TVjj<`6dy^2j`V`)mC;cB%* z8{>_%E1^FH!*{>4a7*C1v>~1*@TMcLK{7nEQ!_igZC}ikJ$*<$yHy>7)oy79A~#xE zWavoJOIOC$5b6*q*F_qN1>2#MY)AXVyr$6x4b=$x^*aqF*L?vmj>Mgv+|ITnw_BoW zO?jwHvNy^prH{9$rrik1#fhyU^MpFqF2fYEt(;4`Q&XWOGDH8k6M=%@fics4ajI;st# zCU^r1CK&|jzUhRMv;+W~6N;u<;#DI6cCw-otsc@IsN3MoSD^O`eNflIoR~l4*&-%RBYk@gb^|-JXs&~KuSEmMxB}xSb z@K76cXD=Y|=I&SNC2E+>Zg?R6E%DGCH5J1nU!A|@eX9oS(WPaMm==k2s_ueCqdZw| z&hqHp)47`c{BgwgvY2{xz%OIkY1xDwkw!<0veB#yF4ZKJyabhyyVS`gZepcFIk%e2 zTcrmt2@-8`7i-@5Nz>oQWFuMC_KlroCl(PLSodswHqJ3fn<;gxg9=}~3x_L3P`9Sn zChIf}8vCHvTriz~T2~FamRi?rh?>3bX1j}%bLH+uFX+p&+^aXbOK7clZxdU~6Uxgy z8R=obwO4dL%pmVo*Ktf=lH6hnlz_5k3cG;m8lgaPp~?eD!Yn2kf)tU6PF{kLyn|oI@eQ`F z3IF7~Blqg8-uwUuWZScRKn%c2_}dXB6Dx_&xR*n9M9LXasJhtZdr$vBY!rP{c@=)& z#!?L$2UrkvClwQO>U*fSMs67oSj2mxiJ$t;E|>q%Kh_GzzWWO&3;ufU%2z%ucBU8H z3WIwr$n)cfCXR&>tyB7BcSInK>=ByZA%;cVEJhcg<#6N{aZC4>K41XF>ZgjG`z_u& zGY?;Ad?-sgiOnI`oppF1o1Gurqbi*;#x2>+SSV6|1^G@ooVy@fg?wyf@0Y!UZ4!}nGuLeC^l)6pwkh|oRY`s1Pm$>zZ3u-83T|9 zGaKJIV3_x+u1>cRibsaJpJqhcm%?0-L;2 zitBrdRxNmb0OO2J%Y&Ym(6*`_P3&&5Bw157{o7LFguvxC$4&zTy#U=W*l&(Q2MNO} zfaUwYm{XtILD$3864IA_nn34oVa_g^FRuHL5wdUd)+W-p-iWCKe8m_cMHk+=? zeKX)M?Dt(|{r5t7IenkAXo%&EXIb-i^w+0CX0D=xApC=|Xy(`xy+QG^UyFe z+#J6h_&T5i#sV)hj3D4WN%z;2+jJcZxcI3*CHXGmOF3^)JD5j&wfX)e?-|V0GPuA+ zQFot%aEqGNJJHn$!_}#PaAvQ^{3-Ye7b}rWwrUmX53(|~i0v{}G_sI9uDch_brX&6 zWl5Ndj-AYg(W9CGfQf<6!YmY>Ey)+uYd_JNXH=>|`OH-CDCmcH(0%iD_aLlNHKH z7bcW-^5+QV$jK?R*)wZ>r9t}loM@XN&M-Pw=F#xn(;u3!(3SXXY^@=aoj70;_=QE9 zGghsG3ekq#N||u{4We_25U=y#T*S{4I{++Ku)> zQ!DZW;pVcn>b;&g2;YE#+V`v*Bl&Y-i@X6D*OpNA{G@JAXho&aOk(_j^weW{#3X5Y z%$q_wpb07EYPdmyH(1^09i$ca{O<}7) zRWncXdSPgBE%BM#by!E>tdnc$8RwUJg1*x($6$}ae$e9Knj8gvVZe#bLi!<+&BkFj zg@nOpDneyc+hU9P-;jmOSMN|*H#>^Ez#?;%C3hg_65leSUm;iz)UkW)jX#p)e&S&M z1|a?wDzV5NVnlhRBCd_;F87wp>6c<&nkgvC+!@KGiIqWY4l}=&1w7|r6{oBN8xyzh zG$b#2=RJp_iq6)#t5%yLkKx(0@D=C3w+oiXtSuaQ%I1WIb-eiE$d~!)b@|4XLy!CZ z9p=t=%3ad@Ep+<9003D2KZ5VyP~_n$=;~r&YUg5UZ0KVD&tR1DHy9x)qWtKJp#Kq# zP*8p#W(8JJ_*h_3W}FlvRam?<4Z+-H77^$Lvi+#vmhL9J zJ<1SV45xi;SrO2f=-OB(7#iNA5)x1uNC-yNxUw|!00vcW2PufRm>e~toH;M0Q85MQLWd?3O{i8H+5VkR@l9Dg-ma ze2fZ%>G(u5(k9EHj2L6!;(KZ8%8|*-1V|B#EagbF(rc+5iL_5;Eu)L4Z-V;0HfK4d z*{utLse_rvHZeQ>V5H=f78M3Ntg1BPxFCVD{HbNA6?9*^YIq;B-DJd{Ca2L#)qWP? zvX^NhFmX?CTWw&Ns}lgs;r3i+Bq@y}Ul+U%pzOS0Fcv9~aB(0!>GT0)NO?p=25LjN z2bh>6RhgqD7bQj#k-KOm@JLgMa6>%-ok1WpOe)FS^XOU{c?d5shG(lIn3GiVBxmg`u%-j=)^v&pX1JecJics3&jvPI)mDut52? z3jEA)DM%}BYbxxKrizVYwq?(P&19EXlwD9^-6J+4!}9{ywR9Gk42jjAURAF&EO|~N z)?s>$Da@ikI4|^z0e{r`J8zIs>SpM~Vn^{3fArRu;?+43>lD+^XtUcY1HidJwnR6+ z!;oG2=B6Z_=M%*{z-RaHc(n|1RTKQdNjjV!Pn9lFt^4w|AeN06*j}ZyhqZ^!-=cyGP_ShV1rGxkx8t zB;8`h!S{LD%ot``700d0@Grql(DTt4Awgmi+Yr0@#jbe=2#UkK%rv=OLqF)9D7D1j z!~McAwMYkeaL$~kI~90)5vBhBzWYc3Cj1WI0RS`z000R8-@ET0dA~*r(gSiCJmQMN&4%1D zyVNf0?}sBH8zNbBLn>~(W{d3%@kL_eQ6jEcR{l>C|JK z(R-fA!z|TTRG40|zv}7E@PqCAXP3n`;%|SCQ|ZS%ym$I{`}t3KPL&^l5`3>yah4*6 zifO#{VNz3)?ZL$be;NEaAk9b#{tV?V7 zP|wf5YA*1;s<)9A4~l3BHzG&HH`1xNr#%){4xZ!jq%o=7nN*wMuXlFV{HaiQLJ`5G zBhDi#D(m`Q1pLh@Tq+L;OwuC52RdW7b8}~60WCOK5iYMUad9}7aWBuILb({5=z~YF zt?*Jr5NG+WadM{mDL>GyiByCuR)hd zA=HM?J6l1Xv0Dl+LW@w$OTcEoOda^nFCw*Sy^I@$sSuneMl{4ys)|RY#9&NxW4S)9 zq|%83IpslTLoz~&vTo!Ga@?rj_kw{|k{nv+w&Ku?fyk4Ki4I?);M|5Axm)t+BaE)D zm(`AQ#k^DWrjbuXoJf2{Aj^KT zFb1zMSqxq|vceV+Mf-)$oPflsO$@*A0n0Z!R{&(xh8s}=;t(lIy zv$S8x>m;vQNHuRzoaOo?eiWFe{0;$s`Bc+Osz~}Van${u;g(su`3lJ^TEfo~nERfP z)?aFzpDgnLYiERsKPu|0tq4l2wT)Atr6Qb%m-AUn6HnCue*yWICp7TjW$@sO zm5rm4aTcPQ(rfi7a`xP7cKCFrJD}*&_~xgLyr^-bmsL}y;A5P|al8J3WUoBSjqu%v zxC;mK!g(7r6RRJ852Z~feoC&sD3(6}^5-uLK8o)9{8L_%%rItZK9C){UxB|;G>JbP zsRRtS4-3B*5c+K2kvmgZK8472%l>3cntWUOVHxB|{Ay~aOg5RN;{PJgeVD*H%ac+y!h#wi%o2bF2Ca8IyMyH{>4#{E_8u^@+l-+n=V}Sq?$O z{091@v%Bd*3pk0^2UtiF9Z+(a@wy6 zUdw8J*ze$K#=$48IBi1U%;hmhO>lu!uU;+RS}p&6@rQila7WftH->*A4=5W|Fmtze z)7E}jh@cbmr9iup^i%*(uF%LG&!+Fyl@LFA-}Ca#bxRfDJAiR2dt6644TaYw1Ma79 zt8&DYj31j^5WPNf5P&{)J?WlCe@<3u^78wnd(Ja4^a>{^Tw}W>|Cjt^If|7l^l)^Q zbz|7~CF(k_9~n|h;ysZ+jHzkXf(*O*@5m zLzUmbHp=x!Q|!9NVXyipZ3)^GuIG$k;D)EK!a5=8MFLI_lpf`HPKl=-Ww%z8H_0$j ztJ||IfFG1lE9nmQ0+jPQy zCBdKkjArH@K7jVcMNz);Q(Q^R{d5G?-kk;Uu_IXSyWB)~KGIizZL(^&qF;|1PI7!E zTP`%l)gpX|OFn&)M%txpQ2F!hdA~hX1Cm5)IrdljqzRg!f{mN%G~H1&oqe`5eJCIF zHdD7O;AX-{XEV(a`gBFJ9ews#CVS2y!&>Cm_dm3C8*n3MA*e67(WC?uP@8TXuMroq z{#w$%z@CBIkRM7?}Xib+>hRjy?%G!fiw8! z8(gB+8J~KOU}yO7UGm&1g_MDJ$IXS!`+*b*QW2x)9>K~Y*E&bYMnjl6h!{17_8d!%&9D`a7r&LKZjC<&XOvTRaKJ1 zUY@hl5^R&kZl3lU3njk`3dPzxj$2foOL26r(9zsVF3n_F#v)s5vv3@dgs|lP#eylq62{<-vczqP!RpVBTgI>@O6&sU>W|do17+#OzQ7o5A$ICH z?GqwqnK^n2%LR;$^oZM;)+>$X3s2n}2jZ7CdWIW0lnGK-b#EG01)P@aU`pg}th&J-TrU`tIpb5t((0eu|!u zQz+3ZiOQ^?RxxK4;zs=l8q!-n7X{@jSwK(iqNFiRColuEOg}!7cyZi`iBX4g1pNBj zAPzL?P^Ljhn;1$r8?bc=#n|Ed7wB&oHcw()&*k#SS#h}jO?ZB246EGItsz*;^&tzp zu^YJ0=lwsi`eP_pU8}6JA7MS;9pfD;DsSsLo~ogzMNP70@@;Fm8f0^;>$Z>~}GWRw!W5J3tNX*^2+1f3hz{~rIzJo z6W%J(H!g-eI_J1>0juX$X4Cl6i+3wbc~k146UIX&G22}WE>0ga#WLsn9tY(&29zBvH1$`iWtTe zG2jYl@P!P)eb<5DsR72BdI7-zP&cZNI{7q3e@?N8IKc4DE#UVr->|-ryuJXk^u^>4 z$3wE~=q390;XuOQP~TNoDR?#|NSPJ%sTMInA6*rJ%go|=YjGe!B>z6u$IhgQSwoV* zjy3F2#I>uK{42{&IqP59)Y(1*Z>>#W8rCf4_eVsH)`v!P#^;BgzKDR`ARGEZzkNX+ zJUQu=*-ol=Xqqt5=`=pA@BIn@6a9G8C{c&`i^(i+BxQO9?YZ3iu%$$da&Kb?2kCCo zo7t$UpSFWqmydXf@l3bVJ=%K?SSw)|?srhJ-1ZdFu*5QhL$~-IQS!K1s@XzAtv6*Y zl8@(5BlWYLt1yAWy?rMD&bwze8bC3-GfNH=p zynNFCdxyX?K&G(ZZ)afguQ2|r;XoV^=^(;Cku#qYn4Lus`UeKt6rAlFo_rU`|Rq z&G?~iWMBio<78of-2X(ZYHx~=U0Vz4btyXkctMKdc9UM!vYr~B-(>)(Hc|D zMzkN4!PBg%tZoh+=Gba!0++d193gbMk2&krfDgcbx0jI92cq?FFESVg0D$>F+bil} zY~$)|>1HZsX=5sAZ2WgPB5P=8X#TI+NQ(M~GqyVB53c6IdX=k>Wu@A0Svf5#?uHaF zsYn|koIi3$(%GZ2+G+7Fv^lHTb#5b8sAHSTnL^qWZLM<(1|9|QFw9pnRU{svj}_Al zL)b9>fN{QiA($8peNEJyy`(a{&uh-T4_kdZFIVsKKVM(?05}76EEz?#W za^fiZOAd14IJ4zLX-n7Lq0qlQ^lW8Cvz4UKkV9~P}>sq0?xD3vg+$4vLm~C(+ zM{-3Z#qnZ09bJ>}j?6ry^h+@PfaD7*jZxBEY4)UG&daWb??6)TP+|3#Z&?GL?1i+280CFsE|vIXQbm| zM}Pk!U`U5NsNbyKzkrul-DzwB{X?n3E6?TUHr{M&+R*2%yOiXdW-_2Yd6?38M9Vy^ z*lE%gA{wwoSR~vN0=no}tP2Ul5Gk5M(Xq`$nw#ndFk`tcpd5A=Idue`XZ!FS>Q zG^0w#>P4pPG+*NC9gLP4x2m=cKP}YuS!l^?sHSFftZy{4CoQrb_ z^20(NnG`wAhMI=eq)SsIE~&Gp9Ne0nD4%Xiu|0Fj1UFk?6avDqjdXz{O1nKao*46y zT8~iA%Exu=G#{x=KD;_C&M+Zx4+n`sHT>^>=-1YM;H<72k>$py1?F3#T1*ef9mLZw z5naLQr?n7K;2l+{_uIw*_1nsTn~I|kkCgrn;|G~##hM;9l7Jy$yJfmk+&}W@JeKcF zx@@Woiz8qdi|D%aH3XTx5*wDlbs?dC1_nrFpm^QbG@wM=i2?Zg;$VK!c^Dp8<}BTI zyRhAq@#%2pGV49*Y5_mV4+OICP|%I(dQ7x=6Ob}>EjnB_-_18*xrY?b%-yEDT(wrO z9RY2QT0`_OpGfMObKHV;QLVnrK%mc?$WAdIT`kJQT^n%GuzE7|9@k3ci5fYOh(287 zuIbg!GB3xLg$YN=n)^pHGB0jH+_iIiC=nUcD;G6LuJsjn2VI1cyZx=a?ShCsF==QK z;q~*m&}L<-cb+mDDXzvvrRsybcgQ;Vg21P(uLv5I+eGc7o7tc6`;OA9{soHFOz zT~2?>Ts}gprIX$wRBb4yE>ot<8+*Bv`qbSDv*VtRi|cyWS>)Fjs>fkNOH-+PX&4(~ z&)T8Zam2L6puQl?;5zg9h<}k4#|yH9czHw;1jw-pwBM*O2hUR6yvHATrI%^mvs9q_ z&ccT0>f#eDG<^WG^q@oVqlJrhxH)dcq2cty@l3~|5#UDdExyXUmLQ}f4#;6fI{f^t zDCsgIJ~0`af%YR%Ma5VQq-p21k`vaBu6WE?66+5=XUd%Ay%D$irN>5LhluRWt7 zov-=f>QbMk*G##&DTQyou$s7UqjjW@k6=!I@!k+S{pP8R(2=e@io;N8E`EOB;OGoI zw6Q+{X1_I{OO0HPpBz!X!@`5YQ2)t{+!?M_iH25X(d~-Zx~cXnS9z>u?+If|iNJbx zyFU2d1!ITX64D|lE0Z{dLRqL1Ajj=CCMfC4lD3&mYR_R_VZ>_7_~|<^o*%_&jevU+ zQ4|qzci=0}Jydw|LXLCrOl1_P6Xf@c0$ieK2^7@A9UbF{@V_0p%lqW|L?5k>bVM8|p5v&2g;~r>B8uo<4N+`B zH{J)h;SYiIVx@#jI&p-v3dwL5QNV1oxPr8J%ooezTnLW>i*3Isb49%5i!&ac_dEXv zvXmVUck^QHmyrF8>CGXijC_R-y(Qr{3Zt~EmW)-nC!tiH`wlw5D*W7Pip;T?&j%kX z6DkZX4&}iw>hE(boLyjOoupf6JpvBG8}jIh!!VhnD0>}KSMMo{1#uU6kiFcA04~|7 zVO8eI&x1`g4CZ<2cYUI(n#wz2MtVFHx47yE5eL~8bot~>EHbevSt}LLMQX?odD{Ux zJMnam{d)W4da{l7&y-JrgiU~qY3$~}_F#G7|MxT)e;G{U`In&?`j<5D->}cb{}{T(4DF0BOk-=1195KB-E*o@c?`>y#4=dMtYtSY=&L{!TAjFVcq0y@AH`vH! z$41+u!Ld&}F^COPgL(EE{0X7LY&%D7-(?!kjFF7=qw<;`V{nwWBq<)1QiGJgUc^Vz ztMUlq1bZqKn17|6x6iAHbWc~l1HcmAxr%$Puv!znW)!JiukwIrqQ00|H$Z)OmGG@= zv%A8*4cq}(?qn4rN6o`$Y))(MyXr8R<2S^J+v(wmFmtac!%VOfN?&(8Nr!T@kV`N; z*Q33V3t`^rN&aBiHet)18wy{*wi1=W!B%B-Q6}SCrUl$~Hl{@!95ydml@FK8P=u4s z4e*7gV2s=YxEvskw2Ju!2%{8h01rx-3`NCPc(O zH&J0VH5etNB2KY6k4R@2Wvl^Ck$MoR3=)|SEclT2ccJ!RI9Nuter7u9@;sWf-%um;GfI!=eEIQ2l2p_YWUd{|6EG ze{yO6;lMc>;2tPrsNdi@&1K6(1;|$xe8vLgiouj%QD%gYk`4p{Ktv9|j+!OF-P?@p z;}SV|oIK)iwlBs+`ROXkhd&NK zzo__r!B>tOXpBJMDcv!Mq54P+n4(@dijL^EpO1wdg~q+!DT3lB<>9AANSe!T1XgC=J^)IP0XEZ()_vpu!!3HQyJhwh?r`Ae%Yr~b% zO*NY9t9#qWa@GCPYOF9aron7thfWT`eujS4`t2uG6)~JRTI;f(ZuoRQwjZjp5Pg34 z)rp$)Kr?R+KdJ;IO;pM{$6|2y=k_siqvp%)2||cHTe|b5Ht8&A{wazGNca zX$Ol?H)E_R@SDi~4{d-|8nGFhZPW;Cts1;08TwUvLLv&_2$O6Vt=M)X;g%HUr$&06 zISZb(6)Q3%?;3r~*3~USIg=HcJhFtHhIV(siOwV&QkQe#J%H9&E21!C*d@ln3E@J* zVqRO^<)V^ky-R|%{(9`l-(JXq9J)1r$`uQ8a}$vr9E^nNiI*thK8=&UZ0dsFN_eSl z(q~lnD?EymWLsNa3|1{CRPW60>DSkY9YQ;$4o3W7Ms&@&lv9eH!tk~N&dhqX&>K@} zi1g~GqglxkZ5pEFkllJ)Ta1I^c&Bt6#r(QLQ02yHTaJB~- zCcE=5tmi`UA>@P=1LBfBiqk)HB4t8D?02;9eXj~kVPwv?m{5&!&TFYhu>3=_ zsGmYZ^mo*-j69-42y&Jj0cBLLEulNRZ9vXE)8~mt9C#;tZs;=#M=1*hebkS;7(aGf zcs7zH(I8Eui9UU4L--))yy`&d&$In&VA2?DAEss4LAPCLd>-$i?lpXvn!gu^JJ$(DoUlc6wE98VLZ*z`QGQov5l4Fm_h?V-;mHLYDVOwKz7>e4+%AzeO>P6v}ndPW| zM>m#6Tnp7K?0mbK=>gV}=@k*0Mr_PVAgGMu$j+pWxzq4MAa&jpCDU&-5eH27Iz>m^ zax1?*HhG%pJ((tkR(V(O(L%7v7L%!_X->IjS3H5kuXQT2!ow(;%FDE>16&3r){!ex zhf==oJ!}YU89C9@mfDq!P3S4yx$aGB?rbtVH?sHpg?J5C->!_FHM%Hl3#D4eplxzQ zRA+<@LD%LKSkTk2NyWCg7u=$%F#;SIL44~S_OGR}JqX}X+=bc@swpiClB`Zbz|f!4 z7Ysah7OkR8liXfI`}IIwtEoL}(URrGe;IM8%{>b1SsqXh)~w}P>yiFRaE>}rEnNkT z!HXZUtxUp1NmFm)Dm@-{FI^aRQqpSkz}ZSyKR%Y}YHNzBk)ZIp} zMtS=aMvkgWKm9&oTcU0?S|L~CDqA+sHpOxwnswF-fEG)cXCzUR?ps@tZa$=O)=L+5 zf%m58cq8g_o}3?Bhh+c!w4(7AjxwQ3>WnVi<{{38g7yFboo>q|+7qs<$8CPXUFAN< zG&}BHbbyQ5n|qqSr?U~GY{@GJ{(Jny{bMaOG{|IkUj7tj^9pa9|FB_<+KHLxSxR;@ zHpS$4V)PP+tx}22fWx(Ku9y+}Ap;VZqD0AZW4gCDTPCG=zgJmF{|x;(rvdM|2|9a}cex6xrMkERnkE;}jvU-kmzd%_J50$M`lIPCKf+^*zL=@LW`1SaEc%=m zQ+lT06Gw+wVwvQ9fZ~#qd430v2HndFsBa9WjD0P}K(rZYdAt^5WQIvb%D^Q|pkVE^ zte$&#~zmULFACGfS#g=2OLOnIf2Of-k!(BIHjs77nr!5Q1*I9 z1%?=~#Oss!rV~?-6Gm~BWJiA4mJ5TY&iPm_$)H1_rTltuU1F3I(qTQ^U$S>%$l z)Wx1}R?ij0idp@8w-p!Oz{&*W;v*IA;JFHA9%nUvVDy7Q8woheC#|8QuDZb-L_5@R zOqHwrh|mVL9b=+$nJxM`3eE{O$sCt$UK^2@L$R(r^-_+z?lOo+me-VW=Zw z-Bn>$4ovfWd%SPY`ab-u9{INc*k2h+yH%toDHIyqQ zO68=u`N}RIIs7lsn1D){)~%>ByF<>i@qFb<-axvu(Z+6t7v<^z&gm9McRB~BIaDn$ z#xSGT!rzgad8o>~kyj#h1?7g96tOcCJniQ+*#=b7wPio>|6a1Z?_(TS{)KrPe}(8j z!#&A=k(&Pj^F;r)CI=Z{LVu>uj!_W1q4b`N1}E(i%;BWjbEcnD=mv$FL$l?zS6bW!{$7j1GR5ocn94P2u{ z70tAAcpqtQo<@cXw~@i-@6B23;317|l~S>CB?hR5qJ%J3EFgyBdJd^fHZu7AzHF(BQ!tyAz^L0`X z23S4Fe{2X$W0$zu9gm%rg~A>ijaE#GlYlrF9$ds^QtaszE#4M(OLVP2O-;XdT(XIC zatwzF*)1c+t~c{L=fMG8Z=k5lv>U0;C{caN1NItnuSMp)6G3mbahu>E#sj&oy94KC zpH}8oEw{G@N3pvHhp{^-YaZeH;K+T_1AUv;IKD<=mv^&Ueegrb!yf`4VlRl$M?wsl zZyFol(2|_QM`e_2lYSABpKR{{NlxlDSYQNkS;J66aT#MSiTx~;tUmvs-b*CrR4w=f z8+0;*th6kfZ3|5!Icx3RV11sp=?`0Jy3Fs0N4GZQMN=8HmT6%x9@{Dza)k}UwL6JT zHRDh;%!XwXr6yuuy`4;Xsn0zlR$k%r%9abS1;_v?`HX_hI|+EibVnlyE@3aL5vhQq zlIG?tN^w@0(v9M*&L+{_+RQZw=o|&BRPGB>e5=ys7H`nc8nx)|-g;s7mRc7hg{GJC zAe^vCIJhajmm7C6g! zL&!WAQ~5d_5)00?w_*|*H>3$loHrvFbitw#WvLB!JASO?#5Ig5$Ys10n>e4|3d;tS zELJ0|R4n3Az(Fl3-r^QiV_C;)lQ1_CW{5bKS15U|E9?ZgLec@%kXr84>5jV2a5v=w z?pB1GPdxD$IQL4)G||B_lI+A=08MUFFR4MxfGOu07vfIm+j=z9tp~5i_6jb`tR>qV z$#`=BQ*jpCjm$F0+F)L%xRlnS%#&gro6PiRfu^l!EVan|r3y}AHJQOORGx4~ z&<)3=K-tx518DZyp%|!EqpU!+X3Et7n2AaC5(AtrkW>_57i}$eqs$rupubg0a1+WO zGHZKLN2L0D;ab%{_S1Plm|hx8R?O14*w*f&2&bB050n!R2by zw!@XOQx$SqZ5I<(Qu$V6g>o#A!JVwErWv#(Pjx=KeS0@hxr4?13zj#oWwPS(7Ro|v z>Mp@Kmxo79q|}!5qtX2-O@U&&@6s~!I&)1WQIl?lTnh6UdKT_1R640S4~f=_xoN3- zI+O)$R@RjV$F=>Ti7BlnG1-cFKCC(t|Qjm{SalS~V-tX#+2ekRhwmN zZr`8{QF6y~Z!D|{=1*2D-JUa<(1Z=;!Ei!KiRNH?o{p5o3crFF=_pX9O-YyJchr$~ zRC`+G+8kx~fD2k*ZIiiIGR<8r&M@3H?%JVOfE>)})7ScOd&?OjgAGT@WVNSCZ8N(p zuQG~76GE3%(%h1*vUXg$vH{ua0b`sQ4f0*y=u~lgyb^!#CcPJa2mkSEHGLsnO^kb$ zru5_l#nu=Y{rSMWiYx?nO{8I!gH+?wEj~UM?IrG}E|bRIBUM>UlY<`T1EHpRr36vv zBi&dG8oxS|J$!zoaq{+JpJy+O^W(nt*|#g32bd&K^w-t>!Vu9N!k9eA8r!Xc{utY> zg9aZ(D2E0gL#W0MdjwES-7~Wa8iubPrd?8-$C4BP?*wok&O8+ykOx{P=Izx+G~hM8 z*9?BYz!T8~dzcZr#ux8kS7u7r@A#DogBH8km8Ry4slyie^n|GrTbO|cLhpqgMdsjX zJ_LdmM#I&4LqqsOUIXK8gW;V0B(7^$y#h3h>J0k^WJfAMeYek%Y-Dcb_+0zPJez!GM zAmJ1u;*rK=FNM0Nf}Y!!P9c4)HIkMnq^b;JFd!S3?_Qi2G#LIQ)TF|iHl~WKK6JmK zbv7rPE6VkYr_%_BT}CK8h=?%pk@3cz(UrZ{@h40%XgThP*-Oeo`T0eq9 zA8BnWZKzCy5e&&_GEsU4*;_k}(8l_&al5K-V*BFM=O~;MgRkYsOs%9eOY6s6AtE*<7GQAR2ulC3RAJrG_P1iQK5Z~&B z&f8X<>yJV6)oDGIlS$Y*D^Rj(cszTy5c81a5IwBr`BtnC6_e`ArI8CaTX_%rx7;cn zR-0?J_LFg*?(#n~G8cXut(1nVF0Oka$A$1FGcERU<^ggx;p@CZc?3UB41RY+wLS`LWFNSs~YP zuw1@DNN3lTd|jDL7gjBsd9}wIw}4xT2+8dBQzI00m<@?c2L%>}QLfK5%r!a-iII`p zX@`VEUH)uj^$;7jVUYdADQ2k*!1O3WdfgF?OMtUXNpQ1}QINamBTKDuv19^{$`8A1 zeq%q*O0mi@(%sZU>Xdb0Ru96CFqk9-L3pzLVsMQ`Xpa~N6CR{9Rm2)A|CI21L(%GW zh&)Y$BNHa=FD+=mBw3{qTgw)j0b!Eahs!rZnpu)z!!E$*eXE~##yaXz`KE5(nQM`s zD!$vW9XH)iMxu9R>r$VlLk9oIR%HxpUiW=BK@4U)|1WNQ=mz9a z^!KkO=>GaJ!GBXm{KJj^;kh-MkUlEQ%lza`-G&}C5y1>La1sR6hT=d*NeCnuK%_LV zOXt$}iP6(YJKc9j-Fxq~*ItVUqljQ8?oaysB-EYtFQp9oxZ|5m0^Hq(qV!S+hq#g( z?|i*H2MIr^Kxgz+3vIljQ*Feejy6S4v~jKEPTF~Qhq!(ms5>NGtRgO5vfPPc4Z^AM zTj!`5xEreIN)vaNxa|q6qWdg>+T`Ol0Uz)ckXBXEGvPNEL3R8hB3=C5`@=SYgAju1 z!)UBr{2~=~xa{b8>x2@C7weRAEuatC)3pkRhT#pMPTpSbA|tan%U7NGMvzmF?c!V8 z=pEWxbdXbTAGtWTyI?Fml%lEr-^AE}w#l(<7OIw;ctw}imYax&vR4UYNJZK6P7ZOd zP87XfhnUHxCUHhM@b*NbTi#(-8|wcv%3BGNs#zRCVV(W?1Qj6^PPQa<{yaBwZ`+<`w|;rqUY_C z&AeyKwwf*q#OW-F()lir=T^<^wjK65Lif$puuU5+tk$;e_EJ;Lu+pH>=-8=PDhkBg z8cWt%@$Sc#C6F$Vd+0507;{OOyT7Hs%nKS88q-W!$f~9*WGBpHGgNp}=C*7!RiZ5s zn1L_DbKF@B8kwhDiLKRB@lsXVVLK|ph=w%_`#owlf@s@V(pa`GY$8h%;-#h@TsO|Y8V=n@*!Rog7<7Cid%apR|x zOjhHCyfbIt%+*PCveTEcuiDi%Wx;O;+K=W?OFUV%)%~6;gl?<0%)?snDDqIvkHF{ zyI02)+lI9ov42^hL>ZRrh*HhjF9B$A@=H94iaBESBF=eC_KT$8A@uB^6$~o?3Wm5t1OIaqF^~><2?4e3c&)@wKn9bD? zoeCs;H>b8DL^F&>Xw-xjZEUFFTv>JD^O#1E#)CMBaG4DX9bD(Wtc8Rzq}9soQ8`jf zeSnHOL}<+WVSKp4kkq&?SbETjq6yr@4%SAqOG=9E(3YeLG9dtV+8vmzq+6PFPk{L; z(&d++iu=^F%b+ea$i2UeTC{R*0Isk;vFK!no<;L+(`y`3&H-~VTdKROkdyowo1iqR zbVW(3`+(PQ2>TKY>N!jGmGo7oeoB8O|P_!Ic@ zZ^;3dnuXo;WJ?S+)%P>{Hcg!Jz#2SI(s&dY4QAy_vRlmOh)QHvs_7c&zkJCmJGVvV zX;Mtb>QE+xp`KyciG$Cn*0?AK%-a|=o!+7x&&yzHQOS>8=B*R=niSnta^Pxp1`=md z#;$pS$4WCT?mbiCYU?FcHGZ#)kHVJTTBt^%XE(Q};aaO=Zik0UgLcc0I(tUpt(>|& zcxB_|fxCF7>&~5eJ=Dpn&5Aj{A^cV^^}(7w#p;HG&Q)EaN~~EqrE1qKrMAc&WXIE;>@<&)5;gD2?={Xf@Mvn@OJKw=8Mgn z!JUFMwD+s==JpjhroT&d{$kQAy%+d`a*XxDEVxy3`NHzmITrE`o!;5ClXNPb4t*8P zzAivdr{j_v!=9!^?T3y?gzmqDWX6mkzhIzJ-3S{T5bcCFMr&RPDryMcdwbBuZbsgN zGrp@^i?rcfN7v0NKGzDPGE#4yszxu=I_`MI%Z|10nFjU-UjQXXA?k8Pk|OE<(?ae) zE%vG#eZAlj*E7_3dx#Zz4kMLj>H^;}33UAankJiDy5ZvEhrjr`!9eMD8COp}U*hP+ zF}KIYx@pkccIgyxFm#LNw~G&`;o&5)2`5aogs`1~7cMZQ7zj!%L4E`2yzlQN6REX20&O<9 zKV6fyr)TScJPPzNTC2gL+0x#=u>(({{D7j)c-%tvqls3#Y?Z1m zV5WUE)zdJ{$p>yX;^P!UcXP?UD~YM;IRa#Rs5~l+*$&nO(;Ers`G=0D!twR(0GF@c zHl9E5DQI}Oz74n zfKP>&$q0($T4y$6w(p=ERAFh+>n%iaeRA%!T%<^+pg?M)@ucY<&59$x9M#n+V&>}=nO9wCV{O~lg&v#+jcUj(tQ z`0u1YH)-`U$15a{pBkGyPL0THv1P|4e@pf@3IBZS4dVJPo#H>pWq%Lr0YS-SeWash z8R7=jb28KPMI|_lo#GEO|5B?N_e``H*23{~a!AmUJ+fb4HX-%QI@lSEUxKlGV7z7Q zSKw@-TR>@1RL%w{x}dW#k1NgW+q4yt2Xf1J62Bx*O^WG8OJ|FqI4&@d3_o8Id@*)4 zYrk=>@!wv~mh7YWv*bZhxqSmFh2Xq)o=m;%n$I?GSz49l1$xRpPu_^N(vZ>*>Z<04 z2+rP70oM=NDysd!@fQdM2OcyT?3T^Eb@lIC-UG=Bw{BjQ&P`KCv$AcJ;?`vdZ4){d z&gkoUK{$!$$K`3*O-jyM1~p-7T*qb)Ys>Myt^;#1&a%O@x8A+E>! zY8=eD`ZG)LVagDLBeHg>=atOG?Kr%h4B%E6m@J^C+U|y)XX@f z8oyJDW|9g=<#f<{JRr{y#~euMnv)`7j=%cHWLc}ngjq~7k**6%4u>Px&W%4D94(r* z+akunK}O0DC2A%Xo9jyF;DobX?!1I(7%}@7F>i%&nk*LMO)bMGg2N+1iqtg+r(70q zF5{Msgsm5GS7DT`kBsjMvOrkx&|EU!{{~gL4d2MWrAT=KBQ-^zQCUq{5PD1orxlIL zq;CvlWx#f1NWvh`hg011I%?T_s!e38l*lWVt|~z-PO4~~1g)SrJ|>*tXh=QfXT)%( z+ex+inPvD&O4Ur;JGz>$sUOnWdpSLcm1X%aQDw4{dB!cnj`^muI$CJ2%p&-kULVCE z>$eMR36kN$wCPR+OFDM3-U(VOrp9k3)lI&YVFqd;Kpz~K)@Fa&FRw}L(SoD z9B4a+hQzZT-BnVltst&=kq6Y(f^S4hIGNKYBgMxGJ^;2yrO}P3;r)(-I-CZ)26Y6? z&rzHI_1GCvGkgy-t1E;r^3Le30|%$ebDRu2+gdLG)r=A~Qz`}~&L@aGJ{}vVs_GE* zVUjFnzHiXfKQbpv&bR&}l2bzIjAooB)=-XNcYmrGmBh(&iu@o!^hn0^#}m2yZZUK8 zufVm7Gq0y`Mj;9b>`c?&PZkU0j4>IL=UL&-Lp3j&47B5pAW4JceG{!XCA)kT<%2nqCxj<)uy6XR_uws~>_MEKPOpAQ!H zkn>FKh)<9DwwS*|Y(q?$^N!6(51O0 z^JM~Ax{AI1Oj$fs-S5d4T7Z_i1?{%0SsIuQ&r8#(JA=2iLcTN+?>wOL532%&dMYkT z*T5xepC+V6zxhS@vNbMoi|i)=rpli@R9~P!39tWbSSb904ekv7D#quKbgFEMTb48P zuq(VJ+&L8aWU(_FCD$3^uD!YM%O^K(dvy~Wm2hUuh6bD|#(I39Xt>N1Y{ZqXL`Fg6 zKQ?T2htHN!(Bx;tV2bfTtIj7e)liN-29s1kew>v(D^@)#v;}C4-G=7x#;-dM4yRWm zyY`cS21ulzMK{PoaQ6xChEZ}o_#}X-o}<&0)$1#3we?+QeLt;aVCjeA)hn!}UaKt< zat1fHEx13y-rXNMvpUUmCVzocPmN~-Y4(YJvQ#db)4|%B!rBsgAe+*yor~}FrNH08 z3V!97S}D7d$zbSD{$z;@IYMxM6aHdypIuS*pr_U6;#Y!_?0i|&yU*@16l z*dcMqDQgfNBf}?quiu4e>H)yTVfsp#f+Du0@=Kc41QockXkCkvu>FBd6Q+@FL!(Yx z2`YuX#eMEiLEDhp+9uFqME_E^faV&~9qjBHJkIp~%$x^bN=N)K@kvSVEMdDuzA0sn z88CBG?`RX1@#hQNd`o^V{37)!w|nA)QfiYBE^m=yQKv-fQF+UCMcuEe1d4BH7$?>b zJl-r9@0^Ie=)guO1vOd=i$_4sz>y3x^R7n4ED!5oXL3@5**h(xr%Hv)_gILarO46q+MaDOF%ChaymKoI6JU5Pg;7#2n9-18|S1;AK+ zgsn6;k6-%!QD>D?cFy}8F;r@z8H9xN1jsOBw2vQONVqBVEbkiNUqgw~*!^##ht>w0 zUOykwH=$LwX2j&nLy=@{hr)2O&-wm-NyjW7n~Zs9UlH;P7iP3 zI}S(r0YFVYacnKH(+{*)Tbw)@;6>%=&Th=+Z6NHo_tR|JCI8TJiXv2N7ei7M^Q+RM z?9o`meH$5Yi;@9XaNR#jIK^&{N|DYNNbtdb)XW1Lv2k{E>;?F`#Pq|&_;gm~&~Zc9 zf+6ZE%{x4|{YdtE?a^gKyzr}dA>OxQv+pq|@IXL%WS0CiX!V zm$fCePA%lU{%pTKD7|5NJHeXg=I0jL@$tOF@K*MI$)f?om)D63K*M|r`gb9edD1~Y zc|w7N)Y%do7=0{RC|AziW7#am$)9jciRJ?IWl9PE{G3U+$%FcyKs_0Cgq`=K3@ttV z9g;M!3z~f_?P%y3-ph%vBMeS@p7P&Ea8M@97+%XEj*(1E6vHj==d zjsoviB>j^$_^OI_DEPvFkVo(BGRo%cJeD){6Uckei=~1}>sp299|IRjhXe)%?uP0I zF5+>?0#Ye}T^Y$u_rc4=lPcq4K^D(TZG-w30-YiEM=dcK+4#o*>lJ8&JLi+3UcpZk z!^?95S^C0ja^jwP`|{<+3cBVog$(mRdQmadS+Vh~z zS@|P}=|z3P6uS+&@QsMp0no9Od&27O&14zHXGAOEy zh~OKpymK5C%;LLb467@KgIiVwYbYd6wFxI{0-~MOGfTq$nBTB!{SrWmL9Hs}C&l&l#m?s*{tA?BHS4mVKHAVMqm63H<|c5n0~k)-kbg zXidai&9ZUy0~WFYYKT;oe~rytRk?)r8bptITsWj(@HLI;@=v5|XUnSls7$uaxFRL+ zRVMGuL3w}NbV1`^=Pw*0?>bm8+xfeY(1PikW*PB>>Tq(FR`91N0c2&>lL2sZo5=VD zQY{>7dh_TX98L2)n{2OV=T10~*YzX27i2Q7W86M4$?gZIXZaBq#sA*{PH8){|GUi;oM>e?ua7eF4WFuFYZSG| zze?srg|5Ti8Og{O zeFxuw9!U+zhyk?@w zjsA6(oKD=Ka;A>Ca)oPORxK+kxH#O@zhC!!XS4@=swnuMk>t+JmLmFiE^1aX3f<)D@`%K0FGK^gg1a1j>zi z2KhV>sjU7AX3F$SEqrXSC}fRx64GDoc%!u2Yag68Lw@w9v;xOONf@o)Lc|Uh3<21ctTYu-mFZuHk*+R{GjXHIGq3p)tFtQp%TYqD=j1&y)>@zxoxUJ!G@ zgI0XKmP6MNzw>nRxK$-Gbzs}dyfFzt>#5;f6oR27ql!%+{tr+(`(>%51|k`ML} zY4eE)Lxq|JMas(;JibNQds1bUB&r}ydMQXBY4x(^&fY_&LlQC)3hylc$~8&~|06-D z#T+%66rYbHX%^KuqJED_wuGB+=h`nWA!>1n0)3wZrBG3%`b^Ozv6__dNa@%V14|!D zQ?o$z5u0^8`giv%qE!BzZ!3j;BlDlJDk)h@9{nSQeEk!z9RGW) z${RSF3phEM*ce*>Xdp}585vj$|40=&S{S-GTiE?Op*vY&Lvr9}BO$XWy80IF+6@%n z5*2ueT_g@ofP#u5pxb7n*fv^Xtt7&?SRc{*2Ka-*!BuOpf}neHGCiHy$@Ka1^Dint z;DkmIL$-e)rj4o2WQV%Gy;Xg(_Bh#qeOsTM2f@KEe~4kJ8kNLQ+;(!j^bgJMcNhvklP5Z6I+9Fq@c&D~8Fb-4rmDT!MB5QC{Dsb;BharP*O;SF4& zc$wj-7Oep7#$WZN!1nznc@Vb<_Dn%ga-O#J(l=OGB`dy=Sy&$(5-n3zzu%d7E#^8`T@}V+5B;PP8J14#4cCPw-SQTdGa2gWL0*zKM z#DfSXs_iWOMt)0*+Y>Lkd=LlyoHjublNLefhKBv@JoC>P7N1_#> zv=mLWe96%EY;!ZGSQDbZWb#;tzqAGgx~uk+-$+2_8U`!ypbwXl z^2E-FkM1?lY@yt8=J3%QK+xaZ6ok=-y%=KXCD^0r!5vUneW>95PzCkOPO*t}p$;-> ze5j-BLT_;)cZQzR2CEsm@rU7GZfFtdp*a|g4wDr%8?2QkIGasRfDWT-Dvy*U{?IHT z*}wGnzdlSptl#ZF^sf)KT|BJs&kLG91^A6ls{CzFprZ6-Y!V0Xysh%9p%iMd7HLsS zN+^Un$tDV)T@i!v?3o0Fsx2qI(AX_$dDkBzQ@fRM%n zRXk6hb9Py#JXUs+7)w@eo;g%QQ95Yq!K_d=z{0dGS+pToEI6=Bo8+{k$7&Z zo4>PH(`ce8E-Ps&uv`NQ;U$%t;w~|@E3WVOCi~R4oj5wP?%<*1C%}Jq%a^q~T7u>K zML5AKfQDv6>PuT`{SrKHRAF+^&edg6+5R_#H?Lz3iGoWo#PCEd0DS;)2U({{X#zU^ zw_xv{4x7|t!S)>44J;KfA|DC?;uQ($l+5Vp7oeqf7{GBF9356nx|&B~gs+@N^gSdd zvb*>&W)|u#F{Z_b`f#GVtQ`pYv3#||N{xj1NgB<#=Odt6{eB%#9RLt5v zIi|0u70`#ai}9fJjKv7dE!9ZrOIX!3{$z_K5FBd-Kp-&e4(J$LD-)NMTp^_pB`RT; zftVVlK2g@+1Ahv2$D){@Y#cL#dUj9*&%#6 zd2m9{1NYp>)6=oAvqdCn5#cx{AJ%S8skUgMglu2*IAtd+z1>B&`MuEAS(D(<6X#Lj z?f4CFx$)M&$=7*>9v1ER4b6!SIz-m0e{o0BfkySREchp?WdVPpQCh!q$t>?rL!&Jg zd#heM;&~A}VEm8Dvy&P|J*eAV&w!&Nx6HFV&B8jJFVTmgLaswn!cx$&%JbTsloz!3 zMEz1d`k==`Ueub_JAy_&`!ogbwx27^ZXgFNAbx=g_I~5nO^r)}&myw~+yY*cJl4$I znNJ32M&K=0(2Dj_>@39`3=FX!v3nZHno_@q^!y}%(yw0PqOo=);6Y@&ylVe>nMOZ~ zd>j#QQSBn3oaWd;qy$&5(5H$Ayi)0haAYO6TH>FR?rhqHmNOO+(})NB zLI@B@v0)eq!ug`>G<@htRlp3n!EpU|n+G+AvXFrWSUsLMBfL*ZB`CRsIVHNTR&b?K zxBgsN0BjfB>UVcJ|x%=-zb%OV7lmZc& zxiupadZVF7)6QuhoY;;FK2b*qL0J-Rn-8!X4ZY$-ZSUXV5DFd7`T41c(#lAeLMoeT z4%g655v@7AqT!i@)Edt5JMbN(=Q-6{=L4iG8RA%}w;&pKmtWvI4?G9pVRp|RTw`g0 zD5c12B&A2&P6Ng~8WM2eIW=wxd?r7A*N+&!Be7PX3s|7~z=APxm=A?5 zt>xB4WG|*Td@VX{Rs)PV0|yK`oI3^xn(4c_j&vgxk_Y3o(-`_5o`V zRTghg6%l@(qodXN;dB#+OKJEEvhfcnc#BeO2|E(5df-!fKDZ!%9!^BJ_4)9P+9Dq5 zK1=(v?KmIp34r?z{NEWnLB3Px{XYwy-akun4F7xTRr2^zeYW{gcK9)>aJDdU5;w5@ zak=<+-PLH-|04pelTb%ULpuuuJC7DgyT@D|p{!V!0v3KpDnRjANN12q6SUR3mb9<- z>2r~IApQGhstZ!3*?5V z8#)hJ0TdZg0M-BK#nGFP>$i=qk82DO z7h;Ft!D5E15OgW)&%lej*?^1~2=*Z5$2VX>V{x8SC+{i10BbtUk9@I#Vi&hX)q

Q!LwySI{Bnv%Sm)yh{^sSVJ8&h_D-BJ_YZe5eCaAWU9b$O2c z$T|{vWVRtOL!xC0DTc(Qbe`ItNtt5hr<)VijD0{U;T#bUEp381_y`%ZIav?kuYG{iyYdEBPW=*xNSc;Rlt6~F4M`5G+VtOjc z*0qGzCb@gME5udTjJA-9O<&TWd~}ysBd(eVT1-H82-doyH9RST)|+Pb{o*;$j9Tjs zhU!IlsPsj8=(x3bAKJTopW3^6AKROHR^7wZ185wJGVhA~hEc|LP;k7NEz-@4p5o}F z`AD6naG3(n=NF9HTH81=F+Q|JOz$7wm9I<+#BSmB@o_cLt2GkW9|?7mM;r!JZp89l zbo!Hp8=n!XH1{GwaDU+k)pGp`C|cXkCU5%vcH)+v@0eK>%7gWxmuMu9YLlChA|_D@ zi#5zovN_!a-0?~pUV-Rj*1P)KwdU-LguR>YM&*Nen+ln8Q$?WFCJg%DY%K}2!!1FE zDv-A%Cbwo^p(lzac&_TZ-l#9kq`mhLcY3h9ZTUVCM(Ad&=EriQY5{jJv<5K&g|*Lk zgV%ILnf1%8V2B0E&;Sp4sYbYOvvMebLwYwzkRQ#F8GpTQq#uv=J`uaSJ34OWITeSGo6+-8Xw znCk*n{kdDEi)Hi&u^)~cs@iyCkFWB2SWZU|Uc%^43ZIZQ-vWNExCCtDWjqHs;;tWf$v{}0{p0Rvxkq``)*>+Akq%|Na zA`@~-Vfe|+(AIlqru+7Ceh4nsVmO9p9jc8}HX^W&ViBDXT+uXbT#R#idPn&L>+#b6 zflC-4C5-X;kUnR~L>PSLh*gvL68}RBsu#2l`s_9KjUWRhiqF`j)`y`2`YU(>3bdBj z?>iyjEhe-~$^I5!nn%B6Wh+I`FvLNvauve~eX<+Ipl&04 zT}};W&1a3%W?dJ2=N#0t?e+aK+%t}5q%jSLvp3jZ%?&F}nOOWr>+{GFIa%wO_2`et z=JzoRR~}iKuuR+azPI8;Gf9)z3kyA4EIOSl!sRR$DlW}0>&?GbgPojmjmnln;cTqCt=ADbE zZ8GAnoM+S1(5$i8^O4t`ue;vO4i}z0wz-QEIVe5_u03;}-!G1NyY8;h^}y;tzY}i5 zqQr#Ur3Fy8sSa$Q0ys+f`!`+>9WbvU_I`Sj;$4{S>O3?#inLHCrtLy~!s#WXV=oVP zeE93*Nc`PBi4q@%Ao$x4lw9vLHM!6mn3-b_cebF|n-2vt-zYVF_&sDE--J-P;2WHo z+@n2areE0o$LjvjlV2X7ZU@j+`{*8zq`JR3gKF#EW|#+{nMyo-a>nFFTg&vhyT=b} zDa8+v0(Dgx0yRL@ZXOYIlVSZ0|MFizy0VPW8;AfA5|pe!#j zX}Py^8fl5SyS4g1WSKKtnyP+_PoOwMMwu`(i@Z)diJp~U54*-miOchy7Z35eL>^M z4p<-aIxH4VUZgS783@H%M7P9hX>t{|RU7$n4T(brCG#h9e9p! z+o`i;EGGq3&pF;~5V~eBD}lC)>if$w%Vf}AFxGqO88|ApfHf&Bvu+xdG)@vuF}Yvk z)o;~k-%+0K0g+L`Wala!$=ZV|z$e%>f0%XoLib%)!R^RoS+{!#X?h-6uu zF&&KxORdZU&EwQFITIRLo(7TA3W}y6X{?Y%y2j0It!ekU#<)$qghZtpcS>L3uh`Uj z7GY;6f$9qKynP#oS3$$a{p^{D+0oJQ71`1?OAn_m8)UGZmj3l*ZI)`V-a>MKGGFG< z&^jg#Ok%(hhm>hSrZ5;Qga4u(?^i>GiW_j9%_7M>j(^|Om$#{k+^*ULnEgzW_1gCICtAD^WpC`A z{9&DXkG#01Xo)U$OC(L5Y$DQ|Q4C6CjUKk1UkPj$nXH##J{c8e#K|&{mA*;b$r0E4 zUNo0jthwA(c&N1l=PEe8Rw_8cEl|-eya9z&H3#n`B$t#+aJ03RFMzrV@gowbe8v(c zIFM60^0&lCFO10NU4w@|61xiZ4CVXeaKjd;d?sv52XM*lS8XiVjgWpRB;&U_C0g+`6B5V&w|O6B*_q zsATxL!M}+$He)1eOWECce#eS@2n^xhlB4<_Nn?yCVEQWDs(r`|@2GqLe<#(|&P0U? z$7V5IgpWf09uIf_RazRwC?qEqRaHyL?iiS05UiGesJy%^>-C{{ypTBI&B0-iUYhk> zIk<5xpsuV@g|z(AZD+C-;A!fTG=df1=<%nxy(a(IS+U{ME4ZbDEBtcD_3V=icT6*_ z)>|J?>&6%nvHhZERBtjK+s4xnut*@>GAmA5m*OTp$!^CHTr}vM4n(X1Q*;{e-Rd2BCF-u@1ZGm z!S8hJ6L=Gl4T_SDa7Xx|-{4mxveJg=ctf`BJ*fy!yF6Dz&?w(Q_6B}WQVtNI!BVBC zKfX<>7vd6C96}XAQmF-Jd?1Q4eTfRB3q7hCh0f!(JkdWT5<{iAE#dKy*Jxq&3a1@~ z8C||Dn2mFNyrUV|<-)C^_y7@8c2Fz+2jrae9deBDu;U}tJ{^xAdxCD248(k;dCJ%o z`y3sADe>U%suxwwv~8A1+R$VB=Q?%U?4joI$um;aH+eCrBqpn- z%79D_7rb;R-;-9RTrwi9dPlg8&@tfWhhZ(Vx&1PQ+6(huX`;M9x~LrW~~#3{j0Bh2kDU$}@!fFQej4VGkJv?M4rU^x!RU zEwhu$!CA_iDjFjrJa`aocySDX16?~;+wgav;}Zut6Mg%C4>}8FL?8)Kgwc(Qlj{@#2Pt0?G`$h7P#M+qoXtlV@d}%c&OzO+QYKK`kyXaK{U(O^2DyIXCZlNQjt0^8~8JzNGrIxhj}}M z&~QZlbx%t;MJ(Vux;2tgNKGlAqphLq%pd}JG9uoVHUo?|hN{pLQ6Em%r*+7t^<);X zm~6=qChlNAVXNN*Sow->*4;}T;l;D1I-5T{Bif@4_}=>l`tK;qqDdt5zvisCKhMAH z#r}`)7VW?LZqfdmXQ%zo5bJ00{Xb9^YKrk0Nf|oIW*K@(=`o2Vndz}ZDyk{!u}PVx zzd--+_WC*U{~DH3{?GI64IB+@On&@9X>EUAo&L+G{L^dozaI4C3G#2wr~hseW@K&g zKWs{uHu-9Je!3;4pE>eBltKUXb^*hG8I&413)$J&{D4N%7PcloU6bn%jPxJyQL?g* z9g+YFFEDiE`8rW^laCNzQmi7CTnPfwyg3VDHRAl>h=In6jeaVOP@!-CP60j3+#vpL zEYmh_oP0{-gTe7Or`L6x)6w?77QVi~jD8lWN@3RHcm80iV%M1A!+Y6iHM)05iC64tb$X2lV_%Txk@0l^hZqi^%Z?#- zE;LE0uFx)R08_S-#(wC=dS&}vj6P4>5ZWjhthP=*Hht&TdLtKDR;rXEX4*z0h74FA zMCINqrh3Vq;s%3MC1YL`{WjIAPkVL#3rj^9Pj9Ss7>7duy!9H0vYF%>1jh)EPqvlr6h%R%CxDsk| z!BACz7E%j?bm=pH6Eaw{+suniuY7C9Ut~1cWfOX9KW9=H><&kQlinPV3h9R>3nJvK z4L9(DRM=x;R&d#a@oFY7mB|m8h4692U5eYfcw|QKwqRsshN(q^v$4$)HgPpAJDJ`I zkqjq(8Cd!K!+wCd=d@w%~e$=gdUgD&wj$LQ1r>-E=O@c ze+Z$x{>6(JA-fNVr)X;*)40Eym1TtUZI1Pwwx1hUi+G1Jlk~vCYeXMNYtr)1?qwyg zsX_e*$h?380O00ou?0R@7-Fc59o$UvyVs4cUbujHUA>sH!}L54>`e` zHUx#Q+Hn&Og#YVOuo*niy*GU3rH;%f``nk#NN5-xrZ34NeH$l`4@t);4(+0|Z#I>Y z)~Kzs#exIAaf--65L0UHT_SvV8O2WYeD>Mq^Y6L!Xu8%vnpofG@w!}R7M28?i1*T&zp3X4^OMCY6(Dg<-! zXmcGQrRgHXGYre7GfTJ)rhl|rs%abKT_Nt24_Q``XH{88NVPW+`x4ZdrMuO0iZ0g` z%p}y};~T5gbb9SeL8BSc`SO#ixC$@QhXxZ=B}L`tP}&k?1oSPS=4%{UOHe0<_XWln zwbl5cn(j-qK`)vGHY5B5C|QZd5)W7c@{bNVXqJ!!n$^ufc?N9C-BF2QK1(kv++h!>$QbAjq)_b$$PcJdV+F7hz0Hu@ zqj+}m0qn{t^tD3DfBb~0B36|Q`bs*xs|$i^G4uNUEBl4g;op-;Wl~iThgga?+dL7s zUP(8lMO?g{GcYpDS{NM!UA8Hco?#}eNEioRBHy4`mq!Pd-9@-97|k$hpEX>xoX+dY zDr$wfm^P&}Wu{!%?)U_(%Mn79$(ywvu*kJ9r4u|MyYLI_67U7%6Gd_vb##Nerf@>& z8W11z$$~xEZt$dPG}+*IZky+os5Ju2eRi;1=rUEeIn>t-AzC_IGM-IXWK3^6QNU+2pe=MBn4I*R@A%-iLDCOHTE-O^wo$sL_h{dcPl=^muAQb`_BRm};=cy{qSkui;`WSsj9%c^+bIDQ z0`_?KX0<-=o!t{u(Ln)v>%VGL z0pC=GB7*AQ?N7N{ut*a%MH-tdtNmNC+Yf$|KS)BW(gQJ*z$d{+{j?(e&hgTy^2|AR9vx1Xre2fagGv0YXWqtNkg*v%40v?BJBt|f9wX5 z{QTlCM}b-0{mV?IG>TW_BdviUKhtosrBqdfq&Frdz>cF~yK{P@(w{Vr7z2qKFwLhc zQuogKO@~YwyS9%+d-zD7mJG~@?EFJLSn!a&mhE5$_4xBl&6QHMzL?CdzEnC~C3$X@ zvY!{_GR06ep5;<#cKCSJ%srxX=+pn?ywDwtJ2{TV;0DKBO2t++B(tIO4)Wh`rD13P z4fE$#%zkd=UzOB74gi=-*CuID&Z3zI^-`4U^S?dHxK8fP*;fE|a(KYMgMUo`THIS1f!*6dOI2 zFjC3O=-AL`6=9pp;`CYPTdVX z8(*?V&%QoipuH0>WKlL8A*zTKckD!paN@~hh zmXzm~qZhMGVdQGd=AG8&20HW0RGV8X{$9LldFZYm zE?}`Q3i?xJRz43S?VFMmqRyvWaS#(~Lempg9nTM$EFDP(Gzx#$r)W&lpFKqcAoJh-AxEw$-bjW>`_+gEi z2w`99#UbFZGiQjS8kj~@PGqpsPX`T{YOj`CaEqTFag;$jY z8_{Wzz>HXx&G*Dx<5skhpETxIdhKH?DtY@b9l8$l?UkM#J-Snmts7bd7xayKTFJ(u zyAT&@6cAYcs{PBfpqZa%sxhJ5nSZBPji?Zlf&}#L?t)vC4X5VLp%~fz2Sx<*oN<7` z?ge=k<=X7r<~F7Tvp9#HB{!mA!QWBOf%EiSJ6KIF8QZNjg&x~-%e*tflL(ji_S^sO ztmib1rp09uon}RcsFi#k)oLs@$?vs(i>5k3YN%$T(5Or(TZ5JW9mA6mIMD08=749$ z!d+l*iu{Il7^Yu}H;lgw=En1sJpCKPSqTCHy4(f&NPelr31^*l%KHq^QE>z>Ks_bH zjbD?({~8Din7IvZeJ>8Ey=e;I?thpzD=zE5UHeO|neioJwG;IyLk?xOz(yO&0DTU~ z^#)xcs|s>Flgmp;SmYJ4g(|HMu3v7#;c*Aa8iF#UZo7CvDq4>8#qLJ|YdZ!AsH%^_7N1IQjCro

K7UpUK$>l@ zw`1S}(D?mUXu_C{wupRS-jiX~w=Uqqhf|Vb3Cm9L=T+w91Cu^ z*&Ty%sN?x*h~mJc4g~k{xD4ZmF%FXZNC;oVDwLZ_WvrnzY|{v8hc1nmx4^}Z;yriXsAf+Lp+OFLbR!&Ox?xABwl zu8w&|5pCxmu#$?Cv2_-Vghl2LZ6m7}VLEfR5o2Ou$x02uA-%QB2$c(c1rH3R9hesc zfpn#oqpbKuVsdfV#cv@5pV4^f_!WS+F>SV6N0JQ9E!T90EX((_{bSSFv9ld%I0&}9 zH&Jd4MEX1e0iqDtq~h?DBrxQX1iI0lIs<|kB$Yrh&cpeK0-^K%=FBsCBT46@h#yi!AyDq1V(#V}^;{{V*@T4WJ&U-NTq43w=|K>z8%pr_nC>%C(Wa_l78Ufib$r8Od)IIN=u>417 z`Hl{9A$mI5A(;+-Q&$F&h-@;NR>Z<2U;Y21>>Z;s@0V@SbkMQQj%_;~+qTuQ?c|AV zcWm3XZQHhP&R%QWarS%mJ!9R^&!_)*s(v+VR@I#QrAT}`17Y+l<`b-nvmDNW`De%y zrwTZ9EJrj1AFA>B`1jYDow}~*dfPs}IZMO3=a{Fy#IOILc8F0;JS4x(k-NSpbN@qM z`@aE_e}5{!$v3+qVs7u?sOV(y@1Os*Fgu`fCW9=G@F_#VQ%xf$hj0~wnnP0$hFI+@ zkQj~v#V>xn)u??YutKsX>pxKCl^p!C-o?+9;!Nug^ z{rP!|+KsP5%uF;ZCa5F;O^9TGac=M|=V z_H(PfkV1rz4jl?gJ(ArXMyWT4y(86d3`$iI4^l9`vLdZkzpznSd5Ikfrs8qcSy&>z zTIZgWZGXw0n9ibQxYWE@gI0(3#KA-dAdPcsL_|hg2@~C!VZDM}5;v_Nykfq!*@*Zf zE_wVgx82GMDryKO{U{D>vSzSc%B~|cjDQrt5BN=Ugpsf8H8f1lR4SGo#hCuXPL;QQ z#~b?C4MoepT3X`qdW2dNn& zo8)K}%Lpu>0tQei+{>*VGErz|qjbK#9 zvtd8rcHplw%YyQCKR{kyo6fgg!)6tHUYT(L>B7er5)41iG`j$qe*kSh$fY!PehLcD zWeKZHn<492B34*JUQh=CY1R~jT9Jt=k=jCU2=SL&&y5QI2uAG2?L8qd2U(^AW#{(x zThSy=C#>k+QMo^7caQcpU?Qn}j-`s?1vXuzG#j8(A+RUAY})F@=r&F(8nI&HspAy4 z4>(M>hI9c7?DCW8rw6|23?qQMSq?*Vx?v30U%luBo)B-k2mkL)Ljk5xUha3pK>EEj z@(;tH|M@xkuN?gsz;*bygizwYR!6=(Xgcg^>WlGtRYCozY<rFX2E>kaZo)O<^J7a`MX8Pf`gBd4vrtD|qKn&B)C&wp0O-x*@-|m*0egT=-t@%dD zgP2D+#WPptnc;_ugD6%zN}Z+X4=c61XNLb7L1gWd8;NHrBXwJ7s0ce#lWnnFUMTR& z1_R9Fin4!d17d4jpKcfh?MKRxxQk$@)*hradH2$3)nyXep5Z;B z?yX+-Bd=TqO2!11?MDtG0n(*T^!CIiF@ZQymqq1wPM_X$Iu9-P=^}v7npvvPBu!d$ z7K?@CsA8H38+zjA@{;{kG)#AHME>Ix<711_iQ@WWMObXyVO)a&^qE1GqpP47Q|_AG zP`(AD&r!V^MXQ^e+*n5~Lp9!B+#y3#f8J^5!iC@3Y@P`;FoUH{G*pj*q7MVV)29+j z>BC`a|1@U_v%%o9VH_HsSnM`jZ-&CDvbiqDg)tQEnV>b%Ptm)T|1?TrpIl)Y$LnG_ zzKi5j2Fx^K^PG1=*?GhK;$(UCF-tM~^=Z*+Wp{FSuy7iHt9#4n(sUuHK??@v+6*|10Csdnyg9hAsC5_OrSL;jVkLlf zHXIPukLqbhs~-*oa^gqgvtpgTk_7GypwH><53riYYL*M=Q@F-yEPLqQ&1Sc zZB%w}T~RO|#jFjMWcKMZccxm-SL)s_ig?OC?y_~gLFj{n8D$J_Kw%{r0oB8?@dWzn zB528d-wUBQzrrSSLq?fR!K%59Zv9J4yCQhhDGwhptpA5O5U?Hjqt>8nOD zi{)0CI|&Gu%zunGI*XFZh(ix)q${jT8wnnzbBMPYVJc4HX*9d^mz|21$=R$J$(y7V zo0dxdbX3N#=F$zjstTf*t8vL)2*{XH!+<2IJ1VVFa67|{?LP&P41h$2i2;?N~RA30LV`BsUcj zfO9#Pg1$t}7zpv#&)8`mis3~o+P(DxOMgz-V*(?wWaxi?R=NhtW}<#^Z?(BhSwyar zG|A#Q7wh4OfK<|DAcl9THc-W4*>J4nTevsD%dkj`U~wSUCh15?_N@uMdF^Kw+{agk zJ`im^wDqj`Ev)W3k3stasP`88-M0ZBs7;B6{-tSm3>I@_e-QfT?7|n0D~0RRqDb^G zyHb=is;IwuQ&ITzL4KsP@Z`b$d%B0Wuhioo1CWttW8yhsER1ZUZzA{F*K=wmi-sb#Ju+j z-l@In^IKnb{bQG}Ps>+Vu_W#grNKNGto+yjA)?>0?~X`4I3T@5G1)RqGUZuP^NJCq&^HykuYtMDD8qq+l8RcZNJsvN(10{ zQ1$XcGt}QH-U^WU!-wRR1d--{B$%vY{JLWIV%P4-KQuxxDeJaF#{eu&&r!3Qu{w}0f--8^H|KwE>)ORrcR+2Qf zb})DRcH>k0zWK8@{RX}NYvTF;E~phK{+F;MkIP$)T$93Ba2R2TvKc>`D??#mv9wg$ zd~|-`Qx5LwwsZ2hb*Rt4S9dsF%Cny5<1fscy~)d;0m2r$f=83<->c~!GNyb!U)PA; zq^!`@@)UaG)Ew(9V?5ZBq#c%dCWZrplmuM`o~TyHjAIMh0*#1{B>K4po-dx$Tk-Cq z=WZDkP5x2W&Os`N8KiYHRH#UY*n|nvd(U>yO=MFI-2BEp?x@=N<~CbLJBf6P)}vLS?xJXYJ2^<3KJUdrwKnJnTp{ zjIi|R=L7rn9b*D#Xxr4*R<3T5AuOS+#U8hNlfo&^9JO{VbH!v9^JbK=TCGR-5EWR@ zN8T-_I|&@A}(hKeL4_*eb!1G8p~&_Im8|wc>Cdir+gg90n1dw?QaXcx6Op_W1r=axRw>4;rM*UOpT#Eb9xU1IiWo@h?|5uP zka>-XW0Ikp@dIe;MN8B01a7+5V@h3WN{J=HJ*pe0uwQ3S&MyWFni47X32Q7SyCTNQ z+sR!_9IZa5!>f&V$`q!%H8ci!a|RMx5}5MA_kr+bhtQy{-^)(hCVa@I!^TV4RBi zAFa!Nsi3y37I5EK;0cqu|9MRj<^r&h1lF}u0KpKQD^5Y+LvFEwM zLU@@v4_Na#Axy6tn3P%sD^5P#<7F;sd$f4a7LBMk zGU^RZHBcxSA%kCx*eH&wgA?Qwazm8>9SCSz_!;MqY-QX<1@p$*T8lc?@`ikEqJ>#w zcG``^CoFMAhdEXT9qt47g0IZkaU)4R7wkGs^Ax}usqJ5HfDYAV$!=6?>J6+Ha1I<5 z|6=9soU4>E))tW$<#>F ziZ$6>KJf0bPfbx_)7-}tMINlc=}|H+$uX)mhC6-Hz+XZxsKd^b?RFB6et}O#+>Wmw9Ec9) z{q}XFWp{3@qmyK*Jvzpyqv57LIR;hPXKsrh{G?&dRjF%Zt5&m20Ll?OyfUYC3WRn{cgQ?^V~UAv+5 z&_m#&nIwffgX1*Z2#5^Kl4DbE#NrD&Hi4|7SPqZ}(>_+JMz=s|k77aEL}<=0Zfb)a z%F(*L3zCA<=xO)2U3B|pcTqDbBoFp>QyAEU(jMu8(jLA61-H!ucI804+B!$E^cQQa z)_ERrW3g!B9iLb3nn3dlkvD7KsY?sRvls3QC0qPi>o<)GHx%4Xb$5a3GBTJ(k@`e@ z$RUa^%S15^1oLEmA=sayrP5;9qtf!Z1*?e$ORVPsXpL{jL<6E)0sj&swP3}NPmR%FM?O>SQgN5XfHE< zo(4#Cv11(%Nnw_{_Ro}r6=gKd{k?NebJ~<~Kv0r(r0qe4n3LFx$5%x(BKvrz$m?LG zjLIc;hbj0FMdb9aH9Lpsof#yG$(0sG2%RL;d(n>;#jb!R_+dad+K;Ccw!|RY?uS(a zj~?=&M!4C(5LnlH6k%aYvz@7?xRa^2gml%vn&eKl$R_lJ+e|xsNfXzr#xuh(>`}9g zLHSyiFwK^-p!;p$yt7$F|3*IfO3Mlu9e>Dpx8O`37?fA`cj`C0B-m9uRhJjs^mRp# zWB;Aj6|G^1V6`jg7#7V9UFvnB4((nIwG?k%c7h`?0tS8J3Bn0t#pb#SA}N-|45$-j z$R>%7cc2ebAClXc(&0UtHX<>pd)akR3Kx_cK+n<}FhzmTx!8e9^u2e4%x{>T6pQ`6 zO182bh$-W5A3^wos0SV_TgPmF4WUP-+D25KjbC{y_6W_9I2_vNKwU(^qSdn&>^=*t z&uvp*@c8#2*paD!ZMCi3;K{Na;I4Q35zw$YrW5U@Kk~)&rw;G?d7Q&c9|x<Hg|CNMsxovmfth*|E*GHezPTWa^Hd^F4!B3sF;)? z(NaPyAhocu1jUe(!5Cy|dh|W2=!@fNmuNOzxi^tE_jAtzNJ0JR-avc_H|ve#KO}#S z#a(8secu|^Tx553d4r@3#6^MHbH)vmiBpn0X^29xEv!Vuh1n(Sr5I0V&`jA2;WS|Y zbf0e}X|)wA-Pf5gBZ>r4YX3Mav1kKY(ulAJ0Q*jB)YhviHK)w!TJsi3^dMa$L@^{` z_De`fF4;M87vM3Ph9SzCoCi$#Fsd38u!^0#*sPful^p5oI(xGU?yeYjn;Hq1!wzFk zG&2w}W3`AX4bxoVm03y>ts{KaDf!}b&7$(P4KAMP=vK5?1In^-YYNtx1f#}+2QK@h zeSeAI@E6Z8a?)>sZ`fbq9_snl6LCu6g>o)rO;ijp3|$vig+4t} zylEo7$SEW<_U+qgVcaVhk+4k+C9THI5V10qV*dOV6pPtAI$)QN{!JRBKh-D zk2^{j@bZ}yqW?<#VVuI_27*cI-V~sJiqQv&m07+10XF+#ZnIJdr8t`9s_EE;T2V;B z4UnQUH9EdX%zwh-5&wflY#ve!IWt0UE-My3?L#^Bh%kcgP1q{&26eXLn zTkjJ*w+(|_>Pq0v8{%nX$QZbf)tbJaLY$03;MO=Ic-uqYUmUCuXD>J>o6BCRF=xa% z3R4SK9#t1!K4I_d>tZgE>&+kZ?Q}1qo4&h%U$GfY058s%*=!kac{0Z+4Hwm!)pFLR zJ+5*OpgWUrm0FPI2ib4NPJ+Sk07j(`diti^i#kh&f}i>P4~|d?RFb#!JN)~D@)beox}bw?4VCf^y*`2{4`-@%SFTry2h z>9VBc9#JxEs1+0i2^LR@B1J`B9Ac=#FW=(?2;5;#U$0E0UNag_!jY$&2diQk_n)bT zl5Me_SUvqUjwCqmVcyb`igygB_4YUB*m$h5oeKv3uIF0sk}~es!{D>4r%PC*F~FN3owq5e0|YeUTSG#Vq%&Gk7uwW z0lDo#_wvflqHeRm*}l?}o;EILszBt|EW*zNPmq#?4A+&i0xx^?9obLyY4xx=Y9&^G;xYXYPxG)DOpPg!i_Ccl#3L}6xAAZzNhPK1XaC_~ z!A|mlo?Be*8Nn=a+FhgpOj@G7yYs(Qk(8&|h@_>w8Y^r&5nCqe0V60rRz?b5%J;GYeBqSAjo|K692GxD4` zRZyM2FdI+-jK2}WAZTZ()w_)V{n5tEb@>+JYluDozCb$fA4H)$bzg(Ux{*hXurjO^ zwAxc+UXu=&JV*E59}h3kzQPG4M)X8E*}#_&}w*KEgtX)cU{vm9b$atHa;s>| z+L6&cn8xUL*OSjx4YGjf6{Eq+Q3{!ZyhrL&^6Vz@jGbI%cAM9GkmFlamTbcQGvOlL zmJ?(FI)c86=JEs|*;?h~o)88>12nXlpMR4@yh%qdwFNpct;vMlc=;{FSo*apJ;p}! zAX~t;3tb~VuP|ZW;z$=IHf->F@Ml)&-&Bnb{iQyE#;GZ@C$PzEf6~q}4D>9jic@mTO5x76ulDz@+XAcm35!VSu zT*Gs>;f0b2TNpjU_BjHZ&S6Sqk6V1370+!eppV2H+FY!q*n=GHQ!9Rn6MjY!Jc77A zG7Y!lFp8?TIHN!LXO?gCnsYM-gQxsm=Ek**VmZu7vnuufD7K~GIxfxbsQ@qv2T zPa`tvHB$fFCyZl>3oYg?_wW)C>^_iDOc^B7klnTOoytQH18WkOk)L2BSD0r%xgRSW zQS9elF^?O=_@|58zKLK;(f77l-Zzu}4{fXed2saq!5k#UZAoDBqYQS{sn@j@Vtp|$ zG%gnZ$U|9@u#w1@11Sjl8ze^Co=)7yS(}=;68a3~g;NDe_X^}yJj;~s8xq9ahQ5_r zxAlTMnep*)w1e(TG%tWsjo3RR;yVGPEO4V{Zp?=a_0R#=V^ioQu4YL=BO4r0$$XTX zZfnw#_$V}sDAIDrezGQ+h?q24St0QNug_?{s-pI(^jg`#JRxM1YBV;a@@JQvH8*>> zIJvku74E0NlXkYe_624>znU0J@L<-c=G#F3k4A_)*;ky!C(^uZfj%WB3-*{*B$?9+ zDm$WFp=0(xnt6`vDQV3Jl5f&R(Mp};;q8d3I%Kn>Kx=^;uSVCw0L=gw53%Bp==8Sw zxtx=cs!^-_+i{2OK`Q;913+AXc_&Z5$@z3<)So0CU3;JAv=H?@Zpi~riQ{z-zLtVL z!oF<}@IgJp)Iyz1zVJ42!SPHSkjYNS4%ulVVIXdRuiZ@5Mx8LJS}J#qD^Zi_xQ@>DKDr-_e#>5h3dtje*NcwH_h;i{Sx7}dkdpuW z(yUCjckQsagv*QGMSi9u1`Z|V^}Wjf7B@q%j2DQXyd0nOyqg%m{CK_lAoKlJ7#8M} z%IvR?Vh$6aDWK2W!=i?*<77q&B8O&3?zP(Cs@kapc)&p7En?J;t-TX9abGT#H?TW? ztO5(lPKRuC7fs}zwcUKbRh=7E8wzTsa#Z{a`WR}?UZ%!HohN}d&xJ=JQhpO1PI#>X zHkb>pW04pU%Bj_mf~U}1F1=wxdBZu1790>3Dm44bQ#F=T4V3&HlOLsGH)+AK$cHk6 zia$=$kog?)07HCL*PI6}DRhpM^*%I*kHM<#1Se+AQ!!xyhcy6j7`iDX7Z-2i73_n# zas*?7LkxS-XSqv;YBa zW_n*32D(HTYQ0$feV_Fru1ZxW0g&iwqixPX3=9t4o)o|kOo79V$?$uh?#8Q8e>4e)V6;_(x&ViUVxma+i25qea;d-oK7ouuDsB^ab{ zu1qjQ%`n56VtxBE#0qAzb7lph`Eb-}TYpXB!H-}3Ykqyp`otprp7{VEuW*^IR2n$Fb99*nAtqT&oOFIf z@w*6>YvOGw@Ja?Pp1=whZqydzx@9X4n^2!n83C5{C?G@|E?&$?p*g68)kNvUTJ)I6 z1Q|(#UuP6pj78GUxq11m-GSszc+)X{C2eo-?8ud9sB=3(D47v?`JAa{V(IF zPZQ_0AY*9M97>Jf<o%#O_%Wq}8>YM=q0|tGY+hlXcpE=Z4Od z`NT7Hu2hnvRoqOw@g1f=bv`+nba{GwA$Ak0INlqI1k<9!x_!sL()h?hEWoWrdU3w` zZ%%)VR+Bc@_v!C#koM1p-3v_^L6)_Ktj4HE>aUh%2XZE@JFMOn)J~c`_7VWNb9c-N z2b|SZMR4Z@E7j&q&9(6H3yjEu6HV7{2!1t0lgizD;mZ9$r(r7W5G$ky@w(T_dFnOD z*p#+z$@pKE+>o@%eT(2-p_C}wbQ5s(%Sn_{$HDN@MB+Ev?t@3dPy`%TZ!z}AThZSu zN<1i$siJhXFdjV zP*y|V<`V8t=h#XTRUR~5`c`Z9^-`*BZf?WAehGdg)E2Je)hqFa!k{V(u+(hTf^Yq& zoruUh2(^3pe)2{bvt4&4Y9CY3js)PUHtd4rVG57}uFJL)D(JfSIo^{P=7liFXG zq5yqgof0V8paQcP!gy+;^pp-DA5pj=gbMN0eW=-eY+N8~y+G>t+x}oa!5r>tW$xhI zPQSv=pi;~653Gvf6~*JcQ%t1xOrH2l3Zy@8AoJ+wz@daW@m7?%LXkr!bw9GY@ns3e zSfuWF_gkWnesv?s3I`@}NgE2xwgs&rj?kH-FEy82=O8`+szN ziHch`vvS`zNfap14!&#i9H@wF7}yIPm=UB%(o(}F{wsZ(wA0nJ2aD^@B41>>o-_U6 zUqD~vdo48S8~FTb^+%#zcbQiiYoDKYcj&$#^;Smmb+Ljp(L=1Kt_J!;0s%1|JK}Wi z;={~oL!foo5n8=}rs6MmUW~R&;SIJO3TL4Ky?kh+b2rT9B1Jl4>#Uh-Bec z`Hsp<==#UEW6pGPhNk8H!!DUQR~#F9jEMI6T*OWfN^Ze&X(4nV$wa8QUJ>oTkruH# zm~O<`J7Wxseo@FqaZMl#Y(mrFW9AHM9Kb|XBMqaZ2a)DvJgYipkDD_VUF_PKd~dT7 z#02}bBfPn9a!X!O#83=lbJSK#E}K&yx-HI#T6ua)6o0{|={*HFusCkHzs|Fn&|C3H zBck1cmfcWVUN&i>X$YU^Sn6k2H;r3zuXbJFz)r5~3$d$tUj(l1?o={MM){kjgqXRO zc5R*#{;V7AQh|G|)jLM@wGAK&rm2~@{Pewv#06pHbKn#wL0P6F1!^qw9g&cW3Z=9} zj)POhOlwsh@eF=>z?#sIs*C-Nl(yU!#DaiaxhEs#iJqQ8w%(?+6lU02MYSeDkr!B- zPjMv+on6OLXgGnAtl(ao>|X2Y8*Hb}GRW5}-IzXnoo-d0!m4Vy$GS!XOLy>3_+UGs z2D|YcQx@M#M|}TDOetGi{9lGo9m-=0-^+nKE^*?$^uHkxZh}I{#UTQd;X!L+W@jm( zDg@N4+lUqI92o_rNk{3P>1gxAL=&O;x)ZT=q1mk0kLlE$WeWuY_$0`0jY-Kkt zP*|m3AF}Ubd=`<>(Xg0har*_@x2YH}bn0Wk*OZz3*e5;Zc;2uBdnl8?&XjupbkOeNZsNh6pvsq_ydmJI+*z**{I{0K)-;p1~k8cpJXL$^t!-`E}=*4G^-E8>H!LjTPxSx zcF+cS`ommfKMhNSbas^@YbTpH1*RFrBuATUR zt{oFWSk^$xU&kbFQ;MCX22RAN5F6eq9UfR$ut`Jw--p2YX)A*J69m^!oYfj2y7NYcH6&r+0~_sH^c^nzeN1AU4Ga7=FlR{S|Mm~MpzY0$Z+p2W(a={b-pR9EO1Rs zB%KY|@wLcAA@)KXi!d2_BxrkhDn`DT1=Dec}V!okd{$+wK z4E{n8R*xKyci1(CnNdhf$Dp2(Jpof0-0%-38X=Dd9PQgT+w%Lshx9+loPS~MOm%ZT zt%2B2iL_KU_ita%N>xjB!#71_3=3c}o zgeW~^U_ZTJQ2!PqXulQd=3b=XOQhwATK$y(9$#1jOQ4}4?~l#&nek)H(04f(Sr=s| zWv7Lu1=%WGk4FSw^;;!8&YPM)pQDCY9DhU`hMty1@sq1=Tj7bFsOOBZOFlpR`W>-J$-(kezWJj;`?x-v>ev{*8V z8p|KXJPV$HyQr1A(9LVrM47u-XpcrIyO`yWvx1pVYc&?154aneRpLqgx)EMvRaa#|9?Wwqs2+W8n5~79G z(}iCiLk;?enn}ew`HzhG+tu+Ru@T+K5juvZN)wY;x6HjvqD!&!)$$;1VAh~7fg0K| zEha#aN=Yv|3^~YFH}cc38ovVb%L|g@9W6fo(JtT6$fa?zf@Ct88e}m?i)b*Jgc{fl zExfdvw-BYDmH6>(4QMt#p0;FUIQqkhD}aH?a7)_%JtA~soqj{ppP_82yi9kaxuK>~ ze_)Zt>1?q=ZH*kF{1iq9sr*tVuy=u>Zev}!gEZx@O6-fjyu9X00gpIl-fS_pzjpqJ z1yqBmf9NF!jaF<+YxgH6oXBdK)sH(>VZ)1siyA$P<#KDt;8NT*l_0{xit~5j1P)FN zI8hhYKhQ)i z37^aP13B~u65?sg+_@2Kr^iWHN=U;EDSZ@2W2!5ALhGNWXnFBY%7W?1 z=HI9JzQ-pLKZDYTv<0-lt|6c-RwhxZ)mU2Os{bsX_i^@*fKUj8*aDO5pks=qn3Dv6 zwggpKLuyRCTVPwmw1r}B#AS}?X7b837UlXwp~E2|PJw2SGVueL7){Y&z!jL!XN=0i zU^Eig`S2`{+gU$68aRdWx?BZ{sU_f=8sn~>s~M?GU~`fH5kCc; z8ICp+INM3(3{#k32RZdv6b9MQYdZXNuk7ed8;G?S2nT+NZBG=Tar^KFl2SvhW$bGW#kdWL-I)s_IqVnCDDM9fm8g;P;8 z7t4yZn3^*NQfx7SwmkzP$=fwdC}bafQSEF@pd&P8@H#`swGy_rz;Z?Ty5mkS%>m#% zp_!m9e<()sfKiY(nF<1zBz&&`ZlJf6QLvLhl`_``%RW&{+O>Xhp;lwSsyRqGf=RWd zpftiR`={2(siiPAS|p}@q=NhVc0ELprt%=fMXO3B)4ryC2LT(o=sLM7hJC!}T1@)E zA3^J$3&1*M6Xq>03FX`R&w*NkrZE?FwU+Muut;>qNhj@bX17ZJxnOlPSZ=Zeiz~T_ zOu#yc3t6ONHB;?|r4w+pI)~KGN;HOGC)txxiUN8#mexj+W(cz%9a4sx|IRG=}ia zuEBuba3AHsV2feqw-3MvuL`I+2|`Ud4~7ZkN=JZ;L20|Oxna5vx1qbIh#k2O4$RQF zo`tL()zxaqibg^GbB+BS5#U{@K;WWQj~GcB1zb}zJkPwH|5hZ9iH2308!>_;%msji zJHSL~s)YHBR=Koa1mLEOHos*`gp=s8KA-C zu0aE+W!#iJ*0xqKm3A`fUGy#O+X+5W36myS>Uh2!R*s$aCU^`K&KKLCCDkejX2p=5 z%o7-fl03x`gaSNyr?3_JLv?2RLS3F*8ub>Jd@^Cc17)v8vYEK4aqo?OS@W9mt%ITJ z9=S2%R8M){CugT@k~~0x`}Vl!svYqX=E)c_oU6o}#Hb^%G1l3BudxA{F*tbjG;W_>=xV73pKY53v%>I)@D36I_@&p$h|Aw zonQS`07z_F#@T-%@-Tb|)7;;anoD_WH>9ewFy(ZcEOM$#Y)8>qi7rCnsH9GO-_7zF zu*C87{Df1P4TEOsnzZ@H%&lvV(3V@;Q!%+OYRp`g05PjY^gL$^$-t0Y>H*CDDs?FZly*oZ&dxvsxaUWF!{em4{A>n@vpXg$dwvt@_rgmHF z-MER`ABa8R-t_H*kv>}CzOpz;!>p^^9ztHMsHL|SRnS<-y5Z*r(_}c4=fXF`l^-i}>e7v!qs_jv zqvWhX^F=2sDNWA9c@P0?lUlr6ecrTKM%pNQ^?*Lq?p-0~?_j50xV%^(+H>sMul#Tw zeciF*1=?a7cI(}352%>LO96pD+?9!fNyl^9v3^v&Y4L)mNGK0FN43&Xf8jUlxW1Bw zyiu2;qW-aGNhs=zbuoxnxiwZ3{PFZM#Kw)9H@(hgX23h(`Wm~m4&TvoZoYp{plb^> z_#?vXcxd>r7K+1HKJvhed>gtK`TAbJUazUWQY6T~t2af%#<+Veyr%7-#*A#@&*;@g58{i|E%6yC_InGXCOd{L0;$)z#?n7M`re zh!kO{6=>7I?*}czyF7_frt#)s1CFJ_XE&VrDA?Dp3XbvF{qsEJgb&OLSNz_5g?HpK z9)8rsr4JN!Af3G9!#Qn(6zaUDqLN(g2g8*M)Djap?WMK9NKlkC)E2|-g|#-rp%!Gz zAHd%`iq|81efi93m3yTBw3g0j#;Yb2X{mhRAI?&KDmbGqou(2xiRNb^sV}%%Wu0?< z?($L>(#BO*)^)rSgyNRni$i`R4v;GhlCZ8$@e^ROX(p=2_v6Y!%^As zu022)fHdv_-~Yu_H6WVPLpHQx!W%^6j)cBhS`O3QBW#x(eX54d&I22op(N59b*&$v zFiSRY6rOc^(dgSV1>a7-5C;(5S5MvKcM2Jm-LD9TGqDpP097%52V+0>Xqq!! zq4e3vj53SE6i8J`XcQB|MZPP8j;PAOnpGnllH6#Ku~vS42xP*Nz@~y%db7Xi8s09P z1)e%8ys6&M8D=Dt6&t`iKG_4X=!kgRQoh%Z`dc&mlOUqXk-k`jKv9@(a^2-Upw>?< zt5*^DV~6Zedbec4NVl($2T{&b)zA@b#dUyd>`2JC0=xa_fIm8{5um zr-!ApXZhC8@=vC2WyxO|!@0Km)h8ep*`^he92$@YwP>VcdoS5OC^s38e#7RPsg4j+ zbVGG}WRSET&ZfrcR(x~k8n1rTP%CnfUNKUonD$P?FtNFF#cn!wEIab-;jU=B1dHK@ z(;(yAQJ`O$sMn>h;pf^8{JISW%d+@v6@CnXh9n5TXGC}?FI9i-D0OMaIg&mAg=0Kn zNJ7oz5*ReJukD55fUsMuaP+H4tDN&V9zfqF@ zr=#ecUk9wu{0;!+gl;3Bw=Vn^)z$ahVhhw)io!na&9}LmWurLb0zubxK=UEnU*{5P z+SP}&*(iBKSO4{alBHaY^)5Q=mZ+2OwIooJ7*Q5XJ+2|q`9#f?6myq!&oz?klihLq z4C)$XP!BNS0G_Z1&TM>?Jk{S~{F3n83ioli=IO6f%wkvCl(RFFw~j0tb{GvXTx>*sB0McY0s&SNvj4+^h`9nJ_wM>F!Uc>X}9PifQekn0sKI2SAJP!a4h z5cyGTuCj3ZBM^&{dRelIlT^9zcfaAuL5Y~bl!ppSf`wZbK$z#6U~rdclk``e+!qhe z6Qspo*%<)eu6?C;Bp<^VuW6JI|Ncvyn+LlSl;Mp22Bl7ARQ0Xc24%29(ZrdsIPw&-=yHQ7_Vle|5h>AST0 zUGX2Zk34vp?U~IHT|;$U86T+UUHl_NE4m|}>E~6q``7hccCaT^#y+?wD##Q%HwPd8 zV3x4L4|qqu`B$4(LXqDJngNy-{&@aFBvVsywt@X^}iH7P%>bR?ciC$I^U-4Foa`YKI^qDyGK7k%E%c_P=yzAi`YnxGA%DeNd++j3*h^ z=rn>oBd0|~lZ<6YvmkKY*ZJlJ;Im0tqgWu&E92eqt;+NYdxx`eS(4Hw_Jb5|yVvBg z*tbdY^!AN;luEyN4VRhS@-_DC{({ziH{&Z}iGElSV~qvT>L-8G%+yEL zX#MFOhj{InyKG=mvW-<1B@c-}x$vA(nU?>S>0*eN#!SLzQ)Ex7fvQ)S4D<8|I#N$3 zT5Ei`Z?cxBODHX8(Xp73v`IsAYC@9b;t}z0wxVuQSY1J^GRwDPN@qbM-ZF48T$GZ< z8WU+;Pqo?{ghI-KZ-i*ydXu`Ep0Xw^McH_KE9J0S7G;x8Fe`DVG?j3Pv=0YzJ}yZR z%2=oqHiUjvuk0~Ca>Kol4CFi0_xQT~;_F?=u+!kIDl-9g`#ZNZ9HCy17Ga1v^Jv9# z{T4Kb1-AzUxq*MutfOWWZgD*HnFfyYg0&e9f(5tZ>krPF6{VikNeHoc{linPPt#Si z&*g>(c54V8rT_AX!J&bNm-!umPvOR}vDai#`CX___J#=zeB*{4<&2WpaDncZsOkp* zsg<%@@rbrMkR_ux9?LsQxzoBa1s%$BBn6vk#{&&zUwcfzeCBJUwFYSF$08qDsB;gWQN*g!p8pxjofWbqNSZOEKOaTx@+* zwdt5*Q47@EOZ~EZL9s?1o?A%9TJT=Ob_13yyugvPg*e&ZU(r6^k4=2+D-@n=Hv5vu zSXG|hM(>h9^zn=eQ=$6`JO&70&2|%V5Lsx>)(%#;pcOfu>*nk_3HB_BNaH$`jM<^S zcSftDU1?nL;jy)+sfonQN}(}gUW?d_ikr*3=^{G)=tjBtEPe>TO|0ddVB zTklrSHiW+!#26frPXQQ(YN8DG$PZo?(po(QUCCf_OJC`pw*uey00%gmH!`WJkrKXj2!#6?`T25mTu9OJp2L8z3! z=arrL$ZqxuE{%yV)14Kd>k}j7pxZ6#$Dz8$@WV5p8kTqN<-7W)Q7Gt2{KoOPK_tZ| zf2WG~O5@{qPI+W<4f_;reuFVdO^5`ADC1!JQE|N`s3cq@(0WB!n0uh@*c{=LAd;~} zyGK@hbF-Oo+!nN)@i*O(`@FA#u?o=~e{`4O#5}z&=UkU*50fOrzi11D^&FOqe>wii z?*k+2|EcUs;Gx{!@KBT~>PAwLrIDT7Th=Utu?~?np@t^gFs?zgX=D${RwOY^WGh-+ z+#4$066ISh8eYW#FXWp~S`<*%O^ZuItL1Tyqt8#tZ zY120E;^VG`!lZn&3sPd$RkdHpU#|w+bYV)pJC|SH9g%|5IkxVTQcBA4CL0}$&}ef@ zW^Vtj%M;;_1xxP9x#ex17&4N*{ksO*_4O}xYu(p*JkL#yr}@7b)t5X?%CY<+s5_MJ zuiqt+N_;A(_)%lumoyRFixWa-M7qK_9s6<1X?JDa9fP!+_6u~~M$5L=ipB=7(j#f< zZ34J%=bs549%~_mA(|={uZNs_0?o7;-LBP(ZRnkd{-^|2|=4vUTmtByHL8 zEph`(LSEzQj68a+`d$V<45J7cyv^#|^|%fD#si1Nx!4NW*`l*{->HEWNh6-|g>-=r zXmQ|-i}Ku$ndUeHQ^&ieT!Lf}vf6GaqW9$DJ2NWrqwPY%%4nip$@vK$nRp*_C-v<| zuKz~ZyN&<%!NS26&x?jhy+@awJipMQ-8(X4#Ae5??U<1QMt1l9R=w9fAnEF}NYu$2 z>6}Vkc zIb*A?G*z8^IvibmBKn_u^5&T_1oey0gZS2~obf(#xk=erZGTEdQnt3DMGM+0oPwss zj5zXD;(oWhB_T@~Ig#9@v)AKtXu3>Inmgf@A|-lD-1U>cNyl3h?ADD9)GG4}zUGPk zZzaXe!~Kf?<~@$G?Uql3t8jy9{2!doq4=J}j9ktTxss{p6!9UdjyDERlA*xZ!=Q)KDs5O)phz>Vq3BNGoM(H|=1*Q4$^2fTZw z(%nq1P|5Rt81}SYJpEEzMPl5VJsV5&4e)ZWKDyoZ>1EwpkHx-AQVQc8%JMz;{H~p{=FXV>jIxvm4X*qv52e?Y-f%DJ zxEA165GikEASQ^fH6K#d!Tpu2HP{sFs%E=e$gYd$aj$+xue6N+Wc(rAz~wUsk2`(b z8Kvmyz%bKQxpP}~baG-rwYcYCvkHOi zlkR<=>ZBTU*8RF_d#Bl@zZsRIhx<%~Z@Z=ik z>adw3!DK(8R|q$vy{FTxw%#xliD~6qXmY^7_9kthVPTF~Xy1CfBqbU~?1QmxmU=+k z(ggxvEuA;0e&+ci-zQR{-f7aO{O(Pz_OsEjLh_K>MbvoZ4nxtk5u{g@nPv)cgW_R} z9}EA4K4@z0?7ue}Z(o~R(X&FjejUI2g~08PH1E4w>9o{)S(?1>Z0XMvTb|;&EuyOE zGvWNpYX)Nv<8|a^;1>bh#&znEcl-r!T#pn= z4$?Yudha6F%4b>*8@=BdtXXY4N+`U4Dmx$}>HeVJk-QdTG@t!tVT#0(LeV0gvqyyw z2sEp^9eY0N`u10Tm4n8No&A=)IeEC|gnmEXoNSzu!1<4R<%-9kY_8~5Ej?zRegMn78wuMs#;i&eUA0Zk_RXQ3b&TT} z;SCI=7-FUB@*&;8|n>(_g^HGf3@QODE3LpmX~ELnymQm{Sx9xrKS zK29p~?v@R$0=v6Dr5aW>-!{+h@?Q58|Kz8{{W`%J+lDAdb&M5VHrX_mDY;1-JLnf)ezmPau$)1;=`-FU=-r-83tX=C`S#}GZufju zQ>sXNT0Ny=k@nc%cFnvA_i4SC)?_ORXHq8B4D%el1uPX`c~uG#S1M7C+*MMqLw78E zhY2dI8@+N^qrMI1+;TUda(vGqGSRyU{Fnm`aqrr7bz42c5xsOO-~oZpkzorD1g}Y<6rk&3>PsSGy}W?MtqFky@A(X# zIuNZK0cK?^=;PUAu>j0#HtjbHCV*6?jzA&OoE$*Jlga*}LF`SF?WLhv1O|zqC<>*> zYB;#lsYKx0&kH@BFpW8n*yDcc6?;_zaJs<-jPSkCsSX-!aV=P5kUgF@Nu<{a%#K*F z134Q{9|YX7X(v$62_cY3^G%t~rD>Q0z@)1|zs)vjJ6Jq9;7#Ki`w+eS**En?7;n&7 zu==V3T&eFboN3ZiMx3D8qYc;VjFUk_H-WWCau(VFXSQf~viH0L$gwD$UfFHqNcgN`x}M+YQ6RnN<+@t>JUp#)9YOkqst-Ga?{FsDpEeX0(5v{0J~SEbWiL zXC2}M4?UH@u&|;%0y`eb33ldo4~z-x8zY!oVmV=c+f$m?RfDC35mdQ2E>Pze7KWP- z>!Bh<&57I+O_^s}9Tg^k)h7{xx@0a0IA~GAOt2yy!X%Q$1rt~LbTB6@Du!_0%HV>N zlf)QI1&gvERKwso23mJ!Ou6ZS#zCS5W`gxE5T>C#E|{i<1D35C222I33?Njaz`On7 zi<+VWFP6D{e-{yiN#M|Jgk<44u1TiMI78S5W`Sdb5f+{zu34s{CfWN7a3Cf^@L%!& zN$?|!!9j2c)j$~+R6n#891w-z8(!oBpL2K=+%a$r2|~8-(vQj5_XT`<0Ksf;oP+tz z9CObS!0m)Tgg`K#xBM8B(|Z)Wb&DYL{WTYv`;A=q6~Nnx2+!lTIXtj8J7dZE!P_{z z#f8w6F}^!?^KE#+ZDv+xd5O&3EmomZzsv?>E-~ygGum45fk!SBN&|eo1rKw^?aZJ4 E2O(~oYXATM literal 0 HcmV?d00001 diff --git a/android/SherpaOnnxJavaDemo/gradle/wrapper/gradle-wrapper.properties b/android/SherpaOnnxJavaDemo/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..489dbeed1 --- /dev/null +++ b/android/SherpaOnnxJavaDemo/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Tue Oct 22 10:59:18 CST 2024 +distributionBase=GRADLE_USER_HOME +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip +distributionPath=wrapper/dists +zipStorePath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME diff --git a/android/SherpaOnnxJavaDemo/gradlew b/android/SherpaOnnxJavaDemo/gradlew new file mode 100644 index 000000000..4f906e0c8 --- /dev/null +++ b/android/SherpaOnnxJavaDemo/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/android/SherpaOnnxJavaDemo/gradlew.bat b/android/SherpaOnnxJavaDemo/gradlew.bat new file mode 100644 index 000000000..107acd32c --- /dev/null +++ b/android/SherpaOnnxJavaDemo/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/android/SherpaOnnxJavaDemo/settings.gradle b/android/SherpaOnnxJavaDemo/settings.gradle new file mode 100644 index 000000000..a1c973b67 --- /dev/null +++ b/android/SherpaOnnxJavaDemo/settings.gradle @@ -0,0 +1,17 @@ +pluginManagement { + repositories { + gradlePluginPortal() + google() + mavenCentral() + } +} +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { + google() + mavenCentral() + } +} +rootProject.name = "SherpaOnnxJavaDemo" +include ':app' +include ':sherpa' diff --git a/android/SherpaOnnxJavaDemo/sherpa/.gitignore b/android/SherpaOnnxJavaDemo/sherpa/.gitignore new file mode 100644 index 000000000..42afabfd2 --- /dev/null +++ b/android/SherpaOnnxJavaDemo/sherpa/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/android/SherpaOnnxJavaDemo/sherpa/build.gradle b/android/SherpaOnnxJavaDemo/sherpa/build.gradle new file mode 100644 index 000000000..a171d3e45 --- /dev/null +++ b/android/SherpaOnnxJavaDemo/sherpa/build.gradle @@ -0,0 +1,43 @@ +plugins { + id 'com.android.library' +} + +android { + namespace 'com.k2fsa.sherpa' + compileSdk 34 + + defaultConfig { + minSdk 26 + targetSdk 27 + versionCode 1 + versionName "1.0" + missingDimensionStrategy 'base', 'feature1' + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + } + + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } + compileOptions { + sourceCompatibility JavaVersion.VERSION_1_8 + targetCompatibility JavaVersion.VERSION_1_8 + } +} + +dependencies { + +// implementation "androidx.appcompat" +// implementation libs.material +// testImplementation libs.junit +// androidTestImplementation libs.androidx.test.ext.junit +// androidTestImplementation libs.espresso.core + implementation 'androidx.appcompat:appcompat:1.6.1' + implementation 'com.google.android.material:material:1.9.0' + testImplementation 'junit:junit:4.13.2' + androidTestImplementation 'androidx.test.ext:junit:1.1.5' + androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1' + +} \ No newline at end of file diff --git a/android/SherpaOnnxJavaDemo/sherpa/proguard-rules.pro b/android/SherpaOnnxJavaDemo/sherpa/proguard-rules.pro new file mode 100644 index 000000000..481bb4348 --- /dev/null +++ b/android/SherpaOnnxJavaDemo/sherpa/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile \ No newline at end of file diff --git a/android/SherpaOnnxJavaDemo/sherpa/src/main/AndroidManifest.xml b/android/SherpaOnnxJavaDemo/sherpa/src/main/AndroidManifest.xml new file mode 100644 index 000000000..1aab3af56 --- /dev/null +++ b/android/SherpaOnnxJavaDemo/sherpa/src/main/AndroidManifest.xml @@ -0,0 +1,9 @@ + + + + + + \ No newline at end of file diff --git a/android/SherpaOnnxJavaDemo/sherpa/src/main/java/com b/android/SherpaOnnxJavaDemo/sherpa/src/main/java/com new file mode 120000 index 000000000..4b54cb296 --- /dev/null +++ b/android/SherpaOnnxJavaDemo/sherpa/src/main/java/com @@ -0,0 +1 @@ +../../../../../../../sherpa-onnx/sherpa-onnx/java-api/src/com \ No newline at end of file diff --git a/android/SherpaOnnxJavaDemo/sherpa/src/main/res/values/strings.xml b/android/SherpaOnnxJavaDemo/sherpa/src/main/res/values/strings.xml new file mode 100644 index 000000000..7c5b69c5f --- /dev/null +++ b/android/SherpaOnnxJavaDemo/sherpa/src/main/res/values/strings.xml @@ -0,0 +1,3 @@ + + sherpa + \ No newline at end of file From effd5ef2be885f9de2e280faa9798778746e3ae4 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 23 Oct 2024 12:07:43 +0800 Subject: [PATCH 030/183] Add C++ API for streaming ASR. (#1455) It is a wrapper around the C API. --- .github/scripts/test-cxx-api.sh | 21 ++ .../workflows/aarch64-linux-gnu-shared.yaml | 2 +- .../workflows/aarch64-linux-gnu-static.yaml | 2 +- .github/workflows/android.yaml | 2 +- .github/workflows/apk-asr-2pass.yaml | 2 +- .github/workflows/apk-asr.yaml | 2 +- .../workflows/apk-audio-tagging-wearos.yaml | 2 +- .github/workflows/apk-audio-tagging.yaml | 2 +- .github/workflows/apk-kws.yaml | 2 +- .../workflows/apk-speaker-diarization.yaml | 2 +- .../workflows/apk-speaker-identification.yaml | 2 +- .../apk-spoken-language-identification.yaml | 2 +- .github/workflows/apk-tts-engine.yaml | 2 +- .github/workflows/apk-tts.yaml | 2 +- .github/workflows/apk-vad-asr.yaml | 2 +- .github/workflows/apk-vad.yaml | 2 +- .github/workflows/arm-linux-gnueabihf.yaml | 2 +- .github/workflows/build-wheels-aarch64.yaml | 2 +- .github/workflows/build-wheels-armv7l.yaml | 2 +- .../workflows/build-wheels-linux-cuda.yaml | 2 +- .github/workflows/build-wheels-linux.yaml | 2 +- .../workflows/build-wheels-macos-arm64.yaml | 2 +- .../build-wheels-macos-universal2.yaml | 2 +- .github/workflows/build-wheels-macos-x64.yaml | 2 +- .github/workflows/build-wheels-win32.yaml | 2 +- .../workflows/build-wheels-win64-cuda.yaml | 2 +- .github/workflows/build-wheels-win64.yaml | 2 +- .github/workflows/build-xcframework.yaml | 2 +- .github/workflows/c-api.yaml | 2 - .github/workflows/cxx-api.yaml | 117 ++++++++++++ .github/workflows/dot-net.yaml | 2 +- .../workflows/export-3dspeaker-to-onnx.yaml | 2 +- .github/workflows/export-ced-to-onnx.yaml | 2 +- .github/workflows/export-libriheavy.yaml | 4 +- .../workflows/export-melo-tts-to-onnx.yaml | 2 +- ...r-hybrid-transducer-ctc-non-streaming.yaml | 2 +- ...d-transducer-transducer-non-streaming.yaml | 2 +- ...ort-nemo-speaker-verification-to-onnx.yaml | 2 +- .../export-pyannote-segmentation-to-onnx.yaml | 2 +- .../export-revai-segmentation-to-onnx.yaml | 2 +- .../workflows/export-sense-voice-to-onnx.yaml | 2 +- .github/workflows/export-telespeech-ctc.yaml | 4 +- .github/workflows/export-wenet-to-onnx.yaml | 12 +- .../workflows/export-wespeaker-to-onnx.yaml | 2 +- .github/workflows/export-whisper-to-onnx.yaml | 2 +- .github/workflows/flutter-android.yaml | 2 +- .github/workflows/flutter-linux.yaml | 2 +- .github/workflows/flutter-macos.yaml | 4 +- .github/workflows/flutter-windows-x64.yaml | 4 +- .github/workflows/lazarus.yaml | 2 +- .github/workflows/linux.yaml | 14 ++ .github/workflows/macos.yaml | 14 ++ .github/workflows/sanitizer.yaml | 9 +- .github/workflows/windows-x64.yaml | 13 +- .github/workflows/windows-x86.yaml | 13 +- CMakeLists.txt | 9 +- .../streaming-ctc-buffered-tokens-c-api.c | 2 +- ...reaming-paraformer-buffered-tokens-c-api.c | 2 +- c-api-examples/streaming-paraformer-c-api.c | 2 +- ...zipformer-buffered-tokens-hotwords-c-api.c | 2 +- c-api-examples/streaming-zipformer-c-api.c | 2 +- cmake/cmake_extension.py | 1 + cxx-api-examples/CMakeLists.txt | 4 + .../streaming-zipformer-cxx-api.cc | 91 +++++++++ ffmpeg-examples/sherpa-onnx-ffmpeg.c | 4 +- .../StreamingSpeechRecognitionDlg.h | 6 +- scripts/node-addon-api/src/streaming-asr.cc | 5 +- sherpa-onnx/c-api/CMakeLists.txt | 23 ++- sherpa-onnx/c-api/c-api.cc | 3 +- sherpa-onnx/c-api/c-api.h | 3 +- sherpa-onnx/c-api/cxx-api.cc | 159 ++++++++++++++++ sherpa-onnx/c-api/cxx-api.h | 179 ++++++++++++++++++ 72 files changed, 729 insertions(+), 83 deletions(-) create mode 100755 .github/scripts/test-cxx-api.sh create mode 100644 .github/workflows/cxx-api.yaml create mode 100644 cxx-api-examples/CMakeLists.txt create mode 100644 cxx-api-examples/streaming-zipformer-cxx-api.cc create mode 100644 sherpa-onnx/c-api/cxx-api.cc create mode 100644 sherpa-onnx/c-api/cxx-api.h diff --git a/.github/scripts/test-cxx-api.sh b/.github/scripts/test-cxx-api.sh new file mode 100755 index 000000000..89b2f1dd6 --- /dev/null +++ b/.github/scripts/test-cxx-api.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -ex + +log() { + # This function is from espnet + local fname=${BASH_SOURCE[1]##*/} + echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" +} + +echo "CXX_STREAMING_ZIPFORMER_EXE is $CXX_STREAMING_ZIPFORMER_EXE" +echo "PATH: $PATH" + +log "------------------------------------------------------------" +log "Test streaming zipformer CXX API" +log "------------------------------------------------------------" +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 +tar xvf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 +rm sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 +$CXX_STREAMING_ZIPFORMER_EXE +rm -rf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 diff --git a/.github/workflows/aarch64-linux-gnu-shared.yaml b/.github/workflows/aarch64-linux-gnu-shared.yaml index 5e82d9b3a..dbba7c132 100644 --- a/.github/workflows/aarch64-linux-gnu-shared.yaml +++ b/.github/workflows/aarch64-linux-gnu-shared.yaml @@ -193,7 +193,7 @@ jobs: rm -rf huggingface export GIT_CLONE_PROTECTION_ACTIVE=false - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface + GIT_LFS_SKIP_SMUDGE=1 git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs huggingface cd huggingface mkdir -p aarch64 diff --git a/.github/workflows/aarch64-linux-gnu-static.yaml b/.github/workflows/aarch64-linux-gnu-static.yaml index 765e2422f..6cbfc0e27 100644 --- a/.github/workflows/aarch64-linux-gnu-static.yaml +++ b/.github/workflows/aarch64-linux-gnu-static.yaml @@ -184,7 +184,7 @@ jobs: rm -rf huggingface export GIT_CLONE_PROTECTION_ACTIVE=false - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface + GIT_LFS_SKIP_SMUDGE=1 git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs huggingface cd huggingface mkdir -p aarch64 diff --git a/.github/workflows/android.yaml b/.github/workflows/android.yaml index 35dfd6b26..ebe9cd90f 100644 --- a/.github/workflows/android.yaml +++ b/.github/workflows/android.yaml @@ -121,7 +121,7 @@ jobs: rm -rf huggingface export GIT_CLONE_PROTECTION_ACTIVE=false - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface + GIT_LFS_SKIP_SMUDGE=1 git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs huggingface cd huggingface diff --git a/.github/workflows/apk-asr-2pass.yaml b/.github/workflows/apk-asr-2pass.yaml index bbe61060a..bfbc2f170 100644 --- a/.github/workflows/apk-asr-2pass.yaml +++ b/.github/workflows/apk-asr-2pass.yaml @@ -163,7 +163,7 @@ jobs: SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/apk-asr.yaml b/.github/workflows/apk-asr.yaml index fc1cd1f5d..c51ce1e20 100644 --- a/.github/workflows/apk-asr.yaml +++ b/.github/workflows/apk-asr.yaml @@ -163,7 +163,7 @@ jobs: SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/apk-audio-tagging-wearos.yaml b/.github/workflows/apk-audio-tagging-wearos.yaml index 0ed823076..bfe9f9ac7 100644 --- a/.github/workflows/apk-audio-tagging-wearos.yaml +++ b/.github/workflows/apk-audio-tagging-wearos.yaml @@ -163,7 +163,7 @@ jobs: SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/apk-audio-tagging.yaml b/.github/workflows/apk-audio-tagging.yaml index f6b85c3b2..c11180c4a 100644 --- a/.github/workflows/apk-audio-tagging.yaml +++ b/.github/workflows/apk-audio-tagging.yaml @@ -160,7 +160,7 @@ jobs: export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false - git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/apk-kws.yaml b/.github/workflows/apk-kws.yaml index 524622de8..43cdef49e 100644 --- a/.github/workflows/apk-kws.yaml +++ b/.github/workflows/apk-kws.yaml @@ -160,7 +160,7 @@ jobs: SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/apk-speaker-diarization.yaml b/.github/workflows/apk-speaker-diarization.yaml index 8e422e13f..90bcc7323 100644 --- a/.github/workflows/apk-speaker-diarization.yaml +++ b/.github/workflows/apk-speaker-diarization.yaml @@ -163,7 +163,7 @@ jobs: SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/apk-speaker-identification.yaml b/.github/workflows/apk-speaker-identification.yaml index e32ad3bc9..c88718d6e 100644 --- a/.github/workflows/apk-speaker-identification.yaml +++ b/.github/workflows/apk-speaker-identification.yaml @@ -163,7 +163,7 @@ jobs: SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/apk-spoken-language-identification.yaml b/.github/workflows/apk-spoken-language-identification.yaml index 3cb9c83b2..cc7525cd4 100644 --- a/.github/workflows/apk-spoken-language-identification.yaml +++ b/.github/workflows/apk-spoken-language-identification.yaml @@ -163,7 +163,7 @@ jobs: SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/apk-tts-engine.yaml b/.github/workflows/apk-tts-engine.yaml index d251483e4..b8614cb76 100644 --- a/.github/workflows/apk-tts-engine.yaml +++ b/.github/workflows/apk-tts-engine.yaml @@ -164,7 +164,7 @@ jobs: SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/apk-tts.yaml b/.github/workflows/apk-tts.yaml index dd0aa3f77..1609739c6 100644 --- a/.github/workflows/apk-tts.yaml +++ b/.github/workflows/apk-tts.yaml @@ -164,7 +164,7 @@ jobs: SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/apk-vad-asr.yaml b/.github/workflows/apk-vad-asr.yaml index 8310043a9..4c587f1be 100644 --- a/.github/workflows/apk-vad-asr.yaml +++ b/.github/workflows/apk-vad-asr.yaml @@ -163,7 +163,7 @@ jobs: SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/apk-vad.yaml b/.github/workflows/apk-vad.yaml index d9af75477..f1a4364fc 100644 --- a/.github/workflows/apk-vad.yaml +++ b/.github/workflows/apk-vad.yaml @@ -160,7 +160,7 @@ jobs: SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/arm-linux-gnueabihf.yaml b/.github/workflows/arm-linux-gnueabihf.yaml index a56b2cdad..6a2874910 100644 --- a/.github/workflows/arm-linux-gnueabihf.yaml +++ b/.github/workflows/arm-linux-gnueabihf.yaml @@ -205,7 +205,7 @@ jobs: git config --global user.name "Fangjun Kuang" rm -rf huggingface - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface + GIT_LFS_SKIP_SMUDGE=1 git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs huggingface cd huggingface mkdir -p arm32 diff --git a/.github/workflows/build-wheels-aarch64.yaml b/.github/workflows/build-wheels-aarch64.yaml index 9d4ac571e..860ccdcda 100644 --- a/.github/workflows/build-wheels-aarch64.yaml +++ b/.github/workflows/build-wheels-aarch64.yaml @@ -99,7 +99,7 @@ jobs: d=cpu/$SHERPA_ONNX_VERSION - git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/build-wheels-armv7l.yaml b/.github/workflows/build-wheels-armv7l.yaml index 05c3b196d..584c64687 100644 --- a/.github/workflows/build-wheels-armv7l.yaml +++ b/.github/workflows/build-wheels-armv7l.yaml @@ -102,7 +102,7 @@ jobs: d=cpu/$SHERPA_ONNX_VERSION - git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/build-wheels-linux-cuda.yaml b/.github/workflows/build-wheels-linux-cuda.yaml index b1ee89825..3f3f66dda 100644 --- a/.github/workflows/build-wheels-linux-cuda.yaml +++ b/.github/workflows/build-wheels-linux-cuda.yaml @@ -113,7 +113,7 @@ jobs: d=cuda/$SHERPA_ONNX_VERSION - git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/build-wheels-linux.yaml b/.github/workflows/build-wheels-linux.yaml index e16f5bb9a..5ca10639a 100644 --- a/.github/workflows/build-wheels-linux.yaml +++ b/.github/workflows/build-wheels-linux.yaml @@ -96,7 +96,7 @@ jobs: d=cpu/$SHERPA_ONNX_VERSION - git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/build-wheels-macos-arm64.yaml b/.github/workflows/build-wheels-macos-arm64.yaml index ce899c5d1..181246e8b 100644 --- a/.github/workflows/build-wheels-macos-arm64.yaml +++ b/.github/workflows/build-wheels-macos-arm64.yaml @@ -68,7 +68,7 @@ jobs: d=cpu/$SHERPA_ONNX_VERSION - git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/build-wheels-macos-universal2.yaml b/.github/workflows/build-wheels-macos-universal2.yaml index 4578d370e..88c51b3a0 100644 --- a/.github/workflows/build-wheels-macos-universal2.yaml +++ b/.github/workflows/build-wheels-macos-universal2.yaml @@ -68,7 +68,7 @@ jobs: d=cpu/$SHERPA_ONNX_VERSION - git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/build-wheels-macos-x64.yaml b/.github/workflows/build-wheels-macos-x64.yaml index b7bf6ff54..695186958 100644 --- a/.github/workflows/build-wheels-macos-x64.yaml +++ b/.github/workflows/build-wheels-macos-x64.yaml @@ -83,7 +83,7 @@ jobs: d=cpu/$SHERPA_ONNX_VERSION - git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/build-wheels-win32.yaml b/.github/workflows/build-wheels-win32.yaml index 256084783..04a02d45f 100644 --- a/.github/workflows/build-wheels-win32.yaml +++ b/.github/workflows/build-wheels-win32.yaml @@ -67,7 +67,7 @@ jobs: d=cpu/$SHERPA_ONNX_VERSION - git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/build-wheels-win64-cuda.yaml b/.github/workflows/build-wheels-win64-cuda.yaml index f0a17da8c..5224f20f0 100644 --- a/.github/workflows/build-wheels-win64-cuda.yaml +++ b/.github/workflows/build-wheels-win64-cuda.yaml @@ -75,7 +75,7 @@ jobs: d=cuda/$SHERPA_ONNX_VERSION - git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/build-wheels-win64.yaml b/.github/workflows/build-wheels-win64.yaml index 14e3e2ac4..0bae16c4a 100644 --- a/.github/workflows/build-wheels-win64.yaml +++ b/.github/workflows/build-wheels-win64.yaml @@ -73,7 +73,7 @@ jobs: d=cpu/$SHERPA_ONNX_VERSION - git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/build-xcframework.yaml b/.github/workflows/build-xcframework.yaml index 2afd95cab..90953010e 100644 --- a/.github/workflows/build-xcframework.yaml +++ b/.github/workflows/build-xcframework.yaml @@ -135,7 +135,7 @@ jobs: rm -rf huggingface export GIT_CLONE_PROTECTION_ACTIVE=false - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface + GIT_LFS_SKIP_SMUDGE=1 git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs huggingface cd huggingface diff --git a/.github/workflows/c-api.yaml b/.github/workflows/c-api.yaml index 589bda71f..94a52a4bc 100644 --- a/.github/workflows/c-api.yaml +++ b/.github/workflows/c-api.yaml @@ -4,8 +4,6 @@ on: push: branches: - master - tags: - - 'v[0-9]+.[0-9]+.[0-9]+*' paths: - '.github/workflows/c-api.yaml' - 'CMakeLists.txt' diff --git a/.github/workflows/cxx-api.yaml b/.github/workflows/cxx-api.yaml new file mode 100644 index 000000000..1c882f2ff --- /dev/null +++ b/.github/workflows/cxx-api.yaml @@ -0,0 +1,117 @@ +name: cxx-api + +on: + push: + branches: + - master + paths: + - '.github/workflows/cxx-api.yaml' + - 'CMakeLists.txt' + - 'cmake/**' + - 'sherpa-onnx/csrc/*' + - 'sherpa-onnx/c-api/*' + - 'cxx-api-examples/**' + pull_request: + branches: + - master + paths: + - '.github/workflows/cxx-api.yaml' + - 'CMakeLists.txt' + - 'cmake/**' + - 'sherpa-onnx/csrc/*' + - 'sherpa-onnx/c-api/*' + - 'cxx-api-examples/**' + + workflow_dispatch: + +concurrency: + group: cxx-api-${{ github.ref }} + cancel-in-progress: true + +jobs: + cxx_api: + name: ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ matrix.os }}-cxx-api-shared + + - name: Build sherpa-onnx + shell: bash + run: | + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + cmake --version + + mkdir build + cd build + + cmake \ + -D CMAKE_BUILD_TYPE=Release \ + -D BUILD_SHARED_LIBS=ON \ + -D CMAKE_INSTALL_PREFIX=./install \ + -D SHERPA_ONNX_ENABLE_BINARY=OFF \ + .. + + make -j2 install + + ls -lh install/lib + ls -lh install/include + + if [[ ${{ matrix.os }} == ubuntu-latest ]]; then + ldd ./install/lib/libsherpa-onnx-c-api.so + ldd ./install/lib/libsherpa-onnx-cxx-api.so + echo "---" + readelf -d ./install/lib/libsherpa-onnx-c-api.so + readelf -d ./install/lib/libsherpa-onnx-cxx-api.so + fi + + if [[ ${{ matrix.os }} == macos-latest ]]; then + otool -L ./install/lib/libsherpa-onnx-c-api.dylib + otool -L ./install/lib/libsherpa-onnx-cxx-api.dylib + fi + + - name: Test streaming zipformer + shell: bash + run: | + g++ -std=c++17 -o streaming-zipformer-cxx-api ./cxx-api-examples/streaming-zipformer-cxx-api.cc \ + -I ./build/install/include \ + -L ./build/install/lib/ \ + -l sherpa-onnx-cxx-api \ + -l sherpa-onnx-c-api \ + -l onnxruntime + + ls -lh streaming-zipformer-cxx-api + + if [[ ${{ matrix.os }} == ubuntu-latest ]]; then + ldd ./streaming-zipformer-cxx-api + echo "----" + readelf -d ./streaming-zipformer-cxx-api + fi + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 + tar xvf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 + rm sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 + + ls -lh sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 + echo "---" + ls -lh sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/test_wavs + + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH + export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH + + ./streaming-zipformer-cxx-api + + rm -rf sherpa-onnx-streaming-zipformer-* + rm ./streaming-zipformer-cxx-api diff --git a/.github/workflows/dot-net.yaml b/.github/workflows/dot-net.yaml index 36637a9e2..4c6e44133 100644 --- a/.github/workflows/dot-net.yaml +++ b/.github/workflows/dot-net.yaml @@ -90,7 +90,7 @@ jobs: export GIT_CLONE_PROTECTION_ACTIVE=false export GIT_LFS_SKIP_SMUDGE=1 - git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs huggingface cd huggingface git fetch diff --git a/.github/workflows/export-3dspeaker-to-onnx.yaml b/.github/workflows/export-3dspeaker-to-onnx.yaml index a3fa98760..e62d42784 100644 --- a/.github/workflows/export-3dspeaker-to-onnx.yaml +++ b/.github/workflows/export-3dspeaker-to-onnx.yaml @@ -59,7 +59,7 @@ jobs: d=speaker-embedding-models export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false - git clone https://huggingface.co/csukuangfj/$d huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$d huggingface mv -v ./*.onnx ./huggingface cd huggingface git lfs track "*.onnx" diff --git a/.github/workflows/export-ced-to-onnx.yaml b/.github/workflows/export-ced-to-onnx.yaml index 70c4cc5fb..2f714bb80 100644 --- a/.github/workflows/export-ced-to-onnx.yaml +++ b/.github/workflows/export-ced-to-onnx.yaml @@ -66,7 +66,7 @@ jobs: export GIT_LFS_SKIP_SMUDGE=1 d=sherpa-onnx-ced-$m-audio-tagging-2024-04-19 export GIT_CLONE_PROTECTION_ACTIVE=false - git clone https://huggingface.co/k2-fsa/$d huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/k2-fsa/$d huggingface mv -v $d/* huggingface cd huggingface git lfs track "*.onnx" diff --git a/.github/workflows/export-libriheavy.yaml b/.github/workflows/export-libriheavy.yaml index cfe0a28d2..69c22ef24 100644 --- a/.github/workflows/export-libriheavy.yaml +++ b/.github/workflows/export-libriheavy.yaml @@ -56,7 +56,7 @@ jobs: src=sherpa-onnx-zipformer-en-libriheavy-20230926-$m echo "Process $src" - git clone https://huggingface.co/csukuangfj/$src huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$src huggingface cd huggingface git fetch git pull @@ -100,7 +100,7 @@ jobs: src=sherpa-onnx-zipformer-en-libriheavy-20230830-$m-punct-case echo "Process $src" - git clone https://huggingface.co/csukuangfj/$src huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$src huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/export-melo-tts-to-onnx.yaml b/.github/workflows/export-melo-tts-to-onnx.yaml index 0dc9bfe9d..42cc28d6a 100644 --- a/.github/workflows/export-melo-tts-to-onnx.yaml +++ b/.github/workflows/export-melo-tts-to-onnx.yaml @@ -56,7 +56,7 @@ jobs: export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false - git clone https://huggingface.co/csukuangfj/vits-melo-tts-zh_en huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/vits-melo-tts-zh_en huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/export-nemo-fast-conformer-hybrid-transducer-ctc-non-streaming.yaml b/.github/workflows/export-nemo-fast-conformer-hybrid-transducer-ctc-non-streaming.yaml index 138c708ad..bbabfb60c 100644 --- a/.github/workflows/export-nemo-fast-conformer-hybrid-transducer-ctc-non-streaming.yaml +++ b/.github/workflows/export-nemo-fast-conformer-hybrid-transducer-ctc-non-streaming.yaml @@ -67,7 +67,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false - git clone https://huggingface.co/csukuangfj/$m huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$m huggingface cp -av $m/* huggingface cd huggingface git lfs track "*.onnx" diff --git a/.github/workflows/export-nemo-fast-conformer-hybrid-transducer-transducer-non-streaming.yaml b/.github/workflows/export-nemo-fast-conformer-hybrid-transducer-transducer-non-streaming.yaml index 7a7b7fc4e..4a7e2339e 100644 --- a/.github/workflows/export-nemo-fast-conformer-hybrid-transducer-transducer-non-streaming.yaml +++ b/.github/workflows/export-nemo-fast-conformer-hybrid-transducer-transducer-non-streaming.yaml @@ -67,7 +67,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false - git clone https://huggingface.co/csukuangfj/$m huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$m huggingface cp -av $m/* huggingface cd huggingface git lfs track "*.onnx" diff --git a/.github/workflows/export-nemo-speaker-verification-to-onnx.yaml b/.github/workflows/export-nemo-speaker-verification-to-onnx.yaml index a9bd5a788..505966413 100644 --- a/.github/workflows/export-nemo-speaker-verification-to-onnx.yaml +++ b/.github/workflows/export-nemo-speaker-verification-to-onnx.yaml @@ -59,7 +59,7 @@ jobs: d=speaker-embedding-models export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false - git clone https://huggingface.co/csukuangfj/$d huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$d huggingface mv -v ./*.onnx ./huggingface cd huggingface git lfs track "*.onnx" diff --git a/.github/workflows/export-pyannote-segmentation-to-onnx.yaml b/.github/workflows/export-pyannote-segmentation-to-onnx.yaml index ece0ffa28..53f8dac7d 100644 --- a/.github/workflows/export-pyannote-segmentation-to-onnx.yaml +++ b/.github/workflows/export-pyannote-segmentation-to-onnx.yaml @@ -75,7 +75,7 @@ jobs: d=sherpa-onnx-pyannote-segmentation-3-0 export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false - git clone https://huggingface.co/csukuangfj/$d huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$d huggingface cp -v $d/* ./huggingface cd huggingface git lfs track "*.onnx" diff --git a/.github/workflows/export-revai-segmentation-to-onnx.yaml b/.github/workflows/export-revai-segmentation-to-onnx.yaml index f0f1594c6..d82f7c4e0 100644 --- a/.github/workflows/export-revai-segmentation-to-onnx.yaml +++ b/.github/workflows/export-revai-segmentation-to-onnx.yaml @@ -75,7 +75,7 @@ jobs: d=sherpa-onnx-reverb-diarization-v1 export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false - git clone https://huggingface.co/csukuangfj/$d huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$d huggingface cp -v $d/* ./huggingface cd huggingface git lfs track "*.onnx" diff --git a/.github/workflows/export-sense-voice-to-onnx.yaml b/.github/workflows/export-sense-voice-to-onnx.yaml index 41a9a31a6..1c3e91729 100644 --- a/.github/workflows/export-sense-voice-to-onnx.yaml +++ b/.github/workflows/export-sense-voice-to-onnx.yaml @@ -66,7 +66,7 @@ jobs: export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false - git clone https://huggingface.co/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17 huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17 huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/export-telespeech-ctc.yaml b/.github/workflows/export-telespeech-ctc.yaml index 102c3884e..4f66d7ca4 100644 --- a/.github/workflows/export-telespeech-ctc.yaml +++ b/.github/workflows/export-telespeech-ctc.yaml @@ -60,7 +60,7 @@ jobs: export GIT_CLONE_PROTECTION_ACTIVE=false - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-telespeech-ctc-zh-2024-06-04 hf + GIT_LFS_SKIP_SMUDGE=1 git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-telespeech-ctc-zh-2024-06-04 hf cp -a $src/* hf/ cd hf git lfs track "*.pdf" @@ -84,7 +84,7 @@ jobs: export GIT_CLONE_PROTECTION_ACTIVE=false rm -rf hf - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04 hf + GIT_LFS_SKIP_SMUDGE=1 git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04 hf cp -a $src/* hf/ cd hf git lfs track "*.pdf" diff --git a/.github/workflows/export-wenet-to-onnx.yaml b/.github/workflows/export-wenet-to-onnx.yaml index 626f477e6..7ef3a54b6 100644 --- a/.github/workflows/export-wenet-to-onnx.yaml +++ b/.github/workflows/export-wenet-to-onnx.yaml @@ -49,7 +49,7 @@ jobs: export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false - git clone https://huggingface.co/csukuangfj/sherpa-onnx-zh-wenet-aishell huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-zh-wenet-aishell huggingface cd huggingface git fetch git pull @@ -98,7 +98,7 @@ jobs: export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false - git clone https://huggingface.co/csukuangfj/sherpa-onnx-zh-wenet-aishell2 huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-zh-wenet-aishell2 huggingface cd huggingface git fetch git pull @@ -147,7 +147,7 @@ jobs: export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false - git clone https://huggingface.co/csukuangfj/sherpa-onnx-zh-wenet-multi-cn huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-zh-wenet-multi-cn huggingface cd huggingface git fetch git pull @@ -196,7 +196,7 @@ jobs: export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false - git clone https://huggingface.co/csukuangfj/sherpa-onnx-zh-wenet-wenetspeech huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-zh-wenet-wenetspeech huggingface cd huggingface git fetch git pull @@ -245,7 +245,7 @@ jobs: export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false - git clone https://huggingface.co/csukuangfj/sherpa-onnx-en-wenet-librispeech huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-en-wenet-librispeech huggingface cd huggingface git fetch git pull @@ -295,7 +295,7 @@ jobs: export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false - git clone https://huggingface.co/csukuangfj/sherpa-onnx-en-wenet-gigaspeech huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-en-wenet-gigaspeech huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/export-wespeaker-to-onnx.yaml b/.github/workflows/export-wespeaker-to-onnx.yaml index 764f77ca7..05694f693 100644 --- a/.github/workflows/export-wespeaker-to-onnx.yaml +++ b/.github/workflows/export-wespeaker-to-onnx.yaml @@ -64,7 +64,7 @@ jobs: d=speaker-embedding-models export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false - git clone https://huggingface.co/csukuangfj/$d huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$d huggingface mv -v ./*.onnx ./huggingface cd huggingface git lfs track "*.onnx" diff --git a/.github/workflows/export-whisper-to-onnx.yaml b/.github/workflows/export-whisper-to-onnx.yaml index a50aa99d7..53aebdd3b 100644 --- a/.github/workflows/export-whisper-to-onnx.yaml +++ b/.github/workflows/export-whisper-to-onnx.yaml @@ -145,7 +145,7 @@ jobs: export GIT_LFS_SKIP_SMUDGE=1 - git clone https://huggingface.co/csukuangfj/sherpa-onnx-whisper-${{ matrix.model }} huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-whisper-${{ matrix.model }} huggingface rm -rf huggingface/* diff --git a/.github/workflows/flutter-android.yaml b/.github/workflows/flutter-android.yaml index 9752a82c6..c2b1d01db 100644 --- a/.github/workflows/flutter-android.yaml +++ b/.github/workflows/flutter-android.yaml @@ -214,7 +214,7 @@ jobs: SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - git clone https://huggingface.co/csukuangfj/sherpa-onnx-flutter huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-flutter huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/flutter-linux.yaml b/.github/workflows/flutter-linux.yaml index b6b1fb9c8..f1fdd5ec7 100644 --- a/.github/workflows/flutter-linux.yaml +++ b/.github/workflows/flutter-linux.yaml @@ -261,7 +261,7 @@ jobs: SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - git clone https://huggingface.co/csukuangfj/sherpa-onnx-flutter huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-flutter huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/flutter-macos.yaml b/.github/workflows/flutter-macos.yaml index 7c8a38e4c..e85ff1644 100644 --- a/.github/workflows/flutter-macos.yaml +++ b/.github/workflows/flutter-macos.yaml @@ -101,7 +101,7 @@ jobs: SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - git clone https://huggingface.co/csukuangfj/sherpa-onnx-flutter huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-flutter huggingface cd huggingface git fetch git pull @@ -207,7 +207,7 @@ jobs: SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - git clone https://huggingface.co/csukuangfj/sherpa-onnx-flutter huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-flutter huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/flutter-windows-x64.yaml b/.github/workflows/flutter-windows-x64.yaml index f4d296b70..59f6a6af9 100644 --- a/.github/workflows/flutter-windows-x64.yaml +++ b/.github/workflows/flutter-windows-x64.yaml @@ -94,7 +94,7 @@ jobs: SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - git clone https://huggingface.co/csukuangfj/sherpa-onnx-flutter huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-flutter huggingface cd huggingface git fetch git pull @@ -192,7 +192,7 @@ jobs: SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - git clone https://huggingface.co/csukuangfj/sherpa-onnx-flutter huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-flutter huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/lazarus.yaml b/.github/workflows/lazarus.yaml index 11df53644..e7cef16ef 100644 --- a/.github/workflows/lazarus.yaml +++ b/.github/workflows/lazarus.yaml @@ -355,7 +355,7 @@ jobs: SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - git clone https://huggingface.co/csukuangfj/sherpa-onnx-bin huggingface + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-bin huggingface cd huggingface git fetch git pull diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml index 1d3e8dc7b..e21c452c0 100644 --- a/.github/workflows/linux.yaml +++ b/.github/workflows/linux.yaml @@ -19,6 +19,8 @@ on: - '.github/scripts/test-offline-punctuation.sh' - '.github/scripts/test-online-punctuation.sh' - '.github/scripts/test-speaker-diarization.sh' + - '.github/scripts/test-c-api.sh' + - '.github/scripts/test-cxx-api.sh' - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' @@ -40,6 +42,8 @@ on: - '.github/scripts/test-offline-punctuation.sh' - '.github/scripts/test-online-punctuation.sh' - '.github/scripts/test-speaker-diarization.sh' + - '.github/scripts/test-c-api.sh' + - '.github/scripts/test-cxx-api.sh' - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' @@ -145,6 +149,16 @@ jobs: name: release-${{ matrix.build_type }}-with-shared-lib-${{ matrix.shared_lib }}-with-tts-${{ matrix.with_tts }} path: install/* + - name: Test C++ API + shell: bash + run: | + du -h -d1 . + export PATH=$PWD/build/bin:$PATH + export CXX_STREAMING_ZIPFORMER_EXE=streaming-zipformer-cxx-api + + .github/scripts/test-cxx-api.sh + du -h -d1 . + - name: Test offline speaker diarization shell: bash run: | diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml index f3d70f583..7b01846a7 100644 --- a/.github/workflows/macos.yaml +++ b/.github/workflows/macos.yaml @@ -19,6 +19,8 @@ on: - '.github/scripts/test-offline-punctuation.sh' - '.github/scripts/test-online-punctuation.sh' - '.github/scripts/test-speaker-diarization.sh' + - '.github/scripts/test-c-api.sh' + - '.github/scripts/test-cxx-api.sh' - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' @@ -39,6 +41,8 @@ on: - '.github/scripts/test-offline-punctuation.sh' - '.github/scripts/test-online-punctuation.sh' - '.github/scripts/test-speaker-diarization.sh' + - '.github/scripts/test-c-api.sh' + - '.github/scripts/test-cxx-api.sh' - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' @@ -117,6 +121,16 @@ jobs: otool -L build/bin/sherpa-onnx otool -l build/bin/sherpa-onnx + - name: Test C++ API + shell: bash + run: | + du -h -d1 . + export PATH=$PWD/build/bin:$PATH + export CXX_STREAMING_ZIPFORMER_EXE=streaming-zipformer-cxx-api + + .github/scripts/test-cxx-api.sh + du -h -d1 . + - name: Test offline speaker diarization shell: bash run: | diff --git a/.github/workflows/sanitizer.yaml b/.github/workflows/sanitizer.yaml index 7fce3834a..f12bafa9a 100644 --- a/.github/workflows/sanitizer.yaml +++ b/.github/workflows/sanitizer.yaml @@ -76,6 +76,14 @@ jobs: otool -L build/bin/sherpa-onnx otool -l build/bin/sherpa-onnx + - name: Test C++ API + shell: bash + run: | + export PATH=$PWD/build/bin:$PATH + export CXX_STREAMING_ZIPFORMER_EXE=streaming-zipformer-cxx-api + + .github/scripts/test-cxx-api.sh + - name: Test online punctuation shell: bash run: | @@ -109,7 +117,6 @@ jobs: .github/scripts/test-online-ctc.sh - - name: Test C API shell: bash run: | diff --git a/.github/workflows/windows-x64.yaml b/.github/workflows/windows-x64.yaml index 758935918..f07d6c78c 100644 --- a/.github/workflows/windows-x64.yaml +++ b/.github/workflows/windows-x64.yaml @@ -18,6 +18,8 @@ on: - '.github/scripts/test-offline-punctuation.sh' - '.github/scripts/test-online-punctuation.sh' - '.github/scripts/test-speaker-diarization.sh' + - '.github/scripts/test-c-api.sh' + - '.github/scripts/test-cxx-api.sh' - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' @@ -36,6 +38,8 @@ on: - '.github/scripts/test-offline-punctuation.sh' - '.github/scripts/test-online-punctuation.sh' - '.github/scripts/test-speaker-diarization.sh' + - '.github/scripts/test-c-api.sh' + - '.github/scripts/test-cxx-api.sh' - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' @@ -89,10 +93,17 @@ jobs: name: release-windows-x64-${{ matrix.shared_lib }}-${{ matrix.with_tts }} path: build/install/* + - name: Test C++ API + shell: bash + run: | + export PATH=$PWD/build/bin/Release:$PATH + export CXX_STREAMING_ZIPFORMER_EXE=streaming-zipformer-cxx-api.exe + + .github/scripts/test-cxx-api.sh + - name: Test offline speaker diarization shell: bash run: | - du -h -d1 . export PATH=$PWD/build/bin/Release:$PATH export EXE=sherpa-onnx-offline-speaker-diarization.exe diff --git a/.github/workflows/windows-x86.yaml b/.github/workflows/windows-x86.yaml index b9e473184..bacbdff01 100644 --- a/.github/workflows/windows-x86.yaml +++ b/.github/workflows/windows-x86.yaml @@ -18,6 +18,8 @@ on: - '.github/scripts/test-offline-punctuation.sh' - '.github/scripts/test-online-punctuation.sh' - '.github/scripts/test-speaker-diarization.sh' + - '.github/scripts/test-c-api.sh' + - '.github/scripts/test-cxx-api.sh' - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' @@ -36,6 +38,8 @@ on: - '.github/scripts/test-offline-punctuation.sh' - '.github/scripts/test-online-punctuation.sh' - '.github/scripts/test-speaker-diarization.sh' + - '.github/scripts/test-c-api.sh' + - '.github/scripts/test-cxx-api.sh' - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' @@ -89,10 +93,17 @@ jobs: name: release-windows-x86-${{ matrix.shared_lib }}-${{ matrix.with_tts }} path: build/install/* + - name: Test C++ API + shell: bash + run: | + export PATH=$PWD/build/bin/Release:$PATH + export CXX_STREAMING_ZIPFORMER_EXE=streaming-zipformer-cxx-api.exe + + .github/scripts/test-cxx-api.sh + - name: Test offline speaker diarization shell: bash run: | - du -h -d1 . export PATH=$PWD/build/bin/Release:$PATH export EXE=sherpa-onnx-offline-speaker-diarization.exe diff --git a/CMakeLists.txt b/CMakeLists.txt index 578dc78d8..4e1850c44 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,9 +51,11 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") -set(CMAKE_SKIP_BUILD_RPATH FALSE) -set(BUILD_RPATH_USE_ORIGIN TRUE) -set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) +if(NOT WIN32) + set(CMAKE_SKIP_BUILD_RPATH FALSE) + set(BUILD_RPATH_USE_ORIGIN TRUE) + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) +endif() if(NOT APPLE) set(SHERPA_ONNX_RPATH_ORIGIN "$ORIGIN") @@ -399,6 +401,7 @@ add_subdirectory(sherpa-onnx) if(SHERPA_ONNX_ENABLE_C_API AND SHERPA_ONNX_ENABLE_BINARY AND SHERPA_ONNX_BUILD_C_API_EXAMPLES) set(SHERPA_ONNX_PKG_WITH_CARGS "-lcargs") add_subdirectory(c-api-examples) + add_subdirectory(cxx-api-examples) endif() if(SHERPA_ONNX_ENABLE_WASM) diff --git a/c-api-examples/streaming-ctc-buffered-tokens-c-api.c b/c-api-examples/streaming-ctc-buffered-tokens-c-api.c index 33690e008..eb834fb70 100644 --- a/c-api-examples/streaming-ctc-buffered-tokens-c-api.c +++ b/c-api-examples/streaming-ctc-buffered-tokens-c-api.c @@ -95,7 +95,7 @@ int32_t main() { recognizer_config.decoding_method = "greedy_search"; recognizer_config.model_config = online_model_config; - SherpaOnnxOnlineRecognizer *recognizer = + const SherpaOnnxOnlineRecognizer *recognizer = SherpaOnnxCreateOnlineRecognizer(&recognizer_config); free((void *)tokens_buf); diff --git a/c-api-examples/streaming-paraformer-buffered-tokens-c-api.c b/c-api-examples/streaming-paraformer-buffered-tokens-c-api.c index a597374df..be08f4149 100644 --- a/c-api-examples/streaming-paraformer-buffered-tokens-c-api.c +++ b/c-api-examples/streaming-paraformer-buffered-tokens-c-api.c @@ -96,7 +96,7 @@ int32_t main() { recognizer_config.decoding_method = "greedy_search"; recognizer_config.model_config = online_model_config; - SherpaOnnxOnlineRecognizer *recognizer = + const SherpaOnnxOnlineRecognizer *recognizer = SherpaOnnxCreateOnlineRecognizer(&recognizer_config); free((void *)tokens_buf); diff --git a/c-api-examples/streaming-paraformer-c-api.c b/c-api-examples/streaming-paraformer-c-api.c index b54116f08..11748e084 100644 --- a/c-api-examples/streaming-paraformer-c-api.c +++ b/c-api-examples/streaming-paraformer-c-api.c @@ -57,7 +57,7 @@ int32_t main() { recognizer_config.decoding_method = "greedy_search"; recognizer_config.model_config = online_model_config; - SherpaOnnxOnlineRecognizer *recognizer = + const SherpaOnnxOnlineRecognizer *recognizer = SherpaOnnxCreateOnlineRecognizer(&recognizer_config); if (recognizer == NULL) { diff --git a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c index c991d4999..c8afca15a 100644 --- a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c +++ b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c @@ -116,7 +116,7 @@ int32_t main() { recognizer_config.hotwords_buf = hotwords_buf; recognizer_config.hotwords_buf_size = hotwords_buf_size; - SherpaOnnxOnlineRecognizer *recognizer = + const SherpaOnnxOnlineRecognizer *recognizer = SherpaOnnxCreateOnlineRecognizer(&recognizer_config); free((void *)tokens_buf); diff --git a/c-api-examples/streaming-zipformer-c-api.c b/c-api-examples/streaming-zipformer-c-api.c index e1417639d..a38d22f02 100644 --- a/c-api-examples/streaming-zipformer-c-api.c +++ b/c-api-examples/streaming-zipformer-c-api.c @@ -63,7 +63,7 @@ int32_t main() { recognizer_config.decoding_method = "greedy_search"; recognizer_config.model_config = online_model_config; - SherpaOnnxOnlineRecognizer *recognizer = + const SherpaOnnxOnlineRecognizer *recognizer = SherpaOnnxCreateOnlineRecognizer(&recognizer_config); if (recognizer == NULL) { diff --git a/cmake/cmake_extension.py b/cmake/cmake_extension.py index c49c32555..3d0dbef8e 100644 --- a/cmake/cmake_extension.py +++ b/cmake/cmake_extension.py @@ -80,6 +80,7 @@ def get_binaries(): binaries += [ "onnxruntime.dll", "sherpa-onnx-c-api.dll", + "sherpa-onnx-cxx-api.dll", ] return binaries diff --git a/cxx-api-examples/CMakeLists.txt b/cxx-api-examples/CMakeLists.txt new file mode 100644 index 000000000..b51d2e50d --- /dev/null +++ b/cxx-api-examples/CMakeLists.txt @@ -0,0 +1,4 @@ +include_directories(${CMAKE_SOURCE_DIR}) + +add_executable(streaming-zipformer-cxx-api ./streaming-zipformer-cxx-api.cc) +target_link_libraries(streaming-zipformer-cxx-api sherpa-onnx-cxx-api) diff --git a/cxx-api-examples/streaming-zipformer-cxx-api.cc b/cxx-api-examples/streaming-zipformer-cxx-api.cc new file mode 100644 index 000000000..4f38647f4 --- /dev/null +++ b/cxx-api-examples/streaming-zipformer-cxx-api.cc @@ -0,0 +1,91 @@ +// cxx-api-examples/streaming-zipformer-cxx-api.cc +// Copyright (c) 2024 Xiaomi Corporation + +// +// This file demonstrates how to use streaming Zipformer +// with sherpa-onnx's C++ API. +// +// clang-format off +// +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 +// tar xvf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 +// rm sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 +// +// clang-format on + +#include // NOLINT +#include +#include + +#include "sherpa-onnx/c-api/cxx-api.h" + +int32_t main() { + using namespace sherpa_onnx::cxx; + OnlineRecognizerConfig config; + + // please see + // https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#csukuangfj-sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-bilingual-chinese-english + config.model_config.transducer.encoder = + "./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/" + "encoder-epoch-99-avg-1.int8.onnx"; + + // Note: We recommend not using int8.onnx for the decoder. + config.model_config.transducer.decoder = + "./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/" + "decoder-epoch-99-avg-1.onnx"; + + config.model_config.transducer.joiner = + "./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/" + "joiner-epoch-99-avg-1.int8.onnx"; + + config.model_config.tokens = + "./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt"; + + config.model_config.num_threads = 1; + + std::cout << "Loading model\n"; + OnlineRecognizer recongizer = OnlineRecognizer::Create(config); + if (!recongizer.Get()) { + std::cerr << "Please check your config\n"; + return -1; + } + std::cout << "Loading model done\n"; + + std::string wave_filename = + "./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/test_wavs/" + "0.wav"; + Wave wave = ReadWave(wave_filename); + if (wave.samples.empty()) { + std::cerr << "Failed to read: '" << wave_filename << "'\n"; + return -1; + } + + std::cout << "Start recognition\n"; + const auto begin = std::chrono::steady_clock::now(); + + OnlineStream stream = recongizer.CreateStream(); + stream.AcceptWaveform(wave.sample_rate, wave.samples.data(), + wave.samples.size()); + while (recongizer.IsReady(&stream)) { + recongizer.Decode(&stream); + } + + OnlineRecognizerResult result = recongizer.GetResult(&stream); + + const auto end = std::chrono::steady_clock::now(); + const float elapsed_seconds = + std::chrono::duration_cast(end - begin) + .count() / + 1000.; + float duration = wave.samples.size() / static_cast(wave.sample_rate); + float rtf = elapsed_seconds / duration; + + std::cout << "text: " << result.text << "\n"; + printf("Number of threads: %d\n", config.model_config.num_threads); + printf("Duration: %.3fs\n", duration); + printf("Elapsed seconds: %.3fs\n", elapsed_seconds); + printf("(Real time factor) RTF = %.3f / %.3f = %.3f\n", elapsed_seconds, + duration, rtf); + + return 0; +} diff --git a/ffmpeg-examples/sherpa-onnx-ffmpeg.c b/ffmpeg-examples/sherpa-onnx-ffmpeg.c index f99ac0bdc..31e7491f0 100644 --- a/ffmpeg-examples/sherpa-onnx-ffmpeg.c +++ b/ffmpeg-examples/sherpa-onnx-ffmpeg.c @@ -290,7 +290,7 @@ int main(int argc, char **argv) { } SherpaOnnxOnlineRecognizerConfig config; - memset(&config, 0, sizeof(config)); + memset(&config, 0, sizeof(config)); config.model_config.tokens = argv[1]; config.model_config.transducer.encoder = argv[2]; config.model_config.transducer.decoder = argv[3]; @@ -318,7 +318,7 @@ int main(int argc, char **argv) { config.rule2_min_trailing_silence = 1.2; config.rule3_min_utterance_length = 300; - SherpaOnnxOnlineRecognizer *recognizer = + const SherpaOnnxOnlineRecognizer *recognizer = SherpaOnnxCreateOnlineRecognizer(&config); SherpaOnnxOnlineStream *stream = SherpaOnnxCreateOnlineStream(recognizer); const SherpaOnnxDisplay *display = SherpaOnnxCreateDisplay(50); diff --git a/mfc-examples/StreamingSpeechRecognition/StreamingSpeechRecognitionDlg.h b/mfc-examples/StreamingSpeechRecognition/StreamingSpeechRecognitionDlg.h index 36b3d9cad..39acbcfe4 100644 --- a/mfc-examples/StreamingSpeechRecognition/StreamingSpeechRecognitionDlg.h +++ b/mfc-examples/StreamingSpeechRecognition/StreamingSpeechRecognitionDlg.h @@ -45,7 +45,7 @@ class CStreamingSpeechRecognitionDlg : public CDialogEx { private: Microphone mic_; - SherpaOnnxOnlineRecognizer *recognizer_ = nullptr; + const SherpaOnnxOnlineRecognizer *recognizer_ = nullptr; PaStream *pa_stream_ = nullptr; RecognizerThread *thread_ = nullptr; @@ -54,7 +54,7 @@ class CStreamingSpeechRecognitionDlg : public CDialogEx { public: bool started_ = false; - SherpaOnnxOnlineStream *stream_ = nullptr; + const SherpaOnnxOnlineStream *stream_ = nullptr; public: int RunThread(); @@ -79,4 +79,4 @@ class RecognizerThread : public CWinThread { private: CStreamingSpeechRecognitionDlg *dlg_; -}; \ No newline at end of file +}; diff --git a/scripts/node-addon-api/src/streaming-asr.cc b/scripts/node-addon-api/src/streaming-asr.cc index 6057cade3..cb3aaac86 100644 --- a/scripts/node-addon-api/src/streaming-asr.cc +++ b/scripts/node-addon-api/src/streaming-asr.cc @@ -199,7 +199,8 @@ static Napi::External CreateOnlineRecognizerWrapper( c.ctc_fst_decoder_config = GetCtcFstDecoderConfig(o); - SherpaOnnxOnlineRecognizer *recognizer = SherpaOnnxCreateOnlineRecognizer(&c); + const SherpaOnnxOnlineRecognizer *recognizer = + SherpaOnnxCreateOnlineRecognizer(&c); if (c.model_config.transducer.encoder) { delete[] c.model_config.transducer.encoder; @@ -281,7 +282,7 @@ static Napi::External CreateOnlineRecognizerWrapper( } return Napi::External::New( - env, recognizer, + env, const_cast(recognizer), [](Napi::Env env, SherpaOnnxOnlineRecognizer *recognizer) { SherpaOnnxDestroyOnlineRecognizer(recognizer); }); diff --git a/sherpa-onnx/c-api/CMakeLists.txt b/sherpa-onnx/c-api/CMakeLists.txt index d87f849aa..9c8a59771 100644 --- a/sherpa-onnx/c-api/CMakeLists.txt +++ b/sherpa-onnx/c-api/CMakeLists.txt @@ -3,12 +3,25 @@ add_library(sherpa-onnx-c-api c-api.cc) target_link_libraries(sherpa-onnx-c-api sherpa-onnx-core) if(BUILD_SHARED_LIBS) - target_compile_definitions(sherpa-onnx-c-api PRIVATE SHERPA_ONNX_BUILD_SHARED_LIBS=1) - target_compile_definitions(sherpa-onnx-c-api PRIVATE SHERPA_ONNX_BUILD_MAIN_LIB=1) + target_compile_definitions(sherpa-onnx-c-api PUBLIC SHERPA_ONNX_BUILD_SHARED_LIBS=1) + target_compile_definitions(sherpa-onnx-c-api PUBLIC SHERPA_ONNX_BUILD_MAIN_LIB=1) endif() -install(TARGETS sherpa-onnx-c-api DESTINATION lib) +add_library(sherpa-onnx-cxx-api cxx-api.cc) +target_link_libraries(sherpa-onnx-cxx-api sherpa-onnx-c-api) -install(FILES c-api.h - DESTINATION include/sherpa-onnx/c-api +install( + TARGETS + sherpa-onnx-c-api + sherpa-onnx-cxx-api + DESTINATION + lib +) + +install( + FILES + c-api.h + cxx-api.h + DESTINATION + include/sherpa-onnx/c-api ) diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index 4ba0a4a60..653355a66 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -5,6 +5,7 @@ #include "sherpa-onnx/c-api/c-api.h" #include +#include #include #include #include @@ -51,7 +52,7 @@ struct SherpaOnnxDisplay { #define SHERPA_ONNX_OR(x, y) (x ? x : y) -SherpaOnnxOnlineRecognizer *SherpaOnnxCreateOnlineRecognizer( +const SherpaOnnxOnlineRecognizer *SherpaOnnxCreateOnlineRecognizer( const SherpaOnnxOnlineRecognizerConfig *config) { sherpa_onnx::OnlineRecognizerConfig recognizer_config; diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index 4b41a81a9..0a01379d4 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -205,7 +205,8 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOnlineStream SherpaOnnxOnlineStream; /// @param config Config for the recognizer. /// @return Return a pointer to the recognizer. The user has to invoke // SherpaOnnxDestroyOnlineRecognizer() to free it to avoid memory leak. -SHERPA_ONNX_API SherpaOnnxOnlineRecognizer *SherpaOnnxCreateOnlineRecognizer( +SHERPA_ONNX_API const SherpaOnnxOnlineRecognizer * +SherpaOnnxCreateOnlineRecognizer( const SherpaOnnxOnlineRecognizerConfig *config); /// Free a pointer returned by SherpaOnnxCreateOnlineRecognizer() diff --git a/sherpa-onnx/c-api/cxx-api.cc b/sherpa-onnx/c-api/cxx-api.cc new file mode 100644 index 000000000..243e99738 --- /dev/null +++ b/sherpa-onnx/c-api/cxx-api.cc @@ -0,0 +1,159 @@ +// sherpa-onnx/c-api/cxx-api.cc +// +// Copyright (c) 2024 Xiaomi Corporation +#include "sherpa-onnx/c-api/cxx-api.h" + +#include +#include + +namespace sherpa_onnx::cxx { + +Wave ReadWave(const std::string &filename) { + auto p = SherpaOnnxReadWave(filename.c_str()); + + Wave ans; + if (p) { + ans.samples.resize(p->num_samples); + + std::copy(p->samples, p->samples + p->num_samples, ans.samples.data()); + + ans.sample_rate = p->sample_rate; + SherpaOnnxFreeWave(p); + } + + return ans; +} + +OnlineStream::OnlineStream(const SherpaOnnxOnlineStream *p) + : MoveOnly(p) {} + +void OnlineStream::Destroy(const SherpaOnnxOnlineStream *p) const { + SherpaOnnxDestroyOnlineStream(p); +} + +void OnlineStream::AcceptWaveform(int32_t sample_rate, const float *samples, + int32_t n) const { + SherpaOnnxOnlineStreamAcceptWaveform(p_, sample_rate, samples, n); +} + +OnlineRecognizer OnlineRecognizer::Create( + const OnlineRecognizerConfig &config) { + struct SherpaOnnxOnlineRecognizerConfig c; + memset(&c, 0, sizeof(c)); + + c.feat_config.sample_rate = config.feat_config.sample_rate; + c.feat_config.feature_dim = config.feat_config.feature_dim; + + c.model_config.transducer.encoder = + config.model_config.transducer.encoder.c_str(); + c.model_config.transducer.decoder = + config.model_config.transducer.decoder.c_str(); + c.model_config.transducer.joiner = + config.model_config.transducer.joiner.c_str(); + + c.model_config.paraformer.encoder = + config.model_config.paraformer.encoder.c_str(); + c.model_config.paraformer.decoder = + config.model_config.paraformer.decoder.c_str(); + + c.model_config.zipformer2_ctc.model = + config.model_config.zipformer2_ctc.model.c_str(); + + c.model_config.tokens = config.model_config.tokens.c_str(); + c.model_config.num_threads = config.model_config.num_threads; + c.model_config.provider = config.model_config.provider.c_str(); + c.model_config.debug = config.model_config.debug; + c.model_config.model_type = config.model_config.model_type.c_str(); + c.model_config.modeling_unit = config.model_config.modeling_unit.c_str(); + c.model_config.bpe_vocab = config.model_config.bpe_vocab.c_str(); + c.model_config.tokens_buf = config.model_config.tokens_buf.c_str(); + c.model_config.tokens_buf_size = config.model_config.tokens_buf.size(); + + c.decoding_method = config.decoding_method.c_str(); + c.max_active_paths = config.max_active_paths; + c.enable_endpoint = config.enable_endpoint; + c.rule1_min_trailing_silence = config.rule1_min_trailing_silence; + c.rule2_min_trailing_silence = config.rule2_min_trailing_silence; + c.rule3_min_utterance_length = config.rule3_min_utterance_length; + c.hotwords_file = config.hotwords_file.c_str(); + c.hotwords_score = config.hotwords_score; + + c.ctc_fst_decoder_config.graph = config.ctc_fst_decoder_config.graph.c_str(); + c.ctc_fst_decoder_config.max_active = + config.ctc_fst_decoder_config.max_active; + + c.rule_fsts = config.rule_fsts.c_str(); + c.rule_fars = config.rule_fars.c_str(); + + c.blank_penalty = config.blank_penalty; + + c.hotwords_buf = config.hotwords_buf.c_str(); + c.hotwords_buf_size = config.hotwords_buf.size(); + + auto p = SherpaOnnxCreateOnlineRecognizer(&c); + return OnlineRecognizer(p); +} + +OnlineRecognizer::OnlineRecognizer(const SherpaOnnxOnlineRecognizer *p) + : MoveOnly(p) {} + +void OnlineRecognizer::Destroy(const SherpaOnnxOnlineRecognizer *p) const { + SherpaOnnxDestroyOnlineRecognizer(p); +} + +OnlineStream OnlineRecognizer::CreateStream() const { + auto s = SherpaOnnxCreateOnlineStream(p_); + return OnlineStream{s}; +} + +OnlineStream OnlineRecognizer::CreateStream(const std::string &hotwords) const { + auto s = SherpaOnnxCreateOnlineStreamWithHotwords(p_, hotwords.c_str()); + return OnlineStream{s}; +} + +bool OnlineRecognizer::IsReady(const OnlineStream *s) const { + return SherpaOnnxIsOnlineStreamReady(p_, s->Get()); +} + +void OnlineRecognizer::Decode(const OnlineStream *s) const { + SherpaOnnxDecodeOnlineStream(p_, s->Get()); +} + +void OnlineRecognizer::Decode(const OnlineStream *ss, int32_t n) const { + if (n <= 0) { + return; + } + + std::vector streams(n); + for (int32_t i = 0; i != n; ++n) { + streams[i] = ss[i].Get(); + } + + SherpaOnnxDecodeMultipleOnlineStreams(p_, streams.data(), n); +} + +OnlineRecognizerResult OnlineRecognizer::GetResult( + const OnlineStream *s) const { + auto r = SherpaOnnxGetOnlineStreamResult(p_, s->Get()); + + OnlineRecognizerResult ans; + ans.text = r->text; + + ans.tokens.resize(r->count); + for (int32_t i = 0; i != r->count; ++i) { + ans.tokens[i] = r->tokens_arr[i]; + } + + if (r->timestamps) { + ans.timestamps.resize(r->count); + std::copy(r->timestamps, r->timestamps + r->count, ans.timestamps.data()); + } + + ans.json = r->json; + + SherpaOnnxDestroyOnlineRecognizerResult(r); + + return ans; +} + +} // namespace sherpa_onnx::cxx diff --git a/sherpa-onnx/c-api/cxx-api.h b/sherpa-onnx/c-api/cxx-api.h new file mode 100644 index 000000000..416333b7f --- /dev/null +++ b/sherpa-onnx/c-api/cxx-api.h @@ -0,0 +1,179 @@ +// sherpa-onnx/c-api/cxx-api.h +// +// Copyright (c) 2024 Xiaomi Corporation + +// C++ Wrapper of the C API for sherpa-onnx +#ifndef SHERPA_ONNX_C_API_CXX_API_H_ +#define SHERPA_ONNX_C_API_CXX_API_H_ + +#include +#include + +#include "sherpa-onnx/c-api/c-api.h" + +namespace sherpa_onnx::cxx { + +struct SHERPA_ONNX_API OnlineTransducerModelConfig { + std::string encoder; + std::string decoder; + std::string joiner; +}; + +struct SHERPA_ONNX_API OnlineParaformerModelConfig { + std::string encoder; + std::string decoder; +}; + +struct SHERPA_ONNX_API OnlineZipformer2CtcModelConfig { + std::string model; +}; + +struct SHERPA_ONNX_API OnlineModelConfig { + OnlineTransducerModelConfig transducer; + OnlineParaformerModelConfig paraformer; + OnlineZipformer2CtcModelConfig zipformer2_ctc; + std::string tokens; + int32_t num_threads = 1; + std::string provider = "cpu"; + bool debug = false; + std::string model_type; + std::string modeling_unit = "cjkchar"; + std::string bpe_vocab; + std::string tokens_buf; +}; + +struct SHERPA_ONNX_API FeatureConfig { + int32_t sample_rate = 16000; + int32_t feature_dim = 80; +}; + +struct SHERPA_ONNX_API OnlineCtcFstDecoderConfig { + std::string graph; + int32_t max_active = 3000; +}; + +struct SHERPA_ONNX_API OnlineRecognizerConfig { + FeatureConfig feat_config; + OnlineModelConfig model_config; + + std::string decoding_method = "greedy_search"; + + int32_t max_active_paths = 4; + + bool enable_endpoint = false; + + float rule1_min_trailing_silence = 2.4; + + float rule2_min_trailing_silence = 1.2; + + float rule3_min_utterance_length = 20; + + std::string hotwords_file; + + float hotwords_score = 1.5; + + OnlineCtcFstDecoderConfig ctc_fst_decoder_config; + std::string rule_fsts; + std::string rule_fars; + float blank_penalty = 0; + + std::string hotwords_buf; +}; + +struct SHERPA_ONNX_API OnlineRecognizerResult { + std::string text; + std::vector tokens; + std::vector timestamps; + std::string json; +}; + +struct SHERPA_ONNX_API Wave { + std::vector samples; + int32_t sample_rate; +}; + +SHERPA_ONNX_API Wave ReadWave(const std::string &filename); + +template +class SHERPA_ONNX_API MoveOnly { + public: + explicit MoveOnly(const T *p) : p_(p) {} + + ~MoveOnly() { Destroy(); } + + MoveOnly(const MoveOnly &) = delete; + + MoveOnly &operator=(const MoveOnly &) = delete; + + MoveOnly(MoveOnly &&other) : p_(other.Release()) {} + + MoveOnly &operator=(MoveOnly &&other) { + if (&other == this) { + return *this; + } + + Destroy(); + + p_ = other.Release(); + } + + const T *Get() const { return p_; } + + const T *Release() { + const T *p = p_; + p_ = nullptr; + return p; + } + + private: + void Destroy() { + if (p_ == nullptr) { + return; + } + + static_cast(this)->Destroy(p_); + + p_ = nullptr; + } + + protected: + const T *p_ = nullptr; +}; + +class SHERPA_ONNX_API OnlineStream + : public MoveOnly { + public: + explicit OnlineStream(const SherpaOnnxOnlineStream *p); + + void AcceptWaveform(int32_t sample_rate, const float *samples, + int32_t n) const; + + void Destroy(const SherpaOnnxOnlineStream *p) const; +}; + +class SHERPA_ONNX_API OnlineRecognizer + : public MoveOnly { + public: + static OnlineRecognizer Create(const OnlineRecognizerConfig &config); + + void Destroy(const SherpaOnnxOnlineRecognizer *p) const; + + OnlineStream CreateStream() const; + + OnlineStream CreateStream(const std::string &hotwords) const; + + bool IsReady(const OnlineStream *s) const; + + void Decode(const OnlineStream *s) const; + + void Decode(const OnlineStream *ss, int32_t n) const; + + OnlineRecognizerResult GetResult(const OnlineStream *s) const; + + private: + explicit OnlineRecognizer(const SherpaOnnxOnlineRecognizer *p); +}; + +} // namespace sherpa_onnx::cxx + +#endif // SHERPA_ONNX_C_API_CXX_API_H_ From ceb69ebd946a6f022d8349ba793d173fd0f5d204 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 23 Oct 2024 16:40:12 +0800 Subject: [PATCH 031/183] Add C++ API for non-streaming ASR (#1456) --- .github/scripts/test-cxx-api.sh | 21 +++ .github/workflows/cxx-api.yaml | 69 +++++++++ .github/workflows/linux.yaml | 2 + .github/workflows/macos.yaml | 2 + .github/workflows/sanitizer.yaml | 1 + .github/workflows/windows-x64.yaml | 2 + .github/workflows/windows-x86.yaml | 2 + c-api-examples/paraformer-c-api.c | 5 +- c-api-examples/sense-voice-c-api.c | 5 +- .../streaming-ctc-buffered-tokens-c-api.c | 3 +- ...reaming-paraformer-buffered-tokens-c-api.c | 3 +- c-api-examples/streaming-paraformer-c-api.c | 3 +- ...zipformer-buffered-tokens-hotwords-c-api.c | 3 +- c-api-examples/streaming-zipformer-c-api.c | 3 +- c-api-examples/telespeech-c-api.c | 5 +- c-api-examples/vad-sense-voice-c-api.c | 9 +- c-api-examples/whisper-c-api.c | 5 +- c-api-examples/zipformer-c-api.c | 5 +- cxx-api-examples/CMakeLists.txt | 6 + cxx-api-examples/sense-voice-cxx-api.cc | 78 ++++++++++ .../streaming-zipformer-cxx-api.cc | 2 + cxx-api-examples/whisper-cxx-api.cc | 76 +++++++++ ffmpeg-examples/sherpa-onnx-ffmpeg.c | 3 +- .../NonStreamingSpeechRecognitionDlg.cpp | 2 +- .../NonStreamingSpeechRecognitionDlg.h | 2 +- .../node-addon-api/src/non-streaming-asr.cc | 10 +- sherpa-onnx/c-api/c-api.cc | 14 +- sherpa-onnx/c-api/c-api.h | 15 +- sherpa-onnx/c-api/cxx-api.cc | 146 ++++++++++++++++++ sherpa-onnx/c-api/cxx-api.h | 129 ++++++++++++++++ sherpa-onnx/csrc/online-recognizer-impl.cc | 16 +- 31 files changed, 604 insertions(+), 43 deletions(-) create mode 100644 cxx-api-examples/sense-voice-cxx-api.cc create mode 100644 cxx-api-examples/whisper-cxx-api.cc diff --git a/.github/scripts/test-cxx-api.sh b/.github/scripts/test-cxx-api.sh index 89b2f1dd6..aedf16133 100755 --- a/.github/scripts/test-cxx-api.sh +++ b/.github/scripts/test-cxx-api.sh @@ -9,6 +9,8 @@ log() { } echo "CXX_STREAMING_ZIPFORMER_EXE is $CXX_STREAMING_ZIPFORMER_EXE" +echo "CXX_WHISPER_EXE is $CXX_WHISPER_EXE" +echo "CXX_SENSE_VOICE_EXE is $CXX_SENSE_VOICE_EXE" echo "PATH: $PATH" log "------------------------------------------------------------" @@ -19,3 +21,22 @@ tar xvf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 rm sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 $CXX_STREAMING_ZIPFORMER_EXE rm -rf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 + +log "------------------------------------------------------------" +log "Test Whisper CXX API" +log "------------------------------------------------------------" +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2 +tar xvf sherpa-onnx-whisper-tiny.en.tar.bz2 +rm sherpa-onnx-whisper-tiny.en.tar.bz2 +$CXX_WHISPER_EXE +rm -rf sherpa-onnx-whisper-tiny.en + +log "------------------------------------------------------------" +log "Test SenseVoice CXX API" +log "------------------------------------------------------------" +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 +tar xvf sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 +rm sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 + +$CXX_SENSE_VOICE_EXE +rm -rf sherpa-onnx-sense-voice-* diff --git a/.github/workflows/cxx-api.yaml b/.github/workflows/cxx-api.yaml index 1c882f2ff..357aaa227 100644 --- a/.github/workflows/cxx-api.yaml +++ b/.github/workflows/cxx-api.yaml @@ -4,6 +4,7 @@ on: push: branches: - master + - cxx-api-asr-non-streaming paths: - '.github/workflows/cxx-api.yaml' - 'CMakeLists.txt' @@ -82,6 +83,74 @@ jobs: otool -L ./install/lib/libsherpa-onnx-cxx-api.dylib fi + - name: Test whisper + shell: bash + run: | + g++ -std=c++17 -o whisper-cxx-api ./cxx-api-examples/whisper-cxx-api.cc \ + -I ./build/install/include \ + -L ./build/install/lib/ \ + -l sherpa-onnx-cxx-api \ + -l sherpa-onnx-c-api \ + -l onnxruntime + + ls -lh whisper-cxx-api + + if [[ ${{ matrix.os }} == ubuntu-latest ]]; then + ldd ./whisper-cxx-api + echo "----" + readelf -d ./whisper-cxx-api + fi + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2 + tar xvf sherpa-onnx-whisper-tiny.en.tar.bz2 + rm sherpa-onnx-whisper-tiny.en.tar.bz2 + + ls -lh sherpa-onnx-whisper-tiny.en + echo "---" + ls -lh sherpa-onnx-whisper-tiny.en/test_wavs + + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH + export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH + + ./whisper-cxx-api + + rm -rf sherpa-onnx-whisper-* + rm ./whisper-cxx-api + + - name: Test SenseVoice + shell: bash + run: | + g++ -std=c++17 -o sense-voice-cxx-api ./cxx-api-examples/sense-voice-cxx-api.cc \ + -I ./build/install/include \ + -L ./build/install/lib/ \ + -l sherpa-onnx-cxx-api \ + -l sherpa-onnx-c-api \ + -l onnxruntime + + ls -lh sense-voice-cxx-api + + if [[ ${{ matrix.os }} == ubuntu-latest ]]; then + ldd ./sense-voice-cxx-api + echo "----" + readelf -d ./sense-voice-cxx-api + fi + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 + tar xvf sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 + rm sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 + + ls -lh sherpa-onnx-sense-voice-* + echo "---" + ls -lh sherpa-onnx-sense-voice-*/test_wavs + + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH + export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH + + ./sense-voice-cxx-api + + rm -rf sherpa-onnx-sense-voice-* + rm ./sense-voice-cxx-api + - name: Test streaming zipformer shell: bash run: | diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml index e21c452c0..a37f48618 100644 --- a/.github/workflows/linux.yaml +++ b/.github/workflows/linux.yaml @@ -155,6 +155,8 @@ jobs: du -h -d1 . export PATH=$PWD/build/bin:$PATH export CXX_STREAMING_ZIPFORMER_EXE=streaming-zipformer-cxx-api + export CXX_WHISPER_EXE=whisper-cxx-api + export CXX_SENSE_VOICE_EXE=sense-voice-cxx-api .github/scripts/test-cxx-api.sh du -h -d1 . diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml index 7b01846a7..849631015 100644 --- a/.github/workflows/macos.yaml +++ b/.github/workflows/macos.yaml @@ -127,6 +127,8 @@ jobs: du -h -d1 . export PATH=$PWD/build/bin:$PATH export CXX_STREAMING_ZIPFORMER_EXE=streaming-zipformer-cxx-api + export CXX_WHISPER_EXE=whisper-cxx-api + export CXX_SENSE_VOICE_EXE=sense-voice-cxx-api .github/scripts/test-cxx-api.sh du -h -d1 . diff --git a/.github/workflows/sanitizer.yaml b/.github/workflows/sanitizer.yaml index f12bafa9a..7cda96899 100644 --- a/.github/workflows/sanitizer.yaml +++ b/.github/workflows/sanitizer.yaml @@ -81,6 +81,7 @@ jobs: run: | export PATH=$PWD/build/bin:$PATH export CXX_STREAMING_ZIPFORMER_EXE=streaming-zipformer-cxx-api + export CXX_WHISPER_EXE=whisper-cxx-api .github/scripts/test-cxx-api.sh diff --git a/.github/workflows/windows-x64.yaml b/.github/workflows/windows-x64.yaml index f07d6c78c..9435dcefd 100644 --- a/.github/workflows/windows-x64.yaml +++ b/.github/workflows/windows-x64.yaml @@ -98,6 +98,8 @@ jobs: run: | export PATH=$PWD/build/bin/Release:$PATH export CXX_STREAMING_ZIPFORMER_EXE=streaming-zipformer-cxx-api.exe + export CXX_WHISPER_EXE=whisper-cxx-api.exe + export CXX_SENSE_VOICE_EXE=sense-voice-cxx-api.exe .github/scripts/test-cxx-api.sh diff --git a/.github/workflows/windows-x86.yaml b/.github/workflows/windows-x86.yaml index bacbdff01..36089b2dd 100644 --- a/.github/workflows/windows-x86.yaml +++ b/.github/workflows/windows-x86.yaml @@ -98,6 +98,8 @@ jobs: run: | export PATH=$PWD/build/bin/Release:$PATH export CXX_STREAMING_ZIPFORMER_EXE=streaming-zipformer-cxx-api.exe + export CXX_WHISPER_EXE=whisper-cxx-api.exe + export CXX_SENSE_VOICE_EXE=sense-voice-cxx-api.exe .github/scripts/test-cxx-api.sh diff --git a/c-api-examples/paraformer-c-api.c b/c-api-examples/paraformer-c-api.c index 345aed555..98d38c789 100644 --- a/c-api-examples/paraformer-c-api.c +++ b/c-api-examples/paraformer-c-api.c @@ -54,7 +54,7 @@ int32_t main() { recognizer_config.decoding_method = "greedy_search"; recognizer_config.model_config = offline_model_config; - SherpaOnnxOfflineRecognizer *recognizer = + const SherpaOnnxOfflineRecognizer *recognizer = SherpaOnnxCreateOfflineRecognizer(&recognizer_config); if (recognizer == NULL) { @@ -63,7 +63,8 @@ int32_t main() { return -1; } - SherpaOnnxOfflineStream *stream = SherpaOnnxCreateOfflineStream(recognizer); + const SherpaOnnxOfflineStream *stream = + SherpaOnnxCreateOfflineStream(recognizer); SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate, wave->samples, wave->num_samples); diff --git a/c-api-examples/sense-voice-c-api.c b/c-api-examples/sense-voice-c-api.c index 06e890636..25d58219e 100644 --- a/c-api-examples/sense-voice-c-api.c +++ b/c-api-examples/sense-voice-c-api.c @@ -56,7 +56,7 @@ int32_t main() { recognizer_config.decoding_method = "greedy_search"; recognizer_config.model_config = offline_model_config; - SherpaOnnxOfflineRecognizer *recognizer = + const SherpaOnnxOfflineRecognizer *recognizer = SherpaOnnxCreateOfflineRecognizer(&recognizer_config); if (recognizer == NULL) { @@ -65,7 +65,8 @@ int32_t main() { return -1; } - SherpaOnnxOfflineStream *stream = SherpaOnnxCreateOfflineStream(recognizer); + const SherpaOnnxOfflineStream *stream = + SherpaOnnxCreateOfflineStream(recognizer); SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate, wave->samples, wave->num_samples); diff --git a/c-api-examples/streaming-ctc-buffered-tokens-c-api.c b/c-api-examples/streaming-ctc-buffered-tokens-c-api.c index eb834fb70..98f5b4a60 100644 --- a/c-api-examples/streaming-ctc-buffered-tokens-c-api.c +++ b/c-api-examples/streaming-ctc-buffered-tokens-c-api.c @@ -107,7 +107,8 @@ int32_t main() { return -1; } - SherpaOnnxOnlineStream *stream = SherpaOnnxCreateOnlineStream(recognizer); + const SherpaOnnxOnlineStream *stream = + SherpaOnnxCreateOnlineStream(recognizer); const SherpaOnnxDisplay *display = SherpaOnnxCreateDisplay(50); int32_t segment_id = 0; diff --git a/c-api-examples/streaming-paraformer-buffered-tokens-c-api.c b/c-api-examples/streaming-paraformer-buffered-tokens-c-api.c index be08f4149..0c382cc94 100644 --- a/c-api-examples/streaming-paraformer-buffered-tokens-c-api.c +++ b/c-api-examples/streaming-paraformer-buffered-tokens-c-api.c @@ -108,7 +108,8 @@ int32_t main() { return -1; } - SherpaOnnxOnlineStream *stream = SherpaOnnxCreateOnlineStream(recognizer); + const SherpaOnnxOnlineStream *stream = + SherpaOnnxCreateOnlineStream(recognizer); const SherpaOnnxDisplay *display = SherpaOnnxCreateDisplay(50); int32_t segment_id = 0; diff --git a/c-api-examples/streaming-paraformer-c-api.c b/c-api-examples/streaming-paraformer-c-api.c index 11748e084..384ea411b 100644 --- a/c-api-examples/streaming-paraformer-c-api.c +++ b/c-api-examples/streaming-paraformer-c-api.c @@ -66,7 +66,8 @@ int32_t main() { return -1; } - SherpaOnnxOnlineStream *stream = SherpaOnnxCreateOnlineStream(recognizer); + const SherpaOnnxOnlineStream *stream = + SherpaOnnxCreateOnlineStream(recognizer); const SherpaOnnxDisplay *display = SherpaOnnxCreateDisplay(50); int32_t segment_id = 0; diff --git a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c index c8afca15a..bd76ea8ab 100644 --- a/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c +++ b/c-api-examples/streaming-zipformer-buffered-tokens-hotwords-c-api.c @@ -130,7 +130,8 @@ int32_t main() { return -1; } - SherpaOnnxOnlineStream *stream = SherpaOnnxCreateOnlineStream(recognizer); + const SherpaOnnxOnlineStream *stream = + SherpaOnnxCreateOnlineStream(recognizer); const SherpaOnnxDisplay *display = SherpaOnnxCreateDisplay(50); int32_t segment_id = 0; diff --git a/c-api-examples/streaming-zipformer-c-api.c b/c-api-examples/streaming-zipformer-c-api.c index a38d22f02..6011186ea 100644 --- a/c-api-examples/streaming-zipformer-c-api.c +++ b/c-api-examples/streaming-zipformer-c-api.c @@ -72,7 +72,8 @@ int32_t main() { return -1; } - SherpaOnnxOnlineStream *stream = SherpaOnnxCreateOnlineStream(recognizer); + const SherpaOnnxOnlineStream *stream = + SherpaOnnxCreateOnlineStream(recognizer); const SherpaOnnxDisplay *display = SherpaOnnxCreateDisplay(50); int32_t segment_id = 0; diff --git a/c-api-examples/telespeech-c-api.c b/c-api-examples/telespeech-c-api.c index fa7824c3b..9bf34b1a8 100644 --- a/c-api-examples/telespeech-c-api.c +++ b/c-api-examples/telespeech-c-api.c @@ -49,7 +49,7 @@ int32_t main() { recognizer_config.decoding_method = "greedy_search"; recognizer_config.model_config = offline_model_config; - SherpaOnnxOfflineRecognizer *recognizer = + const SherpaOnnxOfflineRecognizer *recognizer = SherpaOnnxCreateOfflineRecognizer(&recognizer_config); if (recognizer == NULL) { @@ -58,7 +58,8 @@ int32_t main() { return -1; } - SherpaOnnxOfflineStream *stream = SherpaOnnxCreateOfflineStream(recognizer); + const SherpaOnnxOfflineStream *stream = + SherpaOnnxCreateOfflineStream(recognizer); SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate, wave->samples, wave->num_samples); diff --git a/c-api-examples/vad-sense-voice-c-api.c b/c-api-examples/vad-sense-voice-c-api.c index 172ec0a79..3049c9572 100644 --- a/c-api-examples/vad-sense-voice-c-api.c +++ b/c-api-examples/vad-sense-voice-c-api.c @@ -66,7 +66,7 @@ int32_t main() { recognizer_config.decoding_method = "greedy_search"; recognizer_config.model_config = offline_model_config; - SherpaOnnxOfflineRecognizer *recognizer = + const SherpaOnnxOfflineRecognizer *recognizer = SherpaOnnxCreateOfflineRecognizer(&recognizer_config); if (recognizer == NULL) { @@ -108,8 +108,9 @@ int32_t main() { const SherpaOnnxSpeechSegment *segment = SherpaOnnxVoiceActivityDetectorFront(vad); - SherpaOnnxOfflineStream *stream = + const SherpaOnnxOfflineStream *stream = SherpaOnnxCreateOfflineStream(recognizer); + SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate, segment->samples, segment->n); @@ -138,7 +139,9 @@ int32_t main() { const SherpaOnnxSpeechSegment *segment = SherpaOnnxVoiceActivityDetectorFront(vad); - SherpaOnnxOfflineStream *stream = SherpaOnnxCreateOfflineStream(recognizer); + const SherpaOnnxOfflineStream *stream = + SherpaOnnxCreateOfflineStream(recognizer); + SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate, segment->samples, segment->n); diff --git a/c-api-examples/whisper-c-api.c b/c-api-examples/whisper-c-api.c index 3a71bcb03..2e795b025 100644 --- a/c-api-examples/whisper-c-api.c +++ b/c-api-examples/whisper-c-api.c @@ -58,7 +58,7 @@ int32_t main() { recognizer_config.decoding_method = "greedy_search"; recognizer_config.model_config = offline_model_config; - SherpaOnnxOfflineRecognizer *recognizer = + const SherpaOnnxOfflineRecognizer *recognizer = SherpaOnnxCreateOfflineRecognizer(&recognizer_config); if (recognizer == NULL) { @@ -69,7 +69,8 @@ int32_t main() { return -1; } - SherpaOnnxOfflineStream *stream = SherpaOnnxCreateOfflineStream(recognizer); + const SherpaOnnxOfflineStream *stream = + SherpaOnnxCreateOfflineStream(recognizer); SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate, wave->samples, wave->num_samples); diff --git a/c-api-examples/zipformer-c-api.c b/c-api-examples/zipformer-c-api.c index 35393b19c..4db22fc38 100644 --- a/c-api-examples/zipformer-c-api.c +++ b/c-api-examples/zipformer-c-api.c @@ -60,7 +60,7 @@ int32_t main() { recognizer_config.decoding_method = "greedy_search"; recognizer_config.model_config = offline_model_config; - SherpaOnnxOfflineRecognizer *recognizer = + const SherpaOnnxOfflineRecognizer *recognizer = SherpaOnnxCreateOfflineRecognizer(&recognizer_config); if (recognizer == NULL) { @@ -69,7 +69,8 @@ int32_t main() { return -1; } - SherpaOnnxOfflineStream *stream = SherpaOnnxCreateOfflineStream(recognizer); + const SherpaOnnxOfflineStream *stream = + SherpaOnnxCreateOfflineStream(recognizer); SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate, wave->samples, wave->num_samples); diff --git a/cxx-api-examples/CMakeLists.txt b/cxx-api-examples/CMakeLists.txt index b51d2e50d..7c9853080 100644 --- a/cxx-api-examples/CMakeLists.txt +++ b/cxx-api-examples/CMakeLists.txt @@ -2,3 +2,9 @@ include_directories(${CMAKE_SOURCE_DIR}) add_executable(streaming-zipformer-cxx-api ./streaming-zipformer-cxx-api.cc) target_link_libraries(streaming-zipformer-cxx-api sherpa-onnx-cxx-api) + +add_executable(whisper-cxx-api ./whisper-cxx-api.cc) +target_link_libraries(whisper-cxx-api sherpa-onnx-cxx-api) + +add_executable(sense-voice-cxx-api ./sense-voice-cxx-api.cc) +target_link_libraries(sense-voice-cxx-api sherpa-onnx-cxx-api) diff --git a/cxx-api-examples/sense-voice-cxx-api.cc b/cxx-api-examples/sense-voice-cxx-api.cc new file mode 100644 index 000000000..15d752058 --- /dev/null +++ b/cxx-api-examples/sense-voice-cxx-api.cc @@ -0,0 +1,78 @@ +// cxx-api-examples/sense-voice-cxx-api.cc +// Copyright (c) 2024 Xiaomi Corporation + +// +// This file demonstrates how to use sense voice with sherpa-onnx's C++ API. +// +// clang-format off +// +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 +// tar xvf sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 +// rm sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 +// +// clang-format on + +#include // NOLINT +#include +#include + +#include "sherpa-onnx/c-api/cxx-api.h" + +int32_t main() { + using namespace sherpa_onnx::cxx; + OfflineRecognizerConfig config; + + config.model_config.sense_voice.model = + "./sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/model.int8.onnx"; + config.model_config.sense_voice.use_itn = true; + config.model_config.sense_voice.language = "auto"; + config.model_config.tokens = + "./sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/tokens.txt"; + + config.model_config.num_threads = 1; + + std::cout << "Loading model\n"; + OfflineRecognizer recongizer = OfflineRecognizer::Create(config); + if (!recongizer.Get()) { + std::cerr << "Please check your config\n"; + return -1; + } + std::cout << "Loading model done\n"; + + std::string wave_filename = + "./sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/test_wavs/en.wav"; + + Wave wave = ReadWave(wave_filename); + if (wave.samples.empty()) { + std::cerr << "Failed to read: '" << wave_filename << "'\n"; + return -1; + } + + std::cout << "Start recognition\n"; + const auto begin = std::chrono::steady_clock::now(); + + OfflineStream stream = recongizer.CreateStream(); + stream.AcceptWaveform(wave.sample_rate, wave.samples.data(), + wave.samples.size()); + + recongizer.Decode(&stream); + + OfflineRecognizerResult result = recongizer.GetResult(&stream); + + const auto end = std::chrono::steady_clock::now(); + const float elapsed_seconds = + std::chrono::duration_cast(end - begin) + .count() / + 1000.; + float duration = wave.samples.size() / static_cast(wave.sample_rate); + float rtf = elapsed_seconds / duration; + + std::cout << "text: " << result.text << "\n"; + printf("Number of threads: %d\n", config.model_config.num_threads); + printf("Duration: %.3fs\n", duration); + printf("Elapsed seconds: %.3fs\n", elapsed_seconds); + printf("(Real time factor) RTF = %.3f / %.3f = %.3f\n", elapsed_seconds, + duration, rtf); + + return 0; +} diff --git a/cxx-api-examples/streaming-zipformer-cxx-api.cc b/cxx-api-examples/streaming-zipformer-cxx-api.cc index 4f38647f4..5a49dcfc9 100644 --- a/cxx-api-examples/streaming-zipformer-cxx-api.cc +++ b/cxx-api-examples/streaming-zipformer-cxx-api.cc @@ -66,6 +66,8 @@ int32_t main() { OnlineStream stream = recongizer.CreateStream(); stream.AcceptWaveform(wave.sample_rate, wave.samples.data(), wave.samples.size()); + stream.InputFinished(); + while (recongizer.IsReady(&stream)) { recongizer.Decode(&stream); } diff --git a/cxx-api-examples/whisper-cxx-api.cc b/cxx-api-examples/whisper-cxx-api.cc new file mode 100644 index 000000000..82f0ddb53 --- /dev/null +++ b/cxx-api-examples/whisper-cxx-api.cc @@ -0,0 +1,76 @@ +// cxx-api-examples/whisper-cxx-api.cc +// Copyright (c) 2024 Xiaomi Corporation + +// +// This file demonstrates how to use whisper with sherpa-onnx's C++ API. +// +// clang-format off +// +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2 +// tar xvf sherpa-onnx-whisper-tiny.en.tar.bz2 +// rm sherpa-onnx-whisper-tiny.en.tar.bz2 +// +// clang-format on + +#include // NOLINT +#include +#include + +#include "sherpa-onnx/c-api/cxx-api.h" + +int32_t main() { + using namespace sherpa_onnx::cxx; + OfflineRecognizerConfig config; + + config.model_config.whisper.encoder = + "./sherpa-onnx-whisper-tiny.en/tiny.en-encoder.int8.onnx"; + config.model_config.whisper.decoder = + "./sherpa-onnx-whisper-tiny.en/tiny.en-decoder.int8.onnx"; + config.model_config.tokens = + "./sherpa-onnx-whisper-tiny.en/tiny.en-tokens.txt"; + + config.model_config.num_threads = 1; + + std::cout << "Loading model\n"; + OfflineRecognizer recongizer = OfflineRecognizer::Create(config); + if (!recongizer.Get()) { + std::cerr << "Please check your config\n"; + return -1; + } + std::cout << "Loading model done\n"; + + std::string wave_filename = "./sherpa-onnx-whisper-tiny.en/test_wavs/0.wav"; + Wave wave = ReadWave(wave_filename); + if (wave.samples.empty()) { + std::cerr << "Failed to read: '" << wave_filename << "'\n"; + return -1; + } + + std::cout << "Start recognition\n"; + const auto begin = std::chrono::steady_clock::now(); + + OfflineStream stream = recongizer.CreateStream(); + stream.AcceptWaveform(wave.sample_rate, wave.samples.data(), + wave.samples.size()); + + recongizer.Decode(&stream); + + OfflineRecognizerResult result = recongizer.GetResult(&stream); + + const auto end = std::chrono::steady_clock::now(); + const float elapsed_seconds = + std::chrono::duration_cast(end - begin) + .count() / + 1000.; + float duration = wave.samples.size() / static_cast(wave.sample_rate); + float rtf = elapsed_seconds / duration; + + std::cout << "text: " << result.text << "\n"; + printf("Number of threads: %d\n", config.model_config.num_threads); + printf("Duration: %.3fs\n", duration); + printf("Elapsed seconds: %.3fs\n", elapsed_seconds); + printf("(Real time factor) RTF = %.3f / %.3f = %.3f\n", elapsed_seconds, + duration, rtf); + + return 0; +} diff --git a/ffmpeg-examples/sherpa-onnx-ffmpeg.c b/ffmpeg-examples/sherpa-onnx-ffmpeg.c index 31e7491f0..cf3614518 100644 --- a/ffmpeg-examples/sherpa-onnx-ffmpeg.c +++ b/ffmpeg-examples/sherpa-onnx-ffmpeg.c @@ -320,7 +320,8 @@ int main(int argc, char **argv) { const SherpaOnnxOnlineRecognizer *recognizer = SherpaOnnxCreateOnlineRecognizer(&config); - SherpaOnnxOnlineStream *stream = SherpaOnnxCreateOnlineStream(recognizer); + const SherpaOnnxOnlineStream *stream = + SherpaOnnxCreateOnlineStream(recognizer); const SherpaOnnxDisplay *display = SherpaOnnxCreateDisplay(50); int32_t segment_id = 0; diff --git a/mfc-examples/NonStreamingSpeechRecognition/NonStreamingSpeechRecognitionDlg.cpp b/mfc-examples/NonStreamingSpeechRecognition/NonStreamingSpeechRecognitionDlg.cpp index c559c9321..462959247 100644 --- a/mfc-examples/NonStreamingSpeechRecognition/NonStreamingSpeechRecognitionDlg.cpp +++ b/mfc-examples/NonStreamingSpeechRecognition/NonStreamingSpeechRecognitionDlg.cpp @@ -256,7 +256,7 @@ void CNonStreamingSpeechRecognitionDlg::OnBnClickedOk() { } pa_stream_ = nullptr; - SherpaOnnxOfflineStream *stream = SherpaOnnxCreateOfflineStream(recognizer_); + const SherpaOnnxOfflineStream *stream = SherpaOnnxCreateOfflineStream(recognizer_); SherpaOnnxAcceptWaveformOffline(stream, config_.feat_config.sample_rate, samples_.data(), static_cast(samples_.size())); diff --git a/mfc-examples/NonStreamingSpeechRecognition/NonStreamingSpeechRecognitionDlg.h b/mfc-examples/NonStreamingSpeechRecognition/NonStreamingSpeechRecognitionDlg.h index 77a8992e9..19ab83880 100644 --- a/mfc-examples/NonStreamingSpeechRecognition/NonStreamingSpeechRecognitionDlg.h +++ b/mfc-examples/NonStreamingSpeechRecognition/NonStreamingSpeechRecognitionDlg.h @@ -48,7 +48,7 @@ class CNonStreamingSpeechRecognitionDlg : public CDialogEx { private: Microphone mic_; - SherpaOnnxOfflineRecognizer *recognizer_ = nullptr; + const SherpaOnnxOfflineRecognizer *recognizer_ = nullptr; SherpaOnnxOfflineRecognizerConfig config_; PaStream *pa_stream_ = nullptr; diff --git a/scripts/node-addon-api/src/non-streaming-asr.cc b/scripts/node-addon-api/src/non-streaming-asr.cc index b24a37beb..86badc0ff 100644 --- a/scripts/node-addon-api/src/non-streaming-asr.cc +++ b/scripts/node-addon-api/src/non-streaming-asr.cc @@ -203,7 +203,7 @@ CreateOfflineRecognizerWrapper(const Napi::CallbackInfo &info) { SHERPA_ONNX_ASSIGN_ATTR_STR(rule_fars, ruleFars); SHERPA_ONNX_ASSIGN_ATTR_FLOAT(blank_penalty, blankPenalty); - SherpaOnnxOfflineRecognizer *recognizer = + const SherpaOnnxOfflineRecognizer *recognizer = SherpaOnnxCreateOfflineRecognizer(&c); if (c.model_config.transducer.encoder) { @@ -306,7 +306,7 @@ CreateOfflineRecognizerWrapper(const Napi::CallbackInfo &info) { } return Napi::External::New( - env, recognizer, + env, const_cast(recognizer), [](Napi::Env env, SherpaOnnxOfflineRecognizer *recognizer) { SherpaOnnxDestroyOfflineRecognizer(recognizer); }); @@ -336,10 +336,12 @@ static Napi::External CreateOfflineStreamWrapper( SherpaOnnxOfflineRecognizer *recognizer = info[0].As>().Data(); - SherpaOnnxOfflineStream *stream = SherpaOnnxCreateOfflineStream(recognizer); + const SherpaOnnxOfflineStream *stream = + SherpaOnnxCreateOfflineStream(recognizer); return Napi::External::New( - env, stream, [](Napi::Env env, SherpaOnnxOfflineStream *stream) { + env, const_cast(stream), + [](Napi::Env env, SherpaOnnxOfflineStream *stream) { SherpaOnnxDestroyOfflineStream(stream); }); } diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index 653355a66..d7fa383be 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -168,14 +168,14 @@ void SherpaOnnxDestroyOnlineRecognizer( delete recognizer; } -SherpaOnnxOnlineStream *SherpaOnnxCreateOnlineStream( +const SherpaOnnxOnlineStream *SherpaOnnxCreateOnlineStream( const SherpaOnnxOnlineRecognizer *recognizer) { SherpaOnnxOnlineStream *stream = new SherpaOnnxOnlineStream(recognizer->impl->CreateStream()); return stream; } -SherpaOnnxOnlineStream *SherpaOnnxCreateOnlineStreamWithHotwords( +const SherpaOnnxOnlineStream *SherpaOnnxCreateOnlineStreamWithHotwords( const SherpaOnnxOnlineRecognizer *recognizer, const char *hotwords) { SherpaOnnxOnlineStream *stream = new SherpaOnnxOnlineStream(recognizer->impl->CreateStream(hotwords)); @@ -351,7 +351,7 @@ struct SherpaOnnxOfflineStream { static sherpa_onnx::OfflineRecognizerConfig convertConfig( const SherpaOnnxOfflineRecognizerConfig *config); -SherpaOnnxOfflineRecognizer *SherpaOnnxCreateOfflineRecognizer( +const SherpaOnnxOfflineRecognizer *SherpaOnnxCreateOfflineRecognizer( const SherpaOnnxOfflineRecognizerConfig *config) { sherpa_onnx::OfflineRecognizerConfig recognizer_config = convertConfig(config); @@ -490,11 +490,11 @@ void SherpaOnnxOfflineRecognizerSetConfig( } void SherpaOnnxDestroyOfflineRecognizer( - SherpaOnnxOfflineRecognizer *recognizer) { + const SherpaOnnxOfflineRecognizer *recognizer) { delete recognizer; } -SherpaOnnxOfflineStream *SherpaOnnxCreateOfflineStream( +const SherpaOnnxOfflineStream *SherpaOnnxCreateOfflineStream( const SherpaOnnxOfflineRecognizer *recognizer) { SherpaOnnxOfflineStream *stream = new SherpaOnnxOfflineStream(recognizer->impl->CreateStream()); @@ -518,8 +518,8 @@ void SherpaOnnxDecodeOfflineStream( } void SherpaOnnxDecodeMultipleOfflineStreams( - SherpaOnnxOfflineRecognizer *recognizer, SherpaOnnxOfflineStream **streams, - int32_t n) { + const SherpaOnnxOfflineRecognizer *recognizer, + const SherpaOnnxOfflineStream **streams, int32_t n) { std::vector ss(n); for (int32_t i = 0; i != n; ++i) { ss[i] = streams[i]->impl.get(); diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index 0a01379d4..e5fc92eb1 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -220,7 +220,7 @@ SHERPA_ONNX_API void SherpaOnnxDestroyOnlineRecognizer( /// @param recognizer A pointer returned by SherpaOnnxCreateOnlineRecognizer() /// @return Return a pointer to an OnlineStream. The user has to invoke /// SherpaOnnxDestroyOnlineStream() to free it to avoid memory leak. -SHERPA_ONNX_API SherpaOnnxOnlineStream *SherpaOnnxCreateOnlineStream( +SHERPA_ONNX_API const SherpaOnnxOnlineStream *SherpaOnnxCreateOnlineStream( const SherpaOnnxOnlineRecognizer *recognizer); /// Create an online stream for accepting wave samples with the specified hot @@ -229,7 +229,7 @@ SHERPA_ONNX_API SherpaOnnxOnlineStream *SherpaOnnxCreateOnlineStream( /// @param recognizer A pointer returned by SherpaOnnxCreateOnlineRecognizer() /// @return Return a pointer to an OnlineStream. The user has to invoke /// SherpaOnnxDestroyOnlineStream() to free it to avoid memory leak. -SHERPA_ONNX_API SherpaOnnxOnlineStream * +SHERPA_ONNX_API const SherpaOnnxOnlineStream * SherpaOnnxCreateOnlineStreamWithHotwords( const SherpaOnnxOnlineRecognizer *recognizer, const char *hotwords); @@ -453,7 +453,8 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOfflineStream SherpaOnnxOfflineStream; /// @return Return a pointer to the recognizer. The user has to invoke // SherpaOnnxDestroyOfflineRecognizer() to free it to avoid memory // leak. -SHERPA_ONNX_API SherpaOnnxOfflineRecognizer *SherpaOnnxCreateOfflineRecognizer( +SHERPA_ONNX_API const SherpaOnnxOfflineRecognizer * +SherpaOnnxCreateOfflineRecognizer( const SherpaOnnxOfflineRecognizerConfig *config); /// @param config Config for the recognizer. @@ -465,14 +466,14 @@ SHERPA_ONNX_API void SherpaOnnxOfflineRecognizerSetConfig( /// /// @param p A pointer returned by SherpaOnnxCreateOfflineRecognizer() SHERPA_ONNX_API void SherpaOnnxDestroyOfflineRecognizer( - SherpaOnnxOfflineRecognizer *recognizer); + const SherpaOnnxOfflineRecognizer *recognizer); /// Create an offline stream for accepting wave samples. /// /// @param recognizer A pointer returned by SherpaOnnxCreateOfflineRecognizer() /// @return Return a pointer to an OfflineStream. The user has to invoke /// SherpaOnnxDestroyOfflineStream() to free it to avoid memory leak. -SHERPA_ONNX_API SherpaOnnxOfflineStream *SherpaOnnxCreateOfflineStream( +SHERPA_ONNX_API const SherpaOnnxOfflineStream *SherpaOnnxCreateOfflineStream( const SherpaOnnxOfflineRecognizer *recognizer); /// Destroy an offline stream. @@ -518,8 +519,8 @@ SHERPA_ONNX_API void SherpaOnnxDecodeOfflineStream( /// by SherpaOnnxCreateOfflineStream(). /// @param n Number of entries in the given streams. SHERPA_ONNX_API void SherpaOnnxDecodeMultipleOfflineStreams( - SherpaOnnxOfflineRecognizer *recognizer, SherpaOnnxOfflineStream **streams, - int32_t n); + const SherpaOnnxOfflineRecognizer *recognizer, + const SherpaOnnxOfflineStream **streams, int32_t n); SHERPA_ONNX_API typedef struct SherpaOnnxOfflineRecognizerResult { const char *text; diff --git a/sherpa-onnx/c-api/cxx-api.cc b/sherpa-onnx/c-api/cxx-api.cc index 243e99738..262ad3cc9 100644 --- a/sherpa-onnx/c-api/cxx-api.cc +++ b/sherpa-onnx/c-api/cxx-api.cc @@ -36,6 +36,10 @@ void OnlineStream::AcceptWaveform(int32_t sample_rate, const float *samples, SherpaOnnxOnlineStreamAcceptWaveform(p_, sample_rate, samples, n); } +void OnlineStream::InputFinished() const { + SherpaOnnxOnlineStreamInputFinished(p_); +} + OnlineRecognizer OnlineRecognizer::Create( const OnlineRecognizerConfig &config) { struct SherpaOnnxOnlineRecognizerConfig c; @@ -119,6 +123,14 @@ void OnlineRecognizer::Decode(const OnlineStream *s) const { SherpaOnnxDecodeOnlineStream(p_, s->Get()); } +void OnlineRecognizer::Reset(const OnlineStream *s) const { + SherpaOnnxOnlineStreamReset(p_, s->Get()); +} + +bool OnlineRecognizer::IsEndpoint(const OnlineStream *s) const { + return SherpaOnnxOnlineStreamIsEndpoint(p_, s->Get()); +} + void OnlineRecognizer::Decode(const OnlineStream *ss, int32_t n) const { if (n <= 0) { return; @@ -156,4 +168,138 @@ OnlineRecognizerResult OnlineRecognizer::GetResult( return ans; } +// ============================================================================ +// Non-streaming ASR +// ============================================================================ +OfflineStream::OfflineStream(const SherpaOnnxOfflineStream *p) + : MoveOnly(p) {} + +void OfflineStream::Destroy(const SherpaOnnxOfflineStream *p) const { + SherpaOnnxDestroyOfflineStream(p); +} + +void OfflineStream::AcceptWaveform(int32_t sample_rate, const float *samples, + int32_t n) const { + SherpaOnnxAcceptWaveformOffline(p_, sample_rate, samples, n); +} + +OfflineRecognizer OfflineRecognizer::Create( + const OfflineRecognizerConfig &config) { + struct SherpaOnnxOfflineRecognizerConfig c; + memset(&c, 0, sizeof(c)); + + c.feat_config.sample_rate = config.feat_config.sample_rate; + c.feat_config.feature_dim = config.feat_config.feature_dim; + c.model_config.transducer.encoder = + config.model_config.transducer.encoder.c_str(); + c.model_config.transducer.decoder = + config.model_config.transducer.decoder.c_str(); + c.model_config.transducer.joiner = + config.model_config.transducer.joiner.c_str(); + + c.model_config.paraformer.model = + config.model_config.paraformer.model.c_str(); + + c.model_config.nemo_ctc.model = config.model_config.nemo_ctc.model.c_str(); + + c.model_config.whisper.encoder = config.model_config.whisper.encoder.c_str(); + c.model_config.whisper.decoder = config.model_config.whisper.decoder.c_str(); + c.model_config.whisper.language = + config.model_config.whisper.language.c_str(); + c.model_config.whisper.task = config.model_config.whisper.task.c_str(); + c.model_config.whisper.tail_paddings = + config.model_config.whisper.tail_paddings; + + c.model_config.tdnn.model = config.model_config.tdnn.model.c_str(); + + c.model_config.tokens = config.model_config.tokens.c_str(); + c.model_config.num_threads = config.model_config.num_threads; + c.model_config.debug = config.model_config.debug; + c.model_config.provider = config.model_config.provider.c_str(); + c.model_config.model_type = config.model_config.model_type.c_str(); + c.model_config.modeling_unit = config.model_config.modeling_unit.c_str(); + c.model_config.bpe_vocab = config.model_config.bpe_vocab.c_str(); + c.model_config.telespeech_ctc = config.model_config.telespeech_ctc.c_str(); + + c.model_config.sense_voice.model = + config.model_config.sense_voice.model.c_str(); + c.model_config.sense_voice.language = + config.model_config.sense_voice.language.c_str(); + c.model_config.sense_voice.use_itn = config.model_config.sense_voice.use_itn; + + c.lm_config.model = config.lm_config.model.c_str(); + c.lm_config.scale = config.lm_config.scale; + + c.decoding_method = config.decoding_method.c_str(); + c.max_active_paths = config.max_active_paths; + c.hotwords_file = config.hotwords_file.c_str(); + c.hotwords_score = config.hotwords_score; + + c.rule_fsts = config.rule_fsts.c_str(); + c.rule_fars = config.rule_fars.c_str(); + + c.blank_penalty = config.blank_penalty; + + auto p = SherpaOnnxCreateOfflineRecognizer(&c); + return OfflineRecognizer(p); +} + +OfflineRecognizer::OfflineRecognizer(const SherpaOnnxOfflineRecognizer *p) + : MoveOnly(p) {} + +void OfflineRecognizer::Destroy(const SherpaOnnxOfflineRecognizer *p) const { + SherpaOnnxDestroyOfflineRecognizer(p_); +} + +OfflineStream OfflineRecognizer::CreateStream() const { + auto p = SherpaOnnxCreateOfflineStream(p_); + return OfflineStream{p}; +} + +void OfflineRecognizer::Decode(const OfflineStream *s) const { + SherpaOnnxDecodeOfflineStream(p_, s->Get()); +} + +void OfflineRecognizer::Decode(const OfflineStream *ss, int32_t n) const { + if (n <= 0) { + return; + } + + std::vector streams(n); + for (int32_t i = 0; i != n; ++i) { + streams[i] = ss[i].Get(); + } + + SherpaOnnxDecodeMultipleOfflineStreams(p_, streams.data(), n); +} + +OfflineRecognizerResult OfflineRecognizer::GetResult( + const OfflineStream *s) const { + auto r = SherpaOnnxGetOfflineStreamResult(s->Get()); + + OfflineRecognizerResult ans; + if (r) { + ans.text = r->text; + + if (r->timestamps) { + ans.timestamps.resize(r->count); + std::copy(r->timestamps, r->timestamps + r->count, ans.timestamps.data()); + } + + ans.tokens.resize(r->count); + for (int32_t i = 0; i != r->count; ++i) { + ans.tokens[i] = r->tokens_arr[i]; + } + + ans.json = r->json; + ans.lang = r->lang ? r->lang : ""; + ans.emotion = r->emotion ? r->emotion : ""; + ans.event = r->event ? r->event : ""; + } + + SherpaOnnxDestroyOfflineRecognizerResult(r); + + return ans; +} + } // namespace sherpa_onnx::cxx diff --git a/sherpa-onnx/c-api/cxx-api.h b/sherpa-onnx/c-api/cxx-api.h index 416333b7f..b078f5c73 100644 --- a/sherpa-onnx/c-api/cxx-api.h +++ b/sherpa-onnx/c-api/cxx-api.h @@ -13,6 +13,9 @@ namespace sherpa_onnx::cxx { +// ============================================================================ +// Streaming ASR +// ============================================================================ struct SHERPA_ONNX_API OnlineTransducerModelConfig { std::string encoder; std::string decoder; @@ -148,6 +151,8 @@ class SHERPA_ONNX_API OnlineStream void AcceptWaveform(int32_t sample_rate, const float *samples, int32_t n) const; + void InputFinished() const; + void Destroy(const SherpaOnnxOnlineStream *p) const; }; @@ -170,10 +175,134 @@ class SHERPA_ONNX_API OnlineRecognizer OnlineRecognizerResult GetResult(const OnlineStream *s) const; + void Reset(const OnlineStream *s) const; + + bool IsEndpoint(const OnlineStream *s) const; + private: explicit OnlineRecognizer(const SherpaOnnxOnlineRecognizer *p); }; +// ============================================================================ +// Non-streaming ASR +// ============================================================================ +struct SHERPA_ONNX_API OfflineTransducerModelConfig { + std::string encoder; + std::string decoder; + std::string joiner; +}; + +struct SHERPA_ONNX_API OfflineParaformerModelConfig { + std::string model; +}; + +struct SHERPA_ONNX_API OfflineNemoEncDecCtcModelConfig { + std::string model; +}; + +struct SHERPA_ONNX_API OfflineWhisperModelConfig { + std::string encoder; + std::string decoder; + std::string language; + std::string task = "transcribe"; + int32_t tail_paddings = -1; +}; + +struct SHERPA_ONNX_API OfflineTdnnModelConfig { + std::string model; +}; + +struct SHERPA_ONNX_API SherpaOnnxOfflineLMConfig { + std::string model; + float scale = 1.0; +}; + +struct SHERPA_ONNX_API OfflineSenseVoiceModelConfig { + std::string model; + std::string language; + bool use_itn = false; +}; + +struct SHERPA_ONNX_API OfflineModelConfig { + OfflineTransducerModelConfig transducer; + OfflineParaformerModelConfig paraformer; + OfflineNemoEncDecCtcModelConfig nemo_ctc; + OfflineWhisperModelConfig whisper; + OfflineTdnnModelConfig tdnn; + + std::string tokens; + int32_t num_threads = 1; + bool debug = false; + std::string provider = "cpu"; + std::string model_type; + std::string modeling_unit = "cjkchar"; + std::string bpe_vocab; + std::string telespeech_ctc; + OfflineSenseVoiceModelConfig sense_voice; +}; + +struct SHERPA_ONNX_API OfflineLMConfig { + std::string model; + float scale = 1.0; +}; + +struct SHERPA_ONNX_API OfflineRecognizerConfig { + FeatureConfig feat_config; + OfflineModelConfig model_config; + OfflineLMConfig lm_config; + + std::string decoding_method = "greedy_search"; + int32_t max_active_paths = 4; + + std::string hotwords_file; + + float hotwords_score = 1.5; + std::string rule_fsts; + std::string rule_fars; + float blank_penalty = 0; +}; + +struct SHERPA_ONNX_API OfflineRecognizerResult { + std::string text; + std::vector timestamps; + std::vector tokens; + std::string json; + std::string lang; + std::string emotion; + std::string event; +}; + +class SHERPA_ONNX_API OfflineStream + : public MoveOnly { + public: + explicit OfflineStream(const SherpaOnnxOfflineStream *p); + + void AcceptWaveform(int32_t sample_rate, const float *samples, + int32_t n) const; + + void Destroy(const SherpaOnnxOfflineStream *p) const; +}; + +class SHERPA_ONNX_API OfflineRecognizer + : public MoveOnly { + public: + static OfflineRecognizer Create(const OfflineRecognizerConfig &config); + + void Destroy(const SherpaOnnxOfflineRecognizer *p) const; + + OfflineStream CreateStream() const; + + void Decode(const OfflineStream *s) const; + + void Decode(const OfflineStream *ss, int32_t n) const; + + OfflineRecognizerResult GetResult(const OfflineStream *s) const; + + private: + explicit OfflineRecognizer(const SherpaOnnxOfflineRecognizer *p); +}; + } // namespace sherpa_onnx::cxx #endif // SHERPA_ONNX_C_API_CXX_API_H_ + // diff --git a/sherpa-onnx/csrc/online-recognizer-impl.cc b/sherpa-onnx/csrc/online-recognizer-impl.cc index 399dab49e..cf59cb539 100644 --- a/sherpa-onnx/csrc/online-recognizer-impl.cc +++ b/sherpa-onnx/csrc/online-recognizer-impl.cc @@ -30,9 +30,13 @@ std::unique_ptr OnlineRecognizerImpl::Create( if (!config.model_config.transducer.encoder.empty()) { Ort::Env env(ORT_LOGGING_LEVEL_ERROR); + Ort::SessionOptions sess_opts; + sess_opts.SetIntraOpNumThreads(1); + sess_opts.SetInterOpNumThreads(1); + auto decoder_model = ReadFile(config.model_config.transducer.decoder); - auto sess = std::make_unique( - env, decoder_model.data(), decoder_model.size(), Ort::SessionOptions{}); + auto sess = std::make_unique(env, decoder_model.data(), + decoder_model.size(), sess_opts); size_t node_count = sess->GetOutputCount(); @@ -63,9 +67,13 @@ std::unique_ptr OnlineRecognizerImpl::Create( if (!config.model_config.transducer.encoder.empty()) { Ort::Env env(ORT_LOGGING_LEVEL_ERROR); + Ort::SessionOptions sess_opts; + sess_opts.SetIntraOpNumThreads(1); + sess_opts.SetInterOpNumThreads(1); + auto decoder_model = ReadFile(mgr, config.model_config.transducer.decoder); - auto sess = std::make_unique( - env, decoder_model.data(), decoder_model.size(), Ort::SessionOptions{}); + auto sess = std::make_unique(env, decoder_model.data(), + decoder_model.size(), sess_opts); size_t node_count = sess->GetOutputCount(); From b3e05f6dc42d52823d8dd99fe61e52643cb454f0 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 24 Oct 2024 11:15:08 +0800 Subject: [PATCH 032/183] Fix style issues (#1458) --- cmake/asio.cmake | 2 +- cmake/cargs.cmake | 2 +- cmake/cppjieba.cmake | 2 +- cmake/eigen.cmake | 2 +- cmake/espeak-ng-for-piper.cmake | 2 +- cmake/googletest.cmake | 2 +- cmake/hclust-cpp.cmake | 2 + cmake/kaldi-decoder.cmake | 2 +- cmake/kaldi-native-fbank.cmake | 2 +- cmake/kaldifst.cmake | 2 +- cmake/onnxruntime-linux-aarch64-static.cmake | 2 +- cmake/onnxruntime-linux-aarch64.cmake | 2 +- cmake/onnxruntime-linux-arm-static.cmake | 2 +- cmake/onnxruntime-linux-arm.cmake | 2 +- cmake/onnxruntime-linux-riscv64-static.cmake | 2 +- cmake/onnxruntime-linux-riscv64.cmake | 2 +- cmake/onnxruntime-linux-x86_64-gpu.cmake | 2 +- cmake/onnxruntime-linux-x86_64-static.cmake | 2 +- cmake/onnxruntime-linux-x86_64.cmake | 2 +- cmake/onnxruntime-osx-arm64-static.cmake | 2 +- cmake/onnxruntime-osx-arm64.cmake | 2 +- cmake/onnxruntime-osx-universal-static.cmake | 2 +- cmake/onnxruntime-osx-universal.cmake | 2 +- cmake/onnxruntime-osx-x86_64-static.cmake | 2 +- cmake/onnxruntime-osx-x86_64.cmake | 2 +- cmake/onnxruntime-wasm-simd.cmake | 2 +- cmake/onnxruntime-win-arm64.cmake | 2 +- cmake/onnxruntime-win-x64-directml.cmake | 4 +- cmake/onnxruntime-win-x64-gpu.cmake | 2 +- cmake/onnxruntime-win-x64-static-debug.cmake | 2 +- cmake/onnxruntime-win-x64-static.cmake | 2 +- cmake/onnxruntime-win-x64.cmake | 2 +- cmake/onnxruntime-win-x86-static-debug.cmake | 2 +- cmake/onnxruntime-win-x86-static.cmake | 2 +- cmake/onnxruntime-win-x86.cmake | 2 +- cmake/openfst.cmake | 2 +- cmake/piper-phonemize.cmake | 2 +- cmake/pybind11.cmake | 2 +- cmake/simple-sentencepiece.cmake | 2 +- cmake/websocketpp.cmake | 2 +- ffmpeg-examples/sherpa-onnx-ffmpeg.c | 4 +- .../node-addon-api/src/non-streaming-asr.cc | 2 +- scripts/node-addon-api/src/streaming-asr.cc | 6 ++- sherpa-onnx/c-api/cxx-api.h | 20 +++---- sherpa-onnx/csrc/CMakeLists.txt | 1 + sherpa-onnx/csrc/fst-utils.cc | 53 +++++++++++++++++++ sherpa-onnx/csrc/fst-utils.h | 18 +++++++ sherpa-onnx/csrc/jieba-lexicon.cc | 8 +-- sherpa-onnx/csrc/lexicon.h | 7 +++ sherpa-onnx/csrc/melo-tts-lexicon.cc | 8 +-- sherpa-onnx/csrc/offline-ctc-fst-decoder.cc | 43 +-------------- .../offline-speaker-diarization-result.cc | 9 ++-- .../csrc/offline-speaker-diarization.cc | 3 +- sherpa-onnx/csrc/online-ctc-fst-decoder.cc | 4 +- sherpa-onnx/csrc/online-recognizer.cc | 4 ++ 55 files changed, 155 insertions(+), 117 deletions(-) create mode 100644 sherpa-onnx/csrc/fst-utils.cc create mode 100644 sherpa-onnx/csrc/fst-utils.h diff --git a/cmake/asio.cmake b/cmake/asio.cmake index eaa262acb..9e3ce8d23 100644 --- a/cmake/asio.cmake +++ b/cmake/asio.cmake @@ -2,7 +2,7 @@ function(download_asio) include(FetchContent) set(asio_URL "https://github.com/chriskohlhoff/asio/archive/refs/tags/asio-1-24-0.tar.gz") - set(asio_URL2 "https://hub.nuaa.cf/chriskohlhoff/asio/archive/refs/tags/asio-1-24-0.tar.gz") + set(asio_URL2 "https://hf-mirror.com/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/asio-asio-1-24-0.tar.gz") set(asio_HASH "SHA256=cbcaaba0f66722787b1a7c33afe1befb3a012b5af3ad7da7ff0f6b8c9b7a8a5b") # If you don't have access to the Internet, diff --git a/cmake/cargs.cmake b/cmake/cargs.cmake index 54487a6f0..d7c605508 100644 --- a/cmake/cargs.cmake +++ b/cmake/cargs.cmake @@ -2,7 +2,7 @@ function(download_cargs) include(FetchContent) set(cargs_URL "https://github.com/likle/cargs/archive/refs/tags/v1.0.3.tar.gz") - set(cargs_URL2 "https://hub.nuaa.cf/likle/cargs/archive/refs/tags/v1.0.3.tar.gz") + set(cargs_URL2 "https://hf-mirror.com/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/cargs-1.0.3.tar.gz") set(cargs_HASH "SHA256=ddba25bd35e9c6c75bc706c126001b8ce8e084d40ef37050e6aa6963e836eb8b") # If you don't have access to the Internet, diff --git a/cmake/cppjieba.cmake b/cmake/cppjieba.cmake index 9ad27d7b5..167da338f 100644 --- a/cmake/cppjieba.cmake +++ b/cmake/cppjieba.cmake @@ -2,7 +2,7 @@ function(download_cppjieba) include(FetchContent) set(cppjieba_URL "https://github.com/csukuangfj/cppjieba/archive/refs/tags/sherpa-onnx-2024-04-19.tar.gz") - set(cppjieba_URL2 "https://hub.nuaa.cf/csukuangfj/cppjieba/archive/refs/tags/sherpa-onnx-2024-04-19.tar.gz") + set(cppjieba_URL2 "https://hf-mirror.com/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/cppjieba-sherpa-onnx-2024-04-19.tar.gz") set(cppjieba_HASH "SHA256=03e5264687f0efaef05487a07d49c3f4c0f743347bfbf825df4b30cc75ac5288") # If you don't have access to the Internet, diff --git a/cmake/eigen.cmake b/cmake/eigen.cmake index 154cdd4c2..9aef9abc8 100644 --- a/cmake/eigen.cmake +++ b/cmake/eigen.cmake @@ -2,7 +2,7 @@ function(download_eigen) include(FetchContent) set(eigen_URL "https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.tar.gz") - set(eigen_URL2 "https://huggingface.co/csukuangfj/kaldi-hmm-gmm-cmake-deps/resolve/main/eigen-3.4.0.tar.gz") + set(eigen_URL2 "https://hf-mirror.com/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/eigen-3.4.0.tar.gz") set(eigen_HASH "SHA256=8586084f71f9bde545ee7fa6d00288b264a2b7ac3607b974e54d13e7162c1c72") # If you don't have access to the Internet, diff --git a/cmake/espeak-ng-for-piper.cmake b/cmake/espeak-ng-for-piper.cmake index b54a0a6bd..0ef825306 100644 --- a/cmake/espeak-ng-for-piper.cmake +++ b/cmake/espeak-ng-for-piper.cmake @@ -2,7 +2,7 @@ function(download_espeak_ng_for_piper) include(FetchContent) set(espeak_ng_URL "https://github.com/csukuangfj/espeak-ng/archive/f6fed6c58b5e0998b8e68c6610125e2d07d595a7.zip") - set(espeak_ng_URL2 "https://hub.nuaa.cf/csukuangfj/espeak-ng/archive/f6fed6c58b5e0998b8e68c6610125e2d07d595a7.zip") + set(espeak_ng_URL2 "https://hf-mirror.com/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/espeak-ng-f6fed6c58b5e0998b8e68c6610125e2d07d595a7.zip") set(espeak_ng_HASH "SHA256=70cbf4050e7a014aae19140b05e57249da4720f56128459fbe3a93beaf971ae6") set(BUILD_ESPEAK_NG_TESTS OFF CACHE BOOL "" FORCE) diff --git a/cmake/googletest.cmake b/cmake/googletest.cmake index cf5fa10cc..a9bfd443b 100644 --- a/cmake/googletest.cmake +++ b/cmake/googletest.cmake @@ -2,7 +2,7 @@ function(download_googltest) include(FetchContent) set(googletest_URL "https://github.com/google/googletest/archive/refs/tags/v1.13.0.tar.gz") - set(googletest_URL2 "https://hub.nuaa.cf/google/googletest/archive/refs/tags/v1.13.0.tar.gz") + set(googletest_URL2 "https://hf-mirror.com/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/googletest-1.13.0.tar.gz") set(googletest_HASH "SHA256=ad7fdba11ea011c1d925b3289cf4af2c66a352e18d4c7264392fead75e919363") # If you don't have access to the Internet, diff --git a/cmake/hclust-cpp.cmake b/cmake/hclust-cpp.cmake index 904081525..c84ccafc8 100644 --- a/cmake/hclust-cpp.cmake +++ b/cmake/hclust-cpp.cmake @@ -3,6 +3,7 @@ function(download_hclust_cpp) # The latest commit as of 2024.09.29 set(hclust_cpp_URL "https://github.com/csukuangfj/hclust-cpp/archive/refs/tags/2024-09-29.tar.gz") + set(hclust_cpp_URL2 "https://hf-mirror.com/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/hclust-cpp-2024-09-29.tar.gz") set(hclust_cpp_HASH "SHA256=abab51448a3cb54272aae07522970306e0b2cc6479d59d7b19e7aee4d6cedd33") # If you don't have access to the Internet, @@ -20,6 +21,7 @@ function(download_hclust_cpp) set(hclust_cpp_URL "${f}") file(TO_CMAKE_PATH "${hclust_cpp_URL}" hclust_cpp_URL) message(STATUS "Found local downloaded hclust_cpp: ${hclust_cpp_URL}") + set(hclust_cpp_URL2) break() endif() endforeach() diff --git a/cmake/kaldi-decoder.cmake b/cmake/kaldi-decoder.cmake index d3d7ec2d5..91202342a 100644 --- a/cmake/kaldi-decoder.cmake +++ b/cmake/kaldi-decoder.cmake @@ -2,7 +2,7 @@ function(download_kaldi_decoder) include(FetchContent) set(kaldi_decoder_URL "https://github.com/k2-fsa/kaldi-decoder/archive/refs/tags/v0.2.6.tar.gz") - set(kaldi_decoder_URL2 "https://hub.nuaa.cf/k2-fsa/kaldi-decoder/archive/refs/tags/v0.2.6.tar.gz") + set(kaldi_decoder_URL2 "https://hf-mirror.com/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/kaldi-decoder-0.2.6.tar.gz") set(kaldi_decoder_HASH "SHA256=b13c78b37495cafc6ef3f8a7b661b349c55a51abbd7f7f42f389408dcf86a463") set(KALDI_DECODER_BUILD_PYTHON OFF CACHE BOOL "" FORCE) diff --git a/cmake/kaldi-native-fbank.cmake b/cmake/kaldi-native-fbank.cmake index 2d87b6a8b..8f6803c88 100644 --- a/cmake/kaldi-native-fbank.cmake +++ b/cmake/kaldi-native-fbank.cmake @@ -2,7 +2,7 @@ function(download_kaldi_native_fbank) include(FetchContent) set(kaldi_native_fbank_URL "https://github.com/csukuangfj/kaldi-native-fbank/archive/refs/tags/v1.20.0.tar.gz") - set(kaldi_native_fbank_URL2 "https://hub.nuaa.cf/csukuangfj/kaldi-native-fbank/archive/refs/tags/v1.20.0.tar.gz") + set(kaldi_native_fbank_URL2 "https://hf-mirror.com/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/kaldi-native-fbank-1.20.0.tar.gz") set(kaldi_native_fbank_HASH "SHA256=c6195b3cf374eef824644061d3c04f6b2a9267ae554169cbaa9865c89c1fe4f9") set(KALDI_NATIVE_FBANK_BUILD_TESTS OFF CACHE BOOL "" FORCE) diff --git a/cmake/kaldifst.cmake b/cmake/kaldifst.cmake index 765e2571a..034d8c444 100644 --- a/cmake/kaldifst.cmake +++ b/cmake/kaldifst.cmake @@ -2,7 +2,7 @@ function(download_kaldifst) include(FetchContent) set(kaldifst_URL "https://github.com/k2-fsa/kaldifst/archive/refs/tags/v1.7.11.tar.gz") - set(kaldifst_URL2 "https://hub.nuaa.cf/k2-fsa/kaldifst/archive/refs/tags/v1.7.11.tar.gz") + set(kaldifst_URL2 "https://hf-mirror.com/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/kaldifst-1.7.11.tar.gz") set(kaldifst_HASH "SHA256=b43b3332faa2961edc730e47995a58cd4e22ead21905d55b0c4a41375b4a525f") # If you don't have access to the Internet, diff --git a/cmake/onnxruntime-linux-aarch64-static.cmake b/cmake/onnxruntime-linux-aarch64-static.cmake index 9606c79db..4752e0102 100644 --- a/cmake/onnxruntime-linux-aarch64-static.cmake +++ b/cmake/onnxruntime-linux-aarch64-static.cmake @@ -15,7 +15,7 @@ if(BUILD_SHARED_LIBS) endif() set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-linux-aarch64-static_lib-1.17.1.zip") -set(onnxruntime_URL2 "https://hub.nuaa.cf/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-linux-aarch64-static_lib-1.17.1.zip") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-linux-aarch64-static_lib-1.17.1.zip") set(onnxruntime_HASH "SHA256=831b9a3869501040b4399de85f34c4f170e2bcbd41881edaeb553f8dc4080985") # If you don't have access to the Internet, diff --git a/cmake/onnxruntime-linux-aarch64.cmake b/cmake/onnxruntime-linux-aarch64.cmake index 096318e0b..a18f59e51 100644 --- a/cmake/onnxruntime-linux-aarch64.cmake +++ b/cmake/onnxruntime-linux-aarch64.cmake @@ -15,7 +15,7 @@ if(NOT BUILD_SHARED_LIBS) endif() set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-linux-aarch64-glibc2_17-Release-1.17.1-patched.zip") -set(onnxruntime_URL2 "https://hub.nuaa.cf/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-linux-aarch64-glibc2_17-Release-1.17.1-patched.zip") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-linux-aarch64-glibc2_17-Release-1.17.1-patched.zip") set(onnxruntime_HASH "SHA256=6e0e68985f8dd1f643e5a4dbe7cd54c9e176a0cc62249c6bee0699b87fc6d4fb") # If you don't have access to the Internet, diff --git a/cmake/onnxruntime-linux-arm-static.cmake b/cmake/onnxruntime-linux-arm-static.cmake index cf2269afb..fa9170e34 100644 --- a/cmake/onnxruntime-linux-arm-static.cmake +++ b/cmake/onnxruntime-linux-arm-static.cmake @@ -15,7 +15,7 @@ if(BUILD_SHARED_LIBS) endif() set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-linux-arm-static_lib-1.17.1.zip") -set(onnxruntime_URL2 "https://hub.nuaa.cf/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-linux-arm-static_lib-1.17.1.zip") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-linux-arm-static_lib-1.17.1.zip") set(onnxruntime_HASH "SHA256=3f2ba38156d2facfb732c0fe53bc1eaaf2791d9a91dd240380e3d53716798b09") # If you don't have access to the Internet, diff --git a/cmake/onnxruntime-linux-arm.cmake b/cmake/onnxruntime-linux-arm.cmake index a3adfaebd..28bd42686 100644 --- a/cmake/onnxruntime-linux-arm.cmake +++ b/cmake/onnxruntime-linux-arm.cmake @@ -15,7 +15,7 @@ if(NOT BUILD_SHARED_LIBS) endif() set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-linux-arm-1.17.1-patched.zip") -set(onnxruntime_URL2 "https://hub.nuaa.cf/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-linux-arm-1.17.1-patched.zip") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-linux-arm-1.17.1-patched.zip") set(onnxruntime_HASH "SHA256=4ec00f7adc7341c068babea3d0f607349655e598222d4212115ae4f52619efdb") # If you don't have access to the Internet, diff --git a/cmake/onnxruntime-linux-riscv64-static.cmake b/cmake/onnxruntime-linux-riscv64-static.cmake index b400c4741..dec7cf1bb 100644 --- a/cmake/onnxruntime-linux-riscv64-static.cmake +++ b/cmake/onnxruntime-linux-riscv64-static.cmake @@ -15,7 +15,7 @@ if(BUILD_SHARED_LIBS) endif() set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.18.0/onnxruntime-linux-riscv64-static_lib-1.18.0.zip") -set(onnxruntime_URL2 "https://hub.nuaa.cf/csukuangfj/onnxruntime-libs/releases/download/v1.18.0/onnxruntime-linux-riscv64-static_lib-1.18.0.zip") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-linux-riscv64-static_lib-1.18.0.zip") set(onnxruntime_HASH "SHA256=77ecc51d8caf0953755db6edcdec2fc03bce3f6d379bedd635be50bb95f88da5") # If you don't have access to the Internet, diff --git a/cmake/onnxruntime-linux-riscv64.cmake b/cmake/onnxruntime-linux-riscv64.cmake index c773e5ecb..121459326 100644 --- a/cmake/onnxruntime-linux-riscv64.cmake +++ b/cmake/onnxruntime-linux-riscv64.cmake @@ -15,7 +15,7 @@ if(NOT BUILD_SHARED_LIBS) endif() set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.14.1/onnxruntime-linux-riscv64-glibc2_17-Release-1.14.1.zip") -set(onnxruntime_URL2 "https://hub.nuaa.cf/csukuangfj/onnxruntime-libs/releases/download/v1.14.1/onnxruntime-linux-riscv64-glibc2_17-Release-1.14.1.zip") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-linux-riscv64-glibc2_17-Release-1.14.1.zip") set(onnxruntime_HASH "SHA256=c2cbc5af081ff82f46640befd85433811486daaf28e702163c6e4e75020fde81") # If you don't have access to the Internet, diff --git a/cmake/onnxruntime-linux-x86_64-gpu.cmake b/cmake/onnxruntime-linux-x86_64-gpu.cmake index 5407a0b82..7aed2526f 100644 --- a/cmake/onnxruntime-linux-x86_64-gpu.cmake +++ b/cmake/onnxruntime-linux-x86_64-gpu.cmake @@ -20,7 +20,7 @@ endif() set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-linux-x64-gpu-1.17.1-patched.zip") -set(onnxruntime_URL2 "https://hub.nuaa.cf/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-linux-x64-gpu-1.17.1-patched.zip") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-linux-x64-gpu-1.17.1-patched.zip") set(onnxruntime_HASH "SHA256=1261de176e8d9d4d2019f8fa8c732c6d11494f3c6e73168ab6d2cc0903f22551") # If you don't have access to the Internet, diff --git a/cmake/onnxruntime-linux-x86_64-static.cmake b/cmake/onnxruntime-linux-x86_64-static.cmake index c6bb867b2..f72f9ad5e 100644 --- a/cmake/onnxruntime-linux-x86_64-static.cmake +++ b/cmake/onnxruntime-linux-x86_64-static.cmake @@ -15,7 +15,7 @@ if(BUILD_SHARED_LIBS) endif() set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-linux-x64-static_lib-1.17.1-glibc2_17.zip") -set(onnxruntime_URL2 "https://hub.nuaa.cf/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-linux-x64-static_lib-1.17.1-glibc2_17.zip") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-linux-x64-static_lib-1.17.1-glibc2_17.zip") set(onnxruntime_HASH "SHA256=b646beeb983de843a267096d4457d832f93089f5e7264fd54b48cff207cb2068") # If you don't have access to the Internet, diff --git a/cmake/onnxruntime-linux-x86_64.cmake b/cmake/onnxruntime-linux-x86_64.cmake index eaa6f7608..361f4d0d8 100644 --- a/cmake/onnxruntime-linux-x86_64.cmake +++ b/cmake/onnxruntime-linux-x86_64.cmake @@ -15,7 +15,7 @@ if(NOT BUILD_SHARED_LIBS) endif() set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-linux-x64-glibc2_17-Release-1.17.1-patched.zip") -set(onnxruntime_URL2 "https://hub.nuaa.cf/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-linux-x64-glibc2_17-Release-1.17.1-patched.zip") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-linux-x64-glibc2_17-Release-1.17.1-patched.zip") set(onnxruntime_HASH "SHA256=cb90c51a195bdd453aaf1582f3ef63b466dafbb15c4b8a552ca4dce3769e1d1e") # If you don't have access to the Internet, diff --git a/cmake/onnxruntime-osx-arm64-static.cmake b/cmake/onnxruntime-osx-arm64-static.cmake index 494e263ff..7cd5a63ba 100644 --- a/cmake/onnxruntime-osx-arm64-static.cmake +++ b/cmake/onnxruntime-osx-arm64-static.cmake @@ -13,7 +13,7 @@ if(BUILD_SHARED_LIBS) endif() set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-osx-arm64-static_lib-1.17.1.zip") -set(onnxruntime_URL2 "https://hub.nuaa.cf/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-osx-arm64-static_lib-1.17.1.zip") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-osx-arm64-static_lib-1.17.1.zip") set(onnxruntime_HASH "SHA256=b88a4017251c159fea005aefe836bd0cf4d0bc7454e2810784f84a42143f17eb") # If you don't have access to the Internet, diff --git a/cmake/onnxruntime-osx-arm64.cmake b/cmake/onnxruntime-osx-arm64.cmake index 3998cc8b4..e3c986a44 100644 --- a/cmake/onnxruntime-osx-arm64.cmake +++ b/cmake/onnxruntime-osx-arm64.cmake @@ -13,7 +13,7 @@ if(NOT BUILD_SHARED_LIBS) endif() set(onnxruntime_URL "https://github.com/microsoft/onnxruntime/releases/download/v1.17.1/onnxruntime-osx-arm64-1.17.1.tgz") -set(onnxruntime_URL2 "https://hub.nuaa.cf/microsoft/onnxruntime/releases/download/v1.17.1/onnxruntime-osx-arm64-1.17.1.tgz") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-osx-arm64-1.17.1.tgz") set(onnxruntime_HASH "SHA256=89566f424624a7ad9a7d9d5e413c44b9639a994d7171cf409901d125b16e2bb3") # If you don't have access to the Internet, diff --git a/cmake/onnxruntime-osx-universal-static.cmake b/cmake/onnxruntime-osx-universal-static.cmake index 2abcf46b4..5bf635b8e 100644 --- a/cmake/onnxruntime-osx-universal-static.cmake +++ b/cmake/onnxruntime-osx-universal-static.cmake @@ -14,7 +14,7 @@ if(BUILD_SHARED_LIBS) endif() set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-osx-universal2-static_lib-1.17.1.zip") -set(onnxruntime_URL2 "https://hub.nuaa.cf/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-osx-universal2-static_lib-1.17.1.zip") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-osx-universal2-static_lib-1.17.1.zip") set(onnxruntime_HASH "SHA256=45599dbd2fb9dd52d6505930c0e82ca165391e222a68f5606b9ea9d4f3922e15") # If you don't have access to the Internet, diff --git a/cmake/onnxruntime-osx-universal.cmake b/cmake/onnxruntime-osx-universal.cmake index 2b0fbb110..fe5a53a63 100644 --- a/cmake/onnxruntime-osx-universal.cmake +++ b/cmake/onnxruntime-osx-universal.cmake @@ -14,7 +14,7 @@ if(NOT BUILD_SHARED_LIBS) endif() set(onnxruntime_URL "https://github.com/microsoft/onnxruntime/releases/download/v1.17.1/onnxruntime-osx-universal2-1.17.1.tgz") -set(onnxruntime_URL2 "https://hub.nuaa.cf/microsoft/onnxruntime/releases/download/v1.17.1/onnxruntime-osx-universal2-1.17.1.tgz") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-osx-universal2-1.17.1.tgz") set(onnxruntime_HASH "SHA256=9fa57fa6f202a373599377ef75064ae568fda8da838632b26a86024c7378d306") # If you don't have access to the Internet, diff --git a/cmake/onnxruntime-osx-x86_64-static.cmake b/cmake/onnxruntime-osx-x86_64-static.cmake index 259ec4d01..a3c98e709 100644 --- a/cmake/onnxruntime-osx-x86_64-static.cmake +++ b/cmake/onnxruntime-osx-x86_64-static.cmake @@ -13,7 +13,7 @@ if(BUILD_SHARED_LIBS) endif() set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-osx-x86_64-static_lib-1.17.1.zip") -set(onnxruntime_URL2 "https://hub.nuaa.cf/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-osx-x86_64-static_lib-1.17.1.zip") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-osx-x86_64-static_lib-1.17.1.zip") set(onnxruntime_HASH "SHA256=5ff8efb97e50e257943c6c588328d2c57b649278098d3b468036f02755b60903") # If you don't have access to the Internet, diff --git a/cmake/onnxruntime-osx-x86_64.cmake b/cmake/onnxruntime-osx-x86_64.cmake index 81b78991a..4ca967460 100644 --- a/cmake/onnxruntime-osx-x86_64.cmake +++ b/cmake/onnxruntime-osx-x86_64.cmake @@ -13,7 +13,7 @@ if(NOT BUILD_SHARED_LIBS) endif() set(onnxruntime_URL "https://github.com/microsoft/onnxruntime/releases/download/v1.17.1/onnxruntime-osx-x86_64-1.17.1.tgz") -set(onnxruntime_URL2 "https://hub.nuaa.cf/microsoft/onnxruntime/releases/download/v1.17.1/onnxruntime-osx-x86_64-1.17.1.tgz") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-osx-x86_64-1.17.1.tgz") set(onnxruntime_HASH "SHA256=86c6b6896434084ff5086eebc4e9ea90be1ed4d46743f92864f46ee43e7b5059") # If you don't have access to the Internet, diff --git a/cmake/onnxruntime-wasm-simd.cmake b/cmake/onnxruntime-wasm-simd.cmake index dcc8fb5dd..19ac0411c 100644 --- a/cmake/onnxruntime-wasm-simd.cmake +++ b/cmake/onnxruntime-wasm-simd.cmake @@ -11,7 +11,7 @@ if(BUILD_SHARED_LIBS) endif() set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-wasm-static_lib-simd-1.17.1.zip") -set(onnxruntime_URL2 "https://hub.nuaa.cf/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-wasm-static_lib-simd-1.17.1.zip") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-wasm-static_lib-simd-1.17.1.zip") set(onnxruntime_HASH "SHA256=8f07778e4233cf5a61a9d0795d90c5497177fbe8a46b701fda2d8d4e2b11cef8") # If you don't have access to the Internet, diff --git a/cmake/onnxruntime-win-arm64.cmake b/cmake/onnxruntime-win-arm64.cmake index 0705b6451..a4f247e34 100644 --- a/cmake/onnxruntime-win-arm64.cmake +++ b/cmake/onnxruntime-win-arm64.cmake @@ -16,7 +16,7 @@ if(NOT BUILD_SHARED_LIBS) endif() set(onnxruntime_URL "https://github.com/microsoft/onnxruntime/releases/download/v1.17.1/onnxruntime-win-arm64-1.17.1.zip") -set(onnxruntime_URL2 "https://hub.nuaa.cf/microsoft/onnxruntime/releases/download/v1.17.1/onnxruntime-win-arm64-1.17.1.zip") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-win-arm64-1.17.1.zip") set(onnxruntime_HASH "SHA256=47782cebcab0fd7a1f0a3f0676b088c1bc0f4fbf21666f6fe57570dc362fa5a8") # If you don't have access to the Internet, diff --git a/cmake/onnxruntime-win-x64-directml.cmake b/cmake/onnxruntime-win-x64-directml.cmake index 9648ffecc..a171a69a7 100644 --- a/cmake/onnxruntime-win-x64-directml.cmake +++ b/cmake/onnxruntime-win-x64-directml.cmake @@ -20,7 +20,7 @@ if(NOT SHERPA_ONNX_ENABLE_DIRECTML) endif() set(onnxruntime_URL "https://globalcdn.nuget.org/packages/microsoft.ml.onnxruntime.directml.1.14.1.nupkg") -set(onnxruntime_URL2 "https://huggingface.co/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/microsoft.ml.onnxruntime.directml.1.14.1.nupkg") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/microsoft.ml.onnxruntime.directml.1.14.1.nupkg") set(onnxruntime_HASH "SHA256=c8ae7623385b19cd5de968d0df5383e13b97d1b3a6771c9177eac15b56013a5a") # If you don't have access to the Internet, @@ -158,4 +158,4 @@ file(GLOB directml_lib_files "${directml_SOURCE_DIR}/bin/x64-win/DirectML.*") message(STATUS "DirectML lib files: ${directml_lib_files}") install(FILES ${directml_lib_files} DESTINATION lib) -install(FILES ${directml_lib_files} DESTINATION bin) \ No newline at end of file +install(FILES ${directml_lib_files} DESTINATION bin) diff --git a/cmake/onnxruntime-win-x64-gpu.cmake b/cmake/onnxruntime-win-x64-gpu.cmake index 18b64d01f..5265653a5 100644 --- a/cmake/onnxruntime-win-x64-gpu.cmake +++ b/cmake/onnxruntime-win-x64-gpu.cmake @@ -20,7 +20,7 @@ if(NOT SHERPA_ONNX_ENABLE_GPU) endif() set(onnxruntime_URL "https://github.com/microsoft/onnxruntime/releases/download/v1.17.1/onnxruntime-win-x64-gpu-1.17.1.zip") -set(onnxruntime_URL2 "https://hub.nuaa.cf/microsoft/onnxruntime/releases/download/v1.17.1/onnxruntime-win-x64-gpu-1.17.1.zip") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-win-x64-gpu-1.17.1.zip") set(onnxruntime_HASH "SHA256=b7a66f50ad146c2ccb43471d2d3b5ad78084c2d4ddbd3ea82d65f86c867408b2") # If you don't have access to the Internet, diff --git a/cmake/onnxruntime-win-x64-static-debug.cmake b/cmake/onnxruntime-win-x64-static-debug.cmake index 3281f4989..211873cf3 100644 --- a/cmake/onnxruntime-win-x64-static-debug.cmake +++ b/cmake/onnxruntime-win-x64-static-debug.cmake @@ -16,7 +16,7 @@ if(BUILD_SHARED_LIBS) endif() set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-win-x64-static_lib-${CMAKE_BUILD_TYPE}-1.17.1.tar.bz2") -set(onnxruntime_URL2 "https://hub.nuaa.cf/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-win-x64-static_lib-${CMAKE_BUILD_TYPE}-1.17.1.tar.bz2") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-win-x64-static_lib-${CMAKE_BUILD_TYPE}-1.17.1.tar.bz2") if(CMAKE_BUILD_TYPE STREQUAL Debug) set(onnxruntime_HASH "SHA256=ecc68d914541c3b6ebc36148af63fe2a6af0f4f955b35199d612698d23169fa5") elseif(CMAKE_BUILD_TYPE STREQUAL RelWithDebInfo) diff --git a/cmake/onnxruntime-win-x64-static.cmake b/cmake/onnxruntime-win-x64-static.cmake index 009390872..811d64753 100644 --- a/cmake/onnxruntime-win-x64-static.cmake +++ b/cmake/onnxruntime-win-x64-static.cmake @@ -20,7 +20,7 @@ if(NOT CMAKE_BUILD_TYPE STREQUAL Release) endif() set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-win-x64-static_lib-1.17.1.tar.bz2") -set(onnxruntime_URL2 "https://hub.nuaa.cf/github.com/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-win-x64-static_lib-1.17.1.tar.bz2") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-win-x64-static_lib-1.17.1.tar.bz2") set(onnxruntime_HASH "SHA256=42a0c02fda945d1d72433b2a7cdb2187d51cb4d7f3af462c6ae07b25314d5fb3") # If you don't have access to the Internet, diff --git a/cmake/onnxruntime-win-x64.cmake b/cmake/onnxruntime-win-x64.cmake index 26f96fdb0..4dbe0caa6 100644 --- a/cmake/onnxruntime-win-x64.cmake +++ b/cmake/onnxruntime-win-x64.cmake @@ -16,7 +16,7 @@ if(NOT BUILD_SHARED_LIBS) endif() set(onnxruntime_URL "https://github.com/microsoft/onnxruntime/releases/download/v1.17.1/onnxruntime-win-x64-1.17.1.zip") -set(onnxruntime_URL2 "https://hub.nuaa.cf/microsoft/onnxruntime/releases/download/v1.17.1/onnxruntime-win-x64-1.17.1.zip") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-win-x64-1.17.1.zip") set(onnxruntime_HASH "SHA256=4802af9598db02153d7da39432a48823ff69b2fb4b59155461937f20782aa91c") # If you don't have access to the Internet, diff --git a/cmake/onnxruntime-win-x86-static-debug.cmake b/cmake/onnxruntime-win-x86-static-debug.cmake index a8d6858c6..8f00f2a50 100644 --- a/cmake/onnxruntime-win-x86-static-debug.cmake +++ b/cmake/onnxruntime-win-x86-static-debug.cmake @@ -17,7 +17,7 @@ endif() set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-win-x86-static_lib-${CMAKE_BUILD_TYPE}-1.17.1.tar.bz2") -set(onnxruntime_URL2 "https://hub.nuaa.cf/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-win-x86-static_lib-${CMAKE_BUILD_TYPE}-1.17.1.tar.bz2") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-win-x86-static_lib-${CMAKE_BUILD_TYPE}-1.17.1.tar.bz2") if(CMAKE_BUILD_TYPE STREQUAL Debug) set(onnxruntime_HASH "SHA256=b08b223fe09a5640472eec487ff42e4df6bf726e8aba9de40f443a1fabea3334") elseif(CMAKE_BUILD_TYPE STREQUAL RelWithDebInfo) diff --git a/cmake/onnxruntime-win-x86-static.cmake b/cmake/onnxruntime-win-x86-static.cmake index 7e291a616..ce424ee8c 100644 --- a/cmake/onnxruntime-win-x86-static.cmake +++ b/cmake/onnxruntime-win-x86-static.cmake @@ -20,7 +20,7 @@ if(NOT CMAKE_BUILD_TYPE STREQUAL Release) endif() set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-win-x86-static_lib-1.17.1.tar.bz2") -set(onnxruntime_URL2 "https://hub.nuaa.cf/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-win-x86-static_lib-1.17.1.tar.bz2") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-win-x86-static_lib-1.17.1.tar.bz2") set(onnxruntime_HASH "SHA256=52375d3fabc7b437c955a664bfeb9cb7a6391f5219c4b7d3b87ff690416d4b9e") # If you don't have access to the Internet, diff --git a/cmake/onnxruntime-win-x86.cmake b/cmake/onnxruntime-win-x86.cmake index 99ed71653..cd8248300 100644 --- a/cmake/onnxruntime-win-x86.cmake +++ b/cmake/onnxruntime-win-x86.cmake @@ -16,7 +16,7 @@ if(NOT BUILD_SHARED_LIBS) endif() set(onnxruntime_URL "https://github.com/microsoft/onnxruntime/releases/download/v1.17.1/onnxruntime-win-x86-1.17.1.zip") -set(onnxruntime_URL2 "https://hub.nuaa.cf/microsoft/onnxruntime/releases/download/v1.17.1/onnxruntime-win-x86-1.17.1.zip") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-win-x86-1.17.1.zip") set(onnxruntime_HASH "SHA256=9404130825474bd36b2538ed925d6b5f2cf1fb6a443f3e125054ae3470019291") # If you don't have access to the Internet, diff --git a/cmake/openfst.cmake b/cmake/openfst.cmake index 0f5863b7c..2309c2fbe 100644 --- a/cmake/openfst.cmake +++ b/cmake/openfst.cmake @@ -4,7 +4,7 @@ function(download_openfst) include(FetchContent) set(openfst_URL "https://github.com/csukuangfj/openfst/archive/refs/tags/sherpa-onnx-2024-06-19.tar.gz") - set(openfst_URL2 "https://hub.nuaa.cf/csukuangfj/openfst/archive/refs/tags/sherpa-onnx-2024-06-19.tar.gz") + set(openfst_URL2 "https://hf-mirror.com/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/openfst-sherpa-onnx-2024-06-19.tar.gz") set(openfst_HASH "SHA256=5c98e82cc509c5618502dde4860b8ea04d843850ed57e6d6b590b644b268853d") # If you don't have access to the Internet, diff --git a/cmake/piper-phonemize.cmake b/cmake/piper-phonemize.cmake index 7ecf1791b..bcea4e8ac 100644 --- a/cmake/piper-phonemize.cmake +++ b/cmake/piper-phonemize.cmake @@ -2,7 +2,7 @@ function(download_piper_phonemize) include(FetchContent) set(piper_phonemize_URL "https://github.com/csukuangfj/piper-phonemize/archive/dc6b5f4441bffe521047086930b0fc12686acd56.zip") - set(piper_phonemize_URL2 "https://hub.nuaa.cf/csukuangfj/piper-phonemize/archive/dc6b5f4441bffe521047086930b0fc12686acd56.zip") + set(piper_phonemize_URL2 "https://hf-mirror.com/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/piper-phonemize-dc6b5f4441bffe521047086930b0fc12686acd56.zip") set(piper_phonemize_HASH "SHA256=b9faa04204b1756fa455a962abb1f037041c040133d55be58d11f11ab9b3ce14") # If you don't have access to the Internet, diff --git a/cmake/pybind11.cmake b/cmake/pybind11.cmake index 0d4894eff..fe69ccc5a 100644 --- a/cmake/pybind11.cmake +++ b/cmake/pybind11.cmake @@ -2,7 +2,7 @@ function(download_pybind11) include(FetchContent) set(pybind11_URL "https://github.com/pybind/pybind11/archive/refs/tags/v2.10.2.tar.gz") - set(pybind11_URL2 "https://hub.nuaa.cf/pybind/pybind11/archive/refs/tags/v2.10.2.tar.gz") + set(pybind11_URL2 "https://hf-mirror.com/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/pybind11-2.10.2.tar.gz") set(pybind11_HASH "SHA256=93bd1e625e43e03028a3ea7389bba5d3f9f2596abc074b068e70f4ef9b1314ae") # If you don't have access to the Internet, diff --git a/cmake/simple-sentencepiece.cmake b/cmake/simple-sentencepiece.cmake index 09a640b11..4b6750d0f 100644 --- a/cmake/simple-sentencepiece.cmake +++ b/cmake/simple-sentencepiece.cmake @@ -2,7 +2,7 @@ function(download_simple_sentencepiece) include(FetchContent) set(simple-sentencepiece_URL "https://github.com/pkufool/simple-sentencepiece/archive/refs/tags/v0.7.tar.gz") - set(simple-sentencepiece_URL2 "https://hub.nuaa.cf/pkufool/simple-sentencepiece/archive/refs/tags/v0.7.tar.gz") + set(simple-sentencepiece_URL2 "https://hf-mirror.com/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/simple-sentencepiece-0.7.tar.gz") set(simple-sentencepiece_HASH "SHA256=1748a822060a35baa9f6609f84efc8eb54dc0e74b9ece3d82367b7119fdc75af") # If you don't have access to the Internet, diff --git a/cmake/websocketpp.cmake b/cmake/websocketpp.cmake index 6ae9b89a3..79b0585be 100644 --- a/cmake/websocketpp.cmake +++ b/cmake/websocketpp.cmake @@ -3,7 +3,7 @@ function(download_websocketpp) # The latest commit on the develop branch os as 2022-10-22 set(websocketpp_URL "https://github.com/zaphoyd/websocketpp/archive/b9aeec6eaf3d5610503439b4fae3581d9aff08e8.zip") - set(websocketpp_URL2 "https://hub.nuaa.cf/zaphoyd/websocketpp/archive/b9aeec6eaf3d5610503439b4fae3581d9aff08e8.zip") + set(websocketpp_URL2 "https://hf-mirror.com/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/websocketpp-b9aeec6eaf3d5610503439b4fae3581d9aff08e8.zip") set(websocketpp_HASH "SHA256=1385135ede8191a7fbef9ec8099e3c5a673d48df0c143958216cd1690567f583") # If you don't have access to the Internet, diff --git a/ffmpeg-examples/sherpa-onnx-ffmpeg.c b/ffmpeg-examples/sherpa-onnx-ffmpeg.c index cf3614518..82cff1173 100644 --- a/ffmpeg-examples/sherpa-onnx-ffmpeg.c +++ b/ffmpeg-examples/sherpa-onnx-ffmpeg.c @@ -214,8 +214,8 @@ static int init_filters(const char *filters_descr) { } static void sherpa_decode_frame(const AVFrame *frame, - SherpaOnnxOnlineRecognizer *recognizer, - SherpaOnnxOnlineStream *stream, + const SherpaOnnxOnlineRecognizer *recognizer, + const SherpaOnnxOnlineStream *stream, const SherpaOnnxDisplay *display, int32_t *segment_id) { #define N 3200 // 100s. Sample rate is fixed to 16 kHz diff --git a/scripts/node-addon-api/src/non-streaming-asr.cc b/scripts/node-addon-api/src/non-streaming-asr.cc index 86badc0ff..28c0b31ed 100644 --- a/scripts/node-addon-api/src/non-streaming-asr.cc +++ b/scripts/node-addon-api/src/non-streaming-asr.cc @@ -340,7 +340,7 @@ static Napi::External CreateOfflineStreamWrapper( SherpaOnnxCreateOfflineStream(recognizer); return Napi::External::New( - env, const_cast(stream), + env, const_cast(stream), [](Napi::Env env, SherpaOnnxOfflineStream *stream) { SherpaOnnxDestroyOfflineStream(stream); }); diff --git a/scripts/node-addon-api/src/streaming-asr.cc b/scripts/node-addon-api/src/streaming-asr.cc index cb3aaac86..4652be5f5 100644 --- a/scripts/node-addon-api/src/streaming-asr.cc +++ b/scripts/node-addon-api/src/streaming-asr.cc @@ -312,10 +312,12 @@ static Napi::External CreateOnlineStreamWrapper( SherpaOnnxOnlineRecognizer *recognizer = info[0].As>().Data(); - SherpaOnnxOnlineStream *stream = SherpaOnnxCreateOnlineStream(recognizer); + const SherpaOnnxOnlineStream *stream = + SherpaOnnxCreateOnlineStream(recognizer); return Napi::External::New( - env, stream, [](Napi::Env env, SherpaOnnxOnlineStream *stream) { + env, const_cast(stream), + [](Napi::Env env, SherpaOnnxOnlineStream *stream) { SherpaOnnxDestroyOnlineStream(stream); }); } diff --git a/sherpa-onnx/c-api/cxx-api.h b/sherpa-onnx/c-api/cxx-api.h index b078f5c73..17727f817 100644 --- a/sherpa-onnx/c-api/cxx-api.h +++ b/sherpa-onnx/c-api/cxx-api.h @@ -16,22 +16,22 @@ namespace sherpa_onnx::cxx { // ============================================================================ // Streaming ASR // ============================================================================ -struct SHERPA_ONNX_API OnlineTransducerModelConfig { +struct OnlineTransducerModelConfig { std::string encoder; std::string decoder; std::string joiner; }; -struct SHERPA_ONNX_API OnlineParaformerModelConfig { +struct OnlineParaformerModelConfig { std::string encoder; std::string decoder; }; -struct SHERPA_ONNX_API OnlineZipformer2CtcModelConfig { +struct OnlineZipformer2CtcModelConfig { std::string model; }; -struct SHERPA_ONNX_API OnlineModelConfig { +struct OnlineModelConfig { OnlineTransducerModelConfig transducer; OnlineParaformerModelConfig paraformer; OnlineZipformer2CtcModelConfig zipformer2_ctc; @@ -45,17 +45,17 @@ struct SHERPA_ONNX_API OnlineModelConfig { std::string tokens_buf; }; -struct SHERPA_ONNX_API FeatureConfig { +struct FeatureConfig { int32_t sample_rate = 16000; int32_t feature_dim = 80; }; -struct SHERPA_ONNX_API OnlineCtcFstDecoderConfig { +struct OnlineCtcFstDecoderConfig { std::string graph; int32_t max_active = 3000; }; -struct SHERPA_ONNX_API OnlineRecognizerConfig { +struct OnlineRecognizerConfig { FeatureConfig feat_config; OnlineModelConfig model_config; @@ -83,14 +83,14 @@ struct SHERPA_ONNX_API OnlineRecognizerConfig { std::string hotwords_buf; }; -struct SHERPA_ONNX_API OnlineRecognizerResult { +struct OnlineRecognizerResult { std::string text; std::vector tokens; std::vector timestamps; std::string json; }; -struct SHERPA_ONNX_API Wave { +struct Wave { std::vector samples; int32_t sample_rate; }; @@ -118,6 +118,8 @@ class SHERPA_ONNX_API MoveOnly { Destroy(); p_ = other.Release(); + + return *this; } const T *Get() const { return p_; } diff --git a/sherpa-onnx/csrc/CMakeLists.txt b/sherpa-onnx/csrc/CMakeLists.txt index 3e6526563..fafe5de96 100644 --- a/sherpa-onnx/csrc/CMakeLists.txt +++ b/sherpa-onnx/csrc/CMakeLists.txt @@ -18,6 +18,7 @@ set(sources endpoint.cc features.cc file-utils.cc + fst-utils.cc hypothesis.cc keyword-spotter-impl.cc keyword-spotter.cc diff --git a/sherpa-onnx/csrc/fst-utils.cc b/sherpa-onnx/csrc/fst-utils.cc new file mode 100644 index 000000000..5fcf5235a --- /dev/null +++ b/sherpa-onnx/csrc/fst-utils.cc @@ -0,0 +1,53 @@ +// sherpa-onnx/csrc/fst-utils.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include "sherpa-onnx/csrc/fst-utils.h" + +#include "sherpa-onnx/csrc/macros.h" + +namespace sherpa_onnx { + +// This function is copied from kaldi. +// +// @param filename Path to a StdVectorFst or StdConstFst graph +// @return The caller should free the returned pointer using `delete` to +// avoid memory leak. +fst::Fst *ReadGraph(const std::string &filename) { + // read decoding network FST + std::ifstream is(filename, std::ios::binary); + if (!is.good()) { + SHERPA_ONNX_LOGE("Could not open decoding-graph FST %s", filename.c_str()); + } + + fst::FstHeader hdr; + if (!hdr.Read(is, "")) { + SHERPA_ONNX_LOGE("Reading FST: error reading FST header."); + } + + if (hdr.ArcType() != fst::StdArc::Type()) { + SHERPA_ONNX_LOGE("FST with arc type %s not supported", + hdr.ArcType().c_str()); + } + fst::FstReadOptions ropts("", &hdr); + + fst::Fst *decode_fst = nullptr; + + if (hdr.FstType() == "vector") { + decode_fst = fst::VectorFst::Read(is, ropts); + } else if (hdr.FstType() == "const") { + decode_fst = fst::ConstFst::Read(is, ropts); + } else { + SHERPA_ONNX_LOGE("Reading FST: unsupported FST type: %s", + hdr.FstType().c_str()); + } + + if (decode_fst == nullptr) { // fst code will warn. + SHERPA_ONNX_LOGE("Error reading FST (after reading header)."); + return nullptr; + } else { + return decode_fst; + } +} + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/fst-utils.h b/sherpa-onnx/csrc/fst-utils.h new file mode 100644 index 000000000..92e6a7aad --- /dev/null +++ b/sherpa-onnx/csrc/fst-utils.h @@ -0,0 +1,18 @@ +// sherpa-onnx/csrc/fst-utils.h +// +// Copyright (c) 2024 Xiaomi Corporation + +#ifndef SHERPA_ONNX_CSRC_FST_UTILS_H_ +#define SHERPA_ONNX_CSRC_FST_UTILS_H_ + +#include + +#include "fst/fstlib.h" + +namespace sherpa_onnx { + +fst::Fst *ReadGraph(const std::string &filename); + +} + +#endif // SHERPA_ONNX_CSRC_FST_UTILS_H_ diff --git a/sherpa-onnx/csrc/jieba-lexicon.cc b/sherpa-onnx/csrc/jieba-lexicon.cc index e62324103..e995c8cb8 100644 --- a/sherpa-onnx/csrc/jieba-lexicon.cc +++ b/sherpa-onnx/csrc/jieba-lexicon.cc @@ -10,18 +10,12 @@ #include "cppjieba/Jieba.hpp" #include "sherpa-onnx/csrc/file-utils.h" +#include "sherpa-onnx/csrc/lexicon.h" #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/text-utils.h" namespace sherpa_onnx { -// implemented in ./lexicon.cc -std::unordered_map ReadTokens(std::istream &is); - -std::vector ConvertTokensToIds( - const std::unordered_map &token2id, - const std::vector &tokens); - class JiebaLexicon::Impl { public: Impl(const std::string &lexicon, const std::string &tokens, diff --git a/sherpa-onnx/csrc/lexicon.h b/sherpa-onnx/csrc/lexicon.h index 2c71ab7e8..9c2d91ee9 100644 --- a/sherpa-onnx/csrc/lexicon.h +++ b/sherpa-onnx/csrc/lexicon.h @@ -6,6 +6,7 @@ #define SHERPA_ONNX_CSRC_LEXICON_H_ #include +#include #include #include #include @@ -66,6 +67,12 @@ class Lexicon : public OfflineTtsFrontend { bool debug_ = false; }; +std::unordered_map ReadTokens(std::istream &is); + +std::vector ConvertTokensToIds( + const std::unordered_map &token2id, + const std::vector &tokens); + } // namespace sherpa_onnx #endif // SHERPA_ONNX_CSRC_LEXICON_H_ diff --git a/sherpa-onnx/csrc/melo-tts-lexicon.cc b/sherpa-onnx/csrc/melo-tts-lexicon.cc index e379b9c2f..d7332b36c 100644 --- a/sherpa-onnx/csrc/melo-tts-lexicon.cc +++ b/sherpa-onnx/csrc/melo-tts-lexicon.cc @@ -10,18 +10,12 @@ #include "cppjieba/Jieba.hpp" #include "sherpa-onnx/csrc/file-utils.h" +#include "sherpa-onnx/csrc/lexicon.h" #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/text-utils.h" namespace sherpa_onnx { -// implemented in ./lexicon.cc -std::unordered_map ReadTokens(std::istream &is); - -std::vector ConvertTokensToIds( - const std::unordered_map &token2id, - const std::vector &tokens); - class MeloTtsLexicon::Impl { public: Impl(const std::string &lexicon, const std::string &tokens, diff --git a/sherpa-onnx/csrc/offline-ctc-fst-decoder.cc b/sherpa-onnx/csrc/offline-ctc-fst-decoder.cc index 6c9df3fd3..ca3dcaee9 100644 --- a/sherpa-onnx/csrc/offline-ctc-fst-decoder.cc +++ b/sherpa-onnx/csrc/offline-ctc-fst-decoder.cc @@ -11,52 +11,11 @@ #include "kaldi-decoder/csrc/decodable-ctc.h" #include "kaldi-decoder/csrc/eigen.h" #include "kaldi-decoder/csrc/faster-decoder.h" +#include "sherpa-onnx/csrc/fst-utils.h" #include "sherpa-onnx/csrc/macros.h" namespace sherpa_onnx { -// This function is copied from kaldi. -// -// @param filename Path to a StdVectorFst or StdConstFst graph -// @return The caller should free the returned pointer using `delete` to -// avoid memory leak. -fst::Fst *ReadGraph(const std::string &filename) { - // read decoding network FST - std::ifstream is(filename, std::ios::binary); - if (!is.good()) { - SHERPA_ONNX_LOGE("Could not open decoding-graph FST %s", filename.c_str()); - } - - fst::FstHeader hdr; - if (!hdr.Read(is, "")) { - SHERPA_ONNX_LOGE("Reading FST: error reading FST header."); - } - - if (hdr.ArcType() != fst::StdArc::Type()) { - SHERPA_ONNX_LOGE("FST with arc type %s not supported", - hdr.ArcType().c_str()); - } - fst::FstReadOptions ropts("", &hdr); - - fst::Fst *decode_fst = nullptr; - - if (hdr.FstType() == "vector") { - decode_fst = fst::VectorFst::Read(is, ropts); - } else if (hdr.FstType() == "const") { - decode_fst = fst::ConstFst::Read(is, ropts); - } else { - SHERPA_ONNX_LOGE("Reading FST: unsupported FST type: %s", - hdr.FstType().c_str()); - } - - if (decode_fst == nullptr) { // fst code will warn. - SHERPA_ONNX_LOGE("Error reading FST (after reading header)."); - return nullptr; - } else { - return decode_fst; - } -} - /** * @param decoder * @param p Pointer to a 2-d array of shape (num_frames, vocab_size) diff --git a/sherpa-onnx/csrc/offline-speaker-diarization-result.cc b/sherpa-onnx/csrc/offline-speaker-diarization-result.cc index 8bf83f5d9..596957288 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization-result.cc +++ b/sherpa-onnx/csrc/offline-speaker-diarization-result.cc @@ -5,6 +5,7 @@ #include "sherpa-onnx/csrc/offline-speaker-diarization-result.h" #include +#include #include #include #include @@ -48,11 +49,13 @@ OfflineSpeakerDiarizationSegment::Merge( } std::string OfflineSpeakerDiarizationSegment::ToString() const { - char s[128]; - snprintf(s, sizeof(s), "%.3f -- %.3f speaker_%02d", start_, end_, speaker_); + std::array s{}; + + snprintf(s.data(), s.size(), "%.3f -- %.3f speaker_%02d", start_, end_, + speaker_); std::ostringstream os; - os << s; + os << s.data(); if (!text_.empty()) { os << " " << text_; diff --git a/sherpa-onnx/csrc/offline-speaker-diarization.cc b/sherpa-onnx/csrc/offline-speaker-diarization.cc index f34ea4e0e..a4b021b73 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization.cc +++ b/sherpa-onnx/csrc/offline-speaker-diarization.cc @@ -5,6 +5,7 @@ #include "sherpa-onnx/csrc/offline-speaker-diarization.h" #include +#include #include "sherpa-onnx/csrc/offline-speaker-diarization-impl.h" @@ -94,7 +95,7 @@ OfflineSpeakerDiarizationResult OfflineSpeakerDiarization::Process( const float *audio, int32_t n, OfflineSpeakerDiarizationProgressCallback callback /*= nullptr*/, void *callback_arg /*= nullptr*/) const { - return impl_->Process(audio, n, callback, callback_arg); + return impl_->Process(audio, n, std::move(callback), callback_arg); } } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/online-ctc-fst-decoder.cc b/sherpa-onnx/csrc/online-ctc-fst-decoder.cc index 95e16ba75..dea909183 100644 --- a/sherpa-onnx/csrc/online-ctc-fst-decoder.cc +++ b/sherpa-onnx/csrc/online-ctc-fst-decoder.cc @@ -13,14 +13,12 @@ #include "fst/fstlib.h" #include "kaldi-decoder/csrc/decodable-ctc.h" #include "kaldifst/csrc/fstext-utils.h" +#include "sherpa-onnx/csrc/fst-utils.h" #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/online-stream.h" namespace sherpa_onnx { -// defined in ./offline-ctc-fst-decoder.cc -fst::Fst *ReadGraph(const std::string &filename); - OnlineCtcFstDecoder::OnlineCtcFstDecoder( const OnlineCtcFstDecoderConfig &config, int32_t blank_id) : config_(config), fst_(ReadGraph(config.graph)), blank_id_(blank_id) { diff --git a/sherpa-onnx/csrc/online-recognizer.cc b/sherpa-onnx/csrc/online-recognizer.cc index c6b9399d8..ba7cba8ec 100644 --- a/sherpa-onnx/csrc/online-recognizer.cc +++ b/sherpa-onnx/csrc/online-recognizer.cc @@ -19,6 +19,8 @@ namespace sherpa_onnx { +namespace { + /// Helper for `OnlineRecognizerResult::AsJsonString()` template std::string VecToString(const std::vector &vec, int32_t precision = 6) { @@ -51,6 +53,8 @@ std::string VecToString(const std::vector &vec, return oss.str(); } +} // namespace + std::string OnlineRecognizerResult::AsJsonString() const { std::ostringstream os; os << "{ "; From a5295aad10ea932279b415cd573e57273926a69b Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 24 Oct 2024 14:03:09 +0800 Subject: [PATCH 033/183] Handle NaN embeddings in speaker diarization. (#1461) See also https://github.com/thewh1teagle/sherpa-rs/issues/33 --- cxx-api-examples/sense-voice-cxx-api.cc | 2 +- .../streaming-zipformer-cxx-api.cc | 2 +- cxx-api-examples/whisper-cxx-api.cc | 2 +- scripts/check_style_cpplint.sh | 5 ++- ...ffline-speaker-diarization-pyannote-impl.h | 42 ++++++++++++++++++- .../speaker-embedding-extractor-nemo-impl.h | 2 +- 6 files changed, 48 insertions(+), 7 deletions(-) diff --git a/cxx-api-examples/sense-voice-cxx-api.cc b/cxx-api-examples/sense-voice-cxx-api.cc index 15d752058..ea642b980 100644 --- a/cxx-api-examples/sense-voice-cxx-api.cc +++ b/cxx-api-examples/sense-voice-cxx-api.cc @@ -19,7 +19,7 @@ #include "sherpa-onnx/c-api/cxx-api.h" int32_t main() { - using namespace sherpa_onnx::cxx; + using namespace sherpa_onnx::cxx; // NOLINT OfflineRecognizerConfig config; config.model_config.sense_voice.model = diff --git a/cxx-api-examples/streaming-zipformer-cxx-api.cc b/cxx-api-examples/streaming-zipformer-cxx-api.cc index 5a49dcfc9..ac4abc479 100644 --- a/cxx-api-examples/streaming-zipformer-cxx-api.cc +++ b/cxx-api-examples/streaming-zipformer-cxx-api.cc @@ -20,7 +20,7 @@ #include "sherpa-onnx/c-api/cxx-api.h" int32_t main() { - using namespace sherpa_onnx::cxx; + using namespace sherpa_onnx::cxx; // NOLINT OnlineRecognizerConfig config; // please see diff --git a/cxx-api-examples/whisper-cxx-api.cc b/cxx-api-examples/whisper-cxx-api.cc index 82f0ddb53..348d115bd 100644 --- a/cxx-api-examples/whisper-cxx-api.cc +++ b/cxx-api-examples/whisper-cxx-api.cc @@ -19,7 +19,7 @@ #include "sherpa-onnx/c-api/cxx-api.h" int32_t main() { - using namespace sherpa_onnx::cxx; + using namespace sherpa_onnx::cxx; // NOLINT OfflineRecognizerConfig config; config.model_config.whisper.encoder = diff --git a/scripts/check_style_cpplint.sh b/scripts/check_style_cpplint.sh index eedc9afc1..ea419242a 100755 --- a/scripts/check_style_cpplint.sh +++ b/scripts/check_style_cpplint.sh @@ -71,6 +71,9 @@ function is_source_code_file() { } function check_style() { + if [[ $1 == mfc-example* ]]; then + return + fi python3 $cpplint_src $1 || abort $1 } @@ -99,7 +102,7 @@ function do_check() { ;; 2) echo "Check all files" - files=$(find $sherpa_onnx_dir/sherpa-onnx/csrc $sherpa_onnx_dir/sherpa-onnx/python $sherpa_onnx_dir/scripts/node-addon-api/src $sherpa_onnx_dir/sherpa-onnx/jni $sherpa_onnx_dir/sherpa-onnx/c-api -name "*.h" -o -name "*.cc") + files=$(find $sherpa_onnx_dir/cxx-api-examples $sherpa_onnx_dir/c-api-examples $sherpa_onnx_dir/sherpa-onnx/csrc $sherpa_onnx_dir/sherpa-onnx/python $sherpa_onnx_dir/scripts/node-addon-api/src $sherpa_onnx_dir/sherpa-onnx/jni $sherpa_onnx_dir/sherpa-onnx/c-api -name "*.h" -o -name "*.cc") ;; *) echo "Check last commit" diff --git a/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h b/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h index aaedc3be0..51d712eb8 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h +++ b/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h @@ -5,6 +5,7 @@ #define SHERPA_ONNX_CSRC_OFFLINE_SPEAKER_DIARIZATION_PYANNOTE_IMPL_H_ #include +#include #include #include #include @@ -135,9 +136,32 @@ class OfflineSpeakerDiarizationPyannoteImpl } auto chunk_speaker_samples_list_pair = GetChunkSpeakerSampleIndexes(labels); + + // The embedding model may output NaN. valid_indexes contains indexes + // in chunk_speaker_samples_list_pair.second that don't lead to + // NaN embeddings. + std::vector valid_indexes; + valid_indexes.reserve(chunk_speaker_samples_list_pair.second.size()); + Matrix2D embeddings = ComputeEmbeddings(audio, n, chunk_speaker_samples_list_pair.second, - std::move(callback), callback_arg); + &valid_indexes, std::move(callback), callback_arg); + + if (valid_indexes.size() != chunk_speaker_samples_list_pair.second.size()) { + std::vector chunk_speaker_pair; + std::vector> sample_indexes; + + chunk_speaker_pair.reserve(valid_indexes.size()); + sample_indexes.reserve(valid_indexes.size()); + for (auto i : valid_indexes) { + chunk_speaker_pair.push_back(chunk_speaker_samples_list_pair.first[i]); + sample_indexes.push_back( + std::move(chunk_speaker_samples_list_pair.second[i])); + } + + chunk_speaker_samples_list_pair.first = std::move(chunk_speaker_pair); + chunk_speaker_samples_list_pair.second = std::move(sample_indexes); + } std::vector cluster_labels = clustering_->Cluster( &embeddings(0, 0), embeddings.rows(), embeddings.cols()); @@ -431,13 +455,17 @@ class OfflineSpeakerDiarizationPyannoteImpl Matrix2D ComputeEmbeddings( const float *audio, int32_t n, const std::vector> &sample_indexes, + std::vector *valid_indexes, OfflineSpeakerDiarizationProgressCallback callback, void *callback_arg) const { const auto &meta_data = segmentation_model_.GetModelMetaData(); int32_t sample_rate = meta_data.sample_rate; Matrix2D ans(sample_indexes.size(), embedding_extractor_.Dim()); + auto IsNaNWrapper = [](float f) -> bool { return std::isnan(f); }; + int32_t k = 0; + int32_t cur_row_index = 0; for (const auto &v : sample_indexes) { auto stream = embedding_extractor_.CreateStream(); for (const auto &p : v) { @@ -459,7 +487,12 @@ class OfflineSpeakerDiarizationPyannoteImpl std::vector embedding = embedding_extractor_.Compute(stream.get()); - std::copy(embedding.begin(), embedding.end(), &ans(k, 0)); + if (std::none_of(embedding.begin(), embedding.end(), IsNaNWrapper)) { + // a valid embedding + std::copy(embedding.begin(), embedding.end(), &ans(cur_row_index, 0)); + cur_row_index += 1; + valid_indexes->push_back(k); + } k += 1; @@ -468,6 +501,11 @@ class OfflineSpeakerDiarizationPyannoteImpl } } + if (k != cur_row_index) { + auto seq = Eigen::seqN(0, cur_row_index); + ans = ans(seq, Eigen::all); + } + return ans; } diff --git a/sherpa-onnx/csrc/speaker-embedding-extractor-nemo-impl.h b/sherpa-onnx/csrc/speaker-embedding-extractor-nemo-impl.h index 66ad15af3..7e0883085 100644 --- a/sherpa-onnx/csrc/speaker-embedding-extractor-nemo-impl.h +++ b/sherpa-onnx/csrc/speaker-embedding-extractor-nemo-impl.h @@ -122,7 +122,7 @@ class SpeakerEmbeddingExtractorNeMoImpl : public SpeakerEmbeddingExtractorImpl { auto variance = EX2 - EX.array().pow(2); auto stddev = variance.array().sqrt(); - m = (m.rowwise() - EX).array().rowwise() / stddev.array(); + m = (m.rowwise() - EX).array().rowwise() / (stddev.array() + 1e-5); } private: From 2b40079faf756113f1a415cb39699f9489e1e316 Mon Sep 17 00:00:00 2001 From: Peakyxh <112252066+Peakyxh@users.noreply.github.com> Date: Thu, 24 Oct 2024 22:04:51 +0800 Subject: [PATCH 034/183] Add speaker identification with VAD and non-streaming ASR using ALSA (#1463) --- ...ication-with-vad-non-streaming-asr-alsa.py | 494 ++++++++++++++++++ 1 file changed, 494 insertions(+) create mode 100644 python-api-examples/speaker-identification-with-vad-non-streaming-asr-alsa.py diff --git a/python-api-examples/speaker-identification-with-vad-non-streaming-asr-alsa.py b/python-api-examples/speaker-identification-with-vad-non-streaming-asr-alsa.py new file mode 100644 index 000000000..9be196f2b --- /dev/null +++ b/python-api-examples/speaker-identification-with-vad-non-streaming-asr-alsa.py @@ -0,0 +1,494 @@ +#!/usr/bin/env python3 + +""" +This script works only on Linux. It uses ALSA for recording. + +This script shows how to use Python APIs for speaker identification with +a microphone, a VAD model, and a non-streaming ASR model. + +Please see also ./generate-subtitles.py + +Usage: + +(1) Prepare a text file containing speaker related files. + +Each line in the text file contains two columns. The first column is the +speaker name, while the second column contains the wave file of the speaker. + +If the text file contains multiple wave files for the same speaker, then the +embeddings of these files are averaged. + +An example text file is given below: + + foo /path/to/a.wav + bar /path/to/b.wav + foo /path/to/c.wav + foobar /path/to/d.wav + +Each wave file should contain only a single channel; the sample format +should be int16_t; the sample rate can be arbitrary. + +(2) Download a model for computing speaker embeddings + +Please visit +https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models +to download a model. An example is given below: + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/wespeaker_zh_cnceleb_resnet34.onnx + +Note that `zh` means Chinese, while `en` means English. + +(3) Download the VAD model +Please visit +https://github.com/snakers4/silero-vad/raw/master/src/silero_vad/data/silero_vad.onnx +to download silero_vad.onnx + +For instance, + +wget https://github.com/snakers4/silero-vad/raw/master/src/silero_vad/data/silero_vad.onnx + +(4) Please refer to ./generate-subtitles.py +to download a non-streaming ASR model. + +(5) Run this script + +Assume the filename of the text file is speaker.txt. + +python3 ./python-api-examples/speaker-identification-with-vad-non-streaming-asr.py \ + --silero-vad-model=/path/to/silero_vad.onnx \ + --speaker-file ./speaker.txt \ + --model ./wespeaker_zh_cnceleb_resnet34.onnx +""" +import argparse +from collections import defaultdict +from pathlib import Path +from typing import Dict, List, Tuple + +import numpy as np +import sherpa_onnx +import soundfile as sf + +g_sample_rate = 16000 + + +def register_non_streaming_asr_model_args(parser): + parser.add_argument( + "--tokens", + type=str, + help="Path to tokens.txt", + ) + + parser.add_argument( + "--encoder", + default="", + type=str, + help="Path to the transducer encoder model", + ) + + parser.add_argument( + "--decoder", + default="", + type=str, + help="Path to the transducer decoder model", + ) + + parser.add_argument( + "--joiner", + default="", + type=str, + help="Path to the transducer joiner model", + ) + + parser.add_argument( + "--paraformer", + default="", + type=str, + help="Path to the model.onnx from Paraformer", + ) + + parser.add_argument( + "--wenet-ctc", + default="", + type=str, + help="Path to the CTC model.onnx from WeNet", + ) + + parser.add_argument( + "--whisper-encoder", + default="", + type=str, + help="Path to whisper encoder model", + ) + + parser.add_argument( + "--whisper-decoder", + default="", + type=str, + help="Path to whisper decoder model", + ) + + parser.add_argument( + "--whisper-language", + default="", + type=str, + help="""It specifies the spoken language in the input file. + Example values: en, fr, de, zh, jp. + Available languages for multilingual models can be found at + https://github.com/openai/whisper/blob/main/whisper/tokenizer.py#L10 + If not specified, we infer the language from the input audio file. + """, + ) + + parser.add_argument( + "--whisper-task", + default="transcribe", + choices=["transcribe", "translate"], + type=str, + help="""For multilingual models, if you specify translate, the output + will be in English. + """, + ) + + parser.add_argument( + "--whisper-tail-paddings", + default=-1, + type=int, + help="""Number of tail padding frames. + We have removed the 30-second constraint from whisper, so you need to + choose the amount of tail padding frames by yourself. + Use -1 to use a default value for tail padding. + """, + ) + + parser.add_argument( + "--decoding-method", + type=str, + default="greedy_search", + help="""Valid values are greedy_search and modified_beam_search. + modified_beam_search is valid only for transducer models. + """, + ) + + parser.add_argument( + "--feature-dim", + type=int, + default=80, + help="Feature dimension. Must match the one expected by the model", + ) + + +def get_args(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + register_non_streaming_asr_model_args(parser) + + parser.add_argument( + "--speaker-file", + type=str, + required=True, + help="""Path to the speaker file. Read the help doc at the beginning of this + file for the format.""", + ) + + parser.add_argument( + "--model", + type=str, + required=True, + help="Path to the speaker embedding model file.", + ) + + parser.add_argument( + "--silero-vad-model", + type=str, + required=True, + help="Path to silero_vad.onnx", + ) + + parser.add_argument("--threshold", type=float, default=0.6) + + parser.add_argument( + "--num-threads", + type=int, + default=1, + help="Number of threads for neural network computation", + ) + + parser.add_argument( + "--debug", + type=bool, + default=False, + help="True to show debug messages", + ) + + parser.add_argument( + "--provider", + type=str, + default="cpu", + help="Valid values: cpu, cuda, coreml", + ) + + parser.add_argument( + "--device-name", + type=str, + required=True, + help=""" +The device name specifies which microphone to use in case there are several +on your system. You can use + + arecord -l + +to find all available microphones on your computer. For instance, if it outputs + +**** List of CAPTURE Hardware Devices **** +card 3: UACDemoV10 [UACDemoV1.0], device 0: USB Audio [USB Audio] + Subdevices: 1/1 + Subdevice #0: subdevice #0 + +and if you want to select card 3 and device 0 on that card, please use: + + plughw:3,0 + +as the device_name. + """, + ) + + return parser.parse_args() + + +def assert_file_exists(filename: str): + assert Path(filename).is_file(), ( + f"{filename} does not exist!\n" + "Please refer to " + "https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html to download it" + ) + + +def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: + if args.encoder: + assert len(args.paraformer) == 0, args.paraformer + assert len(args.wenet_ctc) == 0, args.wenet_ctc + assert len(args.whisper_encoder) == 0, args.whisper_encoder + assert len(args.whisper_decoder) == 0, args.whisper_decoder + + assert_file_exists(args.encoder) + assert_file_exists(args.decoder) + assert_file_exists(args.joiner) + + recognizer = sherpa_onnx.OfflineRecognizer.from_transducer( + encoder=args.encoder, + decoder=args.decoder, + joiner=args.joiner, + tokens=args.tokens, + num_threads=args.num_threads, + sample_rate=args.sample_rate, + feature_dim=args.feature_dim, + decoding_method=args.decoding_method, + debug=args.debug, + ) + elif args.paraformer: + assert len(args.wenet_ctc) == 0, args.wenet_ctc + assert len(args.whisper_encoder) == 0, args.whisper_encoder + assert len(args.whisper_decoder) == 0, args.whisper_decoder + + assert_file_exists(args.paraformer) + + recognizer = sherpa_onnx.OfflineRecognizer.from_paraformer( + paraformer=args.paraformer, + tokens=args.tokens, + num_threads=args.num_threads, + sample_rate=g_sample_rate, + feature_dim=args.feature_dim, + decoding_method=args.decoding_method, + debug=args.debug, + ) + elif args.wenet_ctc: + assert len(args.whisper_encoder) == 0, args.whisper_encoder + assert len(args.whisper_decoder) == 0, args.whisper_decoder + + assert_file_exists(args.wenet_ctc) + + recognizer = sherpa_onnx.OfflineRecognizer.from_wenet_ctc( + model=args.wenet_ctc, + tokens=args.tokens, + num_threads=args.num_threads, + sample_rate=args.sample_rate, + feature_dim=args.feature_dim, + decoding_method=args.decoding_method, + debug=args.debug, + ) + elif args.whisper_encoder: + assert_file_exists(args.whisper_encoder) + assert_file_exists(args.whisper_decoder) + + recognizer = sherpa_onnx.OfflineRecognizer.from_whisper( + encoder=args.whisper_encoder, + decoder=args.whisper_decoder, + tokens=args.tokens, + num_threads=args.num_threads, + decoding_method=args.decoding_method, + debug=args.debug, + language=args.whisper_language, + task=args.whisper_task, + tail_paddings=args.whisper_tail_paddings, + ) + else: + raise ValueError("Please specify at least one model") + + return recognizer + + +def load_speaker_embedding_model(args): + config = sherpa_onnx.SpeakerEmbeddingExtractorConfig( + model=args.model, + num_threads=args.num_threads, + debug=args.debug, + provider=args.provider, + ) + if not config.validate(): + raise ValueError(f"Invalid config. {config}") + extractor = sherpa_onnx.SpeakerEmbeddingExtractor(config) + return extractor + + +def load_speaker_file(args) -> Dict[str, List[str]]: + if not Path(args.speaker_file).is_file(): + raise ValueError(f"--speaker-file {args.speaker_file} does not exist") + + ans = defaultdict(list) + with open(args.speaker_file) as f: + for line in f: + line = line.strip() + if not line: + continue + + fields = line.split() + if len(fields) != 2: + raise ValueError(f"Invalid line: {line}. Fields: {fields}") + + speaker_name, filename = fields + ans[speaker_name].append(filename) + return ans + + +def load_audio(filename: str) -> Tuple[np.ndarray, int]: + data, sample_rate = sf.read( + filename, + always_2d=True, + dtype="float32", + ) + data = data[:, 0] # use only the first channel + samples = np.ascontiguousarray(data) + return samples, sample_rate + + +def compute_speaker_embedding( + filenames: List[str], + extractor: sherpa_onnx.SpeakerEmbeddingExtractor, +) -> np.ndarray: + assert len(filenames) > 0, "filenames is empty" + + ans = None + for filename in filenames: + print(f"processing {filename}") + samples, sample_rate = load_audio(filename) + stream = extractor.create_stream() + stream.accept_waveform(sample_rate=sample_rate, waveform=samples) + stream.input_finished() + + assert extractor.is_ready(stream) + embedding = extractor.compute(stream) + embedding = np.array(embedding) + if ans is None: + ans = embedding + else: + ans += embedding + + return ans / len(filenames) + + +def main(): + args = get_args() + print(args) + + device_name = args.device_name + print(f"device_name: {device_name}") + alsa = sherpa_onnx.Alsa(device_name) + + recognizer = create_recognizer(args) + extractor = load_speaker_embedding_model(args) + speaker_file = load_speaker_file(args) + + manager = sherpa_onnx.SpeakerEmbeddingManager(extractor.dim) + for name, filename_list in speaker_file.items(): + embedding = compute_speaker_embedding( + filenames=filename_list, + extractor=extractor, + ) + status = manager.add(name, embedding) + if not status: + raise RuntimeError(f"Failed to register speaker {name}") + + vad_config = sherpa_onnx.VadModelConfig() + vad_config.silero_vad.model = args.silero_vad_model + vad_config.silero_vad.min_silence_duration = 0.25 + vad_config.silero_vad.min_speech_duration = 0.25 + vad_config.sample_rate = g_sample_rate + if not vad_config.validate(): + raise ValueError("Errors in vad config") + + window_size = vad_config.silero_vad.window_size + + vad = sherpa_onnx.VoiceActivityDetector(vad_config, buffer_size_in_seconds=100) + + samples_per_read = int(0.1 * g_sample_rate) # 0.1 second = 100 ms + + print("Started! Please speak") + + idx = 0 + buffer = [] + while True: + samples = alsa.read(samples_per_read) # a blocking read + samples = np.array(samples) + buffer = np.concatenate([buffer, samples]) + while len(buffer) > window_size: + vad.accept_waveform(buffer[:window_size]) + buffer = buffer[window_size:] + + while not vad.empty(): + if len(vad.front.samples) < 0.5 * g_sample_rate: + # this segment is too short, skip it + vad.pop() + continue + stream = extractor.create_stream() + stream.accept_waveform( + sample_rate=g_sample_rate, waveform=vad.front.samples + ) + stream.input_finished() + + embedding = extractor.compute(stream) + embedding = np.array(embedding) + name = manager.search(embedding, threshold=args.threshold) + if not name: + name = "unknown" + + # Now for non-streaming ASR + asr_stream = recognizer.create_stream() + asr_stream.accept_waveform( + sample_rate=g_sample_rate, waveform=vad.front.samples + ) + recognizer.decode_stream(asr_stream) + text = asr_stream.result.text + + vad.pop() + + print(f"\r{idx}-{name}: {text}") + idx += 1 + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print("\nCaught Ctrl + C. Exiting") From b41f6d2c94d377d8716a10d4a3c222ac608f542a Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 25 Oct 2024 10:55:16 +0800 Subject: [PATCH 035/183] Support GigaAM CTC models for Russian ASR (#1464) See also https://github.com/salute-developers/GigaAM --- .github/scripts/test-offline-ctc.sh | 15 ++ .../export-nemo-giga-am-to-onnx.yaml | 88 ++++++++++ .github/workflows/linux.yaml | 20 +-- scripts/apk/generate-vad-asr-apk-script.py | 18 ++ scripts/nemo/GigaAM/README.md | 10 ++ scripts/nemo/GigaAM/export-onnx-ctc.py | 114 +++++++++++++ scripts/nemo/GigaAM/run-ctc.sh | 36 ++++ scripts/nemo/GigaAM/test-onnx-ctc.py | 157 ++++++++++++++++++ sherpa-onnx/csrc/features.cc | 2 + sherpa-onnx/csrc/features.h | 1 + sherpa-onnx/csrc/jieba-lexicon.cc | 2 +- sherpa-onnx/csrc/lexicon.cc | 40 +---- sherpa-onnx/csrc/lexicon.h | 6 - sherpa-onnx/csrc/macros.h | 140 +++++++++------- sherpa-onnx/csrc/melo-tts-lexicon.cc | 2 +- sherpa-onnx/csrc/offline-ctc-model.cc | 19 +-- sherpa-onnx/csrc/offline-ctc-model.h | 4 + .../csrc/offline-nemo-enc-dec-ctc-model.cc | 12 +- .../csrc/offline-nemo-enc-dec-ctc-model.h | 2 + .../csrc/offline-recognizer-ctc-impl.h | 19 ++- sherpa-onnx/csrc/offline-recognizer-impl.cc | 6 +- sherpa-onnx/csrc/symbol-table.cc | 66 +++++--- sherpa-onnx/csrc/symbol-table.h | 12 ++ sherpa-onnx/kotlin-api/OfflineRecognizer.kt | 10 ++ 24 files changed, 641 insertions(+), 160 deletions(-) create mode 100644 .github/workflows/export-nemo-giga-am-to-onnx.yaml create mode 100644 scripts/nemo/GigaAM/README.md create mode 100755 scripts/nemo/GigaAM/export-onnx-ctc.py create mode 100755 scripts/nemo/GigaAM/run-ctc.sh create mode 100755 scripts/nemo/GigaAM/test-onnx-ctc.py diff --git a/.github/scripts/test-offline-ctc.sh b/.github/scripts/test-offline-ctc.sh index 57208e9da..f85b58539 100755 --- a/.github/scripts/test-offline-ctc.sh +++ b/.github/scripts/test-offline-ctc.sh @@ -15,6 +15,21 @@ echo "PATH: $PATH" which $EXE +log "------------------------------------------------------------" +log "Run NeMo GigaAM Russian models" +log "------------------------------------------------------------" +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24.tar.bz2 +tar xvf sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24.tar.bz2 +rm sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24.tar.bz2 + +$EXE \ + --nemo-ctc-model=./sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24/model.int8.onnx \ + --tokens=./sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24/tokens.txt \ + --debug=1 \ + ./sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24/test_wavs/example.wav + +rm -rf sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24 + log "------------------------------------------------------------" log "Run SenseVoice models" log "------------------------------------------------------------" diff --git a/.github/workflows/export-nemo-giga-am-to-onnx.yaml b/.github/workflows/export-nemo-giga-am-to-onnx.yaml new file mode 100644 index 000000000..f48c344e6 --- /dev/null +++ b/.github/workflows/export-nemo-giga-am-to-onnx.yaml @@ -0,0 +1,88 @@ +name: export-nemo-giga-am-to-onnx + +on: + workflow_dispatch: + +concurrency: + group: export-nemo-giga-am-to-onnx-${{ github.ref }} + cancel-in-progress: true + +jobs: + export-nemo-am-giga-to-onnx: + if: github.repository_owner == 'k2-fsa' || github.repository_owner == 'csukuangfj' + name: export nemo GigaAM models to ONNX + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [macos-latest] + python-version: ["3.10"] + + steps: + - uses: actions/checkout@v4 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Run CTC + shell: bash + run: | + pushd scripts/nemo/GigaAM + ./run-ctc.sh + popd + + d=sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24 + mkdir $d + mkdir $d/test_wavs + rm scripts/nemo/GigaAM/model.onnx + mv -v scripts/nemo/GigaAM/*.int8.onnx $d/ + mv -v scripts/nemo/GigaAM/*.md $d/ + mv -v scripts/nemo/GigaAM/*.pdf $d/ + mv -v scripts/nemo/GigaAM/tokens.txt $d/ + mv -v scripts/nemo/GigaAM/*.wav $d/test_wavs/ + mv -v scripts/nemo/GigaAM/run-ctc.sh $d/ + mv -v scripts/nemo/GigaAM/*-ctc.py $d/ + + ls -lh scripts/nemo/GigaAM/ + + ls -lh $d + + tar cjvf ${d}.tar.bz2 $d + + - name: Release + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + file: ./*.tar.bz2 + overwrite: true + repo_name: k2-fsa/sherpa-onnx + repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} + tag: asr-models + + - name: Publish to huggingface (CTC) + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v3 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + + d=sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24 + export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$d huggingface + mv -v $d/* ./huggingface + cd huggingface + git lfs track "*.onnx" + git lfs track "*.wav" + git status + git add . + git status + git commit -m "add models" + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$d main diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml index a37f48618..2ad40e215 100644 --- a/.github/workflows/linux.yaml +++ b/.github/workflows/linux.yaml @@ -149,6 +149,16 @@ jobs: name: release-${{ matrix.build_type }}-with-shared-lib-${{ matrix.shared_lib }}-with-tts-${{ matrix.with_tts }} path: install/* + - name: Test offline CTC + shell: bash + run: | + du -h -d1 . + export PATH=$PWD/build/bin:$PATH + export EXE=sherpa-onnx-offline + + .github/scripts/test-offline-ctc.sh + du -h -d1 . + - name: Test C++ API shell: bash run: | @@ -180,16 +190,6 @@ jobs: .github/scripts/test-offline-transducer.sh du -h -d1 . - - name: Test offline CTC - shell: bash - run: | - du -h -d1 . - export PATH=$PWD/build/bin:$PATH - export EXE=sherpa-onnx-offline - - .github/scripts/test-offline-ctc.sh - du -h -d1 . - - name: Test online punctuation shell: bash run: | diff --git a/scripts/apk/generate-vad-asr-apk-script.py b/scripts/apk/generate-vad-asr-apk-script.py index b6361427f..8217e6ea0 100755 --- a/scripts/apk/generate-vad-asr-apk-script.py +++ b/scripts/apk/generate-vad-asr-apk-script.py @@ -333,6 +333,24 @@ def get_models(): ls -lh + popd + """, + ), + Model( + model_name="sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24", + idx=19, + lang="ru", + short_name="nemo_ctc_giga_am", + cmd=""" + pushd $model_name + + rm -rfv test_wavs + + rm -fv *.sh + rm -fv *.py + + ls -lh + popd """, ), diff --git a/scripts/nemo/GigaAM/README.md b/scripts/nemo/GigaAM/README.md new file mode 100644 index 000000000..583d10a16 --- /dev/null +++ b/scripts/nemo/GigaAM/README.md @@ -0,0 +1,10 @@ +# Introduction + +This folder contains scripts for converting models from +https://github.com/salute-developers/GigaAM +to sherpa-onnx. + +The ASR models are for Russian speech recognition in this folder. + +Please see the license of the models at +https://github.com/salute-developers/GigaAM/blob/main/GigaAM%20License_NC.pdf diff --git a/scripts/nemo/GigaAM/export-onnx-ctc.py b/scripts/nemo/GigaAM/export-onnx-ctc.py new file mode 100755 index 000000000..fbcec518e --- /dev/null +++ b/scripts/nemo/GigaAM/export-onnx-ctc.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +# Copyright 2024 Xiaomi Corp. (authors: Fangjun Kuang) +from typing import Dict + +import onnx +import torch +import torchaudio +from nemo.collections.asr.models import EncDecCTCModel +from nemo.collections.asr.modules.audio_preprocessing import ( + AudioToMelSpectrogramPreprocessor as NeMoAudioToMelSpectrogramPreprocessor, +) +from nemo.collections.asr.parts.preprocessing.features import ( + FilterbankFeaturesTA as NeMoFilterbankFeaturesTA, +) +from onnxruntime.quantization import QuantType, quantize_dynamic + + +class FilterbankFeaturesTA(NeMoFilterbankFeaturesTA): + def __init__(self, mel_scale: str = "htk", wkwargs=None, **kwargs): + if "window_size" in kwargs: + del kwargs["window_size"] + if "window_stride" in kwargs: + del kwargs["window_stride"] + + super().__init__(**kwargs) + + self._mel_spec_extractor: torchaudio.transforms.MelSpectrogram = ( + torchaudio.transforms.MelSpectrogram( + sample_rate=self._sample_rate, + win_length=self.win_length, + hop_length=self.hop_length, + n_mels=kwargs["nfilt"], + window_fn=self.torch_windows[kwargs["window"]], + mel_scale=mel_scale, + norm=kwargs["mel_norm"], + n_fft=kwargs["n_fft"], + f_max=kwargs.get("highfreq", None), + f_min=kwargs.get("lowfreq", 0), + wkwargs=wkwargs, + ) + ) + + +class AudioToMelSpectrogramPreprocessor(NeMoAudioToMelSpectrogramPreprocessor): + def __init__(self, mel_scale: str = "htk", **kwargs): + super().__init__(**kwargs) + kwargs["nfilt"] = kwargs["features"] + del kwargs["features"] + self.featurizer = ( + FilterbankFeaturesTA( # Deprecated arguments; kept for config compatibility + mel_scale=mel_scale, + **kwargs, + ) + ) + + +def add_meta_data(filename: str, meta_data: Dict[str, str]): + """Add meta data to an ONNX model. It is changed in-place. + + Args: + filename: + Filename of the ONNX model to be changed. + meta_data: + Key-value pairs. + """ + model = onnx.load(filename) + while len(model.metadata_props): + model.metadata_props.pop() + + for key, value in meta_data.items(): + meta = model.metadata_props.add() + meta.key = key + meta.value = str(value) + + onnx.save(model, filename) + + +def main(): + model = EncDecCTCModel.from_config_file("./ctc_model_config.yaml") + ckpt = torch.load("./ctc_model_weights.ckpt", map_location="cpu") + model.load_state_dict(ckpt, strict=False) + model.eval() + + with open("tokens.txt", "w", encoding="utf-8") as f: + for i, t in enumerate(model.cfg.labels): + f.write(f"{t} {i}\n") + f.write(f" {i+1}\n") + + filename = "model.onnx" + model.export(filename) + + meta_data = { + "vocab_size": len(model.cfg.labels) + 1, + "normalize_type": "", + "subsampling_factor": 4, + "model_type": "EncDecCTCModel", + "version": "1", + "model_author": "https://github.com/salute-developers/GigaAM", + "license": "https://github.com/salute-developers/GigaAM/blob/main/GigaAM%20License_NC.pdf", + "language": "Russian", + "is_giga_am": 1, + } + add_meta_data(filename, meta_data) + + filename_int8 = "model.int8.onnx" + quantize_dynamic( + model_input=filename, + model_output=filename_int8, + weight_type=QuantType.QUInt8, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/nemo/GigaAM/run-ctc.sh b/scripts/nemo/GigaAM/run-ctc.sh new file mode 100755 index 000000000..499bc452b --- /dev/null +++ b/scripts/nemo/GigaAM/run-ctc.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Copyright 2024 Xiaomi Corp. (authors: Fangjun Kuang) + +set -ex + +function install_nemo() { + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py + python3 get-pip.py + + pip install torch==2.4.0 torchaudio==2.4.0 -f https://download.pytorch.org/whl/torch_stable.html + + pip install -qq wget text-unidecode matplotlib>=3.3.2 onnx onnxruntime pybind11 Cython einops kaldi-native-fbank soundfile librosa + pip install -qq ipython + + # sudo apt-get install -q -y sox libsndfile1 ffmpeg python3-pip ipython + + BRANCH='main' + python3 -m pip install git+https://github.com/NVIDIA/NeMo.git@$BRANCH#egg=nemo_toolkit[asr] + + pip install numpy==1.26.4 +} + +function download_files() { + curl -SL -O https://n-ws-q0bez.s3pd12.sbercloud.ru/b-ws-q0bez-jpv/GigaAM/ctc_model_weights.ckpt + curl -SL -O https://n-ws-q0bez.s3pd12.sbercloud.ru/b-ws-q0bez-jpv/GigaAM/ctc_model_config.yaml + curl -SL -O https://n-ws-q0bez.s3pd12.sbercloud.ru/b-ws-q0bez-jpv/GigaAM/example.wav + curl -SL -O https://n-ws-q0bez.s3pd12.sbercloud.ru/b-ws-q0bez-jpv/GigaAM/long_example.wav + curl -SL -O https://huggingface.co/csukuangfj/tmp-files/resolve/main/GigaAM%20License_NC.pdf +} + +install_nemo +download_files + +python3 ./export-onnx-ctc.py +ls -lh +python3 ./test-onnx-ctc.py diff --git a/scripts/nemo/GigaAM/test-onnx-ctc.py b/scripts/nemo/GigaAM/test-onnx-ctc.py new file mode 100755 index 000000000..5c181e49a --- /dev/null +++ b/scripts/nemo/GigaAM/test-onnx-ctc.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# Copyright 2024 Xiaomi Corp. (authors: Fangjun Kuang) + +# https://github.com/salute-developers/GigaAM + +import kaldi_native_fbank as knf +import librosa +import numpy as np +import onnxruntime as ort +import soundfile as sf +import torch + + +def create_fbank(): + opts = knf.FbankOptions() + opts.frame_opts.dither = 0 + opts.frame_opts.remove_dc_offset = False + opts.frame_opts.preemph_coeff = 0 + opts.frame_opts.window_type = "hann" + + # Even though GigaAM uses 400 for fft, here we use 512 + # since kaldi-native-fbank only support fft for power of 2. + opts.frame_opts.round_to_power_of_two = True + + opts.mel_opts.low_freq = 0 + opts.mel_opts.high_freq = 8000 + opts.mel_opts.num_bins = 64 + + fbank = knf.OnlineFbank(opts) + return fbank + + +def compute_features(audio, fbank) -> np.ndarray: + """ + Args: + audio: (num_samples,), np.float32 + fbank: the fbank extractor + Returns: + features: (num_frames, feat_dim), np.float32 + """ + assert len(audio.shape) == 1, audio.shape + fbank.accept_waveform(16000, audio) + ans = [] + processed = 0 + while processed < fbank.num_frames_ready: + ans.append(np.array(fbank.get_frame(processed))) + processed += 1 + ans = np.stack(ans) + return ans + + +def display(sess): + print("==========Input==========") + for i in sess.get_inputs(): + print(i) + print("==========Output==========") + for i in sess.get_outputs(): + print(i) + + +""" +==========Input========== +NodeArg(name='audio_signal', type='tensor(float)', shape=['audio_signal_dynamic_axes_1', 64, 'audio_signal_dynamic_axes_2']) +NodeArg(name='length', type='tensor(int64)', shape=['length_dynamic_axes_1']) +==========Output========== +NodeArg(name='logprobs', type='tensor(float)', shape=['logprobs_dynamic_axes_1', 'logprobs_dynamic_axes_2', 34]) +""" + + +class OnnxModel: + def __init__( + self, + filename: str, + ): + session_opts = ort.SessionOptions() + session_opts.inter_op_num_threads = 1 + session_opts.intra_op_num_threads = 1 + + self.model = ort.InferenceSession( + filename, + sess_options=session_opts, + providers=["CPUExecutionProvider"], + ) + display(self.model) + + def __call__(self, x: np.ndarray): + # x: (T, C) + x = torch.from_numpy(x) + x = x.t().unsqueeze(0) + # x: [1, C, T] + x_lens = torch.tensor([x.shape[-1]], dtype=torch.int64) + + log_probs = self.model.run( + [ + self.model.get_outputs()[0].name, + ], + { + self.model.get_inputs()[0].name: x.numpy(), + self.model.get_inputs()[1].name: x_lens.numpy(), + }, + )[0] + # [batch_size, T, dim] + return log_probs + + +def main(): + filename = "./model.int8.onnx" + tokens = "./tokens.txt" + wav = "./example.wav" + + model = OnnxModel(filename) + + id2token = dict() + with open(tokens, encoding="utf-8") as f: + for line in f: + fields = line.split() + if len(fields) == 1: + id2token[int(fields[0])] = " " + else: + t, idx = fields + id2token[int(idx)] = t + + fbank = create_fbank() + audio, sample_rate = sf.read(wav, dtype="float32", always_2d=True) + audio = audio[:, 0] # only use the first channel + if sample_rate != 16000: + audio = librosa.resample( + audio, + orig_sr=sample_rate, + target_sr=16000, + ) + sample_rate = 16000 + + features = compute_features(audio, fbank) + print("features.shape", features.shape) + + blank = len(id2token) - 1 + prev = -1 + ans = [] + log_probs = model(features) + print("log_probs", log_probs.shape) + log_probs = torch.from_numpy(log_probs)[0] + ids = torch.argmax(log_probs, dim=1).tolist() + for i in ids: + if i != blank and i != prev: + ans.append(i) + prev = i + + tokens = [id2token[i] for i in ans] + + text = "".join(tokens) + print(wav) + print(text) + + +if __name__ == "__main__": + main() diff --git a/sherpa-onnx/csrc/features.cc b/sherpa-onnx/csrc/features.cc index ed806f392..5b50d5f24 100644 --- a/sherpa-onnx/csrc/features.cc +++ b/sherpa-onnx/csrc/features.cc @@ -193,6 +193,7 @@ class FeatureExtractor::Impl { opts_.frame_opts.frame_shift_ms = config_.frame_shift_ms; opts_.frame_opts.frame_length_ms = config_.frame_length_ms; opts_.frame_opts.remove_dc_offset = config_.remove_dc_offset; + opts_.frame_opts.preemph_coeff = config_.preemph_coeff; opts_.frame_opts.window_type = config_.window_type; opts_.mel_opts.num_bins = config_.feature_dim; @@ -211,6 +212,7 @@ class FeatureExtractor::Impl { mfcc_opts_.frame_opts.frame_shift_ms = config_.frame_shift_ms; mfcc_opts_.frame_opts.frame_length_ms = config_.frame_length_ms; mfcc_opts_.frame_opts.remove_dc_offset = config_.remove_dc_offset; + mfcc_opts_.frame_opts.preemph_coeff = config_.preemph_coeff; mfcc_opts_.frame_opts.window_type = config_.window_type; mfcc_opts_.mel_opts.num_bins = config_.feature_dim; diff --git a/sherpa-onnx/csrc/features.h b/sherpa-onnx/csrc/features.h index afbacd2ec..fb5ff2fe3 100644 --- a/sherpa-onnx/csrc/features.h +++ b/sherpa-onnx/csrc/features.h @@ -57,6 +57,7 @@ struct FeatureExtractorConfig { float frame_length_ms = 25.0f; // in milliseconds. bool is_librosa = false; bool remove_dc_offset = true; // Subtract mean of wave before FFT. + float preemph_coeff = 0.97f; // Preemphasis coefficient. std::string window_type = "povey"; // e.g. Hamming window // For models from NeMo diff --git a/sherpa-onnx/csrc/jieba-lexicon.cc b/sherpa-onnx/csrc/jieba-lexicon.cc index e995c8cb8..a53f057f0 100644 --- a/sherpa-onnx/csrc/jieba-lexicon.cc +++ b/sherpa-onnx/csrc/jieba-lexicon.cc @@ -10,8 +10,8 @@ #include "cppjieba/Jieba.hpp" #include "sherpa-onnx/csrc/file-utils.h" -#include "sherpa-onnx/csrc/lexicon.h" #include "sherpa-onnx/csrc/macros.h" +#include "sherpa-onnx/csrc/symbol-table.h" #include "sherpa-onnx/csrc/text-utils.h" namespace sherpa_onnx { diff --git a/sherpa-onnx/csrc/lexicon.cc b/sherpa-onnx/csrc/lexicon.cc index 24184f180..499f17f68 100644 --- a/sherpa-onnx/csrc/lexicon.cc +++ b/sherpa-onnx/csrc/lexicon.cc @@ -21,6 +21,7 @@ #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/onnx-utils.h" +#include "sherpa-onnx/csrc/symbol-table.h" #include "sherpa-onnx/csrc/text-utils.h" namespace sherpa_onnx { @@ -74,45 +75,6 @@ static std::vector ProcessHeteronyms( return ans; } -// Note: We don't use SymbolTable here since tokens may contain a blank -// in the first column -std::unordered_map ReadTokens(std::istream &is) { - std::unordered_map token2id; - - std::string line; - - std::string sym; - int32_t id = -1; - while (std::getline(is, line)) { - std::istringstream iss(line); - iss >> sym; - if (iss.eof()) { - id = atoi(sym.c_str()); - sym = " "; - } else { - iss >> id; - } - - // eat the trailing \r\n on windows - iss >> std::ws; - if (!iss.eof()) { - SHERPA_ONNX_LOGE("Error: %s", line.c_str()); - exit(-1); - } - -#if 0 - if (token2id.count(sym)) { - SHERPA_ONNX_LOGE("Duplicated token %s. Line %s. Existing ID: %d", - sym.c_str(), line.c_str(), token2id.at(sym)); - exit(-1); - } -#endif - token2id.insert({std::move(sym), id}); - } - - return token2id; -} - std::vector ConvertTokensToIds( const std::unordered_map &token2id, const std::vector &tokens) { diff --git a/sherpa-onnx/csrc/lexicon.h b/sherpa-onnx/csrc/lexicon.h index 9c2d91ee9..fb60cdb7f 100644 --- a/sherpa-onnx/csrc/lexicon.h +++ b/sherpa-onnx/csrc/lexicon.h @@ -67,12 +67,6 @@ class Lexicon : public OfflineTtsFrontend { bool debug_ = false; }; -std::unordered_map ReadTokens(std::istream &is); - -std::vector ConvertTokensToIds( - const std::unordered_map &token2id, - const std::vector &tokens); - } // namespace sherpa_onnx #endif // SHERPA_ONNX_CSRC_LEXICON_H_ diff --git a/sherpa-onnx/csrc/macros.h b/sherpa-onnx/csrc/macros.h index 6bd6f62a6..521739a89 100644 --- a/sherpa-onnx/csrc/macros.h +++ b/sherpa-onnx/csrc/macros.h @@ -41,13 +41,13 @@ auto value = \ meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \ if (!value) { \ - SHERPA_ONNX_LOGE("%s does not exist in the metadata", src_key); \ + SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ exit(-1); \ } \ \ dst = atoi(value.get()); \ if (dst < 0) { \ - SHERPA_ONNX_LOGE("Invalid value %d for %s", dst, src_key); \ + SHERPA_ONNX_LOGE("Invalid value %d for '%s'", dst, src_key); \ exit(-1); \ } \ } while (0) @@ -61,80 +61,80 @@ } else { \ dst = atoi(value.get()); \ if (dst < 0) { \ - SHERPA_ONNX_LOGE("Invalid value %d for %s", dst, src_key); \ + SHERPA_ONNX_LOGE("Invalid value %d for '%s'", dst, src_key); \ exit(-1); \ } \ } \ } while (0) // read a vector of integers -#define SHERPA_ONNX_READ_META_DATA_VEC(dst, src_key) \ - do { \ - auto value = \ - meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \ - if (!value) { \ - SHERPA_ONNX_LOGE("%s does not exist in the metadata", src_key); \ - exit(-1); \ - } \ - \ - bool ret = SplitStringToIntegers(value.get(), ",", true, &dst); \ - if (!ret) { \ - SHERPA_ONNX_LOGE("Invalid value %s for %s", value.get(), src_key); \ - exit(-1); \ - } \ +#define SHERPA_ONNX_READ_META_DATA_VEC(dst, src_key) \ + do { \ + auto value = \ + meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \ + if (!value) { \ + SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ + exit(-1); \ + } \ + \ + bool ret = SplitStringToIntegers(value.get(), ",", true, &dst); \ + if (!ret) { \ + SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'", value.get(), src_key); \ + exit(-1); \ + } \ } while (0) // read a vector of floats -#define SHERPA_ONNX_READ_META_DATA_VEC_FLOAT(dst, src_key) \ - do { \ - auto value = \ - meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \ - if (!value) { \ - SHERPA_ONNX_LOGE("%s does not exist in the metadata", src_key); \ - exit(-1); \ - } \ - \ - bool ret = SplitStringToFloats(value.get(), ",", true, &dst); \ - if (!ret) { \ - SHERPA_ONNX_LOGE("Invalid value %s for %s", value.get(), src_key); \ - exit(-1); \ - } \ +#define SHERPA_ONNX_READ_META_DATA_VEC_FLOAT(dst, src_key) \ + do { \ + auto value = \ + meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \ + if (!value) { \ + SHERPA_ONNX_LOGE("%s does not exist in the metadata", src_key); \ + exit(-1); \ + } \ + \ + bool ret = SplitStringToFloats(value.get(), ",", true, &dst); \ + if (!ret) { \ + SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'", value.get(), src_key); \ + exit(-1); \ + } \ } while (0) // read a vector of strings -#define SHERPA_ONNX_READ_META_DATA_VEC_STRING(dst, src_key) \ - do { \ - auto value = \ - meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \ - if (!value) { \ - SHERPA_ONNX_LOGE("%s does not exist in the metadata", src_key); \ - exit(-1); \ - } \ - SplitStringToVector(value.get(), ",", false, &dst); \ - \ - if (dst.empty()) { \ - SHERPA_ONNX_LOGE("Invalid value %s for %s. Empty vector!", value.get(), \ - src_key); \ - exit(-1); \ - } \ +#define SHERPA_ONNX_READ_META_DATA_VEC_STRING(dst, src_key) \ + do { \ + auto value = \ + meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \ + if (!value) { \ + SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ + exit(-1); \ + } \ + SplitStringToVector(value.get(), ",", false, &dst); \ + \ + if (dst.empty()) { \ + SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'. Empty vector!", \ + value.get(), src_key); \ + exit(-1); \ + } \ } while (0) // read a vector of strings separated by sep -#define SHERPA_ONNX_READ_META_DATA_VEC_STRING_SEP(dst, src_key, sep) \ - do { \ - auto value = \ - meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \ - if (!value) { \ - SHERPA_ONNX_LOGE("%s does not exist in the metadata", src_key); \ - exit(-1); \ - } \ - SplitStringToVector(value.get(), sep, false, &dst); \ - \ - if (dst.empty()) { \ - SHERPA_ONNX_LOGE("Invalid value %s for %s. Empty vector!", value.get(), \ - src_key); \ - exit(-1); \ - } \ +#define SHERPA_ONNX_READ_META_DATA_VEC_STRING_SEP(dst, src_key, sep) \ + do { \ + auto value = \ + meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \ + if (!value) { \ + SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ + exit(-1); \ + } \ + SplitStringToVector(value.get(), sep, false, &dst); \ + \ + if (dst.empty()) { \ + SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'. Empty vector!", \ + value.get(), src_key); \ + exit(-1); \ + } \ } while (0) // Read a string @@ -143,17 +143,29 @@ auto value = \ meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \ if (!value) { \ - SHERPA_ONNX_LOGE("%s does not exist in the metadata", src_key); \ + SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ exit(-1); \ } \ \ dst = value.get(); \ if (dst.empty()) { \ - SHERPA_ONNX_LOGE("Invalid value for %s\n", src_key); \ + SHERPA_ONNX_LOGE("Invalid value for '%s'\n", src_key); \ exit(-1); \ } \ } while (0) +#define SHERPA_ONNX_READ_META_DATA_STR_ALLOW_EMPTY(dst, src_key) \ + do { \ + auto value = \ + meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \ + if (!value) { \ + SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ + exit(-1); \ + } \ + \ + dst = value.get(); \ + } while (0) + #define SHERPA_ONNX_READ_META_DATA_STR_WITH_DEFAULT(dst, src_key, \ default_value) \ do { \ @@ -164,7 +176,7 @@ } else { \ dst = value.get(); \ if (dst.empty()) { \ - SHERPA_ONNX_LOGE("Invalid value for %s\n", src_key); \ + SHERPA_ONNX_LOGE("Invalid value for '%s'\n", src_key); \ exit(-1); \ } \ } \ diff --git a/sherpa-onnx/csrc/melo-tts-lexicon.cc b/sherpa-onnx/csrc/melo-tts-lexicon.cc index d7332b36c..a605e6c38 100644 --- a/sherpa-onnx/csrc/melo-tts-lexicon.cc +++ b/sherpa-onnx/csrc/melo-tts-lexicon.cc @@ -10,8 +10,8 @@ #include "cppjieba/Jieba.hpp" #include "sherpa-onnx/csrc/file-utils.h" -#include "sherpa-onnx/csrc/lexicon.h" #include "sherpa-onnx/csrc/macros.h" +#include "sherpa-onnx/csrc/symbol-table.h" #include "sherpa-onnx/csrc/text-utils.h" namespace sherpa_onnx { diff --git a/sherpa-onnx/csrc/offline-ctc-model.cc b/sherpa-onnx/csrc/offline-ctc-model.cc index 66e67ecf1..9d1e05d9b 100644 --- a/sherpa-onnx/csrc/offline-ctc-model.cc +++ b/sherpa-onnx/csrc/offline-ctc-model.cc @@ -21,6 +21,7 @@ namespace { enum class ModelType : std::uint8_t { kEncDecCTCModelBPE, + kEncDecCTCModel, kEncDecHybridRNNTCTCBPEModel, kTdnn, kZipformerCtc, @@ -75,6 +76,8 @@ static ModelType GetModelType(char *model_data, size_t model_data_length, if (model_type.get() == std::string("EncDecCTCModelBPE")) { return ModelType::kEncDecCTCModelBPE; + } else if (model_type.get() == std::string("EncDecCTCModel")) { + return ModelType::kEncDecCTCModel; } else if (model_type.get() == std::string("EncDecHybridRNNTCTCBPEModel")) { return ModelType::kEncDecHybridRNNTCTCBPEModel; } else if (model_type.get() == std::string("tdnn")) { @@ -121,22 +124,18 @@ std::unique_ptr OfflineCtcModel::Create( switch (model_type) { case ModelType::kEncDecCTCModelBPE: return std::make_unique(config); - break; + case ModelType::kEncDecCTCModel: + return std::make_unique(config); case ModelType::kEncDecHybridRNNTCTCBPEModel: return std::make_unique(config); - break; case ModelType::kTdnn: return std::make_unique(config); - break; case ModelType::kZipformerCtc: return std::make_unique(config); - break; case ModelType::kWenetCtc: return std::make_unique(config); - break; case ModelType::kTeleSpeechCtc: return std::make_unique(config); - break; case ModelType::kUnknown: SHERPA_ONNX_LOGE("Unknown model type in offline CTC!"); return nullptr; @@ -177,23 +176,19 @@ std::unique_ptr OfflineCtcModel::Create( switch (model_type) { case ModelType::kEncDecCTCModelBPE: return std::make_unique(mgr, config); - break; + case ModelType::kEncDecCTCModel: + return std::make_unique(mgr, config); case ModelType::kEncDecHybridRNNTCTCBPEModel: return std::make_unique(mgr, config); - break; case ModelType::kTdnn: return std::make_unique(mgr, config); - break; case ModelType::kZipformerCtc: return std::make_unique(mgr, config); - break; case ModelType::kWenetCtc: return std::make_unique(mgr, config); - break; case ModelType::kTeleSpeechCtc: return std::make_unique(mgr, config); - break; case ModelType::kUnknown: SHERPA_ONNX_LOGE("Unknown model type in offline CTC!"); return nullptr; diff --git a/sherpa-onnx/csrc/offline-ctc-model.h b/sherpa-onnx/csrc/offline-ctc-model.h index f4c7406f6..ead532b48 100644 --- a/sherpa-onnx/csrc/offline-ctc-model.h +++ b/sherpa-onnx/csrc/offline-ctc-model.h @@ -66,6 +66,10 @@ class OfflineCtcModel { // Return true if the model supports batch size > 1 virtual bool SupportBatchProcessing() const { return true; } + + // return true for models from https://github.com/salute-developers/GigaAM + // return false otherwise + virtual bool IsGigaAM() const { return false; } }; } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.cc b/sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.cc index 2d790954b..708cb4b4f 100644 --- a/sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.cc +++ b/sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.cc @@ -72,6 +72,8 @@ class OfflineNemoEncDecCtcModel::Impl { std::string FeatureNormalizationMethod() const { return normalize_type_; } + bool IsGigaAM() const { return is_giga_am_; } + private: void Init(void *model_data, size_t model_data_length) { sess_ = std::make_unique(env_, model_data, model_data_length, @@ -92,7 +94,9 @@ class OfflineNemoEncDecCtcModel::Impl { Ort::AllocatorWithDefaultOptions allocator; // used in the macro below SHERPA_ONNX_READ_META_DATA(vocab_size_, "vocab_size"); SHERPA_ONNX_READ_META_DATA(subsampling_factor_, "subsampling_factor"); - SHERPA_ONNX_READ_META_DATA_STR(normalize_type_, "normalize_type"); + SHERPA_ONNX_READ_META_DATA_STR_ALLOW_EMPTY(normalize_type_, + "normalize_type"); + SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(is_giga_am_, "is_giga_am", 0); } private: @@ -112,6 +116,10 @@ class OfflineNemoEncDecCtcModel::Impl { int32_t vocab_size_ = 0; int32_t subsampling_factor_ = 0; std::string normalize_type_; + + // it is 1 for models from + // https://github.com/salute-developers/GigaAM + int32_t is_giga_am_ = 0; }; OfflineNemoEncDecCtcModel::OfflineNemoEncDecCtcModel( @@ -146,4 +154,6 @@ std::string OfflineNemoEncDecCtcModel::FeatureNormalizationMethod() const { return impl_->FeatureNormalizationMethod(); } +bool OfflineNemoEncDecCtcModel::IsGigaAM() const { return impl_->IsGigaAM(); } + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.h b/sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.h index 6e1ba5855..c961e694e 100644 --- a/sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.h +++ b/sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.h @@ -76,6 +76,8 @@ class OfflineNemoEncDecCtcModel : public OfflineCtcModel { // for details std::string FeatureNormalizationMethod() const override; + bool IsGigaAM() const override; + private: class Impl; std::unique_ptr impl_; diff --git a/sherpa-onnx/csrc/offline-recognizer-ctc-impl.h b/sherpa-onnx/csrc/offline-recognizer-ctc-impl.h index 7bbe6938c..1199ff109 100644 --- a/sherpa-onnx/csrc/offline-recognizer-ctc-impl.h +++ b/sherpa-onnx/csrc/offline-recognizer-ctc-impl.h @@ -104,11 +104,20 @@ class OfflineRecognizerCtcImpl : public OfflineRecognizerImpl { } if (!config_.model_config.nemo_ctc.model.empty()) { - config_.feat_config.low_freq = 0; - config_.feat_config.high_freq = 0; - config_.feat_config.is_librosa = true; - config_.feat_config.remove_dc_offset = false; - config_.feat_config.window_type = "hann"; + if (model_->IsGigaAM()) { + config_.feat_config.low_freq = 0; + config_.feat_config.high_freq = 8000; + config_.feat_config.remove_dc_offset = false; + config_.feat_config.preemph_coeff = 0; + config_.feat_config.window_type = "hann"; + config_.feat_config.feature_dim = 64; + } else { + config_.feat_config.low_freq = 0; + config_.feat_config.high_freq = 0; + config_.feat_config.is_librosa = true; + config_.feat_config.remove_dc_offset = false; + config_.feat_config.window_type = "hann"; + } } if (!config_.model_config.wenet_ctc.model.empty()) { diff --git a/sherpa-onnx/csrc/offline-recognizer-impl.cc b/sherpa-onnx/csrc/offline-recognizer-impl.cc index 5062968cc..a80301ebf 100644 --- a/sherpa-onnx/csrc/offline-recognizer-impl.cc +++ b/sherpa-onnx/csrc/offline-recognizer-impl.cc @@ -172,7 +172,7 @@ std::unique_ptr OfflineRecognizerImpl::Create( return std::make_unique(config); } - if (model_type == "EncDecCTCModelBPE" || + if (model_type == "EncDecCTCModelBPE" || model_type == "EncDecCTCModel" || model_type == "EncDecHybridRNNTCTCBPEModel" || model_type == "tdnn" || model_type == "zipformer2_ctc" || model_type == "wenet_ctc" || model_type == "telespeech_ctc") { @@ -189,6 +189,7 @@ std::unique_ptr OfflineRecognizerImpl::Create( " - Non-streaming transducer models from icefall\n" " - Non-streaming Paraformer models from FunASR\n" " - EncDecCTCModelBPE models from NeMo\n" + " - EncDecCTCModel models from NeMo\n" " - EncDecHybridRNNTCTCBPEModel models from NeMo\n" " - Whisper models\n" " - Tdnn models\n" @@ -343,7 +344,7 @@ std::unique_ptr OfflineRecognizerImpl::Create( return std::make_unique(mgr, config); } - if (model_type == "EncDecCTCModelBPE" || + if (model_type == "EncDecCTCModelBPE" || model_type == "EncDecCTCModel" || model_type == "EncDecHybridRNNTCTCBPEModel" || model_type == "tdnn" || model_type == "zipformer2_ctc" || model_type == "wenet_ctc" || model_type == "telespeech_ctc") { @@ -360,6 +361,7 @@ std::unique_ptr OfflineRecognizerImpl::Create( " - Non-streaming transducer models from icefall\n" " - Non-streaming Paraformer models from FunASR\n" " - EncDecCTCModelBPE models from NeMo\n" + " - EncDecCTCModel models from NeMo\n" " - EncDecHybridRNNTCTCBPEModel models from NeMo\n" " - Whisper models\n" " - Tdnn models\n" diff --git a/sherpa-onnx/csrc/symbol-table.cc b/sherpa-onnx/csrc/symbol-table.cc index 5655c03a8..a71225c38 100644 --- a/sherpa-onnx/csrc/symbol-table.cc +++ b/sherpa-onnx/csrc/symbol-table.cc @@ -7,6 +7,8 @@ #include #include #include +#include +#include #if __ANDROID_API__ >= 9 #include @@ -16,10 +18,54 @@ #endif #include "sherpa-onnx/csrc/base64-decode.h" +#include "sherpa-onnx/csrc/lexicon.h" #include "sherpa-onnx/csrc/onnx-utils.h" namespace sherpa_onnx { +std::unordered_map ReadTokens( + std::istream &is, + std::unordered_map *id2token /*= nullptr*/) { + std::unordered_map token2id; + + std::string line; + + std::string sym; + int32_t id = -1; + while (std::getline(is, line)) { + std::istringstream iss(line); + iss >> sym; + if (iss.eof()) { + id = atoi(sym.c_str()); + sym = " "; + } else { + iss >> id; + } + + // eat the trailing \r\n on windows + iss >> std::ws; + if (!iss.eof()) { + SHERPA_ONNX_LOGE("Error: %s", line.c_str()); + exit(-1); + } + +#if 0 + if (token2id.count(sym)) { + SHERPA_ONNX_LOGE("Duplicated token %s. Line %s. Existing ID: %d", + sym.c_str(), line.c_str(), token2id.at(sym)); + exit(-1); + } +#endif + if (id2token) { + id2token->insert({id, sym}); + } + + token2id.insert({std::move(sym), id}); + } + + return token2id; +} + SymbolTable::SymbolTable(const std::string &filename, bool is_file) { if (is_file) { std::ifstream is(filename); @@ -39,25 +85,7 @@ SymbolTable::SymbolTable(AAssetManager *mgr, const std::string &filename) { } #endif -void SymbolTable::Init(std::istream &is) { - std::string sym; - int32_t id = 0; - while (is >> sym >> id) { -#if 0 - // we disable the test here since for some multi-lingual BPE models - // from NeMo, the same symbol can appear multiple times with different IDs. - if (sym != " ") { - assert(sym2id_.count(sym) == 0); - } -#endif - - assert(id2sym_.count(id) == 0); - - sym2id_.insert({sym, id}); - id2sym_.insert({id, sym}); - } - assert(is.eof()); -} +void SymbolTable::Init(std::istream &is) { sym2id_ = ReadTokens(is, &id2sym_); } std::string SymbolTable::ToString() const { std::ostringstream os; diff --git a/sherpa-onnx/csrc/symbol-table.h b/sherpa-onnx/csrc/symbol-table.h index 2c17b4d5e..75a96144e 100644 --- a/sherpa-onnx/csrc/symbol-table.h +++ b/sherpa-onnx/csrc/symbol-table.h @@ -5,8 +5,10 @@ #ifndef SHERPA_ONNX_CSRC_SYMBOL_TABLE_H_ #define SHERPA_ONNX_CSRC_SYMBOL_TABLE_H_ +#include #include #include +#include #if __ANDROID_API__ >= 9 #include "android/asset_manager.h" @@ -15,6 +17,16 @@ namespace sherpa_onnx { +// The same token can be mapped to different integer IDs, so +// we need an id2token argument here. +std::unordered_map ReadTokens( + std::istream &is, + std::unordered_map *id2token = nullptr); + +std::vector ConvertTokensToIds( + const std::unordered_map &token2id, + const std::vector &tokens); + /// It manages mapping between symbols and integer IDs. class SymbolTable { public: diff --git a/sherpa-onnx/kotlin-api/OfflineRecognizer.kt b/sherpa-onnx/kotlin-api/OfflineRecognizer.kt index 203278cb7..10cdc5179 100644 --- a/sherpa-onnx/kotlin-api/OfflineRecognizer.kt +++ b/sherpa-onnx/kotlin-api/OfflineRecognizer.kt @@ -394,6 +394,16 @@ fun getOfflineModelConfig(type: Int): OfflineModelConfig? { modelType = "transducer", ) } + + 19 -> { + val modelDir = "sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24" + return OfflineModelConfig( + nemo = OfflineNemoEncDecCtcModelConfig( + model = "$modelDir/model.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } } return null } From 707cf792c5797b6086c9befc088a38ecfe422a47 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 25 Oct 2024 15:20:13 +0800 Subject: [PATCH 036/183] Add GigaAM NeMo transducer model for Russian ASR (#1467) --- .../export-nemo-giga-am-to-onnx.yaml | 34 ++- scripts/apk/generate-vad-asr-apk-script.py | 18 ++ scripts/nemo/GigaAM/export-onnx-ctc.py | 1 + scripts/nemo/GigaAM/export-onnx-rnnt.py | 119 ++++++++ scripts/nemo/GigaAM/run-ctc.sh | 14 +- scripts/nemo/GigaAM/run-rnnt.sh | 50 ++++ scripts/nemo/GigaAM/test-onnx-rnnt.py | 270 ++++++++++++++++++ sherpa-onnx/csrc/offline-recognizer-impl.cc | 8 +- .../offline-recognizer-transducer-nemo-impl.h | 26 +- .../csrc/offline-transducer-nemo-model.cc | 9 +- .../csrc/offline-transducer-nemo-model.h | 2 + sherpa-onnx/kotlin-api/OfflineRecognizer.kt | 13 + 12 files changed, 543 insertions(+), 21 deletions(-) create mode 100644 scripts/nemo/GigaAM/export-onnx-rnnt.py create mode 100755 scripts/nemo/GigaAM/run-rnnt.sh create mode 100755 scripts/nemo/GigaAM/test-onnx-rnnt.py diff --git a/.github/workflows/export-nemo-giga-am-to-onnx.yaml b/.github/workflows/export-nemo-giga-am-to-onnx.yaml index f48c344e6..1af754d0b 100644 --- a/.github/workflows/export-nemo-giga-am-to-onnx.yaml +++ b/.github/workflows/export-nemo-giga-am-to-onnx.yaml @@ -38,7 +38,7 @@ jobs: mkdir $d/test_wavs rm scripts/nemo/GigaAM/model.onnx mv -v scripts/nemo/GigaAM/*.int8.onnx $d/ - mv -v scripts/nemo/GigaAM/*.md $d/ + cp -v scripts/nemo/GigaAM/*.md $d/ mv -v scripts/nemo/GigaAM/*.pdf $d/ mv -v scripts/nemo/GigaAM/tokens.txt $d/ mv -v scripts/nemo/GigaAM/*.wav $d/test_wavs/ @@ -51,6 +51,34 @@ jobs: tar cjvf ${d}.tar.bz2 $d + - name: Run Transducer + shell: bash + run: | + pushd scripts/nemo/GigaAM + ./run-rnnt.sh + popd + + d=sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24 + mkdir $d + mkdir $d/test_wavs + + mv -v scripts/nemo/GigaAM/encoder.int8.onnx $d/ + mv -v scripts/nemo/GigaAM/decoder.onnx $d/ + mv -v scripts/nemo/GigaAM/joiner.onnx $d/ + + cp -v scripts/nemo/GigaAM/*.md $d/ + mv -v scripts/nemo/GigaAM/*.pdf $d/ + mv -v scripts/nemo/GigaAM/tokens.txt $d/ + mv -v scripts/nemo/GigaAM/*.wav $d/test_wavs/ + mv -v scripts/nemo/GigaAM/run-rnnt.sh $d/ + mv -v scripts/nemo/GigaAM/*-rnnt.py $d/ + + ls -lh scripts/nemo/GigaAM/ + + ls -lh $d + + tar cjvf ${d}.tar.bz2 $d + - name: Release uses: svenstaro/upload-release-action@v2 with: @@ -61,7 +89,7 @@ jobs: repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} tag: asr-models - - name: Publish to huggingface (CTC) + - name: Publish to huggingface (Transducer) env: HF_TOKEN: ${{ secrets.HF_TOKEN }} uses: nick-fields/retry@v3 @@ -73,7 +101,7 @@ jobs: git config --global user.email "csukuangfj@gmail.com" git config --global user.name "Fangjun Kuang" - d=sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24 + d=sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24/ export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$d huggingface diff --git a/scripts/apk/generate-vad-asr-apk-script.py b/scripts/apk/generate-vad-asr-apk-script.py index 8217e6ea0..7671e975d 100755 --- a/scripts/apk/generate-vad-asr-apk-script.py +++ b/scripts/apk/generate-vad-asr-apk-script.py @@ -351,6 +351,24 @@ def get_models(): ls -lh + popd + """, + ), + Model( + model_name="sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24", + idx=20, + lang="ru", + short_name="nemo_transducer_giga_am", + cmd=""" + pushd $model_name + + rm -rfv test_wavs + + rm -fv *.sh + rm -fv *.py + + ls -lh + popd """, ), diff --git a/scripts/nemo/GigaAM/export-onnx-ctc.py b/scripts/nemo/GigaAM/export-onnx-ctc.py index fbcec518e..81feb3b78 100755 --- a/scripts/nemo/GigaAM/export-onnx-ctc.py +++ b/scripts/nemo/GigaAM/export-onnx-ctc.py @@ -75,6 +75,7 @@ def add_meta_data(filename: str, meta_data: Dict[str, str]): onnx.save(model, filename) +@torch.no_grad() def main(): model = EncDecCTCModel.from_config_file("./ctc_model_config.yaml") ckpt = torch.load("./ctc_model_weights.ckpt", map_location="cpu") diff --git a/scripts/nemo/GigaAM/export-onnx-rnnt.py b/scripts/nemo/GigaAM/export-onnx-rnnt.py new file mode 100644 index 000000000..1ac05ff7f --- /dev/null +++ b/scripts/nemo/GigaAM/export-onnx-rnnt.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# Copyright 2024 Xiaomi Corp. (authors: Fangjun Kuang) + +from typing import Dict + +import onnx +import torch +import torchaudio +from nemo.collections.asr.models import EncDecRNNTBPEModel +from nemo.collections.asr.modules.audio_preprocessing import ( + AudioToMelSpectrogramPreprocessor as NeMoAudioToMelSpectrogramPreprocessor, +) +from nemo.collections.asr.parts.preprocessing.features import ( + FilterbankFeaturesTA as NeMoFilterbankFeaturesTA, +) +from onnxruntime.quantization import QuantType, quantize_dynamic + + +def add_meta_data(filename: str, meta_data: Dict[str, str]): + """Add meta data to an ONNX model. It is changed in-place. + + Args: + filename: + Filename of the ONNX model to be changed. + meta_data: + Key-value pairs. + """ + model = onnx.load(filename) + while len(model.metadata_props): + model.metadata_props.pop() + + for key, value in meta_data.items(): + meta = model.metadata_props.add() + meta.key = key + meta.value = str(value) + + onnx.save(model, filename) + + +class FilterbankFeaturesTA(NeMoFilterbankFeaturesTA): + def __init__(self, mel_scale: str = "htk", wkwargs=None, **kwargs): + if "window_size" in kwargs: + del kwargs["window_size"] + if "window_stride" in kwargs: + del kwargs["window_stride"] + + super().__init__(**kwargs) + + self._mel_spec_extractor: torchaudio.transforms.MelSpectrogram = ( + torchaudio.transforms.MelSpectrogram( + sample_rate=self._sample_rate, + win_length=self.win_length, + hop_length=self.hop_length, + n_mels=kwargs["nfilt"], + window_fn=self.torch_windows[kwargs["window"]], + mel_scale=mel_scale, + norm=kwargs["mel_norm"], + n_fft=kwargs["n_fft"], + f_max=kwargs.get("highfreq", None), + f_min=kwargs.get("lowfreq", 0), + wkwargs=wkwargs, + ) + ) + + +class AudioToMelSpectrogramPreprocessor(NeMoAudioToMelSpectrogramPreprocessor): + def __init__(self, mel_scale: str = "htk", **kwargs): + super().__init__(**kwargs) + kwargs["nfilt"] = kwargs["features"] + del kwargs["features"] + self.featurizer = ( + FilterbankFeaturesTA( # Deprecated arguments; kept for config compatibility + mel_scale=mel_scale, + **kwargs, + ) + ) + + +@torch.no_grad() +def main(): + model = EncDecRNNTBPEModel.from_config_file("./rnnt_model_config.yaml") + ckpt = torch.load("./rnnt_model_weights.ckpt", map_location="cpu") + model.load_state_dict(ckpt, strict=False) + model.eval() + + with open("./tokens.txt", "w", encoding="utf-8") as f: + for i, s in enumerate(model.joint.vocabulary): + f.write(f"{s} {i}\n") + f.write(f" {i+1}\n") + print("Saved to tokens.txt") + + model.encoder.export("encoder.onnx") + model.decoder.export("decoder.onnx") + model.joint.export("joiner.onnx") + + meta_data = { + "vocab_size": model.decoder.vocab_size, # not including the blank + "pred_rnn_layers": model.decoder.pred_rnn_layers, + "pred_hidden": model.decoder.pred_hidden, + "normalize_type": "", + "subsampling_factor": 4, + "model_type": "EncDecRNNTBPEModel", + "version": "1", + "model_author": "https://github.com/salute-developers/GigaAM", + "license": "https://github.com/salute-developers/GigaAM/blob/main/GigaAM%20License_NC.pdf", + "language": "Russian", + "is_giga_am": 1, + } + add_meta_data("encoder.onnx", meta_data) + + quantize_dynamic( + model_input="encoder.onnx", + model_output="encoder.int8.onnx", + weight_type=QuantType.QUInt8, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/nemo/GigaAM/run-ctc.sh b/scripts/nemo/GigaAM/run-ctc.sh index 499bc452b..03acc88e2 100755 --- a/scripts/nemo/GigaAM/run-ctc.sh +++ b/scripts/nemo/GigaAM/run-ctc.sh @@ -21,11 +21,15 @@ function install_nemo() { } function download_files() { - curl -SL -O https://n-ws-q0bez.s3pd12.sbercloud.ru/b-ws-q0bez-jpv/GigaAM/ctc_model_weights.ckpt - curl -SL -O https://n-ws-q0bez.s3pd12.sbercloud.ru/b-ws-q0bez-jpv/GigaAM/ctc_model_config.yaml - curl -SL -O https://n-ws-q0bez.s3pd12.sbercloud.ru/b-ws-q0bez-jpv/GigaAM/example.wav - curl -SL -O https://n-ws-q0bez.s3pd12.sbercloud.ru/b-ws-q0bez-jpv/GigaAM/long_example.wav - curl -SL -O https://huggingface.co/csukuangfj/tmp-files/resolve/main/GigaAM%20License_NC.pdf + # curl -SL -O https://n-ws-q0bez.s3pd12.sbercloud.ru/b-ws-q0bez-jpv/GigaAM/ctc_model_weights.ckpt + # curl -SL -O https://n-ws-q0bez.s3pd12.sbercloud.ru/b-ws-q0bez-jpv/GigaAM/ctc_model_config.yaml + # curl -SL -O https://n-ws-q0bez.s3pd12.sbercloud.ru/b-ws-q0bez-jpv/GigaAM/example.wav + # curl -SL -O https://n-ws-q0bez.s3pd12.sbercloud.ru/b-ws-q0bez-jpv/GigaAM/long_example.wav + curl -SL -O https://huggingface.co/csukuangfj/tmp-files/resolve/main/GigaAM/ctc/ctc_model_weights.ckpt + curl -SL -O https://huggingface.co/csukuangfj/tmp-files/resolve/main/GigaAM/ctc/ctc_model_config.yaml + curl -SL -O https://huggingface.co/csukuangfj/tmp-files/resolve/main/GigaAM/example.wav + curl -SL -O https://huggingface.co/csukuangfj/tmp-files/resolve/main/GigaAM/long_example.wav + curl -SL -O https://huggingface.co/csukuangfj/tmp-files/resolve/main/GigaAM/GigaAM%20License_NC.pdf } install_nemo diff --git a/scripts/nemo/GigaAM/run-rnnt.sh b/scripts/nemo/GigaAM/run-rnnt.sh new file mode 100755 index 000000000..209f4f15d --- /dev/null +++ b/scripts/nemo/GigaAM/run-rnnt.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Copyright 2024 Xiaomi Corp. (authors: Fangjun Kuang) + +set -ex + +function install_nemo() { + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py + python3 get-pip.py + + pip install torch==2.4.0 torchaudio==2.4.0 -f https://download.pytorch.org/whl/torch_stable.html + + pip install -qq wget text-unidecode matplotlib>=3.3.2 onnx onnxruntime pybind11 Cython einops kaldi-native-fbank soundfile librosa + pip install -qq ipython + + # sudo apt-get install -q -y sox libsndfile1 ffmpeg python3-pip ipython + + BRANCH='main' + python3 -m pip install git+https://github.com/NVIDIA/NeMo.git@$BRANCH#egg=nemo_toolkit[asr] + + pip install numpy==1.26.4 +} + +function download_files() { + # curl -SL -O https://n-ws-q0bez.s3pd12.sbercloud.ru/b-ws-q0bez-jpv/GigaAM/rnnt_model_weights.ckpt + # curl -SL -O https://n-ws-q0bez.s3pd12.sbercloud.ru/b-ws-q0bez-jpv/GigaAM/rnnt_model_config.yaml + # curl -SL -O https://n-ws-q0bez.s3pd12.sbercloud.ru/b-ws-q0bez-jpv/GigaAM/example.wav + # curl -SL -O https://n-ws-q0bez.s3pd12.sbercloud.ru/b-ws-q0bez-jpv/GigaAM/long_example.wav + # curl -SL -O https://n-ws-q0bez.s3pd12.sbercloud.ru/b-ws-q0bez-jpv/GigaAM/tokenizer_all_sets.tar + + curl -SL -O https://huggingface.co/csukuangfj/tmp-files/resolve/main/GigaAM/rnnt/rnnt_model_weights.ckpt + curl -SL -O https://huggingface.co/csukuangfj/tmp-files/resolve/main/GigaAM/rnnt/rnnt_model_config.yaml + curl -SL -O https://huggingface.co/csukuangfj/tmp-files/resolve/main/GigaAM/example.wav + curl -SL -O https://huggingface.co/csukuangfj/tmp-files/resolve/main/GigaAM/long_example.wav + curl -SL -O https://huggingface.co/csukuangfj/tmp-files/resolve/main/GigaAM/GigaAM%20License_NC.pdf + curl -SL -O https://huggingface.co/csukuangfj/tmp-files/resolve/main/GigaAM/rnnt/tokenizer_all_sets.tar + tar -xf tokenizer_all_sets.tar && rm tokenizer_all_sets.tar + ls -lh + echo "---" + ls -lh tokenizer_all_sets + echo "---" +} + +install_nemo +download_files + +python3 ./export-onnx-rnnt.py +ls -lh +python3 ./test-onnx-rnnt.py +rm -v encoder.onnx +ls -lh diff --git a/scripts/nemo/GigaAM/test-onnx-rnnt.py b/scripts/nemo/GigaAM/test-onnx-rnnt.py new file mode 100755 index 000000000..85c6a5e94 --- /dev/null +++ b/scripts/nemo/GigaAM/test-onnx-rnnt.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +# Copyright 2024 Xiaomi Corp. (authors: Fangjun Kuang) + +import argparse +from pathlib import Path + +import kaldi_native_fbank as knf +import librosa +import numpy as np +import onnxruntime as ort +import soundfile as sf +import torch + + +def create_fbank(): + opts = knf.FbankOptions() + opts.frame_opts.dither = 0 + opts.frame_opts.remove_dc_offset = False + opts.frame_opts.preemph_coeff = 0 + opts.frame_opts.window_type = "hann" + + # Even though GigaAM uses 400 for fft, here we use 512 + # since kaldi-native-fbank only support fft for power of 2. + opts.frame_opts.round_to_power_of_two = True + + opts.mel_opts.low_freq = 0 + opts.mel_opts.high_freq = 8000 + opts.mel_opts.num_bins = 64 + + fbank = knf.OnlineFbank(opts) + return fbank + + +def compute_features(audio, fbank): + assert len(audio.shape) == 1, audio.shape + fbank.accept_waveform(16000, audio) + ans = [] + processed = 0 + while processed < fbank.num_frames_ready: + ans.append(np.array(fbank.get_frame(processed))) + processed += 1 + ans = np.stack(ans) + return ans + + +def display(sess): + print("==========Input==========") + for i in sess.get_inputs(): + print(i) + print("==========Output==========") + for i in sess.get_outputs(): + print(i) + + +""" +==========Input========== +NodeArg(name='audio_signal', type='tensor(float)', shape=['audio_signal_dynamic_axes_1', 64, 'audio_signal_dynamic_axes_2']) +NodeArg(name='length', type='tensor(int64)', shape=['length_dynamic_axes_1']) +==========Output========== +NodeArg(name='outputs', type='tensor(float)', shape=['outputs_dynamic_axes_1', 768, 'outputs_dynamic_axes_2']) +NodeArg(name='encoded_lengths', type='tensor(int64)', shape=['encoded_lengths_dynamic_axes_1']) +==========Input========== +NodeArg(name='targets', type='tensor(int32)', shape=['targets_dynamic_axes_1', 'targets_dynamic_axes_2']) +NodeArg(name='target_length', type='tensor(int32)', shape=['target_length_dynamic_axes_1']) +NodeArg(name='states.1', type='tensor(float)', shape=[1, 'states.1_dim_1', 320]) +NodeArg(name='onnx::LSTM_3', type='tensor(float)', shape=[1, 1, 320]) +==========Output========== +NodeArg(name='outputs', type='tensor(float)', shape=['outputs_dynamic_axes_1', 320, 'outputs_dynamic_axes_2']) +NodeArg(name='prednet_lengths', type='tensor(int32)', shape=['prednet_lengths_dynamic_axes_1']) +NodeArg(name='states', type='tensor(float)', shape=[1, 'states_dynamic_axes_1', 320]) +NodeArg(name='74', type='tensor(float)', shape=[1, 'states_dynamic_axes_1', 320]) +==========Input========== +NodeArg(name='encoder_outputs', type='tensor(float)', shape=['encoder_outputs_dynamic_axes_1', 768, 'encoder_outputs_dynamic_axes_2']) +NodeArg(name='decoder_outputs', type='tensor(float)', shape=['decoder_outputs_dynamic_axes_1', 320, 'decoder_outputs_dynamic_axes_2']) +==========Output========== +NodeArg(name='outputs', type='tensor(float)', shape=['outputs_dynamic_axes_1', 'outputs_dynamic_axes_2', 'outputs_dynamic_axes_3', 513]) +""" + + +class OnnxModel: + def __init__( + self, + encoder: str, + decoder: str, + joiner: str, + ): + self.init_encoder(encoder) + display(self.encoder) + self.init_decoder(decoder) + display(self.decoder) + self.init_joiner(joiner) + display(self.joiner) + + def init_encoder(self, encoder): + session_opts = ort.SessionOptions() + session_opts.inter_op_num_threads = 1 + session_opts.intra_op_num_threads = 1 + + self.encoder = ort.InferenceSession( + encoder, + sess_options=session_opts, + providers=["CPUExecutionProvider"], + ) + + meta = self.encoder.get_modelmeta().custom_metadata_map + self.normalize_type = meta["normalize_type"] + print(meta) + + self.pred_rnn_layers = int(meta["pred_rnn_layers"]) + self.pred_hidden = int(meta["pred_hidden"]) + + def init_decoder(self, decoder): + session_opts = ort.SessionOptions() + session_opts.inter_op_num_threads = 1 + session_opts.intra_op_num_threads = 1 + + self.decoder = ort.InferenceSession( + decoder, + sess_options=session_opts, + providers=["CPUExecutionProvider"], + ) + + def init_joiner(self, joiner): + session_opts = ort.SessionOptions() + session_opts.inter_op_num_threads = 1 + session_opts.intra_op_num_threads = 1 + + self.joiner = ort.InferenceSession( + joiner, + sess_options=session_opts, + providers=["CPUExecutionProvider"], + ) + + def get_decoder_state(self): + batch_size = 1 + state0 = torch.zeros(self.pred_rnn_layers, batch_size, self.pred_hidden).numpy() + state1 = torch.zeros(self.pred_rnn_layers, batch_size, self.pred_hidden).numpy() + return state0, state1 + + def run_encoder(self, x: np.ndarray): + # x: (T, C) + x = torch.from_numpy(x) + x = x.t().unsqueeze(0) + # x: [1, C, T] + x_lens = torch.tensor([x.shape[-1]], dtype=torch.int64) + + (encoder_out, out_len) = self.encoder.run( + [ + self.encoder.get_outputs()[0].name, + self.encoder.get_outputs()[1].name, + ], + { + self.encoder.get_inputs()[0].name: x.numpy(), + self.encoder.get_inputs()[1].name: x_lens.numpy(), + }, + ) + # [batch_size, dim, T] + return encoder_out + + def run_decoder( + self, + token: int, + state0: np.ndarray, + state1: np.ndarray, + ): + target = torch.tensor([[token]], dtype=torch.int32).numpy() + target_len = torch.tensor([1], dtype=torch.int32).numpy() + + ( + decoder_out, + decoder_out_length, + state0_next, + state1_next, + ) = self.decoder.run( + [ + self.decoder.get_outputs()[0].name, + self.decoder.get_outputs()[1].name, + self.decoder.get_outputs()[2].name, + self.decoder.get_outputs()[3].name, + ], + { + self.decoder.get_inputs()[0].name: target, + self.decoder.get_inputs()[1].name: target_len, + self.decoder.get_inputs()[2].name: state0, + self.decoder.get_inputs()[3].name: state1, + }, + ) + return decoder_out, state0_next, state1_next + + def run_joiner( + self, + encoder_out: np.ndarray, + decoder_out: np.ndarray, + ): + # encoder_out: [batch_size, dim, 1] + # decoder_out: [batch_size, dim, 1] + logit = self.joiner.run( + [ + self.joiner.get_outputs()[0].name, + ], + { + self.joiner.get_inputs()[0].name: encoder_out, + self.joiner.get_inputs()[1].name: decoder_out, + }, + )[0] + # logit: [batch_size, 1, 1, vocab_size] + return logit + + +def main(): + model = OnnxModel("encoder.int8.onnx", "decoder.onnx", "joiner.onnx") + + id2token = dict() + with open("./tokens.txt", encoding="utf-8") as f: + for line in f: + t, idx = line.split() + id2token[int(idx)] = t + + fbank = create_fbank() + audio, sample_rate = sf.read("./example.wav", dtype="float32", always_2d=True) + audio = audio[:, 0] # only use the first channel + if sample_rate != 16000: + audio = librosa.resample( + audio, + orig_sr=sample_rate, + target_sr=16000, + ) + sample_rate = 16000 + + tail_padding = np.zeros(sample_rate * 2) + + audio = np.concatenate([audio, tail_padding]) + + blank = len(id2token) - 1 + ans = [blank] + state0, state1 = model.get_decoder_state() + decoder_out, state0_next, state1_next = model.run_decoder(ans[-1], state0, state1) + + features = compute_features(audio, fbank) + print("audio.shape", audio.shape) + print("features.shape", features.shape) + + encoder_out = model.run_encoder(features) + # encoder_out:[batch_size, dim, T) + for t in range(encoder_out.shape[2]): + encoder_out_t = encoder_out[:, :, t : t + 1] + logits = model.run_joiner(encoder_out_t, decoder_out) + logits = torch.from_numpy(logits) + logits = logits.squeeze() + idx = torch.argmax(logits, dim=-1).item() + if idx != blank: + ans.append(idx) + state0 = state0_next + state1 = state1_next + decoder_out, state0_next, state1_next = model.run_decoder( + ans[-1], state0, state1 + ) + + ans = ans[1:] # remove the first blank + print(ans) + tokens = [id2token[i] for i in ans] + underline = "▁" + # underline = b"\xe2\x96\x81".decode() + text = "".join(tokens).replace(underline, " ").strip() + print("./example.wav") + print(text) + + +if __name__ == "__main__": + main() diff --git a/sherpa-onnx/csrc/offline-recognizer-impl.cc b/sherpa-onnx/csrc/offline-recognizer-impl.cc index a80301ebf..f6c6e247c 100644 --- a/sherpa-onnx/csrc/offline-recognizer-impl.cc +++ b/sherpa-onnx/csrc/offline-recognizer-impl.cc @@ -166,7 +166,8 @@ std::unique_ptr OfflineRecognizerImpl::Create( return std::make_unique(config); } - if (model_type == "EncDecHybridRNNTCTCBPEModel" && + if ((model_type == "EncDecHybridRNNTCTCBPEModel" || + model_type == "EncDecRNNTBPEModel") && !config.model_config.transducer.decoder_filename.empty() && !config.model_config.transducer.joiner_filename.empty()) { return std::make_unique(config); @@ -191,6 +192,7 @@ std::unique_ptr OfflineRecognizerImpl::Create( " - EncDecCTCModelBPE models from NeMo\n" " - EncDecCTCModel models from NeMo\n" " - EncDecHybridRNNTCTCBPEModel models from NeMo\n" + " - EncDecRNNTBPEModel models from NeMO" " - Whisper models\n" " - Tdnn models\n" " - Zipformer CTC models\n" @@ -338,7 +340,8 @@ std::unique_ptr OfflineRecognizerImpl::Create( return std::make_unique(mgr, config); } - if (model_type == "EncDecHybridRNNTCTCBPEModel" && + if ((model_type == "EncDecHybridRNNTCTCBPEModel" || + model_type == "EncDecRNNTBPEModel") && !config.model_config.transducer.decoder_filename.empty() && !config.model_config.transducer.joiner_filename.empty()) { return std::make_unique(mgr, config); @@ -363,6 +366,7 @@ std::unique_ptr OfflineRecognizerImpl::Create( " - EncDecCTCModelBPE models from NeMo\n" " - EncDecCTCModel models from NeMo\n" " - EncDecHybridRNNTCTCBPEModel models from NeMo\n" + " - EncDecRNNTBPEModel models from NeMo\n" " - Whisper models\n" " - Tdnn models\n" " - Zipformer CTC models\n" diff --git a/sherpa-onnx/csrc/offline-recognizer-transducer-nemo-impl.h b/sherpa-onnx/csrc/offline-recognizer-transducer-nemo-impl.h index 2f5b9e2a2..6727b0983 100644 --- a/sherpa-onnx/csrc/offline-recognizer-transducer-nemo-impl.h +++ b/sherpa-onnx/csrc/offline-recognizer-transducer-nemo-impl.h @@ -139,23 +139,29 @@ class OfflineRecognizerTransducerNeMoImpl : public OfflineRecognizerImpl { } } - OfflineRecognizerConfig GetConfig() const override { - return config_; - } + OfflineRecognizerConfig GetConfig() const override { return config_; } private: void PostInit() { config_.feat_config.nemo_normalize_type = model_->FeatureNormalizationMethod(); - config_.feat_config.low_freq = 0; - // config_.feat_config.high_freq = 8000; - config_.feat_config.is_librosa = true; - config_.feat_config.remove_dc_offset = false; - // config_.feat_config.window_type = "hann"; config_.feat_config.dither = 0; - config_.feat_config.nemo_normalize_type = - model_->FeatureNormalizationMethod(); + + if (model_->IsGigaAM()) { + config_.feat_config.low_freq = 0; + config_.feat_config.high_freq = 8000; + config_.feat_config.remove_dc_offset = false; + config_.feat_config.preemph_coeff = 0; + config_.feat_config.window_type = "hann"; + config_.feat_config.feature_dim = 64; + } else { + config_.feat_config.low_freq = 0; + // config_.feat_config.high_freq = 8000; + config_.feat_config.is_librosa = true; + config_.feat_config.remove_dc_offset = false; + // config_.feat_config.window_type = "hann"; + } int32_t vocab_size = model_->VocabSize(); diff --git a/sherpa-onnx/csrc/offline-transducer-nemo-model.cc b/sherpa-onnx/csrc/offline-transducer-nemo-model.cc index f18e57da9..5332a835e 100644 --- a/sherpa-onnx/csrc/offline-transducer-nemo-model.cc +++ b/sherpa-onnx/csrc/offline-transducer-nemo-model.cc @@ -153,6 +153,8 @@ class OfflineTransducerNeMoModel::Impl { std::string FeatureNormalizationMethod() const { return normalize_type_; } + bool IsGigaAM() const { return is_giga_am_; } + private: void InitEncoder(void *model_data, size_t model_data_length) { encoder_sess_ = std::make_unique( @@ -181,9 +183,11 @@ class OfflineTransducerNeMoModel::Impl { vocab_size_ += 1; SHERPA_ONNX_READ_META_DATA(subsampling_factor_, "subsampling_factor"); - SHERPA_ONNX_READ_META_DATA_STR(normalize_type_, "normalize_type"); + SHERPA_ONNX_READ_META_DATA_STR_ALLOW_EMPTY(normalize_type_, + "normalize_type"); SHERPA_ONNX_READ_META_DATA(pred_rnn_layers_, "pred_rnn_layers"); SHERPA_ONNX_READ_META_DATA(pred_hidden_, "pred_hidden"); + SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(is_giga_am_, "is_giga_am", 0); if (normalize_type_ == "NA") { normalize_type_ = ""; @@ -245,6 +249,7 @@ class OfflineTransducerNeMoModel::Impl { std::string normalize_type_; int32_t pred_rnn_layers_ = -1; int32_t pred_hidden_ = -1; + int32_t is_giga_am_ = 0; }; OfflineTransducerNeMoModel::OfflineTransducerNeMoModel( @@ -298,4 +303,6 @@ std::string OfflineTransducerNeMoModel::FeatureNormalizationMethod() const { return impl_->FeatureNormalizationMethod(); } +bool OfflineTransducerNeMoModel::IsGigaAM() const { return impl_->IsGigaAM(); } + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-transducer-nemo-model.h b/sherpa-onnx/csrc/offline-transducer-nemo-model.h index 9ac135916..e4017a4c4 100644 --- a/sherpa-onnx/csrc/offline-transducer-nemo-model.h +++ b/sherpa-onnx/csrc/offline-transducer-nemo-model.h @@ -93,6 +93,8 @@ class OfflineTransducerNeMoModel { // for details std::string FeatureNormalizationMethod() const; + bool IsGigaAM() const; + private: class Impl; std::unique_ptr impl_; diff --git a/sherpa-onnx/kotlin-api/OfflineRecognizer.kt b/sherpa-onnx/kotlin-api/OfflineRecognizer.kt index 10cdc5179..b82436356 100644 --- a/sherpa-onnx/kotlin-api/OfflineRecognizer.kt +++ b/sherpa-onnx/kotlin-api/OfflineRecognizer.kt @@ -404,6 +404,19 @@ fun getOfflineModelConfig(type: Int): OfflineModelConfig? { tokens = "$modelDir/tokens.txt", ) } + + 20 -> { + val modelDir = "sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24" + return OfflineModelConfig( + transducer = OfflineTransducerModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.onnx", + joiner = "$modelDir/joiner.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "nemo_transducer", + ) + } } return null } From d5a2f52413e86d85b06a200f24bf1664cc9a7cad Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 25 Oct 2024 15:50:42 +0800 Subject: [PATCH 037/183] Release v1.10.29 (#1468) --- CHANGELOG.md | 14 ++++++++++++++ CMakeLists.txt | 2 +- dart-api-examples/add-punctuations/pubspec.yaml | 2 +- dart-api-examples/audio-tagging/pubspec.yaml | 2 +- dart-api-examples/keyword-spotter/pubspec.yaml | 2 +- dart-api-examples/non-streaming-asr/pubspec.yaml | 2 +- dart-api-examples/speaker-diarization/pubspec.yaml | 2 +- .../speaker-identification/pubspec.yaml | 2 +- dart-api-examples/streaming-asr/pubspec.yaml | 2 +- dart-api-examples/tts/pubspec.yaml | 2 +- .../vad-with-non-streaming-asr/pubspec.yaml | 2 +- dart-api-examples/vad/pubspec.yaml | 2 +- flutter-examples/streaming_asr/pubspec.yaml | 4 ++-- flutter-examples/tts/pubspec.yaml | 4 ++-- flutter/sherpa_onnx/example/example.md | 2 +- flutter/sherpa_onnx/pubspec.yaml | 12 ++++++------ .../sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec | 2 +- .../macos/sherpa_onnx_macos.podspec | 2 +- new-release.sh | 10 +++++----- nodejs-addon-examples/package.json | 2 +- 20 files changed, 44 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 973f4ab7c..176d8ee23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +## 1.10.29 + +* Add Go API for offline punctuation models (#1434) +* Support https://huggingface.co/Revai/reverb-diarization-v1 (#1437) +* Add more models for speaker diarization (#1440) +* Add Java API example for hotwords. (#1442) +* Add java android demo (#1454) +* Add C++ API for streaming ASR. (#1455) +* Add C++ API for non-streaming ASR (#1456) +* Handle NaN embeddings in speaker diarization. (#1461) +* Add speaker identification with VAD and non-streaming ASR using ALSA (#1463) +* Support GigaAM CTC models for Russian ASR (#1464) +* Add GigaAM NeMo transducer model for Russian ASR (#1467) + ## 1.10.28 * Fix swift example for generating subtitles. (#1362) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e1850c44..c9edbf43d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,7 @@ project(sherpa-onnx) # ./nodejs-addon-examples # ./dart-api-examples/ # ./CHANGELOG.md -set(SHERPA_ONNX_VERSION "1.10.28") +set(SHERPA_ONNX_VERSION "1.10.29") # Disable warning about # diff --git a/dart-api-examples/add-punctuations/pubspec.yaml b/dart-api-examples/add-punctuations/pubspec.yaml index ab1326994..892535c85 100644 --- a/dart-api-examples/add-punctuations/pubspec.yaml +++ b/dart-api-examples/add-punctuations/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.28 + sherpa_onnx: ^1.10.29 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/audio-tagging/pubspec.yaml b/dart-api-examples/audio-tagging/pubspec.yaml index 8df9f066f..ddd5ac256 100644 --- a/dart-api-examples/audio-tagging/pubspec.yaml +++ b/dart-api-examples/audio-tagging/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.28 + sherpa_onnx: ^1.10.29 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/keyword-spotter/pubspec.yaml b/dart-api-examples/keyword-spotter/pubspec.yaml index 8d4c6e650..61da20d70 100644 --- a/dart-api-examples/keyword-spotter/pubspec.yaml +++ b/dart-api-examples/keyword-spotter/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.28 + sherpa_onnx: ^1.10.29 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/non-streaming-asr/pubspec.yaml b/dart-api-examples/non-streaming-asr/pubspec.yaml index 3ea2bcf07..be17050f6 100644 --- a/dart-api-examples/non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.28 + sherpa_onnx: ^1.10.29 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/speaker-diarization/pubspec.yaml b/dart-api-examples/speaker-diarization/pubspec.yaml index 700d7e731..17ce94336 100644 --- a/dart-api-examples/speaker-diarization/pubspec.yaml +++ b/dart-api-examples/speaker-diarization/pubspec.yaml @@ -8,7 +8,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.28 + sherpa_onnx: ^1.10.29 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/speaker-identification/pubspec.yaml b/dart-api-examples/speaker-identification/pubspec.yaml index 12886b025..490c80732 100644 --- a/dart-api-examples/speaker-identification/pubspec.yaml +++ b/dart-api-examples/speaker-identification/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.28 + sherpa_onnx: ^1.10.29 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/streaming-asr/pubspec.yaml b/dart-api-examples/streaming-asr/pubspec.yaml index 690e5b5a0..1dc07924d 100644 --- a/dart-api-examples/streaming-asr/pubspec.yaml +++ b/dart-api-examples/streaming-asr/pubspec.yaml @@ -11,7 +11,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.28 + sherpa_onnx: ^1.10.29 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/tts/pubspec.yaml b/dart-api-examples/tts/pubspec.yaml index 095c0c0cb..0a36a2324 100644 --- a/dart-api-examples/tts/pubspec.yaml +++ b/dart-api-examples/tts/pubspec.yaml @@ -8,7 +8,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.28 + sherpa_onnx: ^1.10.29 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml index 12da20269..76230d26e 100644 --- a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.28 + sherpa_onnx: ^1.10.29 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad/pubspec.yaml b/dart-api-examples/vad/pubspec.yaml index 220db6c01..f0ddcbbc0 100644 --- a/dart-api-examples/vad/pubspec.yaml +++ b/dart-api-examples/vad/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.28 + sherpa_onnx: ^1.10.29 path: ^1.9.0 args: ^2.5.0 diff --git a/flutter-examples/streaming_asr/pubspec.yaml b/flutter-examples/streaming_asr/pubspec.yaml index 894cbfdbd..e080857c7 100644 --- a/flutter-examples/streaming_asr/pubspec.yaml +++ b/flutter-examples/streaming_asr/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' -version: 1.10.28 +version: 1.10.29 topics: - speech-recognition @@ -31,7 +31,7 @@ dependencies: record: ^5.1.0 url_launcher: ^6.2.6 - sherpa_onnx: ^1.10.28 + sherpa_onnx: ^1.10.29 # sherpa_onnx: # path: ../../flutter/sherpa_onnx diff --git a/flutter-examples/tts/pubspec.yaml b/flutter-examples/tts/pubspec.yaml index c8f227cac..211685b01 100644 --- a/flutter-examples/tts/pubspec.yaml +++ b/flutter-examples/tts/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' # Remove this line if you wish to publish to pub.dev -version: 1.10.28 +version: 1.10.29 environment: sdk: ">=2.17.0 <4.0.0" @@ -18,7 +18,7 @@ dependencies: cupertino_icons: ^1.0.6 path_provider: ^2.1.3 path: ^1.9.0 - sherpa_onnx: ^1.10.28 + sherpa_onnx: ^1.10.29 # sherpa_onnx: # path: ../../flutter/sherpa_onnx url_launcher: 6.2.6 diff --git a/flutter/sherpa_onnx/example/example.md b/flutter/sherpa_onnx/example/example.md index 0c24a79b2..9255b7ee9 100644 --- a/flutter/sherpa_onnx/example/example.md +++ b/flutter/sherpa_onnx/example/example.md @@ -4,7 +4,7 @@ | Functions | URL | Supported Platforms| |---|---|---| -|Streaming speech recognition| [Address](https://github.com/k2-fsa/sherpa-onnx/tree/master/flutter-examples/streaming_asr)| Android, macOS, Windows| +|Streaming speech recognition| [Address](https://github.com/k2-fsa/sherpa-onnx/tree/master/flutter-examples/streaming_asr)| Android, iOS, macOS, Windows| |Speech synthesis| [Address](https://github.com/k2-fsa/sherpa-onnx/tree/master/flutter-examples/tts)| Android, iOS, Linux, macOS, Windows| ## Pure dart-examples diff --git a/flutter/sherpa_onnx/pubspec.yaml b/flutter/sherpa_onnx/pubspec.yaml index c89994133..221cc7c3e 100644 --- a/flutter/sherpa_onnx/pubspec.yaml +++ b/flutter/sherpa_onnx/pubspec.yaml @@ -17,7 +17,7 @@ topics: - voice-activity-detection # remember to change the version in ../sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec -version: 1.10.28 +version: 1.10.29 homepage: https://github.com/k2-fsa/sherpa-onnx @@ -30,23 +30,23 @@ dependencies: flutter: sdk: flutter - sherpa_onnx_android: ^1.10.28 + sherpa_onnx_android: ^1.10.29 # sherpa_onnx_android: # path: ../sherpa_onnx_android - sherpa_onnx_macos: ^1.10.28 + sherpa_onnx_macos: ^1.10.29 # sherpa_onnx_macos: # path: ../sherpa_onnx_macos - sherpa_onnx_linux: ^1.10.28 + sherpa_onnx_linux: ^1.10.29 # sherpa_onnx_linux: # path: ../sherpa_onnx_linux - sherpa_onnx_windows: ^1.10.28 + sherpa_onnx_windows: ^1.10.29 # sherpa_onnx_windows: # path: ../sherpa_onnx_windows - sherpa_onnx_ios: ^1.10.28 + sherpa_onnx_ios: ^1.10.29 # sherpa_onnx_ios: # path: ../sherpa_onnx_ios diff --git a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec index 54b68d024..9c5d4bd34 100644 --- a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec +++ b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec @@ -7,7 +7,7 @@ # https://groups.google.com/g/dart-ffi/c/nUATMBy7r0c Pod::Spec.new do |s| s.name = 'sherpa_onnx_ios' - s.version = '1.10.28' + s.version = '1.10.29' s.summary = 'A new Flutter FFI plugin project.' s.description = <<-DESC A new Flutter FFI plugin project. diff --git a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec index f11008bb8..fbf3b5e20 100644 --- a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec +++ b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec @@ -4,7 +4,7 @@ # Pod::Spec.new do |s| s.name = 'sherpa_onnx_macos' - s.version = '1.10.28' + s.version = '1.10.29' s.summary = 'sherpa-onnx Flutter FFI plugin project.' s.description = <<-DESC sherpa-onnx Flutter FFI plugin project. diff --git a/new-release.sh b/new-release.sh index ae4148440..1096e82e2 100755 --- a/new-release.sh +++ b/new-release.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.27/1\.10\.28/g' {} \; -find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.27/1\.10\.28/g' {} \; -find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.27/1\.10\.28/g' {} \; -find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.27/1\.10\.28/g' {} \; -find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.27/1\.10\.28/g' {} \; +find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.28/1\.10\.29/g' {} \; +find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.28/1\.10\.29/g' {} \; +find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.28/1\.10\.29/g' {} \; +find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.28/1\.10\.29/g' {} \; +find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.28/1\.10\.29/g' {} \; diff --git a/nodejs-addon-examples/package.json b/nodejs-addon-examples/package.json index 2944f4da3..88bee9b32 100644 --- a/nodejs-addon-examples/package.json +++ b/nodejs-addon-examples/package.json @@ -1,5 +1,5 @@ { "dependencies": { - "sherpa-onnx-node": "^1.10.28" + "sherpa-onnx-node": "^1.10.29" } } From 3d6344ead33067ae40e61404fd9a452df1e67b1a Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 25 Oct 2024 18:49:33 +0800 Subject: [PATCH 038/183] Fix building node-addon for Windows x86. (#1469) --- .../node-addon-api/src/non-streaming-speaker-diarization.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/node-addon-api/src/non-streaming-speaker-diarization.cc b/scripts/node-addon-api/src/non-streaming-speaker-diarization.cc index 56767476a..a35f7924a 100644 --- a/scripts/node-addon-api/src/non-streaming-speaker-diarization.cc +++ b/scripts/node-addon-api/src/non-streaming-speaker-diarization.cc @@ -238,11 +238,12 @@ static Napi::Array OfflineSpeakerDiarizationProcessWrapper( for (int32_t i = 0; i != num_segments; ++i) { Napi::Object obj = Napi::Object::New(env); + obj.Set(Napi::String::New(env, "start"), segments[i].start); obj.Set(Napi::String::New(env, "end"), segments[i].end); obj.Set(Napi::String::New(env, "speaker"), segments[i].speaker); - ans[i] = obj; + ans.Set(i, obj); } SherpaOnnxOfflineSpeakerDiarizationDestroySegment(segments); From b06b460851ae76e4f9e66fee2796e317a16bee77 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sat, 26 Oct 2024 09:51:16 +0800 Subject: [PATCH 039/183] Begin to support https://github.com/usefulsensors/moonshine (#1470) --- .../workflows/export-moonshine-to-onnx.yaml | 106 +++++++ scripts/moonshine/.gitignore | 1 + scripts/moonshine/README.md | 7 + scripts/moonshine/export-onnx.py | 40 +++ scripts/moonshine/run.sh | 90 ++++++ scripts/moonshine/test.py | 274 ++++++++++++++++++ 6 files changed, 518 insertions(+) create mode 100644 .github/workflows/export-moonshine-to-onnx.yaml create mode 100644 scripts/moonshine/.gitignore create mode 100644 scripts/moonshine/README.md create mode 100755 scripts/moonshine/export-onnx.py create mode 100755 scripts/moonshine/run.sh create mode 100755 scripts/moonshine/test.py diff --git a/.github/workflows/export-moonshine-to-onnx.yaml b/.github/workflows/export-moonshine-to-onnx.yaml new file mode 100644 index 000000000..2e73c2e04 --- /dev/null +++ b/.github/workflows/export-moonshine-to-onnx.yaml @@ -0,0 +1,106 @@ +name: export-moonshine-to-onnx + +on: + workflow_dispatch: + +concurrency: + group: export-moonshine-to-onnx-${{ github.ref }} + cancel-in-progress: true + +jobs: + export-moonshine-to-onnx: + if: github.repository_owner == 'k2-fsa' || github.repository_owner == 'csukuangfj' + name: export moonshine models to ONNX + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [macos-latest] + python-version: ["3.10"] + + steps: + - uses: actions/checkout@v4 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Python dependencies + shell: bash + run: | + pip install -q onnx onnxruntime librosa tokenizers soundfile + + - name: Run + shell: bash + run: | + pushd scripts/moonshine + ./run.sh + popd + + mv -v scripts/moonshine/*.tar.bz2 . + mv -v scripts/moonshine/sherpa-onnx-* ./ + + - name: Release + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + file: ./*.tar.bz2 + overwrite: true + repo_name: k2-fsa/sherpa-onnx + repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} + tag: asr-models + + - name: Publish to huggingface (tiny) + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v3 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + + d=sherpa-onnx-moonshine-tiny-en-int8 + export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$d huggingface + mv -v $d/* ./huggingface + cd huggingface + git lfs track "*.onnx" + git lfs track "*.wav" + git status + git add . + git status + git commit -m "add models" + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$d main + rm -rf huggingface + + - name: Publish to huggingface (base) + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v3 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + + d=sherpa-onnx-moonshine-base-en-int8 + export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$d huggingface + mv -v $d/* ./huggingface + cd huggingface + git lfs track "*.onnx" + git lfs track "*.wav" + git status + git add . + git status + git commit -m "add models" + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/$d main + rm -rf huggingface diff --git a/scripts/moonshine/.gitignore b/scripts/moonshine/.gitignore new file mode 100644 index 000000000..c78219ab0 --- /dev/null +++ b/scripts/moonshine/.gitignore @@ -0,0 +1 @@ +tokenizer.json diff --git a/scripts/moonshine/README.md b/scripts/moonshine/README.md new file mode 100644 index 000000000..b9c5e37fb --- /dev/null +++ b/scripts/moonshine/README.md @@ -0,0 +1,7 @@ +# Introduction + +This directory contains models from +https://github.com/usefulsensors/moonshine + +See its license at +https://github.com/usefulsensors/moonshine/blob/main/LICENSE diff --git a/scripts/moonshine/export-onnx.py b/scripts/moonshine/export-onnx.py new file mode 100755 index 000000000..6b0a34720 --- /dev/null +++ b/scripts/moonshine/export-onnx.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# Copyright 2024 Xiaomi Corp. (authors: Fangjun Kuang) + +from pathlib import Path + +import tokenizers +from onnxruntime.quantization import QuantType, quantize_dynamic + + +def generate_tokens(): + if Path("./tokens.txt").is_file(): + return + print("Generating tokens.txt") + tokenizer = tokenizers.Tokenizer.from_file("./tokenizer.json") + vocab_size = tokenizer.get_vocab_size() + with open("tokens.txt", "w", encoding="utf-8") as f: + for i in range(vocab_size): + s = tokenizer.id_to_token(i).strip() + f.write(f"{s}\t{i}\n") + + +def main(): + generate_tokens() + + # Note(fangjun): Don't use int8 for the preprocessor since it has + # a larger impact on the accuracy + for f in ["uncached_decode", "cached_decode", "encode"]: + if Path(f"{f}.int8.onnx").is_file(): + continue + + print("processing", f) + quantize_dynamic( + model_input=f"{f}.onnx", + model_output=f"{f}.int8.onnx", + weight_type=QuantType.QInt8, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/moonshine/run.sh b/scripts/moonshine/run.sh new file mode 100755 index 000000000..0ad139660 --- /dev/null +++ b/scripts/moonshine/run.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Copyright 2024 Xiaomi Corp. (authors: Fangjun Kuang) +set -ex + +cat >LICENSE <"] + eos = token2id[""] + + tokens = [sos] + + encoder_out = model.run_encode(features) + print("encoder_out.shape", encoder_out.shape) # (1, 413, 288) + + logits, states = model.run_uncached_decode( + token=tokens[-1], + token_len=len(tokens), + encoder_out=encoder_out, + ) + + print("logits.shape", logits.shape) # (1, 1, 32768) + print("len(states)", len(states)) # 24 + + max_len = int((audio.shape[-1] / 16000) * 6) + + for i in range(max_len): + token = logits.squeeze().argmax() + if token == eos: + break + tokens.append(token) + + logits, states = model.run_cached_decode( + token=tokens[-1], + token_len=len(tokens), + encoder_out=encoder_out, + states=states, + ) + + tokens = tokens[1:] # remove sos + words = [id2token[i] for i in tokens] + underline = "▁" + # underline = b"\xe2\x96\x81".decode() + text = "".join(words).replace(underline, " ").strip() + + end_t = dt.datetime.now() + t = (end_t - start_t).total_seconds() + rtf = t * 16000 / audio.shape[-1] + + print(text) + print("RTF:", rtf) + + +if __name__ == "__main__": + main() From 0f2732e4e8bd6995e78b326a84b5ae7b300bf069 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sat, 26 Oct 2024 09:59:18 +0800 Subject: [PATCH 040/183] Publish pre-built JNI libs for Linux aarch64 (#1472) --- .github/workflows/linux-jni-aarch64.yaml | 176 +++++++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 .github/workflows/linux-jni-aarch64.yaml diff --git a/.github/workflows/linux-jni-aarch64.yaml b/.github/workflows/linux-jni-aarch64.yaml new file mode 100644 index 000000000..19d1e09cf --- /dev/null +++ b/.github/workflows/linux-jni-aarch64.yaml @@ -0,0 +1,176 @@ +name: linux-jni-aarch64 + +on: + push: + branches: + - jni + tags: + - 'v[0-9]+.[0-9]+.[0-9]+*' + workflow_dispatch: + +concurrency: + group: linux-jni-aarch64-${{ github.ref }} + cancel-in-progress: true + +jobs: + linux-jni-aarch64: + name: linux jni aarch64 + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + # java-version: ['8', '11', '16', '17', '21'] + java-version: ['21'] + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' # See 'Supported distributions' for available options + java-version: ${{ matrix.java-version }} + + - name: Set up QEMU + if: steps.cache-build-result.outputs.cache-hit != 'true' + uses: docker/setup-qemu-action@v2 + with: + platforms: all + + - name: Display PWD + shell: bash + run: | + echo "pwd: $PWD" + ls -lh + du -h -d1 . + + - name: Build sherpa-onnx + if: matrix.java-version == '21' + uses: addnab/docker-run-action@v3 + with: + image: quay.io/pypa/manylinux2014_aarch64 + options: | + --volume ${{ github.workspace }}/:/home/runner/work/sherpa-onnx/sherpa-onnx + shell: bash + run: | + uname -a + gcc --version + cmake --version + cat /etc/*release + id + pwd + + yum install -y java-11-openjdk-devel + java -version + which java + ls -lh $(which java) + ls -lrt /etc/alternatives/java + + export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-11.0.23.0.9-2.el7_9.aarch64 + echo "JAVA_HOME: $JAVA_HOME" + find $JAVA_HOME -name jni.h + + cd /home/runner/work/sherpa-onnx/sherpa-onnx + + git clone --depth 1 --branch v1.2.12 https://github.com/alsa-project/alsa-lib + pushd alsa-lib + ./gitcompile + popd + + export CPLUS_INCLUDE_PATH=$PWD/alsa-lib/include:$CPLUS_INCLUDE_PATH + export SHERPA_ONNX_ALSA_LIB_DIR=$PWD/alsa-lib/src/.libs + + mkdir build + cd build + + cmake \ + -D SHERPA_ONNX_ENABLE_TTS=ON \ + -D CMAKE_BUILD_TYPE=Release \ + -D BUILD_SHARED_LIBS=ON \ + -D CMAKE_INSTALL_PREFIX=./install \ + -D SHERPA_ONNX_ENABLE_BINARY=OFF \ + -D SHERPA_ONNX_ENABLE_JNI=ON \ + .. + + make -j2 + make install + + ls -lh lib + rm -rf ./install/lib/pkgconfig + rm -rf ./install/lib/share + rm -rf ./install/lib/cargs.h + rm -rf ./install/include/cargs.h + rm -rf ./install/lib/libcargs.so + rm -rf ./install/lib/libsherpa-onnx-c-api.so + + echo "----" + ls -lh install/lib + + echo "----" + + - uses: actions/upload-artifact@v4 + if: matrix.java-version == '21' + with: + name: release-jni-linux-${{ matrix.java-version }} + path: build/install/* + + - name: Copy files + if: matrix.java-version == '21' + shell: bash + run: | + du -h -d1 . + SHERPA_ONNX_VERSION=v$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + + dst=sherpa-onnx-${SHERPA_ONNX_VERSION}-linux-aarch64-jni + mkdir $dst + + cp -a build/install/lib $dst/ + cp -a build/install/include $dst/ + + tree $dst + + tar cjvf ${dst}.tar.bz2 $dst + du -h -d1 . + + - name: Publish to huggingface + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && matrix.java-version == '21' + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v3 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + + rm -rf huggingface + export GIT_CLONE_PROTECTION_ACTIVE=false + GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface + + cd huggingface + mkdir -p jni + + cp -v ../sherpa-onnx-*.tar.bz2 ./jni + cp -v ../*.jar ./jni + + git status + git lfs track "*.bz2" + + git add . + + git commit -m "add more files" + + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs main + + - name: Release pre-compiled binaries and libs for linux aarch64 + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') && matrix.java-version == '21' + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + overwrite: true + file: sherpa-onnx-*.tar.bz2 + From 669f5ef44105138116d9d8dd5fe224ec3f6354a6 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sat, 26 Oct 2024 14:34:07 +0800 Subject: [PATCH 041/183] Add C++ runtime and Python APIs for Moonshine models (#1473) --- .github/scripts/test-offline-moonshine.sh | 50 ++++ .github/scripts/test-python.sh | 10 + .github/workflows/linux.yaml | 13 + .github/workflows/macos.yaml | 11 +- .github/workflows/windows-x64.yaml | 8 + .github/workflows/windows-x86.yaml | 8 + python-api-examples/generate-subtitles.py | 117 +++++++- python-api-examples/non_streaming_server.py | 108 ++++++- .../offline-moonshine-decode-files.py | 82 +++++ .../offline-whisper-decode-files.py | 77 +++++ .../vad-with-non-streaming-asr.py | 83 +++++- sherpa-onnx/csrc/CMakeLists.txt | 3 + sherpa-onnx/csrc/offline-model-config.cc | 6 + sherpa-onnx/csrc/offline-model-config.h | 4 + sherpa-onnx/csrc/offline-moonshine-decoder.h | 34 +++ ...offline-moonshine-greedy-search-decoder.cc | 87 ++++++ .../offline-moonshine-greedy-search-decoder.h | 29 ++ .../csrc/offline-moonshine-model-config.cc | 88 ++++++ .../csrc/offline-moonshine-model-config.h | 37 +++ sherpa-onnx/csrc/offline-moonshine-model.cc | 282 ++++++++++++++++++ sherpa-onnx/csrc/offline-moonshine-model.h | 93 ++++++ sherpa-onnx/csrc/offline-recognizer-impl.cc | 15 + .../csrc/offline-recognizer-moonshine-impl.h | 150 ++++++++++ sherpa-onnx/csrc/offline-stream.cc | 27 +- sherpa-onnx/csrc/offline-stream.h | 12 +- sherpa-onnx/csrc/offline-whisper-model.cc | 9 +- sherpa-onnx/csrc/sherpa-onnx-offline.cc | 19 +- sherpa-onnx/csrc/symbol-table.cc | 2 + sherpa-onnx/python/csrc/CMakeLists.txt | 1 + .../python/csrc/offline-model-config.cc | 7 +- .../csrc/offline-moonshine-model-config.cc | 28 ++ .../csrc/offline-moonshine-model-config.h | 16 + .../python/sherpa_onnx/offline_recognizer.py | 92 +++++- 33 files changed, 1572 insertions(+), 36 deletions(-) create mode 100755 .github/scripts/test-offline-moonshine.sh create mode 100644 python-api-examples/offline-moonshine-decode-files.py create mode 100644 python-api-examples/offline-whisper-decode-files.py create mode 100644 sherpa-onnx/csrc/offline-moonshine-decoder.h create mode 100644 sherpa-onnx/csrc/offline-moonshine-greedy-search-decoder.cc create mode 100644 sherpa-onnx/csrc/offline-moonshine-greedy-search-decoder.h create mode 100644 sherpa-onnx/csrc/offline-moonshine-model-config.cc create mode 100644 sherpa-onnx/csrc/offline-moonshine-model-config.h create mode 100644 sherpa-onnx/csrc/offline-moonshine-model.cc create mode 100644 sherpa-onnx/csrc/offline-moonshine-model.h create mode 100644 sherpa-onnx/csrc/offline-recognizer-moonshine-impl.h create mode 100644 sherpa-onnx/python/csrc/offline-moonshine-model-config.cc create mode 100644 sherpa-onnx/python/csrc/offline-moonshine-model-config.h diff --git a/.github/scripts/test-offline-moonshine.sh b/.github/scripts/test-offline-moonshine.sh new file mode 100755 index 000000000..1768e82ec --- /dev/null +++ b/.github/scripts/test-offline-moonshine.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +set -e + +log() { + # This function is from espnet + local fname=${BASH_SOURCE[1]##*/} + echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" +} + +export GIT_CLONE_PROTECTION_ACTIVE=false + +echo "EXE is $EXE" +echo "PATH: $PATH" + +which $EXE + +names=( +tiny +base +) + +for name in ${names[@]}; do + log "------------------------------------------------------------" + log "Run $name" + log "------------------------------------------------------------" + + repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-$name.tar.bz2 + repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-$name-en-int8.tar.bz2 + curl -SL -O $repo_url + tar xvf sherpa-onnx-moonshine-$name-en-int8.tar.bz2 + rm sherpa-onnx-moonshine-$name-en-int8.tar.bz2 + repo=sherpa-onnx-moonshine-$name-en-int8 + log "Start testing ${repo_url}" + + log "test int8 onnx" + + time $EXE \ + --moonshine-preprocessor=$repo/preprocess.onnx \ + --moonshine-encoder=$repo/encode.int8.onnx \ + --moonshine-uncached-decoder=$repo/uncached_decode.int8.onnx \ + --moonshine-cached-decoder=$repo/cached_decode.int8.onnx \ + --tokens=$repo/tokens.txt \ + --num-threads=2 \ + $repo/test_wavs/0.wav \ + $repo/test_wavs/1.wav \ + $repo/test_wavs/8k.wav + + rm -rf $repo +done diff --git a/.github/scripts/test-python.sh b/.github/scripts/test-python.sh index 8c9d303b0..91f6f66bc 100755 --- a/.github/scripts/test-python.sh +++ b/.github/scripts/test-python.sh @@ -8,6 +8,16 @@ log() { echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" } +log "test offline Moonshine" + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + +python3 ./python-api-examples/offline-moonshine-decode-files.py + +rm -rf sherpa-onnx-moonshine-tiny-en-int8 + log "test offline speaker diarization" curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml index 2ad40e215..98c88e589 100644 --- a/.github/workflows/linux.yaml +++ b/.github/workflows/linux.yaml @@ -149,6 +149,19 @@ jobs: name: release-${{ matrix.build_type }}-with-shared-lib-${{ matrix.shared_lib }}-with-tts-${{ matrix.with_tts }} path: install/* + - name: Test offline Moonshine + if: matrix.build_type != 'Debug' + shell: bash + run: | + du -h -d1 . + export PATH=$PWD/build/bin:$PATH + export EXE=sherpa-onnx-offline + + readelf -d build/bin/sherpa-onnx-offline + + .github/scripts/test-offline-moonshine.sh + du -h -d1 . + - name: Test offline CTC shell: bash run: | diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml index 849631015..9e17491bb 100644 --- a/.github/workflows/macos.yaml +++ b/.github/workflows/macos.yaml @@ -121,6 +121,15 @@ jobs: otool -L build/bin/sherpa-onnx otool -l build/bin/sherpa-onnx + - name: Test offline Moonshine + if: matrix.build_type != 'Debug' + shell: bash + run: | + export PATH=$PWD/build/bin:$PATH + export EXE=sherpa-onnx-offline + + .github/scripts/test-offline-moonshine.sh + - name: Test C++ API shell: bash run: | @@ -243,8 +252,6 @@ jobs: .github/scripts/test-offline-whisper.sh - - - name: Test online transducer shell: bash run: | diff --git a/.github/workflows/windows-x64.yaml b/.github/workflows/windows-x64.yaml index 9435dcefd..50bf014d4 100644 --- a/.github/workflows/windows-x64.yaml +++ b/.github/workflows/windows-x64.yaml @@ -93,6 +93,14 @@ jobs: name: release-windows-x64-${{ matrix.shared_lib }}-${{ matrix.with_tts }} path: build/install/* + - name: Test offline Moonshine for windows x64 + shell: bash + run: | + export PATH=$PWD/build/bin/Release:$PATH + export EXE=sherpa-onnx-offline.exe + + .github/scripts/test-offline-moonshine.sh + - name: Test C++ API shell: bash run: | diff --git a/.github/workflows/windows-x86.yaml b/.github/workflows/windows-x86.yaml index 36089b2dd..8a1370959 100644 --- a/.github/workflows/windows-x86.yaml +++ b/.github/workflows/windows-x86.yaml @@ -93,6 +93,14 @@ jobs: name: release-windows-x86-${{ matrix.shared_lib }}-${{ matrix.with_tts }} path: build/install/* + - name: Test offline Moonshine for windows x86 + shell: bash + run: | + export PATH=$PWD/build/bin/Release:$PATH + export EXE=sherpa-onnx-offline.exe + + .github/scripts/test-offline-moonshine.sh + - name: Test C++ API shell: bash run: | diff --git a/python-api-examples/generate-subtitles.py b/python-api-examples/generate-subtitles.py index 85871016a..bf51f7627 100755 --- a/python-api-examples/generate-subtitles.py +++ b/python-api-examples/generate-subtitles.py @@ -47,7 +47,19 @@ --feature-dim=80 \ /path/to/test.mp4 -(3) For Whisper models +(3) For Moonshine models + +./python-api-examples/generate-subtitles.py \ + --silero-vad-model=/path/to/silero_vad.onnx \ + --moonshine-preprocessor=./sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx \ + --moonshine-encoder=./sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx \ + --moonshine-uncached-decoder=./sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx \ + --moonshine-cached-decoder=./sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx \ + --tokens=./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt \ + --num-threads=2 \ + /path/to/test.mp4 + +(4) For Whisper models ./python-api-examples/generate-subtitles.py \ --silero-vad-model=/path/to/silero_vad.onnx \ @@ -58,7 +70,7 @@ --num-threads=2 \ /path/to/test.mp4 -(4) For SenseVoice CTC models +(5) For SenseVoice CTC models ./python-api-examples/generate-subtitles.py \ --silero-vad-model=/path/to/silero_vad.onnx \ @@ -68,7 +80,7 @@ /path/to/test.mp4 -(5) For WeNet CTC models +(6) For WeNet CTC models ./python-api-examples/generate-subtitles.py \ --silero-vad-model=/path/to/silero_vad.onnx \ @@ -83,6 +95,7 @@ used in this file. """ import argparse +import datetime as dt import shutil import subprocess import sys @@ -157,7 +170,7 @@ def get_args(): parser.add_argument( "--num-threads", type=int, - default=1, + default=2, help="Number of threads for neural network computation", ) @@ -208,6 +221,34 @@ def get_args(): """, ) + parser.add_argument( + "--moonshine-preprocessor", + default="", + type=str, + help="Path to moonshine preprocessor model", + ) + + parser.add_argument( + "--moonshine-encoder", + default="", + type=str, + help="Path to moonshine encoder model", + ) + + parser.add_argument( + "--moonshine-uncached-decoder", + default="", + type=str, + help="Path to moonshine uncached decoder model", + ) + + parser.add_argument( + "--moonshine-cached-decoder", + default="", + type=str, + help="Path to moonshine cached decoder model", + ) + parser.add_argument( "--decoding-method", type=str, @@ -263,6 +304,12 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: assert len(args.wenet_ctc) == 0, args.wenet_ctc assert len(args.whisper_encoder) == 0, args.whisper_encoder assert len(args.whisper_decoder) == 0, args.whisper_decoder + assert len(args.moonshine_preprocessor) == 0, args.moonshine_preprocessor + assert len(args.moonshine_encoder) == 0, args.moonshine_encoder + assert ( + len(args.moonshine_uncached_decoder) == 0 + ), args.moonshine_uncached_decoder + assert len(args.moonshine_cached_decoder) == 0, args.moonshine_cached_decoder assert_file_exists(args.encoder) assert_file_exists(args.decoder) @@ -284,6 +331,12 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: assert len(args.wenet_ctc) == 0, args.wenet_ctc assert len(args.whisper_encoder) == 0, args.whisper_encoder assert len(args.whisper_decoder) == 0, args.whisper_decoder + assert len(args.moonshine_preprocessor) == 0, args.moonshine_preprocessor + assert len(args.moonshine_encoder) == 0, args.moonshine_encoder + assert ( + len(args.moonshine_uncached_decoder) == 0 + ), args.moonshine_uncached_decoder + assert len(args.moonshine_cached_decoder) == 0, args.moonshine_cached_decoder assert_file_exists(args.paraformer) @@ -300,6 +353,12 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: assert len(args.wenet_ctc) == 0, args.wenet_ctc assert len(args.whisper_encoder) == 0, args.whisper_encoder assert len(args.whisper_decoder) == 0, args.whisper_decoder + assert len(args.moonshine_preprocessor) == 0, args.moonshine_preprocessor + assert len(args.moonshine_encoder) == 0, args.moonshine_encoder + assert ( + len(args.moonshine_uncached_decoder) == 0 + ), args.moonshine_uncached_decoder + assert len(args.moonshine_cached_decoder) == 0, args.moonshine_cached_decoder assert_file_exists(args.sense_voice) recognizer = sherpa_onnx.OfflineRecognizer.from_sense_voice( @@ -312,6 +371,12 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: elif args.wenet_ctc: assert len(args.whisper_encoder) == 0, args.whisper_encoder assert len(args.whisper_decoder) == 0, args.whisper_decoder + assert len(args.moonshine_preprocessor) == 0, args.moonshine_preprocessor + assert len(args.moonshine_encoder) == 0, args.moonshine_encoder + assert ( + len(args.moonshine_uncached_decoder) == 0 + ), args.moonshine_uncached_decoder + assert len(args.moonshine_cached_decoder) == 0, args.moonshine_cached_decoder assert_file_exists(args.wenet_ctc) @@ -327,6 +392,12 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: elif args.whisper_encoder: assert_file_exists(args.whisper_encoder) assert_file_exists(args.whisper_decoder) + assert len(args.moonshine_preprocessor) == 0, args.moonshine_preprocessor + assert len(args.moonshine_encoder) == 0, args.moonshine_encoder + assert ( + len(args.moonshine_uncached_decoder) == 0 + ), args.moonshine_uncached_decoder + assert len(args.moonshine_cached_decoder) == 0, args.moonshine_cached_decoder recognizer = sherpa_onnx.OfflineRecognizer.from_whisper( encoder=args.whisper_encoder, @@ -339,6 +410,22 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: task=args.whisper_task, tail_paddings=args.whisper_tail_paddings, ) + elif args.moonshine_preprocessor: + assert_file_exists(args.moonshine_preprocessor) + assert_file_exists(args.moonshine_encoder) + assert_file_exists(args.moonshine_uncached_decoder) + assert_file_exists(args.moonshine_cached_decoder) + + recognizer = sherpa_onnx.OfflineRecognizer.from_moonshine( + preprocessor=args.moonshine_preprocessor, + encoder=args.moonshine_encoder, + uncached_decoder=args.moonshine_uncached_decoder, + cached_decoder=args.moonshine_cached_decoder, + tokens=args.tokens, + num_threads=args.num_threads, + decoding_method=args.decoding_method, + debug=args.debug, + ) else: raise ValueError("Please specify at least one model") @@ -424,28 +511,32 @@ def main(): segment_list = [] print("Started!") + start_t = dt.datetime.now() + num_processed_samples = 0 - is_silence = False + is_eof = False # TODO(fangjun): Support multithreads while True: # *2 because int16_t has two bytes data = process.stdout.read(frames_per_read * 2) if not data: - if is_silence: + if is_eof: break - is_silence = True - # The converted audio file does not have a mute data of 1 second or more at the end, which will result in the loss of the last segment data + is_eof = True + # pad 1 second at the end of the file for the VAD data = np.zeros(1 * args.sample_rate, dtype=np.int16) samples = np.frombuffer(data, dtype=np.int16) samples = samples.astype(np.float32) / 32768 + num_processed_samples += samples.shape[0] + buffer = np.concatenate([buffer, samples]) while len(buffer) > window_size: vad.accept_waveform(buffer[:window_size]) buffer = buffer[window_size:] - if is_silence: + if is_eof: vad.flush() streams = [] @@ -471,6 +562,11 @@ def main(): seg.text = stream.result.text segment_list.append(seg) + end_t = dt.datetime.now() + elapsed_seconds = (end_t - start_t).total_seconds() + duration = num_processed_samples / 16000 + rtf = elapsed_seconds / duration + srt_filename = Path(args.sound_file).with_suffix(".srt") with open(srt_filename, "w", encoding="utf-8") as f: for i, seg in enumerate(segment_list): @@ -479,6 +575,9 @@ def main(): print("", file=f) print(f"Saved to {srt_filename}") + print(f"Audio duration:\t{duration:.3f} s") + print(f"Elapsed:\t{elapsed_seconds:.3f} s") + print(f"RTF = {elapsed_seconds:.3f}/{duration:.3f} = {rtf:.3f}") print("Done!") diff --git a/python-api-examples/non_streaming_server.py b/python-api-examples/non_streaming_server.py index 2194d6f54..3dd12564e 100755 --- a/python-api-examples/non_streaming_server.py +++ b/python-api-examples/non_streaming_server.py @@ -66,7 +66,21 @@ --wenet-ctc ./sherpa-onnx-zh-wenet-wenetspeech/model.onnx \ --tokens ./sherpa-onnx-zh-wenet-wenetspeech/tokens.txt -(5) Use a Whisper model +(5) Use a Moonshine model + +cd /path/to/sherpa-onnx +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + +python3 ./python-api-examples/non_streaming_server.py \ + --moonshine-preprocessor=./sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx \ + --moonshine-encoder=./sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx \ + --moonshine-uncached-decoder=./sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx \ + --moonshine-cached-decoder=./sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx \ + --tokens=./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt + +(6) Use a Whisper model cd /path/to/sherpa-onnx curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2 @@ -78,7 +92,7 @@ --whisper-decoder=./sherpa-onnx-whisper-tiny.en/tiny.en-decoder.onnx \ --tokens=./sherpa-onnx-whisper-tiny.en/tiny.en-tokens.txt -(5) Use a tdnn model of the yesno recipe from icefall +(7) Use a tdnn model of the yesno recipe from icefall cd /path/to/sherpa-onnx @@ -92,7 +106,7 @@ --tdnn-model=./sherpa-onnx-tdnn-yesno/model-epoch-14-avg-2.onnx \ --tokens=./sherpa-onnx-tdnn-yesno/tokens.txt -(6) Use a Non-streaming SenseVoice model +(8) Use a Non-streaming SenseVoice model curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 tar xvf sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 @@ -254,6 +268,36 @@ def add_tdnn_ctc_model_args(parser: argparse.ArgumentParser): ) +def add_moonshine_model_args(parser: argparse.ArgumentParser): + parser.add_argument( + "--moonshine-preprocessor", + default="", + type=str, + help="Path to moonshine preprocessor model", + ) + + parser.add_argument( + "--moonshine-encoder", + default="", + type=str, + help="Path to moonshine encoder model", + ) + + parser.add_argument( + "--moonshine-uncached-decoder", + default="", + type=str, + help="Path to moonshine uncached decoder model", + ) + + parser.add_argument( + "--moonshine-cached-decoder", + default="", + type=str, + help="Path to moonshine cached decoder model", + ) + + def add_whisper_model_args(parser: argparse.ArgumentParser): parser.add_argument( "--whisper-encoder", @@ -311,6 +355,7 @@ def add_model_args(parser: argparse.ArgumentParser): add_wenet_ctc_model_args(parser) add_tdnn_ctc_model_args(parser) add_whisper_model_args(parser) + add_moonshine_model_args(parser) parser.add_argument( "--tokens", @@ -876,6 +921,12 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: assert len(args.whisper_encoder) == 0, args.whisper_encoder assert len(args.whisper_decoder) == 0, args.whisper_decoder assert len(args.tdnn_model) == 0, args.tdnn_model + assert len(args.moonshine_preprocessor) == 0, args.moonshine_preprocessor + assert len(args.moonshine_encoder) == 0, args.moonshine_encoder + assert ( + len(args.moonshine_uncached_decoder) == 0 + ), args.moonshine_uncached_decoder + assert len(args.moonshine_cached_decoder) == 0, args.moonshine_cached_decoder assert_file_exists(args.encoder) assert_file_exists(args.decoder) @@ -903,6 +954,12 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: assert len(args.whisper_encoder) == 0, args.whisper_encoder assert len(args.whisper_decoder) == 0, args.whisper_decoder assert len(args.tdnn_model) == 0, args.tdnn_model + assert len(args.moonshine_preprocessor) == 0, args.moonshine_preprocessor + assert len(args.moonshine_encoder) == 0, args.moonshine_encoder + assert ( + len(args.moonshine_uncached_decoder) == 0 + ), args.moonshine_uncached_decoder + assert len(args.moonshine_cached_decoder) == 0, args.moonshine_cached_decoder assert_file_exists(args.paraformer) @@ -921,6 +978,12 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: assert len(args.whisper_encoder) == 0, args.whisper_encoder assert len(args.whisper_decoder) == 0, args.whisper_decoder assert len(args.tdnn_model) == 0, args.tdnn_model + assert len(args.moonshine_preprocessor) == 0, args.moonshine_preprocessor + assert len(args.moonshine_encoder) == 0, args.moonshine_encoder + assert ( + len(args.moonshine_uncached_decoder) == 0 + ), args.moonshine_uncached_decoder + assert len(args.moonshine_cached_decoder) == 0, args.moonshine_cached_decoder assert_file_exists(args.sense_voice) recognizer = sherpa_onnx.OfflineRecognizer.from_sense_voice( @@ -934,6 +997,12 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: assert len(args.whisper_encoder) == 0, args.whisper_encoder assert len(args.whisper_decoder) == 0, args.whisper_decoder assert len(args.tdnn_model) == 0, args.tdnn_model + assert len(args.moonshine_preprocessor) == 0, args.moonshine_preprocessor + assert len(args.moonshine_encoder) == 0, args.moonshine_encoder + assert ( + len(args.moonshine_uncached_decoder) == 0 + ), args.moonshine_uncached_decoder + assert len(args.moonshine_cached_decoder) == 0, args.moonshine_cached_decoder assert_file_exists(args.nemo_ctc) @@ -950,6 +1019,12 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: assert len(args.whisper_encoder) == 0, args.whisper_encoder assert len(args.whisper_decoder) == 0, args.whisper_decoder assert len(args.tdnn_model) == 0, args.tdnn_model + assert len(args.moonshine_preprocessor) == 0, args.moonshine_preprocessor + assert len(args.moonshine_encoder) == 0, args.moonshine_encoder + assert ( + len(args.moonshine_uncached_decoder) == 0 + ), args.moonshine_uncached_decoder + assert len(args.moonshine_cached_decoder) == 0, args.moonshine_cached_decoder assert_file_exists(args.wenet_ctc) @@ -966,6 +1041,12 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: assert len(args.tdnn_model) == 0, args.tdnn_model assert_file_exists(args.whisper_encoder) assert_file_exists(args.whisper_decoder) + assert len(args.moonshine_preprocessor) == 0, args.moonshine_preprocessor + assert len(args.moonshine_encoder) == 0, args.moonshine_encoder + assert ( + len(args.moonshine_uncached_decoder) == 0 + ), args.moonshine_uncached_decoder + assert len(args.moonshine_cached_decoder) == 0, args.moonshine_cached_decoder recognizer = sherpa_onnx.OfflineRecognizer.from_whisper( encoder=args.whisper_encoder, @@ -980,6 +1061,12 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: ) elif args.tdnn_model: assert_file_exists(args.tdnn_model) + assert len(args.moonshine_preprocessor) == 0, args.moonshine_preprocessor + assert len(args.moonshine_encoder) == 0, args.moonshine_encoder + assert ( + len(args.moonshine_uncached_decoder) == 0 + ), args.moonshine_uncached_decoder + assert len(args.moonshine_cached_decoder) == 0, args.moonshine_cached_decoder recognizer = sherpa_onnx.OfflineRecognizer.from_tdnn_ctc( model=args.tdnn_model, @@ -990,6 +1077,21 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: decoding_method=args.decoding_method, provider=args.provider, ) + elif args.moonshine_preprocessor: + assert_file_exists(args.moonshine_preprocessor) + assert_file_exists(args.moonshine_encoder) + assert_file_exists(args.moonshine_uncached_decoder) + assert_file_exists(args.moonshine_cached_decoder) + + recognizer = sherpa_onnx.OfflineRecognizer.from_moonshine( + preprocessor=args.moonshine_preprocessor, + encoder=args.moonshine_encoder, + uncached_decoder=args.moonshine_uncached_decoder, + cached_decoder=args.moonshine_cached_decoder, + tokens=args.tokens, + num_threads=args.num_threads, + decoding_method=args.decoding_method, + ) else: raise ValueError("Please specify at least one model") diff --git a/python-api-examples/offline-moonshine-decode-files.py b/python-api-examples/offline-moonshine-decode-files.py new file mode 100644 index 000000000..f4d153d87 --- /dev/null +++ b/python-api-examples/offline-moonshine-decode-files.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 + +""" +This file shows how to use a non-streaming Moonshine model from +https://github.com/usefulsensors/moonshine +to decode files. + +Please download model files from +https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models + +For instance, + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +""" + +import datetime as dt +from pathlib import Path + +import sherpa_onnx +import soundfile as sf + + +def create_recognizer(): + preprocessor = "./sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx" + encoder = "./sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx" + uncached_decoder = "./sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx" + cached_decoder = "./sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx" + + tokens = "./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt" + test_wav = "./sherpa-onnx-moonshine-tiny-en-int8/test_wavs/0.wav" + + if not Path(preprocessor).is_file() or not Path(test_wav).is_file(): + raise ValueError( + """Please download model files from + https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models + """ + ) + return ( + sherpa_onnx.OfflineRecognizer.from_moonshine( + preprocessor=preprocessor, + encoder=encoder, + uncached_decoder=uncached_decoder, + cached_decoder=cached_decoder, + tokens=tokens, + debug=True, + ), + test_wav, + ) + + +def main(): + recognizer, wave_filename = create_recognizer() + + audio, sample_rate = sf.read(wave_filename, dtype="float32", always_2d=True) + audio = audio[:, 0] # only use the first channel + + # audio is a 1-D float32 numpy array normalized to the range [-1, 1] + # sample_rate does not need to be 16000 Hz + + start_t = dt.datetime.now() + + stream = recognizer.create_stream() + stream.accept_waveform(sample_rate, audio) + recognizer.decode_stream(stream) + + end_t = dt.datetime.now() + elapsed_seconds = (end_t - start_t).total_seconds() + duration = audio.shape[-1] / sample_rate + rtf = elapsed_seconds / duration + + print(stream.result) + print(wave_filename) + print("Text:", stream.result.text) + print(f"Audio duration:\t{duration:.3f} s") + print(f"Elapsed:\t{elapsed_seconds:.3f} s") + print(f"RTF = {elapsed_seconds:.3f}/{duration:.3f} = {rtf:.3f}") + + +if __name__ == "__main__": + main() diff --git a/python-api-examples/offline-whisper-decode-files.py b/python-api-examples/offline-whisper-decode-files.py new file mode 100644 index 000000000..aa85ccdf9 --- /dev/null +++ b/python-api-examples/offline-whisper-decode-files.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 + +""" +This file shows how to use a non-streaming whisper model from +https://github.com/openai/whisper +to decode files. + +Please download model files from +https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models + +For instance, + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2 +tar xvf sherpa-onnx-whisper-tiny.en.tar.bz2 +rm sherpa-onnx-whisper-tiny.en.tar.bz2 +""" + +import datetime as dt +from pathlib import Path + +import sherpa_onnx +import soundfile as sf + + +def create_recognizer(): + encoder = "./sherpa-onnx-whisper-tiny.en/tiny.en-encoder.int8.onnx" + decoder = "./sherpa-onnx-whisper-tiny.en/tiny.en-decoder.int8.onnx" + tokens = "./sherpa-onnx-whisper-tiny.en/tiny.en-tokens.txt" + test_wav = "./sherpa-onnx-whisper-tiny.en/test_wavs/0.wav" + + if not Path(encoder).is_file() or not Path(test_wav).is_file(): + raise ValueError( + """Please download model files from + https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models + """ + ) + return ( + sherpa_onnx.OfflineRecognizer.from_whisper( + encoder=encoder, + decoder=decoder, + tokens=tokens, + debug=True, + ), + test_wav, + ) + + +def main(): + recognizer, wave_filename = create_recognizer() + + audio, sample_rate = sf.read(wave_filename, dtype="float32", always_2d=True) + audio = audio[:, 0] # only use the first channel + + # audio is a 1-D float32 numpy array normalized to the range [-1, 1] + # sample_rate does not need to be 16000 Hz + + start_t = dt.datetime.now() + + stream = recognizer.create_stream() + stream.accept_waveform(sample_rate, audio) + recognizer.decode_stream(stream) + + end_t = dt.datetime.now() + elapsed_seconds = (end_t - start_t).total_seconds() + duration = audio.shape[-1] / sample_rate + rtf = elapsed_seconds / duration + + print(stream.result) + print(wave_filename) + print("Text:", stream.result.text) + print(f"Audio duration:\t{duration:.3f} s") + print(f"Elapsed:\t{elapsed_seconds:.3f} s") + print(f"RTF = {elapsed_seconds:.3f}/{duration:.3f} = {rtf:.3f}") + + +if __name__ == "__main__": + main() diff --git a/python-api-examples/vad-with-non-streaming-asr.py b/python-api-examples/vad-with-non-streaming-asr.py index 7bb125d1a..f5bde30c6 100755 --- a/python-api-examples/vad-with-non-streaming-asr.py +++ b/python-api-examples/vad-with-non-streaming-asr.py @@ -35,7 +35,18 @@ --sample-rate=16000 \ --feature-dim=80 -(3) For Whisper models +(3) For Moonshine models + +./python-api-examples/vad-with-non-streaming-asr.py \ + --silero-vad-model=/path/to/silero_vad.onnx \ + --moonshine-preprocessor=./sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx \ + --moonshine-encoder=./sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx \ + --moonshine-uncached-decoder=./sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx \ + --moonshine-cached-decoder=./sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx \ + --tokens=./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt \ + --num-threads=2 + +(4) For Whisper models ./python-api-examples/vad-with-non-streaming-asr.py \ --silero-vad-model=/path/to/silero_vad.onnx \ @@ -45,7 +56,7 @@ --whisper-task=transcribe \ --num-threads=2 -(4) For SenseVoice CTC models +(5) For SenseVoice CTC models ./python-api-examples/vad-with-non-streaming-asr.py \ --silero-vad-model=/path/to/silero_vad.onnx \ @@ -192,6 +203,34 @@ def get_args(): """, ) + parser.add_argument( + "--moonshine-preprocessor", + default="", + type=str, + help="Path to moonshine preprocessor model", + ) + + parser.add_argument( + "--moonshine-encoder", + default="", + type=str, + help="Path to moonshine encoder model", + ) + + parser.add_argument( + "--moonshine-uncached-decoder", + default="", + type=str, + help="Path to moonshine uncached decoder model", + ) + + parser.add_argument( + "--moonshine-cached-decoder", + default="", + type=str, + help="Path to moonshine cached decoder model", + ) + parser.add_argument( "--blank-penalty", type=float, @@ -251,6 +290,12 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: assert len(args.sense_voice) == 0, args.sense_voice assert len(args.whisper_encoder) == 0, args.whisper_encoder assert len(args.whisper_decoder) == 0, args.whisper_decoder + assert len(args.moonshine_preprocessor) == 0, args.moonshine_preprocessor + assert len(args.moonshine_encoder) == 0, args.moonshine_encoder + assert ( + len(args.moonshine_uncached_decoder) == 0 + ), args.moonshine_uncached_decoder + assert len(args.moonshine_cached_decoder) == 0, args.moonshine_cached_decoder assert_file_exists(args.encoder) assert_file_exists(args.decoder) @@ -272,6 +317,12 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: assert len(args.sense_voice) == 0, args.sense_voice assert len(args.whisper_encoder) == 0, args.whisper_encoder assert len(args.whisper_decoder) == 0, args.whisper_decoder + assert len(args.moonshine_preprocessor) == 0, args.moonshine_preprocessor + assert len(args.moonshine_encoder) == 0, args.moonshine_encoder + assert ( + len(args.moonshine_uncached_decoder) == 0 + ), args.moonshine_uncached_decoder + assert len(args.moonshine_cached_decoder) == 0, args.moonshine_cached_decoder assert_file_exists(args.paraformer) @@ -287,6 +338,12 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: elif args.sense_voice: assert len(args.whisper_encoder) == 0, args.whisper_encoder assert len(args.whisper_decoder) == 0, args.whisper_decoder + assert len(args.moonshine_preprocessor) == 0, args.moonshine_preprocessor + assert len(args.moonshine_encoder) == 0, args.moonshine_encoder + assert ( + len(args.moonshine_uncached_decoder) == 0 + ), args.moonshine_uncached_decoder + assert len(args.moonshine_cached_decoder) == 0, args.moonshine_cached_decoder assert_file_exists(args.sense_voice) recognizer = sherpa_onnx.OfflineRecognizer.from_sense_voice( @@ -299,6 +356,12 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: elif args.whisper_encoder: assert_file_exists(args.whisper_encoder) assert_file_exists(args.whisper_decoder) + assert len(args.moonshine_preprocessor) == 0, args.moonshine_preprocessor + assert len(args.moonshine_encoder) == 0, args.moonshine_encoder + assert ( + len(args.moonshine_uncached_decoder) == 0 + ), args.moonshine_uncached_decoder + assert len(args.moonshine_cached_decoder) == 0, args.moonshine_cached_decoder recognizer = sherpa_onnx.OfflineRecognizer.from_whisper( encoder=args.whisper_encoder, @@ -311,6 +374,22 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: task=args.whisper_task, tail_paddings=args.whisper_tail_paddings, ) + elif args.moonshine_preprocessor: + assert_file_exists(args.moonshine_preprocessor) + assert_file_exists(args.moonshine_encoder) + assert_file_exists(args.moonshine_uncached_decoder) + assert_file_exists(args.moonshine_cached_decoder) + + recognizer = sherpa_onnx.OfflineRecognizer.from_moonshine( + preprocessor=args.moonshine_preprocessor, + encoder=args.moonshine_encoder, + uncached_decoder=args.moonshine_uncached_decoder, + cached_decoder=args.moonshine_cached_decoder, + tokens=args.tokens, + num_threads=args.num_threads, + decoding_method=args.decoding_method, + debug=args.debug, + ) else: raise ValueError("Please specify at least one model") diff --git a/sherpa-onnx/csrc/CMakeLists.txt b/sherpa-onnx/csrc/CMakeLists.txt index fafe5de96..f34c78609 100644 --- a/sherpa-onnx/csrc/CMakeLists.txt +++ b/sherpa-onnx/csrc/CMakeLists.txt @@ -29,6 +29,9 @@ set(sources offline-lm-config.cc offline-lm.cc offline-model-config.cc + offline-moonshine-greedy-search-decoder.cc + offline-moonshine-model-config.cc + offline-moonshine-model.cc offline-nemo-enc-dec-ctc-model-config.cc offline-nemo-enc-dec-ctc-model.cc offline-paraformer-greedy-search-decoder.cc diff --git a/sherpa-onnx/csrc/offline-model-config.cc b/sherpa-onnx/csrc/offline-model-config.cc index 862e4a60c..787290327 100644 --- a/sherpa-onnx/csrc/offline-model-config.cc +++ b/sherpa-onnx/csrc/offline-model-config.cc @@ -19,6 +19,7 @@ void OfflineModelConfig::Register(ParseOptions *po) { zipformer_ctc.Register(po); wenet_ctc.Register(po); sense_voice.Register(po); + moonshine.Register(po); po->Register("telespeech-ctc", &telespeech_ctc, "Path to model.onnx for telespeech ctc"); @@ -99,6 +100,10 @@ bool OfflineModelConfig::Validate() const { return sense_voice.Validate(); } + if (!moonshine.preprocessor.empty()) { + return moonshine.Validate(); + } + if (!telespeech_ctc.empty() && !FileExists(telespeech_ctc)) { SHERPA_ONNX_LOGE("telespeech_ctc: '%s' does not exist", telespeech_ctc.c_str()); @@ -124,6 +129,7 @@ std::string OfflineModelConfig::ToString() const { os << "zipformer_ctc=" << zipformer_ctc.ToString() << ", "; os << "wenet_ctc=" << wenet_ctc.ToString() << ", "; os << "sense_voice=" << sense_voice.ToString() << ", "; + os << "moonshine=" << moonshine.ToString() << ", "; os << "telespeech_ctc=\"" << telespeech_ctc << "\", "; os << "tokens=\"" << tokens << "\", "; os << "num_threads=" << num_threads << ", "; diff --git a/sherpa-onnx/csrc/offline-model-config.h b/sherpa-onnx/csrc/offline-model-config.h index 8eb725e4e..cfff5eed2 100644 --- a/sherpa-onnx/csrc/offline-model-config.h +++ b/sherpa-onnx/csrc/offline-model-config.h @@ -6,6 +6,7 @@ #include +#include "sherpa-onnx/csrc/offline-moonshine-model-config.h" #include "sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model-config.h" #include "sherpa-onnx/csrc/offline-paraformer-model-config.h" #include "sherpa-onnx/csrc/offline-sense-voice-model-config.h" @@ -26,6 +27,7 @@ struct OfflineModelConfig { OfflineZipformerCtcModelConfig zipformer_ctc; OfflineWenetCtcModelConfig wenet_ctc; OfflineSenseVoiceModelConfig sense_voice; + OfflineMoonshineModelConfig moonshine; std::string telespeech_ctc; std::string tokens; @@ -56,6 +58,7 @@ struct OfflineModelConfig { const OfflineZipformerCtcModelConfig &zipformer_ctc, const OfflineWenetCtcModelConfig &wenet_ctc, const OfflineSenseVoiceModelConfig &sense_voice, + const OfflineMoonshineModelConfig &moonshine, const std::string &telespeech_ctc, const std::string &tokens, int32_t num_threads, bool debug, const std::string &provider, const std::string &model_type, @@ -69,6 +72,7 @@ struct OfflineModelConfig { zipformer_ctc(zipformer_ctc), wenet_ctc(wenet_ctc), sense_voice(sense_voice), + moonshine(moonshine), telespeech_ctc(telespeech_ctc), tokens(tokens), num_threads(num_threads), diff --git a/sherpa-onnx/csrc/offline-moonshine-decoder.h b/sherpa-onnx/csrc/offline-moonshine-decoder.h new file mode 100644 index 000000000..4d0b9ac93 --- /dev/null +++ b/sherpa-onnx/csrc/offline-moonshine-decoder.h @@ -0,0 +1,34 @@ +// sherpa-onnx/csrc/offline-moonshine-decoder.h +// +// Copyright (c) 2023 Xiaomi Corporation + +#ifndef SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_DECODER_H_ +#define SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_DECODER_H_ + +#include + +#include "onnxruntime_cxx_api.h" // NOLINT + +namespace sherpa_onnx { + +struct OfflineMoonshineDecoderResult { + /// The decoded token IDs + std::vector tokens; +}; + +class OfflineMoonshineDecoder { + public: + virtual ~OfflineMoonshineDecoder() = default; + + /** Run beam search given the output from the moonshine encoder model. + * + * @param encoder_out A 3-D tensor of shape (batch_size, T, dim) + * @return Return a vector of size `N` containing the decoded results. + */ + virtual std::vector Decode( + Ort::Value encoder_out) = 0; +}; + +} // namespace sherpa_onnx + +#endif // SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_DECODER_H_ diff --git a/sherpa-onnx/csrc/offline-moonshine-greedy-search-decoder.cc b/sherpa-onnx/csrc/offline-moonshine-greedy-search-decoder.cc new file mode 100644 index 000000000..603bdd0cf --- /dev/null +++ b/sherpa-onnx/csrc/offline-moonshine-greedy-search-decoder.cc @@ -0,0 +1,87 @@ +// sherpa-onnx/csrc/offline-moonshine-greedy-search-decoder.cc +// +// Copyright (c) 2023 Xiaomi Corporation + +#include "sherpa-onnx/csrc/offline-moonshine-greedy-search-decoder.h" + +#include +#include + +#include "sherpa-onnx/csrc/macros.h" +#include "sherpa-onnx/csrc/onnx-utils.h" + +namespace sherpa_onnx { + +std::vector +OfflineMoonshineGreedySearchDecoder::Decode(Ort::Value encoder_out) { + auto encoder_out_shape = encoder_out.GetTensorTypeAndShapeInfo().GetShape(); + if (encoder_out_shape[0] != 1) { + SHERPA_ONNX_LOGE("Support only batch size == 1. Given: %d\n", + static_cast(encoder_out_shape[0])); + return {}; + } + + auto memory_info = + Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault); + + // encoder_out_shape[1] * 384 is the number of audio samples + // 16000 is the sample rate + // + // + // 384 is from the moonshine paper + int32_t max_len = + static_cast(encoder_out_shape[1] * 384 / 16000.0 * 6); + + int32_t sos = 1; + int32_t eos = 2; + int32_t seq_len = 1; + + std::vector tokens; + + std::array token_shape = {1, 1}; + int64_t seq_len_shape = 1; + + Ort::Value token_tensor = Ort::Value::CreateTensor( + memory_info, &sos, 1, token_shape.data(), token_shape.size()); + + Ort::Value seq_len_tensor = + Ort::Value::CreateTensor(memory_info, &seq_len, 1, &seq_len_shape, 1); + + Ort::Value logits{nullptr}; + std::vector states; + + std::tie(logits, states) = model_->ForwardUnCachedDecoder( + std::move(token_tensor), std::move(seq_len_tensor), View(&encoder_out)); + + int32_t vocab_size = logits.GetTensorTypeAndShapeInfo().GetShape()[2]; + + for (int32_t i = 0; i != max_len; ++i) { + const float *p = logits.GetTensorData(); + + int32_t max_token_id = static_cast( + std::distance(p, std::max_element(p, p + vocab_size))); + if (max_token_id == eos) { + break; + } + tokens.push_back(max_token_id); + + seq_len += 1; + + token_tensor = Ort::Value::CreateTensor( + memory_info, &tokens.back(), 1, token_shape.data(), token_shape.size()); + + seq_len_tensor = + Ort::Value::CreateTensor(memory_info, &seq_len, 1, &seq_len_shape, 1); + + std::tie(logits, states) = model_->ForwardCachedDecoder( + std::move(token_tensor), std::move(seq_len_tensor), View(&encoder_out), + std::move(states)); + } + + OfflineMoonshineDecoderResult ans; + ans.tokens = std::move(tokens); + + return {ans}; +} + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-moonshine-greedy-search-decoder.h b/sherpa-onnx/csrc/offline-moonshine-greedy-search-decoder.h new file mode 100644 index 000000000..b215405db --- /dev/null +++ b/sherpa-onnx/csrc/offline-moonshine-greedy-search-decoder.h @@ -0,0 +1,29 @@ +// sherpa-onnx/csrc/offline-moonshine-greedy-search-decoder.h +// +// Copyright (c) 2024 Xiaomi Corporation + +#ifndef SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_GREEDY_SEARCH_DECODER_H_ +#define SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_GREEDY_SEARCH_DECODER_H_ + +#include + +#include "sherpa-onnx/csrc/offline-moonshine-decoder.h" +#include "sherpa-onnx/csrc/offline-moonshine-model.h" + +namespace sherpa_onnx { + +class OfflineMoonshineGreedySearchDecoder : public OfflineMoonshineDecoder { + public: + explicit OfflineMoonshineGreedySearchDecoder(OfflineMoonshineModel *model) + : model_(model) {} + + std::vector Decode( + Ort::Value encoder_out) override; + + private: + OfflineMoonshineModel *model_; // not owned +}; + +} // namespace sherpa_onnx + +#endif // SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_GREEDY_SEARCH_DECODER_H_ diff --git a/sherpa-onnx/csrc/offline-moonshine-model-config.cc b/sherpa-onnx/csrc/offline-moonshine-model-config.cc new file mode 100644 index 000000000..c687507e3 --- /dev/null +++ b/sherpa-onnx/csrc/offline-moonshine-model-config.cc @@ -0,0 +1,88 @@ +// sherpa-onnx/csrc/offline-moonshine-model-config.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include "sherpa-onnx/csrc/offline-moonshine-model-config.h" + +#include "sherpa-onnx/csrc/file-utils.h" +#include "sherpa-onnx/csrc/macros.h" + +namespace sherpa_onnx { + +void OfflineMoonshineModelConfig::Register(ParseOptions *po) { + po->Register("moonshine-preprocessor", &preprocessor, + "Path to onnx preprocessor of moonshine, e.g., preprocess.onnx"); + + po->Register("moonshine-encoder", &encoder, + "Path to onnx encoder of moonshine, e.g., encode.onnx"); + + po->Register( + "moonshine-uncached-decoder", &uncached_decoder, + "Path to onnx uncached_decoder of moonshine, e.g., uncached_decode.onnx"); + + po->Register( + "moonshine-cached-decoder", &cached_decoder, + "Path to onnx cached_decoder of moonshine, e.g., cached_decode.onnx"); +} + +bool OfflineMoonshineModelConfig::Validate() const { + if (preprocessor.empty()) { + SHERPA_ONNX_LOGE("Please provide --moonshine-preprocessor"); + return false; + } + + if (!FileExists(preprocessor)) { + SHERPA_ONNX_LOGE("moonshine preprocessor file '%s' does not exist", + preprocessor.c_str()); + return false; + } + + if (encoder.empty()) { + SHERPA_ONNX_LOGE("Please provide --moonshine-encoder"); + return false; + } + + if (!FileExists(encoder)) { + SHERPA_ONNX_LOGE("moonshine encoder file '%s' does not exist", + encoder.c_str()); + return false; + } + + if (uncached_decoder.empty()) { + SHERPA_ONNX_LOGE("Please provide --moonshine-uncached-decoder"); + return false; + } + + if (!FileExists(uncached_decoder)) { + SHERPA_ONNX_LOGE("moonshine uncached decoder file '%s' does not exist", + uncached_decoder.c_str()); + return false; + } + + if (cached_decoder.empty()) { + SHERPA_ONNX_LOGE("Please provide --moonshine-cached-decoder"); + return false; + } + + if (!FileExists(cached_decoder)) { + SHERPA_ONNX_LOGE("moonshine cached decoder file '%s' does not exist", + cached_decoder.c_str()); + return false; + } + + return true; +} + +std::string OfflineMoonshineModelConfig::ToString() const { + std::ostringstream os; + + os << "OfflineMoonshineModelConfig("; + os << "preprocessor=\"" << preprocessor << "\", "; + os << "encoder=\"" << encoder << "\", "; + os << "uncached_decoder=\"" << uncached_decoder << "\", "; + os << "cached_decoder=\"" << cached_decoder << "\")"; + + return os.str(); +} + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-moonshine-model-config.h b/sherpa-onnx/csrc/offline-moonshine-model-config.h new file mode 100644 index 000000000..829ca520d --- /dev/null +++ b/sherpa-onnx/csrc/offline-moonshine-model-config.h @@ -0,0 +1,37 @@ +// sherpa-onnx/csrc/offline-moonshine-model-config.h +// +// Copyright (c) 2024 Xiaomi Corporation +#ifndef SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_MODEL_CONFIG_H_ +#define SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_MODEL_CONFIG_H_ + +#include + +#include "sherpa-onnx/csrc/parse-options.h" + +namespace sherpa_onnx { + +struct OfflineMoonshineModelConfig { + std::string preprocessor; + std::string encoder; + std::string uncached_decoder; + std::string cached_decoder; + + OfflineMoonshineModelConfig() = default; + OfflineMoonshineModelConfig(const std::string &preprocessor, + const std::string &encoder, + const std::string &uncached_decoder, + const std::string &cached_decoder) + : preprocessor(preprocessor), + encoder(encoder), + uncached_decoder(uncached_decoder), + cached_decoder(cached_decoder) {} + + void Register(ParseOptions *po); + bool Validate() const; + + std::string ToString() const; +}; + +} // namespace sherpa_onnx + +#endif // SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_MODEL_CONFIG_H_ diff --git a/sherpa-onnx/csrc/offline-moonshine-model.cc b/sherpa-onnx/csrc/offline-moonshine-model.cc new file mode 100644 index 000000000..ab71d000f --- /dev/null +++ b/sherpa-onnx/csrc/offline-moonshine-model.cc @@ -0,0 +1,282 @@ +// sherpa-onnx/csrc/offline-moonshine-model.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include "sherpa-onnx/csrc/offline-moonshine-model.h" + +#include +#include +#include + +#include "sherpa-onnx/csrc/macros.h" +#include "sherpa-onnx/csrc/onnx-utils.h" +#include "sherpa-onnx/csrc/session.h" +#include "sherpa-onnx/csrc/text-utils.h" + +namespace sherpa_onnx { + +class OfflineMoonshineModel::Impl { + public: + explicit Impl(const OfflineModelConfig &config) + : config_(config), + env_(ORT_LOGGING_LEVEL_ERROR), + sess_opts_(GetSessionOptions(config)), + allocator_{} { + { + auto buf = ReadFile(config.moonshine.preprocessor); + InitPreprocessor(buf.data(), buf.size()); + } + + { + auto buf = ReadFile(config.moonshine.encoder); + InitEncoder(buf.data(), buf.size()); + } + + { + auto buf = ReadFile(config.moonshine.uncached_decoder); + InitUnCachedDecoder(buf.data(), buf.size()); + } + + { + auto buf = ReadFile(config.moonshine.cached_decoder); + InitCachedDecoder(buf.data(), buf.size()); + } + } + +#if __ANDROID_API__ >= 9 + Impl(AAssetManager *mgr, const OfflineModelConfig &config) + : config_(config), + env_(ORT_LOGGING_LEVEL_ERROR), + sess_opts_(GetSessionOptions(config)), + allocator_{} { + { + auto buf = ReadFile(mgr, config.moonshine.preprocessor); + InitPreprocessor(buf.data(), buf.size()); + } + + { + auto buf = ReadFile(mgr, config.moonshine.encoder); + InitEncoder(buf.data(), buf.size()); + } + + { + auto buf = ReadFile(mgr, config.moonshine.uncached_decoder); + InitUnCachedDecoder(buf.data(), buf.size()); + } + + { + auto buf = ReadFile(mgr, config.moonshine.cached_decoder); + InitCachedDecoder(buf.data(), buf.size()); + } + } +#endif + + Ort::Value ForwardPreprocessor(Ort::Value audio) { + auto features = preprocessor_sess_->Run( + {}, preprocessor_input_names_ptr_.data(), &audio, 1, + preprocessor_output_names_ptr_.data(), + preprocessor_output_names_ptr_.size()); + + return std::move(features[0]); + } + + Ort::Value ForwardEncoder(Ort::Value features, Ort::Value features_len) { + std::array encoder_inputs{std::move(features), + std::move(features_len)}; + auto encoder_out = encoder_sess_->Run( + {}, encoder_input_names_ptr_.data(), encoder_inputs.data(), + encoder_inputs.size(), encoder_output_names_ptr_.data(), + encoder_output_names_ptr_.size()); + + return std::move(encoder_out[0]); + } + + std::pair> ForwardUnCachedDecoder( + Ort::Value tokens, Ort::Value seq_len, Ort::Value encoder_out) { + std::array uncached_decoder_input = { + std::move(tokens), + std::move(encoder_out), + std::move(seq_len), + }; + + auto uncached_decoder_out = uncached_decoder_sess_->Run( + {}, uncached_decoder_input_names_ptr_.data(), + uncached_decoder_input.data(), uncached_decoder_input.size(), + uncached_decoder_output_names_ptr_.data(), + uncached_decoder_output_names_ptr_.size()); + + std::vector states; + states.reserve(uncached_decoder_out.size() - 1); + + int32_t i = -1; + for (auto &s : uncached_decoder_out) { + ++i; + if (i == 0) { + continue; + } + + states.push_back(std::move(s)); + } + + return {std::move(uncached_decoder_out[0]), std::move(states)}; + } + + std::pair> ForwardCachedDecoder( + Ort::Value tokens, Ort::Value seq_len, Ort::Value encoder_out, + std::vector states) { + std::vector cached_decoder_input; + cached_decoder_input.reserve(3 + states.size()); + cached_decoder_input.push_back(std::move(tokens)); + cached_decoder_input.push_back(std::move(encoder_out)); + cached_decoder_input.push_back(std::move(seq_len)); + + for (auto &s : states) { + cached_decoder_input.push_back(std::move(s)); + } + + auto cached_decoder_out = cached_decoder_sess_->Run( + {}, cached_decoder_input_names_ptr_.data(), cached_decoder_input.data(), + cached_decoder_input.size(), cached_decoder_output_names_ptr_.data(), + cached_decoder_output_names_ptr_.size()); + + std::vector next_states; + next_states.reserve(cached_decoder_out.size() - 1); + + int32_t i = -1; + for (auto &s : cached_decoder_out) { + ++i; + if (i == 0) { + continue; + } + + next_states.push_back(std::move(s)); + } + + return {std::move(cached_decoder_out[0]), std::move(next_states)}; + } + + OrtAllocator *Allocator() const { return allocator_; } + + private: + void InitPreprocessor(void *model_data, size_t model_data_length) { + preprocessor_sess_ = std::make_unique( + env_, model_data, model_data_length, sess_opts_); + + GetInputNames(preprocessor_sess_.get(), &preprocessor_input_names_, + &preprocessor_input_names_ptr_); + + GetOutputNames(preprocessor_sess_.get(), &preprocessor_output_names_, + &preprocessor_output_names_ptr_); + } + + void InitEncoder(void *model_data, size_t model_data_length) { + encoder_sess_ = std::make_unique( + env_, model_data, model_data_length, sess_opts_); + + GetInputNames(encoder_sess_.get(), &encoder_input_names_, + &encoder_input_names_ptr_); + + GetOutputNames(encoder_sess_.get(), &encoder_output_names_, + &encoder_output_names_ptr_); + } + + void InitUnCachedDecoder(void *model_data, size_t model_data_length) { + uncached_decoder_sess_ = std::make_unique( + env_, model_data, model_data_length, sess_opts_); + + GetInputNames(uncached_decoder_sess_.get(), &uncached_decoder_input_names_, + &uncached_decoder_input_names_ptr_); + + GetOutputNames(uncached_decoder_sess_.get(), + &uncached_decoder_output_names_, + &uncached_decoder_output_names_ptr_); + } + + void InitCachedDecoder(void *model_data, size_t model_data_length) { + cached_decoder_sess_ = std::make_unique( + env_, model_data, model_data_length, sess_opts_); + + GetInputNames(cached_decoder_sess_.get(), &cached_decoder_input_names_, + &cached_decoder_input_names_ptr_); + + GetOutputNames(cached_decoder_sess_.get(), &cached_decoder_output_names_, + &cached_decoder_output_names_ptr_); + } + + private: + OfflineModelConfig config_; + Ort::Env env_; + Ort::SessionOptions sess_opts_; + Ort::AllocatorWithDefaultOptions allocator_; + + std::unique_ptr preprocessor_sess_; + std::unique_ptr encoder_sess_; + std::unique_ptr uncached_decoder_sess_; + std::unique_ptr cached_decoder_sess_; + + std::vector preprocessor_input_names_; + std::vector preprocessor_input_names_ptr_; + + std::vector preprocessor_output_names_; + std::vector preprocessor_output_names_ptr_; + + std::vector encoder_input_names_; + std::vector encoder_input_names_ptr_; + + std::vector encoder_output_names_; + std::vector encoder_output_names_ptr_; + + std::vector uncached_decoder_input_names_; + std::vector uncached_decoder_input_names_ptr_; + + std::vector uncached_decoder_output_names_; + std::vector uncached_decoder_output_names_ptr_; + + std::vector cached_decoder_input_names_; + std::vector cached_decoder_input_names_ptr_; + + std::vector cached_decoder_output_names_; + std::vector cached_decoder_output_names_ptr_; +}; + +OfflineMoonshineModel::OfflineMoonshineModel(const OfflineModelConfig &config) + : impl_(std::make_unique(config)) {} + +#if __ANDROID_API__ >= 9 +OfflineMoonshineModel::OfflineMoonshineModel(AAssetManager *mgr, + const OfflineModelConfig &config) + : impl_(std::make_unique(mgr, config)) {} +#endif + +OfflineMoonshineModel::~OfflineMoonshineModel() = default; + +Ort::Value OfflineMoonshineModel::ForwardPreprocessor(Ort::Value audio) const { + return impl_->ForwardPreprocessor(std::move(audio)); +} + +Ort::Value OfflineMoonshineModel::ForwardEncoder( + Ort::Value features, Ort::Value features_len) const { + return impl_->ForwardEncoder(std::move(features), std::move(features_len)); +} + +std::pair> +OfflineMoonshineModel::ForwardUnCachedDecoder(Ort::Value token, + Ort::Value seq_len, + Ort::Value encoder_out) const { + return impl_->ForwardUnCachedDecoder(std::move(token), std::move(seq_len), + std::move(encoder_out)); +} + +std::pair> +OfflineMoonshineModel::ForwardCachedDecoder( + Ort::Value token, Ort::Value seq_len, Ort::Value encoder_out, + std::vector states) const { + return impl_->ForwardCachedDecoder(std::move(token), std::move(seq_len), + std::move(encoder_out), std::move(states)); +} + +OrtAllocator *OfflineMoonshineModel::Allocator() const { + return impl_->Allocator(); +} + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-moonshine-model.h b/sherpa-onnx/csrc/offline-moonshine-model.h new file mode 100644 index 000000000..7065b1445 --- /dev/null +++ b/sherpa-onnx/csrc/offline-moonshine-model.h @@ -0,0 +1,93 @@ +// sherpa-onnx/csrc/offline-moonshine-model.h +// +// Copyright (c) 2024 Xiaomi Corporation +#ifndef SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_MODEL_H_ +#define SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_MODEL_H_ + +#include +#include +#include +#include + +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#include "onnxruntime_cxx_api.h" // NOLINT +#include "sherpa-onnx/csrc/offline-model-config.h" + +namespace sherpa_onnx { + +// please see +// https://github.com/k2-fsa/sherpa-onnx/blob/master/scripts/moonshine/test.py +class OfflineMoonshineModel { + public: + explicit OfflineMoonshineModel(const OfflineModelConfig &config); + +#if __ANDROID_API__ >= 9 + OfflineMoonshineModel(AAssetManager *mgr, const OfflineModelConfig &config); +#endif + + ~OfflineMoonshineModel(); + + /** Run the preprocessor model. + * + * @param audio A float32 tensor of shape (batch_size, num_samples) + * + * @return Return a float32 tensor of shape (batch_size, T, dim) that + * can be used as the input of ForwardEncoder() + */ + Ort::Value ForwardPreprocessor(Ort::Value audio) const; + + /** Run the encoder model. + * + * @param features A float32 tensor of shape (batch_size, T, dim) + * @param features_len A int32 tensor of shape (batch_size,) + * @returns A float32 tensor of shape (batch_size, T, dim). + */ + Ort::Value ForwardEncoder(Ort::Value features, Ort::Value features_len) const; + + /** Run the uncached decoder. + * + * @param token A int32 tensor of shape (batch_size, num_tokens) + * @param seq_len A int32 tensor of shape (batch_size,) containing number + * of predicted tokens so far + * @param encoder_out A float32 tensor of shape (batch_size, T, dim) + * + * @returns Return a pair: + * + * - logits, a float32 tensor of shape (batch_size, 1, dim) + * - states, a list of states + */ + std::pair> ForwardUnCachedDecoder( + Ort::Value token, Ort::Value seq_len, Ort::Value encoder_out) const; + + /** Run the cached decoder. + * + * @param token A int32 tensor of shape (batch_size, num_tokens) + * @param seq_len A int32 tensor of shape (batch_size,) containing number + * of predicted tokens so far + * @param encoder_out A float32 tensor of shape (batch_size, T, dim) + * @param states A list of previous states + * + * @returns Return a pair: + * - logits, a float32 tensor of shape (batch_size, 1, dim) + * - states, a list of new states + */ + std::pair> ForwardCachedDecoder( + Ort::Value token, Ort::Value seq_len, Ort::Value encoder_out, + std::vector states) const; + + /** Return an allocator for allocating memory + */ + OrtAllocator *Allocator() const; + + private: + class Impl; + std::unique_ptr impl_; +}; + +} // namespace sherpa_onnx + +#endif // SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_MODEL_H_ diff --git a/sherpa-onnx/csrc/offline-recognizer-impl.cc b/sherpa-onnx/csrc/offline-recognizer-impl.cc index f6c6e247c..07887df60 100644 --- a/sherpa-onnx/csrc/offline-recognizer-impl.cc +++ b/sherpa-onnx/csrc/offline-recognizer-impl.cc @@ -20,6 +20,7 @@ #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/offline-recognizer-ctc-impl.h" +#include "sherpa-onnx/csrc/offline-recognizer-moonshine-impl.h" #include "sherpa-onnx/csrc/offline-recognizer-paraformer-impl.h" #include "sherpa-onnx/csrc/offline-recognizer-sense-voice-impl.h" #include "sherpa-onnx/csrc/offline-recognizer-transducer-impl.h" @@ -51,6 +52,10 @@ std::unique_ptr OfflineRecognizerImpl::Create( return std::make_unique(config); } + if (!config.model_config.moonshine.preprocessor.empty()) { + return std::make_unique(config); + } + // TODO(fangjun): Refactor it. We only need to use model type for the // following models: // 1. transducer and nemo_transducer @@ -67,7 +72,11 @@ std::unique_ptr OfflineRecognizerImpl::Create( model_type == "telespeech_ctc") { return std::make_unique(config); } else if (model_type == "whisper") { + // unreachable return std::make_unique(config); + } else if (model_type == "moonshine") { + // unreachable + return std::make_unique(config); } else { SHERPA_ONNX_LOGE( "Invalid model_type: %s. Trying to load the model to get its type", @@ -225,6 +234,10 @@ std::unique_ptr OfflineRecognizerImpl::Create( return std::make_unique(mgr, config); } + if (!config.model_config.moonshine.preprocessor.empty()) { + return std::make_unique(mgr, config); + } + // TODO(fangjun): Refactor it. We only need to use model type for the // following models: // 1. transducer and nemo_transducer @@ -242,6 +255,8 @@ std::unique_ptr OfflineRecognizerImpl::Create( return std::make_unique(mgr, config); } else if (model_type == "whisper") { return std::make_unique(mgr, config); + } else if (model_type == "moonshine") { + return std::make_unique(mgr, config); } else { SHERPA_ONNX_LOGE( "Invalid model_type: %s. Trying to load the model to get its type", diff --git a/sherpa-onnx/csrc/offline-recognizer-moonshine-impl.h b/sherpa-onnx/csrc/offline-recognizer-moonshine-impl.h new file mode 100644 index 000000000..7d52a41b2 --- /dev/null +++ b/sherpa-onnx/csrc/offline-recognizer-moonshine-impl.h @@ -0,0 +1,150 @@ +// sherpa-onnx/csrc/offline-recognizer-moonshine-impl.h +// +// Copyright (c) 2024 Xiaomi Corporation + +#ifndef SHERPA_ONNX_CSRC_OFFLINE_RECOGNIZER_MOONSHINE_IMPL_H_ +#define SHERPA_ONNX_CSRC_OFFLINE_RECOGNIZER_MOONSHINE_IMPL_H_ + +#include +#include +#include +#include +#include +#include + +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#include "sherpa-onnx/csrc/offline-model-config.h" +#include "sherpa-onnx/csrc/offline-moonshine-decoder.h" +#include "sherpa-onnx/csrc/offline-moonshine-greedy-search-decoder.h" +#include "sherpa-onnx/csrc/offline-moonshine-model.h" +#include "sherpa-onnx/csrc/offline-recognizer-impl.h" +#include "sherpa-onnx/csrc/offline-recognizer.h" +#include "sherpa-onnx/csrc/symbol-table.h" +#include "sherpa-onnx/csrc/transpose.h" + +namespace sherpa_onnx { + +static OfflineRecognitionResult Convert( + const OfflineMoonshineDecoderResult &src, const SymbolTable &sym_table) { + OfflineRecognitionResult r; + r.tokens.reserve(src.tokens.size()); + + std::string text; + for (auto i : src.tokens) { + if (!sym_table.Contains(i)) { + continue; + } + + const auto &s = sym_table[i]; + text += s; + r.tokens.push_back(s); + } + + r.text = text; + + return r; +} + +class OfflineRecognizerMoonshineImpl : public OfflineRecognizerImpl { + public: + explicit OfflineRecognizerMoonshineImpl(const OfflineRecognizerConfig &config) + : OfflineRecognizerImpl(config), + config_(config), + symbol_table_(config_.model_config.tokens), + model_(std::make_unique(config.model_config)) { + Init(); + } + +#if __ANDROID_API__ >= 9 + OfflineRecognizerMoonshineImpl(AAssetManager *mgr, + const OfflineRecognizerConfig &config) + : OfflineRecognizerImpl(mgr, config), + config_(config), + symbol_table_(mgr, config_.model_config.tokens), + model_( + std::make_unique(mgr, config.model_config)) { + Init(); + } + +#endif + + void Init() { + if (config_.decoding_method == "greedy_search") { + decoder_ = + std::make_unique(model_.get()); + } else { + SHERPA_ONNX_LOGE( + "Only greedy_search is supported at present for moonshine. Given %s", + config_.decoding_method.c_str()); + exit(-1); + } + } + + std::unique_ptr CreateStream() const override { + MoonshineTag tag; + return std::make_unique(tag); + } + + void DecodeStreams(OfflineStream **ss, int32_t n) const override { + // batch decoding is not implemented yet + for (int32_t i = 0; i != n; ++i) { + DecodeStream(ss[i]); + } + } + + OfflineRecognizerConfig GetConfig() const override { return config_; } + + private: + void DecodeStream(OfflineStream *s) const { + auto memory_info = + Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault); + + std::vector audio = s->GetFrames(); + + try { + std::array shape{1, static_cast(audio.size())}; + + Ort::Value audio_tensor = Ort::Value::CreateTensor( + memory_info, audio.data(), audio.size(), shape.data(), shape.size()); + + Ort::Value features = + model_->ForwardPreprocessor(std::move(audio_tensor)); + + int32_t features_len = features.GetTensorTypeAndShapeInfo().GetShape()[1]; + + int64_t features_shape = 1; + + Ort::Value features_len_tensor = Ort::Value::CreateTensor( + memory_info, &features_len, 1, &features_shape, 1); + + Ort::Value encoder_out = model_->ForwardEncoder( + std::move(features), std::move(features_len_tensor)); + + auto results = decoder_->Decode(std::move(encoder_out)); + + auto r = Convert(results[0], symbol_table_); + r.text = ApplyInverseTextNormalization(std::move(r.text)); + s->SetResult(r); + } catch (const Ort::Exception &ex) { + SHERPA_ONNX_LOGE( + "\n\nCaught exception:\n\n%s\n\nReturn an empty result. Number of " + "audio samples: %d", + ex.what(), static_cast(audio.size())); + return; + } + } + + private: + OfflineRecognizerConfig config_; + SymbolTable symbol_table_; + std::unique_ptr model_; + std::unique_ptr decoder_; +}; + +} // namespace sherpa_onnx + +#endif // SHERPA_ONNX_CSRC_OFFLINE_RECOGNIZER_MOONSHINE_IMPL_H_ diff --git a/sherpa-onnx/csrc/offline-stream.cc b/sherpa-onnx/csrc/offline-stream.cc index 0f83807dc..6fa0ca6c8 100644 --- a/sherpa-onnx/csrc/offline-stream.cc +++ b/sherpa-onnx/csrc/offline-stream.cc @@ -133,6 +133,10 @@ class OfflineStream::Impl { fbank_ = std::make_unique(opts_); } + explicit Impl(MoonshineTag /*tag*/) : is_moonshine_(true) { + config_.sampling_rate = 16000; + } + void AcceptWaveform(int32_t sampling_rate, const float *waveform, int32_t n) { if (config_.normalize_samples) { AcceptWaveformImpl(sampling_rate, waveform, n); @@ -164,7 +168,9 @@ class OfflineStream::Impl { std::vector samples; resampler->Resample(waveform, n, true, &samples); - if (fbank_) { + if (is_moonshine_) { + samples_.insert(samples_.end(), samples.begin(), samples.end()); + } else if (fbank_) { fbank_->AcceptWaveform(config_.sampling_rate, samples.data(), samples.size()); fbank_->InputFinished(); @@ -181,7 +187,9 @@ class OfflineStream::Impl { return; } // if (sampling_rate != config_.sampling_rate) - if (fbank_) { + if (is_moonshine_) { + samples_.insert(samples_.end(), waveform, waveform + n); + } else if (fbank_) { fbank_->AcceptWaveform(sampling_rate, waveform, n); fbank_->InputFinished(); } else if (mfcc_) { @@ -194,10 +202,18 @@ class OfflineStream::Impl { } int32_t FeatureDim() const { + if (is_moonshine_) { + return samples_.size(); + } + return mfcc_ ? mfcc_opts_.num_ceps : opts_.mel_opts.num_bins; } std::vector GetFrames() const { + if (is_moonshine_) { + return samples_; + } + int32_t n = fbank_ ? fbank_->NumFramesReady() : mfcc_ ? mfcc_->NumFramesReady() : whisper_fbank_->NumFramesReady(); @@ -300,6 +316,10 @@ class OfflineStream::Impl { OfflineRecognitionResult r_; ContextGraphPtr context_graph_; bool is_ced_ = false; + bool is_moonshine_ = false; + + // used only when is_moonshine_== true + std::vector samples_; }; OfflineStream::OfflineStream(const FeatureExtractorConfig &config /*= {}*/, @@ -311,6 +331,9 @@ OfflineStream::OfflineStream(WhisperTag tag) OfflineStream::OfflineStream(CEDTag tag) : impl_(std::make_unique(tag)) {} +OfflineStream::OfflineStream(MoonshineTag tag) + : impl_(std::make_unique(tag)) {} + OfflineStream::~OfflineStream() = default; void OfflineStream::AcceptWaveform(int32_t sampling_rate, const float *waveform, diff --git a/sherpa-onnx/csrc/offline-stream.h b/sherpa-onnx/csrc/offline-stream.h index 95bc80e83..e4bed1115 100644 --- a/sherpa-onnx/csrc/offline-stream.h +++ b/sherpa-onnx/csrc/offline-stream.h @@ -34,7 +34,7 @@ struct OfflineRecognitionResult { // event target of the audio. std::string event; - /// timestamps.size() == tokens.size() + /// timestamps.size() == tokens.size() /// timestamps[i] records the time in seconds when tokens[i] is decoded. std::vector timestamps; @@ -49,6 +49,10 @@ struct WhisperTag { struct CEDTag {}; +// It uses a neural network model, a preprocessor, to convert +// audio samples to features +struct MoonshineTag {}; + class OfflineStream { public: explicit OfflineStream(const FeatureExtractorConfig &config = {}, @@ -56,6 +60,7 @@ class OfflineStream { explicit OfflineStream(WhisperTag tag); explicit OfflineStream(CEDTag tag); + explicit OfflineStream(MoonshineTag tag); ~OfflineStream(); /** @@ -72,7 +77,10 @@ class OfflineStream { void AcceptWaveform(int32_t sampling_rate, const float *waveform, int32_t n) const; - /// Return feature dim of this extractor + /// Return feature dim of this extractor. + /// + /// Note: if it is Moonshine, then it returns the number of audio samples + /// currently received. int32_t FeatureDim() const; // Get all the feature frames of this stream in a 1-D array, which is diff --git a/sherpa-onnx/csrc/offline-whisper-model.cc b/sherpa-onnx/csrc/offline-whisper-model.cc index 6327e5347..485eaf93c 100644 --- a/sherpa-onnx/csrc/offline-whisper-model.cc +++ b/sherpa-onnx/csrc/offline-whisper-model.cc @@ -23,7 +23,6 @@ class OfflineWhisperModel::Impl { explicit Impl(const OfflineModelConfig &config) : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), - debug_(config.debug), sess_opts_(GetSessionOptions(config)), allocator_{} { { @@ -40,7 +39,6 @@ class OfflineWhisperModel::Impl { explicit Impl(const SpokenLanguageIdentificationConfig &config) : lid_config_(config), env_(ORT_LOGGING_LEVEL_ERROR), - debug_(config_.debug), sess_opts_(GetSessionOptions(config)), allocator_{} { { @@ -60,7 +58,6 @@ class OfflineWhisperModel::Impl { env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), allocator_{} { - debug_ = config_.debug; { auto buf = ReadFile(mgr, config.whisper.encoder); InitEncoder(buf.data(), buf.size()); @@ -77,7 +74,6 @@ class OfflineWhisperModel::Impl { env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), allocator_{} { - debug_ = config_.debug; { auto buf = ReadFile(mgr, config.whisper.encoder); InitEncoder(buf.data(), buf.size()); @@ -164,7 +160,7 @@ class OfflineWhisperModel::Impl { } } - if (debug_) { + if (config_.debug) { SHERPA_ONNX_LOGE("Detected language: %s", GetID2Lang().at(lang_id).c_str()); } @@ -237,7 +233,7 @@ class OfflineWhisperModel::Impl { // get meta data Ort::ModelMetadata meta_data = encoder_sess_->GetModelMetadata(); - if (debug_) { + if (config_.debug) { std::ostringstream os; os << "---encoder---\n"; PrintModelMetadata(os, meta_data); @@ -294,7 +290,6 @@ class OfflineWhisperModel::Impl { private: OfflineModelConfig config_; SpokenLanguageIdentificationConfig lid_config_; - bool debug_ = false; Ort::Env env_; Ort::SessionOptions sess_opts_; Ort::AllocatorWithDefaultOptions allocator_; diff --git a/sherpa-onnx/csrc/sherpa-onnx-offline.cc b/sherpa-onnx/csrc/sherpa-onnx-offline.cc index 73e77299a..022f7569b 100644 --- a/sherpa-onnx/csrc/sherpa-onnx-offline.cc +++ b/sherpa-onnx/csrc/sherpa-onnx-offline.cc @@ -43,7 +43,20 @@ See https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-paraformer/in --decoding-method=greedy_search \ /path/to/foo.wav [bar.wav foobar.wav ...] -(3) Whisper models +(3) Moonshine models + +See https://k2-fsa.github.io/sherpa/onnx/moonshine/index.html + + ./bin/sherpa-onnx-offline \ + --moonshine-preprocessor=/Users/fangjun/open-source/sherpa-onnx/scripts/moonshine/preprocess.onnx \ + --moonshine-encoder=/Users/fangjun/open-source/sherpa-onnx/scripts/moonshine/encode.int8.onnx \ + --moonshine-uncached-decoder=/Users/fangjun/open-source/sherpa-onnx/scripts/moonshine/uncached_decode.int8.onnx \ + --moonshine-cached-decoder=/Users/fangjun/open-source/sherpa-onnx/scripts/moonshine/cached_decode.int8.onnx \ + --tokens=/Users/fangjun/open-source/sherpa-onnx/scripts/moonshine/tokens.txt \ + --num-threads=1 \ + /path/to/foo.wav [bar.wav foobar.wav ...] + +(4) Whisper models See https://k2-fsa.github.io/sherpa/onnx/pretrained_models/whisper/tiny.en.html @@ -54,7 +67,7 @@ See https://k2-fsa.github.io/sherpa/onnx/pretrained_models/whisper/tiny.en.html --num-threads=1 \ /path/to/foo.wav [bar.wav foobar.wav ...] -(4) NeMo CTC models +(5) NeMo CTC models See https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-ctc/index.html @@ -68,7 +81,7 @@ See https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-ctc/index.htm ./sherpa-onnx-nemo-ctc-en-conformer-medium/test_wavs/1.wav \ ./sherpa-onnx-nemo-ctc-en-conformer-medium/test_wavs/8k.wav -(5) TDNN CTC model for the yesno recipe from icefall +(6) TDNN CTC model for the yesno recipe from icefall See https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-ctc/yesno/index.html // diff --git a/sherpa-onnx/csrc/symbol-table.cc b/sherpa-onnx/csrc/symbol-table.cc index a71225c38..eed7a1e53 100644 --- a/sherpa-onnx/csrc/symbol-table.cc +++ b/sherpa-onnx/csrc/symbol-table.cc @@ -109,6 +109,8 @@ const std::string SymbolTable::operator[](int32_t id) const { // for byte-level BPE // id 0 is blank, id 1 is sos/eos, id 2 is unk + // + // Note: For moonshine models, 0 is , 1, is , 2 is if (id >= 3 && id <= 258 && sym.size() == 6 && sym[0] == '<' && sym[1] == '0' && sym[2] == 'x' && sym[5] == '>') { std::ostringstream os; diff --git a/sherpa-onnx/python/csrc/CMakeLists.txt b/sherpa-onnx/python/csrc/CMakeLists.txt index 2e971581a..21f77f29d 100644 --- a/sherpa-onnx/python/csrc/CMakeLists.txt +++ b/sherpa-onnx/python/csrc/CMakeLists.txt @@ -11,6 +11,7 @@ set(srcs offline-ctc-fst-decoder-config.cc offline-lm-config.cc offline-model-config.cc + offline-moonshine-model-config.cc offline-nemo-enc-dec-ctc-model-config.cc offline-paraformer-model-config.cc offline-punctuation.cc diff --git a/sherpa-onnx/python/csrc/offline-model-config.cc b/sherpa-onnx/python/csrc/offline-model-config.cc index f498bd7e2..d999486bc 100644 --- a/sherpa-onnx/python/csrc/offline-model-config.cc +++ b/sherpa-onnx/python/csrc/offline-model-config.cc @@ -8,6 +8,7 @@ #include #include "sherpa-onnx/csrc/offline-model-config.h" +#include "sherpa-onnx/python/csrc/offline-moonshine-model-config.h" #include "sherpa-onnx/python/csrc/offline-nemo-enc-dec-ctc-model-config.h" #include "sherpa-onnx/python/csrc/offline-paraformer-model-config.h" #include "sherpa-onnx/python/csrc/offline-sense-voice-model-config.h" @@ -28,6 +29,7 @@ void PybindOfflineModelConfig(py::module *m) { PybindOfflineZipformerCtcModelConfig(m); PybindOfflineWenetCtcModelConfig(m); PybindOfflineSenseVoiceModelConfig(m); + PybindOfflineMoonshineModelConfig(m); using PyClass = OfflineModelConfig; py::class_(*m, "OfflineModelConfig") @@ -39,7 +41,8 @@ void PybindOfflineModelConfig(py::module *m) { const OfflineWhisperModelConfig &, const OfflineTdnnModelConfig &, const OfflineZipformerCtcModelConfig &, const OfflineWenetCtcModelConfig &, - const OfflineSenseVoiceModelConfig &, const std::string &, + const OfflineSenseVoiceModelConfig &, + const OfflineMoonshineModelConfig &, const std::string &, const std::string &, int32_t, bool, const std::string &, const std::string &, const std::string &, const std::string &>(), py::arg("transducer") = OfflineTransducerModelConfig(), @@ -50,6 +53,7 @@ void PybindOfflineModelConfig(py::module *m) { py::arg("zipformer_ctc") = OfflineZipformerCtcModelConfig(), py::arg("wenet_ctc") = OfflineWenetCtcModelConfig(), py::arg("sense_voice") = OfflineSenseVoiceModelConfig(), + py::arg("moonshine") = OfflineMoonshineModelConfig(), py::arg("telespeech_ctc") = "", py::arg("tokens"), py::arg("num_threads"), py::arg("debug") = false, py::arg("provider") = "cpu", py::arg("model_type") = "", @@ -62,6 +66,7 @@ void PybindOfflineModelConfig(py::module *m) { .def_readwrite("zipformer_ctc", &PyClass::zipformer_ctc) .def_readwrite("wenet_ctc", &PyClass::wenet_ctc) .def_readwrite("sense_voice", &PyClass::sense_voice) + .def_readwrite("moonshine", &PyClass::moonshine) .def_readwrite("telespeech_ctc", &PyClass::telespeech_ctc) .def_readwrite("tokens", &PyClass::tokens) .def_readwrite("num_threads", &PyClass::num_threads) diff --git a/sherpa-onnx/python/csrc/offline-moonshine-model-config.cc b/sherpa-onnx/python/csrc/offline-moonshine-model-config.cc new file mode 100644 index 000000000..14bea382b --- /dev/null +++ b/sherpa-onnx/python/csrc/offline-moonshine-model-config.cc @@ -0,0 +1,28 @@ +// sherpa-onnx/python/csrc/offline-moonshine-model-config.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include "sherpa-onnx/csrc/offline-moonshine-model-config.h" + +#include +#include + +#include "sherpa-onnx/python/csrc/offline-moonshine-model-config.h" + +namespace sherpa_onnx { + +void PybindOfflineMoonshineModelConfig(py::module *m) { + using PyClass = OfflineMoonshineModelConfig; + py::class_(*m, "OfflineMoonshineModelConfig") + .def(py::init(), + py::arg("preprocessor"), py::arg("encoder"), + py::arg("uncached_decoder"), py::arg("cached_decoder")) + .def_readwrite("preprocessor", &PyClass::preprocessor) + .def_readwrite("encoder", &PyClass::encoder) + .def_readwrite("uncached_decoder", &PyClass::uncached_decoder) + .def_readwrite("cached_decoder", &PyClass::cached_decoder) + .def("__str__", &PyClass::ToString); +} + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/python/csrc/offline-moonshine-model-config.h b/sherpa-onnx/python/csrc/offline-moonshine-model-config.h new file mode 100644 index 000000000..1b30f9f94 --- /dev/null +++ b/sherpa-onnx/python/csrc/offline-moonshine-model-config.h @@ -0,0 +1,16 @@ +// sherpa-onnx/python/csrc/offline-moonshine-model-config.h +// +// Copyright (c) 2024 Xiaomi Corporation + +#ifndef SHERPA_ONNX_PYTHON_CSRC_OFFLINE_MOONSHINE_MODEL_CONFIG_H_ +#define SHERPA_ONNX_PYTHON_CSRC_OFFLINE_MOONSHINE_MODEL_CONFIG_H_ + +#include "sherpa-onnx/python/csrc/sherpa-onnx.h" + +namespace sherpa_onnx { + +void PybindOfflineMoonshineModelConfig(py::module *m); + +} + +#endif // SHERPA_ONNX_PYTHON_CSRC_OFFLINE_MOONSHINE_MODEL_CONFIG_H_ diff --git a/sherpa-onnx/python/sherpa_onnx/offline_recognizer.py b/sherpa-onnx/python/sherpa_onnx/offline_recognizer.py index e96271a58..391666005 100644 --- a/sherpa-onnx/python/sherpa_onnx/offline_recognizer.py +++ b/sherpa-onnx/python/sherpa_onnx/offline_recognizer.py @@ -8,13 +8,14 @@ OfflineCtcFstDecoderConfig, OfflineLMConfig, OfflineModelConfig, + OfflineMoonshineModelConfig, OfflineNemoEncDecCtcModelConfig, OfflineParaformerModelConfig, - OfflineSenseVoiceModelConfig, ) from _sherpa_onnx import OfflineRecognizer as _Recognizer from _sherpa_onnx import ( OfflineRecognizerConfig, + OfflineSenseVoiceModelConfig, OfflineStream, OfflineTdnnModelConfig, OfflineTransducerModelConfig, @@ -503,12 +504,12 @@ def from_whisper( e.g., tiny, tiny.en, base, base.en, etc. Args: - encoder_model: - Path to the encoder model, e.g., tiny-encoder.onnx, - tiny-encoder.int8.onnx, tiny-encoder.ort, etc. - decoder_model: + encoder: Path to the encoder model, e.g., tiny-encoder.onnx, tiny-encoder.int8.onnx, tiny-encoder.ort, etc. + decoder: + Path to the decoder model, e.g., tiny-decoder.onnx, + tiny-decoder.int8.onnx, tiny-decoder.ort, etc. tokens: Path to ``tokens.txt``. Each line in ``tokens.txt`` contains two columns:: @@ -570,6 +571,87 @@ def from_whisper( self.config = recognizer_config return self + @classmethod + def from_moonshine( + cls, + preprocessor: str, + encoder: str, + uncached_decoder: str, + cached_decoder: str, + tokens: str, + num_threads: int = 1, + decoding_method: str = "greedy_search", + debug: bool = False, + provider: str = "cpu", + rule_fsts: str = "", + rule_fars: str = "", + ): + """ + Please refer to + ``_ + to download pre-trained models for different kinds of moonshine models, + e.g., tiny, base, etc. + + Args: + preprocessor: + Path to the preprocessor model, e.g., preprocess.onnx + encoder: + Path to the encoder model, e.g., encode.int8.onnx + uncached_decoder: + Path to the uncached decoder model, e.g., uncached_decode.int8.onnx, + cached_decoder: + Path to the cached decoder model, e.g., cached_decode.int8.onnx, + tokens: + Path to ``tokens.txt``. Each line in ``tokens.txt`` contains two + columns:: + + symbol integer_id + + num_threads: + Number of threads for neural network computation. + decoding_method: + Valid values: greedy_search. + debug: + True to show debug messages. + provider: + onnxruntime execution providers. Valid values are: cpu, cuda, coreml. + rule_fsts: + If not empty, it specifies fsts for inverse text normalization. + If there are multiple fsts, they are separated by a comma. + rule_fars: + If not empty, it specifies fst archives for inverse text normalization. + If there are multiple archives, they are separated by a comma. + """ + self = cls.__new__(cls) + model_config = OfflineModelConfig( + moonshine=OfflineMoonshineModelConfig( + preprocessor=preprocessor, + encoder=encoder, + uncached_decoder=uncached_decoder, + cached_decoder=cached_decoder, + ), + tokens=tokens, + num_threads=num_threads, + debug=debug, + provider=provider, + ) + + unused_feat_config = FeatureExtractorConfig( + sampling_rate=16000, + feature_dim=80, + ) + + recognizer_config = OfflineRecognizerConfig( + model_config=model_config, + feat_config=unused_feat_config, + decoding_method=decoding_method, + rule_fsts=rule_fsts, + rule_fars=rule_fars, + ) + self.recognizer = _Recognizer(recognizer_config) + self.config = recognizer_config + return self + @classmethod def from_tdnn_ctc( cls, From bd4b223920d492aad8d9e9af1ee44858e8fd8426 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sat, 26 Oct 2024 22:30:29 +0800 Subject: [PATCH 042/183] Add Kotlin and Java API for Moonshine models (#1474) --- .github/workflows/apk-vad-asr.yaml | 5 +- .github/workflows/run-java-test.yaml | 47 +++--- .gitignore | 2 + .../NonStreamingDecodeFileMoonshine.java | 60 +++++++ .../VadFromMicWithNonStreamingMoonshine.java | 152 ++++++++++++++++++ ...run-non-streaming-decode-file-moonshine.sh | 37 +++++ ...un-vad-from-mic-non-streaming-moonshine.sh | 41 +++++ kotlin-api-examples/run.sh | 6 + kotlin-api-examples/test_offline_asr.kt | 3 +- scripts/apk/generate-vad-asr-apk-script.py | 15 ++ sherpa-onnx/java-api/Makefile | 1 + .../k2fsa/sherpa/onnx/OfflineModelConfig.java | 12 ++ .../onnx/OfflineMoonshineModelConfig.java | 70 ++++++++ sherpa-onnx/jni/offline-recognizer.cc | 33 ++++ sherpa-onnx/kotlin-api/OfflineRecognizer.kt | 21 +++ 15 files changed, 480 insertions(+), 25 deletions(-) create mode 100644 java-api-examples/NonStreamingDecodeFileMoonshine.java create mode 100644 java-api-examples/VadFromMicWithNonStreamingMoonshine.java create mode 100755 java-api-examples/run-non-streaming-decode-file-moonshine.sh create mode 100755 java-api-examples/run-vad-from-mic-non-streaming-moonshine.sh create mode 100644 sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineMoonshineModelConfig.java diff --git a/.github/workflows/apk-vad-asr.yaml b/.github/workflows/apk-vad-asr.yaml index 4c587f1be..fe706aa14 100644 --- a/.github/workflows/apk-vad-asr.yaml +++ b/.github/workflows/apk-vad-asr.yaml @@ -23,8 +23,8 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - total: ["5"] - index: ["0", "1", "2", "3", "4"] + total: ["10"] + index: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] steps: - uses: actions/checkout@v4 @@ -165,6 +165,7 @@ jobs: git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface + du -h -d1 . git fetch git pull git merge -m "merge remote" --ff origin main diff --git a/.github/workflows/run-java-test.yaml b/.github/workflows/run-java-test.yaml index 5759ea5d8..cca94422e 100644 --- a/.github/workflows/run-java-test.yaml +++ b/.github/workflows/run-java-test.yaml @@ -107,6 +107,31 @@ jobs: make -j4 ls -lh lib + - name: Run java test (Non-Streaming ASR) + shell: bash + run: | + cd ./java-api-examples + + ./run-non-streaming-decode-file-moonshine.sh + rm -rf sherpa-onnx-moonshine-* + + ./run-non-streaming-decode-file-sense-voice.sh + rm -rf sherpa-onnx-sense-voice-* + + ./run-inverse-text-normalization-paraformer.sh + + ./run-non-streaming-decode-file-paraformer.sh + rm -rf sherpa-onnx-paraformer-zh-* + + ./run-non-streaming-decode-file-transducer.sh + rm -rf sherpa-onnx-zipformer-* + + ./run-non-streaming-decode-file-whisper.sh + rm -rf sherpa-onnx-whisper-* + + ./run-non-streaming-decode-file-nemo.sh + rm -rf sherpa-onnx-nemo-* + - name: Run java test (speaker diarization) shell: bash run: | @@ -206,28 +231,6 @@ jobs: ./run-streaming-decode-file-transducer.sh rm -rf sherpa-onnx-streaming-* - - name: Run java test (Non-Streaming ASR) - shell: bash - run: | - cd ./java-api-examples - - ./run-non-streaming-decode-file-sense-voice.sh - rm -rf sherpa-onnx-sense-voice-* - - ./run-inverse-text-normalization-paraformer.sh - - ./run-non-streaming-decode-file-paraformer.sh - rm -rf sherpa-onnx-paraformer-zh-* - - ./run-non-streaming-decode-file-transducer.sh - rm -rf sherpa-onnx-zipformer-* - - ./run-non-streaming-decode-file-whisper.sh - rm -rf sherpa-onnx-whisper-* - - ./run-non-streaming-decode-file-nemo.sh - rm -rf sherpa-onnx-nemo-* - - name: Run java test (Non-Streaming TTS) shell: bash run: | diff --git a/.gitignore b/.gitignore index 7e6708be4..18178f5fa 100644 --- a/.gitignore +++ b/.gitignore @@ -121,3 +121,5 @@ sherpa-onnx-online-punct-en-2024-08-06 *.mp4 *.mp3 sherpa-onnx-pyannote-segmentation-3-0 +sherpa-onnx-moonshine-tiny-en-int8 +sherpa-onnx-moonshine-base-en-int8 diff --git a/java-api-examples/NonStreamingDecodeFileMoonshine.java b/java-api-examples/NonStreamingDecodeFileMoonshine.java new file mode 100644 index 000000000..232ceb69d --- /dev/null +++ b/java-api-examples/NonStreamingDecodeFileMoonshine.java @@ -0,0 +1,60 @@ +// Copyright 2024 Xiaomi Corporation + +// This file shows how to use an offline Moonshine, +// i.e., non-streaming Moonshine model, +// to decode files. +import com.k2fsa.sherpa.onnx.*; + +public class NonStreamingDecodeFileMoonshine { + public static void main(String[] args) { + // please refer to + // https://k2-fsa.github.io/sherpa/onnx/moonshine/index.html + // to download model files + + String preprocessor = "./sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx"; + String encoder = "./sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx"; + String uncachedDecoder = "./sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx"; + String cachedDecoder = "./sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx"; + + String tokens = "./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt"; + + String waveFilename = "./sherpa-onnx-moonshine-tiny-en-int8/test_wavs/0.wav"; + + WaveReader reader = new WaveReader(waveFilename); + + OfflineMoonshineModelConfig moonshine = + OfflineMoonshineModelConfig.builder() + .setPreprocessor(preprocessor) + .setEncoder(encoder) + .setUncachedDecoder(uncachedDecoder) + .setCachedDecoder(cachedDecoder) + .build(); + + OfflineModelConfig modelConfig = + OfflineModelConfig.builder() + .setMoonshine(moonshine) + .setTokens(tokens) + .setNumThreads(1) + .setDebug(true) + .build(); + + OfflineRecognizerConfig config = + OfflineRecognizerConfig.builder() + .setOfflineModelConfig(modelConfig) + .setDecodingMethod("greedy_search") + .build(); + + OfflineRecognizer recognizer = new OfflineRecognizer(config); + OfflineStream stream = recognizer.createStream(); + stream.acceptWaveform(reader.getSamples(), reader.getSampleRate()); + + recognizer.decode(stream); + + String text = recognizer.getResult(stream).getText(); + + System.out.printf("filename:%s\nresult:%s\n", waveFilename, text); + + stream.release(); + recognizer.release(); + } +} diff --git a/java-api-examples/VadFromMicWithNonStreamingMoonshine.java b/java-api-examples/VadFromMicWithNonStreamingMoonshine.java new file mode 100644 index 000000000..bfedf3fbc --- /dev/null +++ b/java-api-examples/VadFromMicWithNonStreamingMoonshine.java @@ -0,0 +1,152 @@ +// Copyright 2024 Xiaomi Corporation + +// This file shows how to use a silero_vad model with a non-streaming +// Moonshine tiny for speech recognition. + +import com.k2fsa.sherpa.onnx.*; +import javax.sound.sampled.*; + +public class VadFromMicNonStreamingMoonshine { + private static final int sampleRate = 16000; + private static final int windowSize = 512; + + public static Vad createVad() { + // please download ./silero_vad.onnx from + // https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models + String model = "./silero_vad.onnx"; + SileroVadModelConfig sileroVad = + SileroVadModelConfig.builder() + .setModel(model) + .setThreshold(0.5f) + .setMinSilenceDuration(0.25f) + .setMinSpeechDuration(0.5f) + .setWindowSize(windowSize) + .build(); + + VadModelConfig config = + VadModelConfig.builder() + .setSileroVadModelConfig(sileroVad) + .setSampleRate(sampleRate) + .setNumThreads(1) + .setDebug(true) + .setProvider("cpu") + .build(); + + return new Vad(config); + } + + public static OfflineRecognizer createOfflineRecognizer() { + // please refer to + // https://k2-fsa.github.io/sherpa/onnx/moonshine/index.html + // to download model files + + String preprocessor = "./sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx"; + String encoder = "./sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx"; + String uncachedDecoder = "./sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx"; + String cachedDecoder = "./sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx"; + + String tokens = "./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt"; + + OfflineMoonshineModelConfig moonshine = + OfflineMoonshineModelConfig.builder() + .setPreprocessor(preprocessor) + .setEncoder(encoder) + .setUncachedDecoder(uncachedDecoder) + .setCachedDecoder(cachedDecoder) + .build(); + + OfflineModelConfig modelConfig = + OfflineModelConfig.builder() + .setMoonshine(moonshine) + .setTokens(tokens) + .setNumThreads(1) + .setDebug(true) + .build(); + + OfflineRecognizerConfig config = + OfflineRecognizerConfig.builder() + .setOfflineModelConfig(modelConfig) + .setDecodingMethod("greedy_search") + .build(); + + return new OfflineRecognizer(config); + } + + public static void main(String[] args) { + Vad vad = createVad(); + OfflineRecognizer recognizer = createOfflineRecognizer(); + + // https://docs.oracle.com/javase/8/docs/api/javax/sound/sampled/AudioFormat.html + // Linear PCM, 16000Hz, 16-bit, 1 channel, signed, little endian + AudioFormat format = new AudioFormat(sampleRate, 16, 1, true, false); + + // https://docs.oracle.com/javase/8/docs/api/javax/sound/sampled/DataLine.Info.html#Info-java.lang.Class-javax.sound.sampled.AudioFormat-int- + DataLine.Info info = new DataLine.Info(TargetDataLine.class, format); + TargetDataLine targetDataLine; + try { + targetDataLine = (TargetDataLine) AudioSystem.getLine(info); + targetDataLine.open(format); + targetDataLine.start(); + } catch (LineUnavailableException e) { + System.out.println("Failed to open target data line: " + e.getMessage()); + vad.release(); + recognizer.release(); + return; + } + + boolean printed = false; + byte[] buffer = new byte[windowSize * 2]; + float[] samples = new float[windowSize]; + + System.out.println("Started. Please speak"); + boolean running = true; + while (targetDataLine.isOpen() && running) { + int n = targetDataLine.read(buffer, 0, buffer.length); + if (n <= 0) { + System.out.printf("Got %d bytes. Expected %d bytes.\n", n, buffer.length); + continue; + } + for (int i = 0; i != windowSize; ++i) { + short low = buffer[2 * i]; + short high = buffer[2 * i + 1]; + int s = (high << 8) + low; + samples[i] = (float) s / 32768; + } + + vad.acceptWaveform(samples); + if (vad.isSpeechDetected() && !printed) { + System.out.println("Detected speech"); + printed = true; + } + + if (!vad.isSpeechDetected()) { + printed = false; + } + + while (!vad.empty()) { + SpeechSegment segment = vad.front(); + float startTime = segment.getStart() / (float) sampleRate; + float duration = segment.getSamples().length / (float) sampleRate; + + OfflineStream stream = recognizer.createStream(); + stream.acceptWaveform(segment.getSamples(), sampleRate); + recognizer.decode(stream); + String text = recognizer.getResult(stream).getText(); + stream.release(); + + if (!text.isEmpty()) { + System.out.printf("%.3f--%.3f: %s\n", startTime, startTime + duration, text); + } + + if (text.contains("exit the program")) { + running = false; + } + + vad.pop(); + } + } + + vad.release(); + recognizer.release(); + } +} diff --git a/java-api-examples/run-non-streaming-decode-file-moonshine.sh b/java-api-examples/run-non-streaming-decode-file-moonshine.sh new file mode 100755 index 000000000..b0a09db47 --- /dev/null +++ b/java-api-examples/run-non-streaming-decode-file-moonshine.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -ex + +if [[ ! -f ../build/lib/libsherpa-onnx-jni.dylib && ! -f ../build/lib/libsherpa-onnx-jni.so ]]; then + mkdir -p ../build + pushd ../build + cmake \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + -DSHERPA_ONNX_ENABLE_JNI=ON \ + .. + + make -j4 + ls -lh lib + popd +fi + +if [ ! -f ../sherpa-onnx/java-api/build/sherpa-onnx.jar ]; then + pushd ../sherpa-onnx/java-api + make + popd +fi + +if [ ! -f ./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +fi + +java \ + -Djava.library.path=$PWD/../build/lib \ + -cp ../sherpa-onnx/java-api/build/sherpa-onnx.jar \ + NonStreamingDecodeFileMoonshine.java diff --git a/java-api-examples/run-vad-from-mic-non-streaming-moonshine.sh b/java-api-examples/run-vad-from-mic-non-streaming-moonshine.sh new file mode 100755 index 000000000..78147e7fc --- /dev/null +++ b/java-api-examples/run-vad-from-mic-non-streaming-moonshine.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +set -ex + +if [[ ! -f ../build/lib/libsherpa-onnx-jni.dylib && ! -f ../build/lib/libsherpa-onnx-jni.so ]]; then + mkdir -p ../build + pushd ../build + cmake \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + -DSHERPA_ONNX_ENABLE_JNI=ON \ + .. + + make -j4 + ls -lh lib + popd +fi + +if [ ! -f ../sherpa-onnx/java-api/build/sherpa-onnx.jar ]; then + pushd ../sherpa-onnx/java-api + make + popd +fi + +if [ ! -f ./silero_vad.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx +fi + +if [ ! -f ./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +fi + +java \ + -Djava.library.path=$PWD/../build/lib \ + -cp ../sherpa-onnx/java-api/build/sherpa-onnx.jar \ + ./VadFromMicWithNonStreamingMoonshine.java diff --git a/kotlin-api-examples/run.sh b/kotlin-api-examples/run.sh index a2adab418..3b3d15938 100755 --- a/kotlin-api-examples/run.sh +++ b/kotlin-api-examples/run.sh @@ -168,6 +168,12 @@ function testSpokenLanguageIdentification() { } function testOfflineAsr() { + if [ ! -f ./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + fi + if [ ! -f ./sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/tokens.txt ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 tar xvf sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 diff --git a/kotlin-api-examples/test_offline_asr.kt b/kotlin-api-examples/test_offline_asr.kt index d09406563..6c2c10a6a 100644 --- a/kotlin-api-examples/test_offline_asr.kt +++ b/kotlin-api-examples/test_offline_asr.kt @@ -1,7 +1,7 @@ package com.k2fsa.sherpa.onnx fun main() { - val types = arrayOf(0, 2, 5, 6, 15) + val types = arrayOf(0, 2, 5, 6, 15, 21) for (type in types) { test(type) } @@ -16,6 +16,7 @@ fun test(type: Int) { 5 -> "./sherpa-onnx-zipformer-multi-zh-hans-2023-9-2/test_wavs/1.wav" 6 -> "./sherpa-onnx-nemo-ctc-en-citrinet-512/test_wavs/8k.wav" 15 -> "./sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/test_wavs/zh.wav" + 21 -> "./sherpa-onnx-moonshine-tiny-en-int8/test_wavs/0.wav" else -> null } diff --git a/scripts/apk/generate-vad-asr-apk-script.py b/scripts/apk/generate-vad-asr-apk-script.py index 7671e975d..5c3694d69 100755 --- a/scripts/apk/generate-vad-asr-apk-script.py +++ b/scripts/apk/generate-vad-asr-apk-script.py @@ -369,6 +369,21 @@ def get_models(): ls -lh + popd + """, + ), + Model( + model_name="sherpa-onnx-moonshine-tiny-en-int8", + idx=21, + lang="en", + short_name="moonshine_tiny_int8", + cmd=""" + pushd $model_name + + rm -rfv test_wavs + + ls -lh + popd """, ), diff --git a/sherpa-onnx/java-api/Makefile b/sherpa-onnx/java-api/Makefile index 6e4778ae7..8b9278fc0 100644 --- a/sherpa-onnx/java-api/Makefile +++ b/sherpa-onnx/java-api/Makefile @@ -26,6 +26,7 @@ java_files += OnlineRecognizer.java java_files += OfflineTransducerModelConfig.java java_files += OfflineParaformerModelConfig.java java_files += OfflineWhisperModelConfig.java +java_files += OfflineMoonshineModelConfig.java java_files += OfflineNemoEncDecCtcModelConfig.java java_files += OfflineSenseVoiceModelConfig.java java_files += OfflineModelConfig.java diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineModelConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineModelConfig.java index 9e23a3a74..4d0192b6b 100644 --- a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineModelConfig.java +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineModelConfig.java @@ -6,6 +6,7 @@ public class OfflineModelConfig { private final OfflineTransducerModelConfig transducer; private final OfflineParaformerModelConfig paraformer; private final OfflineWhisperModelConfig whisper; + private final OfflineMoonshineModelConfig moonshine; private final OfflineNemoEncDecCtcModelConfig nemo; private final OfflineSenseVoiceModelConfig senseVoice; private final String teleSpeech; @@ -22,6 +23,7 @@ private OfflineModelConfig(Builder builder) { this.transducer = builder.transducer; this.paraformer = builder.paraformer; this.whisper = builder.whisper; + this.moonshine = builder.moonshine; this.nemo = builder.nemo; this.senseVoice = builder.senseVoice; this.teleSpeech = builder.teleSpeech; @@ -50,6 +52,10 @@ public OfflineWhisperModelConfig getZipformer2Ctc() { return whisper; } + public OfflineMoonshineModelConfig getMoonshine() { + return moonshine; + } + public OfflineSenseVoiceModelConfig getSenseVoice() { return senseVoice; } @@ -90,6 +96,7 @@ public static class Builder { private OfflineParaformerModelConfig paraformer = OfflineParaformerModelConfig.builder().build(); private OfflineTransducerModelConfig transducer = OfflineTransducerModelConfig.builder().build(); private OfflineWhisperModelConfig whisper = OfflineWhisperModelConfig.builder().build(); + private OfflineMoonshineModelConfig moonshine = OfflineMoonshineModelConfig.builder().build(); private OfflineNemoEncDecCtcModelConfig nemo = OfflineNemoEncDecCtcModelConfig.builder().build(); private OfflineSenseVoiceModelConfig senseVoice = OfflineSenseVoiceModelConfig.builder().build(); private String teleSpeech = ""; @@ -135,6 +142,11 @@ public Builder setSenseVoice(OfflineSenseVoiceModelConfig senseVoice) { return this; } + public Builder setMoonshine(OfflineMoonshineModelConfig moonshine) { + this.moonshine = moonshine; + return this; + } + public Builder setTokens(String tokens) { this.tokens = tokens; return this; diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineMoonshineModelConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineMoonshineModelConfig.java new file mode 100644 index 000000000..1a324bba2 --- /dev/null +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineMoonshineModelConfig.java @@ -0,0 +1,70 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class OfflineMoonshineModelConfig { + private final String preprocessor; + private final String encoder; + private final String uncachedDecoder; + private final String cachedDecoder; + + private OfflineMoonshineModelConfig(Builder builder) { + this.preprocessor = builder.preprocessor; + this.encoder = builder.encoder; + this.uncachedDecoder = builder.uncachedDecoder; + this.cachedDecoder = builder.cachedDecoder; + } + + public static Builder builder() { + return new Builder(); + } + + public String getPreprocessor() { + return preprocessor; + } + + public String getEncoder() { + return encoder; + } + + public String getUncachedDecoder() { + return uncachedDecoder; + } + + public String getCachedDecoder() { + return cachedDecoder; + } + + public static class Builder { + private String preprocessor = ""; + private String encoder = ""; + private String uncachedDecoder = ""; + private String cachedDecoder = ""; + + public OfflineMoonshineModelConfig build() { + return new OfflineMoonshineModelConfig(this); + } + + public Builder setPreprocessor(String preprocessor) { + this.preprocessor = preprocessor; + return this; + } + + public Builder setEncoder(String encoder) { + this.encoder = encoder; + return this; + } + + public Builder setUncachedDecoder(String uncachedDecoder) { + this.uncachedDecoder = uncachedDecoder; + return this; + } + + public Builder setCachedDecoder(String cachedDecoder) { + this.cachedDecoder = cachedDecoder; + return this; + } + } + + +} diff --git a/sherpa-onnx/jni/offline-recognizer.cc b/sherpa-onnx/jni/offline-recognizer.cc index 5e4b359b6..7df79f346 100644 --- a/sherpa-onnx/jni/offline-recognizer.cc +++ b/sherpa-onnx/jni/offline-recognizer.cc @@ -174,6 +174,39 @@ static OfflineRecognizerConfig GetOfflineConfig(JNIEnv *env, jobject config) { ans.model_config.whisper.tail_paddings = env->GetIntField(whisper_config, fid); + // moonshine + fid = env->GetFieldID(model_config_cls, "moonshine", + "Lcom/k2fsa/sherpa/onnx/OfflineMoonshineModelConfig;"); + jobject moonshine_config = env->GetObjectField(model_config, fid); + jclass moonshine_config_cls = env->GetObjectClass(moonshine_config); + + fid = env->GetFieldID(moonshine_config_cls, "preprocessor", + "Ljava/lang/String;"); + s = (jstring)env->GetObjectField(moonshine_config, fid); + p = env->GetStringUTFChars(s, nullptr); + ans.model_config.moonshine.preprocessor = p; + env->ReleaseStringUTFChars(s, p); + + fid = env->GetFieldID(moonshine_config_cls, "encoder", "Ljava/lang/String;"); + s = (jstring)env->GetObjectField(moonshine_config, fid); + p = env->GetStringUTFChars(s, nullptr); + ans.model_config.moonshine.encoder = p; + env->ReleaseStringUTFChars(s, p); + + fid = env->GetFieldID(moonshine_config_cls, "uncachedDecoder", + "Ljava/lang/String;"); + s = (jstring)env->GetObjectField(moonshine_config, fid); + p = env->GetStringUTFChars(s, nullptr); + ans.model_config.moonshine.uncached_decoder = p; + env->ReleaseStringUTFChars(s, p); + + fid = env->GetFieldID(moonshine_config_cls, "cachedDecoder", + "Ljava/lang/String;"); + s = (jstring)env->GetObjectField(moonshine_config, fid); + p = env->GetStringUTFChars(s, nullptr); + ans.model_config.moonshine.cached_decoder = p; + env->ReleaseStringUTFChars(s, p); + // sense voice fid = env->GetFieldID(model_config_cls, "senseVoice", "Lcom/k2fsa/sherpa/onnx/OfflineSenseVoiceModelConfig;"); diff --git a/sherpa-onnx/kotlin-api/OfflineRecognizer.kt b/sherpa-onnx/kotlin-api/OfflineRecognizer.kt index b82436356..76dc82149 100644 --- a/sherpa-onnx/kotlin-api/OfflineRecognizer.kt +++ b/sherpa-onnx/kotlin-api/OfflineRecognizer.kt @@ -33,6 +33,13 @@ data class OfflineWhisperModelConfig( var tailPaddings: Int = 1000, // Padding added at the end of the samples ) +data class OfflineMoonshineModelConfig( + var preprocessor: String = "", + var encoder: String = "", + var uncachedDecoder: String = "", + var cachedDecoder: String = "", +) + data class OfflineSenseVoiceModelConfig( var model: String = "", var language: String = "", @@ -43,6 +50,7 @@ data class OfflineModelConfig( var transducer: OfflineTransducerModelConfig = OfflineTransducerModelConfig(), var paraformer: OfflineParaformerModelConfig = OfflineParaformerModelConfig(), var whisper: OfflineWhisperModelConfig = OfflineWhisperModelConfig(), + var moonshine: OfflineMoonshineModelConfig = OfflineMoonshineModelConfig(), var nemo: OfflineNemoEncDecCtcModelConfig = OfflineNemoEncDecCtcModelConfig(), var senseVoice: OfflineSenseVoiceModelConfig = OfflineSenseVoiceModelConfig(), var teleSpeech: String = "", @@ -417,6 +425,19 @@ fun getOfflineModelConfig(type: Int): OfflineModelConfig? { modelType = "nemo_transducer", ) } + + 21 -> { + val modelDir = "sherpa-onnx-moonshine-tiny-en-int8" + return OfflineModelConfig( + moonshine = OfflineMoonshineModelConfig( + preprocessor = "$modelDir/preprocess.onnx", + encoder = "$modelDir/encode.int8.onnx", + uncachedDecoder = "$modelDir/uncached_decode.int8.onnx", + cachedDecoder = "$modelDir/cached_decode.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } } return null } From 2ca2985d0496ccf0bb05aa555eb7293dd78d7d48 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sat, 26 Oct 2024 23:24:46 +0800 Subject: [PATCH 043/183] Add C and C++ API for Moonshine models (#1476) --- .github/workflows/c-api.yaml | 74 +++++++++++ .github/workflows/cxx-api.yaml | 22 ++++ c-api-examples/CMakeLists.txt | 9 ++ c-api-examples/moonshine-c-api.c | 83 ++++++++++++ c-api-examples/vad-moonshine-c-api.c | 171 +++++++++++++++++++++++++ c-api-examples/vad-sense-voice-c-api.c | 1 + c-api-examples/vad-whisper-c-api.c | 169 ++++++++++++++++++++++++ cxx-api-examples/CMakeLists.txt | 3 + cxx-api-examples/moonshine-cxx-api.cc | 81 ++++++++++++ sherpa-onnx/c-api/c-api.cc | 12 ++ sherpa-onnx/c-api/c-api.h | 8 ++ sherpa-onnx/c-api/cxx-api.cc | 9 ++ sherpa-onnx/c-api/cxx-api.h | 8 ++ 13 files changed, 650 insertions(+) create mode 100644 c-api-examples/moonshine-c-api.c create mode 100644 c-api-examples/vad-moonshine-c-api.c create mode 100644 c-api-examples/vad-whisper-c-api.c create mode 100644 cxx-api-examples/moonshine-cxx-api.cc diff --git a/.github/workflows/c-api.yaml b/.github/workflows/c-api.yaml index 94a52a4bc..049240d77 100644 --- a/.github/workflows/c-api.yaml +++ b/.github/workflows/c-api.yaml @@ -81,6 +81,80 @@ jobs: otool -L ./install/lib/libsherpa-onnx-c-api.dylib fi + - name: Test vad + Whisper tiny.en + shell: bash + run: | + gcc -o vad-whisper-c-api ./c-api-examples/vad-whisper-c-api.c \ + -I ./build/install/include \ + -L ./build/install/lib/ \ + -l sherpa-onnx-c-api \ + -l onnxruntime + + # Now download models + # + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/Obama.wav + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2 + tar xvf sherpa-onnx-whisper-tiny.en.tar.bz2 + rm sherpa-onnx-whisper-tiny.en.tar.bz2 + + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH + export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH + + ./vad-whisper-c-api + + rm -rf sherpa-onnx-* + rm -rf *.onnx + rm *.wav + + - name: Test vad + Moonshine + shell: bash + run: | + gcc -o vad-moonshine-c-api ./c-api-examples/vad-moonshine-c-api.c \ + -I ./build/install/include \ + -L ./build/install/lib/ \ + -l sherpa-onnx-c-api \ + -l onnxruntime + + # Now download models + # + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/Obama.wav + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH + export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH + + ./vad-moonshine-c-api + + rm -rf sherpa-onnx-* + rm -rf *.onnx + rm *.wav + + - name: Test Moonshine + shell: bash + run: | + gcc -o moonshine-c-api ./c-api-examples/moonshine-c-api.c \ + -I ./build/install/include \ + -L ./build/install/lib/ \ + -l sherpa-onnx-c-api \ + -l onnxruntime + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH + export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH + + ./moonshine-c-api + + rm -rf sherpa-onnx-* + - name: Test ffmpeg if: matrix.os == 'macos-latest' shell: bash diff --git a/.github/workflows/cxx-api.yaml b/.github/workflows/cxx-api.yaml index 357aaa227..8779011a9 100644 --- a/.github/workflows/cxx-api.yaml +++ b/.github/workflows/cxx-api.yaml @@ -83,6 +83,28 @@ jobs: otool -L ./install/lib/libsherpa-onnx-cxx-api.dylib fi + - name: Test Moonshine tiny + shell: bash + run: | + g++ -std=c++17 -o moonshine-cxx-api ./cxx-api-examples/moonshine-cxx-api.cc \ + -I ./build/install/include \ + -L ./build/install/lib/ \ + -l sherpa-onnx-cxx-api \ + -l sherpa-onnx-c-api \ + -l onnxruntime + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH + export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH + + ./moonshine-cxx-api + + rm -rf sherpa-onnx-* + rm ./moonshine-cxx-api + - name: Test whisper shell: bash run: | diff --git a/c-api-examples/CMakeLists.txt b/c-api-examples/CMakeLists.txt index 45ca9a156..c7db2bc27 100644 --- a/c-api-examples/CMakeLists.txt +++ b/c-api-examples/CMakeLists.txt @@ -35,6 +35,9 @@ target_link_libraries(whisper-c-api sherpa-onnx-c-api) add_executable(sense-voice-c-api sense-voice-c-api.c) target_link_libraries(sense-voice-c-api sherpa-onnx-c-api) +add_executable(moonshine-c-api moonshine-c-api.c) +target_link_libraries(moonshine-c-api sherpa-onnx-c-api) + add_executable(zipformer-c-api zipformer-c-api.c) target_link_libraries(zipformer-c-api sherpa-onnx-c-api) @@ -53,6 +56,12 @@ target_link_libraries(telespeech-c-api sherpa-onnx-c-api) add_executable(vad-sense-voice-c-api vad-sense-voice-c-api.c) target_link_libraries(vad-sense-voice-c-api sherpa-onnx-c-api) +add_executable(vad-whisper-c-api vad-whisper-c-api.c) +target_link_libraries(vad-whisper-c-api sherpa-onnx-c-api) + +add_executable(vad-moonshine-c-api vad-moonshine-c-api.c) +target_link_libraries(vad-moonshine-c-api sherpa-onnx-c-api) + add_executable(streaming-zipformer-buffered-tokens-hotwords-c-api streaming-zipformer-buffered-tokens-hotwords-c-api.c) target_link_libraries(streaming-zipformer-buffered-tokens-hotwords-c-api sherpa-onnx-c-api) diff --git a/c-api-examples/moonshine-c-api.c b/c-api-examples/moonshine-c-api.c new file mode 100644 index 000000000..775dd24c9 --- /dev/null +++ b/c-api-examples/moonshine-c-api.c @@ -0,0 +1,83 @@ +// c-api-examples/moonshine-c-api.c +// +// Copyright (c) 2024 Xiaomi Corporation + +// +// This file demonstrates how to use Moonshine tiny with sherpa-onnx's C API. +// clang-format off +// +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +// tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +// rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +// +// clang-format on + +#include +#include +#include + +#include "sherpa-onnx/c-api/c-api.h" + +int32_t main() { + const char *wav_filename = + "./sherpa-onnx-moonshine-tiny-en-int8/test_wavs/0.wav"; + const char *preprocessor = + "./sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx"; + const char *encoder = "./sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx"; + const char *uncached_decoder = + "./sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx"; + const char *cached_decoder = + "./sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx"; + const char *tokens = "./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt"; + + const SherpaOnnxWave *wave = SherpaOnnxReadWave(wav_filename); + if (wave == NULL) { + fprintf(stderr, "Failed to read %s\n", wav_filename); + return -1; + } + + // Offline model config + SherpaOnnxOfflineModelConfig offline_model_config; + memset(&offline_model_config, 0, sizeof(offline_model_config)); + offline_model_config.debug = 1; + offline_model_config.num_threads = 1; + offline_model_config.provider = "cpu"; + offline_model_config.tokens = tokens; + offline_model_config.moonshine.preprocessor = preprocessor; + offline_model_config.moonshine.encoder = encoder; + offline_model_config.moonshine.uncached_decoder = uncached_decoder; + offline_model_config.moonshine.cached_decoder = cached_decoder; + + // Recognizer config + SherpaOnnxOfflineRecognizerConfig recognizer_config; + memset(&recognizer_config, 0, sizeof(recognizer_config)); + recognizer_config.decoding_method = "greedy_search"; + recognizer_config.model_config = offline_model_config; + + const SherpaOnnxOfflineRecognizer *recognizer = + SherpaOnnxCreateOfflineRecognizer(&recognizer_config); + + if (recognizer == NULL) { + fprintf(stderr, "Please check your config!\n"); + SherpaOnnxFreeWave(wave); + return -1; + } + + const SherpaOnnxOfflineStream *stream = + SherpaOnnxCreateOfflineStream(recognizer); + + SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate, wave->samples, + wave->num_samples); + SherpaOnnxDecodeOfflineStream(recognizer, stream); + const SherpaOnnxOfflineRecognizerResult *result = + SherpaOnnxGetOfflineStreamResult(stream); + + fprintf(stderr, "Decoded text: %s\n", result->text); + + SherpaOnnxDestroyOfflineRecognizerResult(result); + SherpaOnnxDestroyOfflineStream(stream); + SherpaOnnxDestroyOfflineRecognizer(recognizer); + SherpaOnnxFreeWave(wave); + + return 0; +} diff --git a/c-api-examples/vad-moonshine-c-api.c b/c-api-examples/vad-moonshine-c-api.c new file mode 100644 index 000000000..1b0d03624 --- /dev/null +++ b/c-api-examples/vad-moonshine-c-api.c @@ -0,0 +1,171 @@ +// c-api-examples/vad-moonshine-c-api.c +// +// Copyright (c) 2024 Xiaomi Corporation + +// +// This file demonstrates how to use VAD + Moonshine with sherpa-onnx's C API. +// clang-format off +// +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/Obama.wav +// +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +// tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +// rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +// +// clang-format on + +#include +#include +#include + +#include "sherpa-onnx/c-api/c-api.h" + +int32_t main() { + const char *wav_filename = "./Obama.wav"; + const char *vad_filename = "./silero_vad.onnx"; + + const char *preprocessor = + "./sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx"; + const char *encoder = "./sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx"; + const char *uncached_decoder = + "./sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx"; + const char *cached_decoder = + "./sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx"; + const char *tokens = "./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt"; + + const SherpaOnnxWave *wave = SherpaOnnxReadWave(wav_filename); + if (wave == NULL) { + fprintf(stderr, "Failed to read %s\n", wav_filename); + return -1; + } + + if (wave->sample_rate != 16000) { + fprintf(stderr, "Expect the sample rate to be 16000. Given: %d\n", + wave->sample_rate); + SherpaOnnxFreeWave(wave); + return -1; + } + + // Offline model config + SherpaOnnxOfflineModelConfig offline_model_config; + memset(&offline_model_config, 0, sizeof(offline_model_config)); + offline_model_config.debug = 0; + offline_model_config.num_threads = 1; + offline_model_config.provider = "cpu"; + offline_model_config.tokens = tokens; + offline_model_config.moonshine.preprocessor = preprocessor; + offline_model_config.moonshine.encoder = encoder; + offline_model_config.moonshine.uncached_decoder = uncached_decoder; + offline_model_config.moonshine.cached_decoder = cached_decoder; + + // Recognizer config + SherpaOnnxOfflineRecognizerConfig recognizer_config; + memset(&recognizer_config, 0, sizeof(recognizer_config)); + recognizer_config.decoding_method = "greedy_search"; + recognizer_config.model_config = offline_model_config; + + const SherpaOnnxOfflineRecognizer *recognizer = + SherpaOnnxCreateOfflineRecognizer(&recognizer_config); + + if (recognizer == NULL) { + fprintf(stderr, "Please check your recognizer config!\n"); + SherpaOnnxFreeWave(wave); + return -1; + } + + SherpaOnnxVadModelConfig vadConfig; + memset(&vadConfig, 0, sizeof(vadConfig)); + vadConfig.silero_vad.model = vad_filename; + vadConfig.silero_vad.threshold = 0.5; + vadConfig.silero_vad.min_silence_duration = 0.5; + vadConfig.silero_vad.min_speech_duration = 0.5; + vadConfig.silero_vad.max_speech_duration = 10; + vadConfig.silero_vad.window_size = 512; + vadConfig.sample_rate = 16000; + vadConfig.num_threads = 1; + vadConfig.debug = 1; + + SherpaOnnxVoiceActivityDetector *vad = + SherpaOnnxCreateVoiceActivityDetector(&vadConfig, 30); + + if (vad == NULL) { + fprintf(stderr, "Please check your recognizer config!\n"); + SherpaOnnxFreeWave(wave); + SherpaOnnxDestroyOfflineRecognizer(recognizer); + return -1; + } + + int32_t window_size = vadConfig.silero_vad.window_size; + int32_t i = 0; + + while (i + window_size < wave->num_samples) { + SherpaOnnxVoiceActivityDetectorAcceptWaveform(vad, wave->samples + i, + window_size); + i += window_size; + + while (!SherpaOnnxVoiceActivityDetectorEmpty(vad)) { + const SherpaOnnxSpeechSegment *segment = + SherpaOnnxVoiceActivityDetectorFront(vad); + + const SherpaOnnxOfflineStream *stream = + SherpaOnnxCreateOfflineStream(recognizer); + + SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate, + segment->samples, segment->n); + + SherpaOnnxDecodeOfflineStream(recognizer, stream); + + const SherpaOnnxOfflineRecognizerResult *result = + SherpaOnnxGetOfflineStreamResult(stream); + + float start = segment->start / 16000.0f; + float duration = segment->n / 16000.0f; + float stop = start + duration; + + fprintf(stderr, "%.3f -- %.3f: %s\n", start, stop, result->text); + + SherpaOnnxDestroyOfflineRecognizerResult(result); + SherpaOnnxDestroyOfflineStream(stream); + + SherpaOnnxDestroySpeechSegment(segment); + SherpaOnnxVoiceActivityDetectorPop(vad); + } + } + + SherpaOnnxVoiceActivityDetectorFlush(vad); + + while (!SherpaOnnxVoiceActivityDetectorEmpty(vad)) { + const SherpaOnnxSpeechSegment *segment = + SherpaOnnxVoiceActivityDetectorFront(vad); + + const SherpaOnnxOfflineStream *stream = + SherpaOnnxCreateOfflineStream(recognizer); + + SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate, segment->samples, + segment->n); + + SherpaOnnxDecodeOfflineStream(recognizer, stream); + + const SherpaOnnxOfflineRecognizerResult *result = + SherpaOnnxGetOfflineStreamResult(stream); + + float start = segment->start / 16000.0f; + float duration = segment->n / 16000.0f; + float stop = start + duration; + + fprintf(stderr, "%.3f -- %.3f: %s\n", start, stop, result->text); + + SherpaOnnxDestroyOfflineRecognizerResult(result); + SherpaOnnxDestroyOfflineStream(stream); + + SherpaOnnxDestroySpeechSegment(segment); + SherpaOnnxVoiceActivityDetectorPop(vad); + } + + SherpaOnnxDestroyOfflineRecognizer(recognizer); + SherpaOnnxDestroyVoiceActivityDetector(vad); + SherpaOnnxFreeWave(wave); + + return 0; +} diff --git a/c-api-examples/vad-sense-voice-c-api.c b/c-api-examples/vad-sense-voice-c-api.c index 3049c9572..ee9504d1a 100644 --- a/c-api-examples/vad-sense-voice-c-api.c +++ b/c-api-examples/vad-sense-voice-c-api.c @@ -81,6 +81,7 @@ int32_t main() { vadConfig.silero_vad.threshold = 0.5; vadConfig.silero_vad.min_silence_duration = 0.5; vadConfig.silero_vad.min_speech_duration = 0.5; + vadConfig.silero_vad.max_speech_duration = 5; vadConfig.silero_vad.window_size = 512; vadConfig.sample_rate = 16000; vadConfig.num_threads = 1; diff --git a/c-api-examples/vad-whisper-c-api.c b/c-api-examples/vad-whisper-c-api.c new file mode 100644 index 000000000..169b4ef12 --- /dev/null +++ b/c-api-examples/vad-whisper-c-api.c @@ -0,0 +1,169 @@ +// c-api-examples/vad-whisper-c-api.c +// +// Copyright (c) 2024 Xiaomi Corporation + +// +// This file demonstrates how to use VAD + Whisper tiny.en with +// sherpa-onnx's C API. +// +// clang-format off +// +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/Obama.wav +// +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2 +// tar xvf sherpa-onnx-whisper-tiny.en.tar.bz2 +// rm sherpa-onnx-whisper-tiny.en.tar.bz2 +// +// clang-format on + +#include +#include +#include + +#include "sherpa-onnx/c-api/c-api.h" + +int32_t main() { + const char *wav_filename = "./Obama.wav"; + const char *vad_filename = "./silero_vad.onnx"; + + const char *encoder = "sherpa-onnx-whisper-tiny.en/tiny.en-encoder.int8.onnx"; + const char *decoder = "sherpa-onnx-whisper-tiny.en/tiny.en-decoder.int8.onnx"; + const char *tokens = "sherpa-onnx-whisper-tiny.en/tiny.en-tokens.txt"; + + const SherpaOnnxWave *wave = SherpaOnnxReadWave(wav_filename); + if (wave == NULL) { + fprintf(stderr, "Failed to read %s\n", wav_filename); + return -1; + } + + if (wave->sample_rate != 16000) { + fprintf(stderr, "Expect the sample rate to be 16000. Given: %d\n", + wave->sample_rate); + SherpaOnnxFreeWave(wave); + return -1; + } + + // Offline model config + SherpaOnnxOfflineModelConfig offline_model_config; + memset(&offline_model_config, 0, sizeof(offline_model_config)); + offline_model_config.debug = 0; + offline_model_config.num_threads = 1; + offline_model_config.provider = "cpu"; + offline_model_config.tokens = tokens; + offline_model_config.whisper.encoder = encoder; + offline_model_config.whisper.decoder = decoder; + offline_model_config.whisper.language = "en"; + offline_model_config.whisper.tail_paddings = 0; + offline_model_config.whisper.task = "transcribe"; + + // Recognizer config + SherpaOnnxOfflineRecognizerConfig recognizer_config; + memset(&recognizer_config, 0, sizeof(recognizer_config)); + recognizer_config.decoding_method = "greedy_search"; + recognizer_config.model_config = offline_model_config; + + const SherpaOnnxOfflineRecognizer *recognizer = + SherpaOnnxCreateOfflineRecognizer(&recognizer_config); + + if (recognizer == NULL) { + fprintf(stderr, "Please check your recognizer config!\n"); + SherpaOnnxFreeWave(wave); + return -1; + } + + SherpaOnnxVadModelConfig vadConfig; + memset(&vadConfig, 0, sizeof(vadConfig)); + vadConfig.silero_vad.model = vad_filename; + vadConfig.silero_vad.threshold = 0.5; + vadConfig.silero_vad.min_silence_duration = 0.5; + vadConfig.silero_vad.min_speech_duration = 0.5; + vadConfig.silero_vad.max_speech_duration = 10; + vadConfig.silero_vad.window_size = 512; + vadConfig.sample_rate = 16000; + vadConfig.num_threads = 1; + vadConfig.debug = 1; + + SherpaOnnxVoiceActivityDetector *vad = + SherpaOnnxCreateVoiceActivityDetector(&vadConfig, 30); + + if (vad == NULL) { + fprintf(stderr, "Please check your recognizer config!\n"); + SherpaOnnxFreeWave(wave); + SherpaOnnxDestroyOfflineRecognizer(recognizer); + return -1; + } + + int32_t window_size = vadConfig.silero_vad.window_size; + int32_t i = 0; + + while (i + window_size < wave->num_samples) { + SherpaOnnxVoiceActivityDetectorAcceptWaveform(vad, wave->samples + i, + window_size); + i += window_size; + + while (!SherpaOnnxVoiceActivityDetectorEmpty(vad)) { + const SherpaOnnxSpeechSegment *segment = + SherpaOnnxVoiceActivityDetectorFront(vad); + + const SherpaOnnxOfflineStream *stream = + SherpaOnnxCreateOfflineStream(recognizer); + + SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate, + segment->samples, segment->n); + + SherpaOnnxDecodeOfflineStream(recognizer, stream); + + const SherpaOnnxOfflineRecognizerResult *result = + SherpaOnnxGetOfflineStreamResult(stream); + + float start = segment->start / 16000.0f; + float duration = segment->n / 16000.0f; + float stop = start + duration; + + fprintf(stderr, "%.3f -- %.3f: %s\n", start, stop, result->text); + + SherpaOnnxDestroyOfflineRecognizerResult(result); + SherpaOnnxDestroyOfflineStream(stream); + + SherpaOnnxDestroySpeechSegment(segment); + SherpaOnnxVoiceActivityDetectorPop(vad); + } + } + + SherpaOnnxVoiceActivityDetectorFlush(vad); + + while (!SherpaOnnxVoiceActivityDetectorEmpty(vad)) { + const SherpaOnnxSpeechSegment *segment = + SherpaOnnxVoiceActivityDetectorFront(vad); + + const SherpaOnnxOfflineStream *stream = + SherpaOnnxCreateOfflineStream(recognizer); + + SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate, segment->samples, + segment->n); + + SherpaOnnxDecodeOfflineStream(recognizer, stream); + + const SherpaOnnxOfflineRecognizerResult *result = + SherpaOnnxGetOfflineStreamResult(stream); + + float start = segment->start / 16000.0f; + float duration = segment->n / 16000.0f; + float stop = start + duration; + + fprintf(stderr, "%.3f -- %.3f: %s\n", start, stop, result->text); + + SherpaOnnxDestroyOfflineRecognizerResult(result); + SherpaOnnxDestroyOfflineStream(stream); + + SherpaOnnxDestroySpeechSegment(segment); + SherpaOnnxVoiceActivityDetectorPop(vad); + } + + SherpaOnnxDestroyOfflineRecognizer(recognizer); + SherpaOnnxDestroyVoiceActivityDetector(vad); + SherpaOnnxFreeWave(wave); + + return 0; +} diff --git a/cxx-api-examples/CMakeLists.txt b/cxx-api-examples/CMakeLists.txt index 7c9853080..e7e722660 100644 --- a/cxx-api-examples/CMakeLists.txt +++ b/cxx-api-examples/CMakeLists.txt @@ -6,5 +6,8 @@ target_link_libraries(streaming-zipformer-cxx-api sherpa-onnx-cxx-api) add_executable(whisper-cxx-api ./whisper-cxx-api.cc) target_link_libraries(whisper-cxx-api sherpa-onnx-cxx-api) +add_executable(moonshine-cxx-api ./moonshine-cxx-api.cc) +target_link_libraries(moonshine-cxx-api sherpa-onnx-cxx-api) + add_executable(sense-voice-cxx-api ./sense-voice-cxx-api.cc) target_link_libraries(sense-voice-cxx-api sherpa-onnx-cxx-api) diff --git a/cxx-api-examples/moonshine-cxx-api.cc b/cxx-api-examples/moonshine-cxx-api.cc new file mode 100644 index 000000000..c2ce565c3 --- /dev/null +++ b/cxx-api-examples/moonshine-cxx-api.cc @@ -0,0 +1,81 @@ +// cxx-api-examples/moonshine-cxx-api.cc +// Copyright (c) 2024 Xiaomi Corporation + +// +// This file demonstrates how to use Moonshine with sherpa-onnx's C++ API. +// +// clang-format off +// +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +// tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +// rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +// +// clang-format on + +#include // NOLINT +#include +#include + +#include "sherpa-onnx/c-api/cxx-api.h" + +int32_t main() { + using namespace sherpa_onnx::cxx; // NOLINT + OfflineRecognizerConfig config; + + config.model_config.moonshine.preprocessor = + "./sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx"; + config.model_config.moonshine.encoder = + "./sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx"; + config.model_config.moonshine.uncached_decoder = + "./sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx"; + config.model_config.moonshine.cached_decoder = + "./sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx"; + config.model_config.tokens = + "./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt"; + + config.model_config.num_threads = 1; + + std::cout << "Loading model\n"; + OfflineRecognizer recongizer = OfflineRecognizer::Create(config); + if (!recongizer.Get()) { + std::cerr << "Please check your config\n"; + return -1; + } + std::cout << "Loading model done\n"; + + std::string wave_filename = + "./sherpa-onnx-moonshine-tiny-en-int8/test_wavs/0.wav"; + Wave wave = ReadWave(wave_filename); + if (wave.samples.empty()) { + std::cerr << "Failed to read: '" << wave_filename << "'\n"; + return -1; + } + + std::cout << "Start recognition\n"; + const auto begin = std::chrono::steady_clock::now(); + + OfflineStream stream = recongizer.CreateStream(); + stream.AcceptWaveform(wave.sample_rate, wave.samples.data(), + wave.samples.size()); + + recongizer.Decode(&stream); + + OfflineRecognizerResult result = recongizer.GetResult(&stream); + + const auto end = std::chrono::steady_clock::now(); + const float elapsed_seconds = + std::chrono::duration_cast(end - begin) + .count() / + 1000.; + float duration = wave.samples.size() / static_cast(wave.sample_rate); + float rtf = elapsed_seconds / duration; + + std::cout << "text: " << result.text << "\n"; + printf("Number of threads: %d\n", config.model_config.num_threads); + printf("Duration: %.3fs\n", duration); + printf("Elapsed seconds: %.3fs\n", elapsed_seconds); + printf("(Real time factor) RTF = %.3f / %.3f = %.3f\n", elapsed_seconds, + duration, rtf); + + return 0; +} diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index d7fa383be..f01b4917f 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -450,6 +450,18 @@ sherpa_onnx::OfflineRecognizerConfig convertConfig( recognizer_config.model_config.sense_voice.use_itn = config->model_config.sense_voice.use_itn; + recognizer_config.model_config.moonshine.preprocessor = + SHERPA_ONNX_OR(config->model_config.moonshine.preprocessor, ""); + + recognizer_config.model_config.moonshine.encoder = + SHERPA_ONNX_OR(config->model_config.moonshine.encoder, ""); + + recognizer_config.model_config.moonshine.uncached_decoder = + SHERPA_ONNX_OR(config->model_config.moonshine.uncached_decoder, ""); + + recognizer_config.model_config.moonshine.cached_decoder = + SHERPA_ONNX_OR(config->model_config.moonshine.cached_decoder, ""); + recognizer_config.lm_config.model = SHERPA_ONNX_OR(config->lm_config.model, ""); recognizer_config.lm_config.scale = diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index e5fc92eb1..8b4b65786 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -389,6 +389,13 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOfflineWhisperModelConfig { int32_t tail_paddings; } SherpaOnnxOfflineWhisperModelConfig; +SHERPA_ONNX_API typedef struct SherpaOnnxOfflineMoonshineModelConfig { + const char *preprocessor; + const char *encoder; + const char *uncached_decoder; + const char *cached_decoder; +} SherpaOnnxOfflineMoonshineModelConfig; + SHERPA_ONNX_API typedef struct SherpaOnnxOfflineTdnnModelConfig { const char *model; } SherpaOnnxOfflineTdnnModelConfig; @@ -424,6 +431,7 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOfflineModelConfig { const char *bpe_vocab; const char *telespeech_ctc; SherpaOnnxOfflineSenseVoiceModelConfig sense_voice; + SherpaOnnxOfflineMoonshineModelConfig moonshine; } SherpaOnnxOfflineModelConfig; SHERPA_ONNX_API typedef struct SherpaOnnxOfflineRecognizerConfig { diff --git a/sherpa-onnx/c-api/cxx-api.cc b/sherpa-onnx/c-api/cxx-api.cc index 262ad3cc9..c66221f0e 100644 --- a/sherpa-onnx/c-api/cxx-api.cc +++ b/sherpa-onnx/c-api/cxx-api.cc @@ -227,6 +227,15 @@ OfflineRecognizer OfflineRecognizer::Create( config.model_config.sense_voice.language.c_str(); c.model_config.sense_voice.use_itn = config.model_config.sense_voice.use_itn; + c.model_config.moonshine.preprocessor = + config.model_config.moonshine.preprocessor.c_str(); + c.model_config.moonshine.encoder = + config.model_config.moonshine.encoder.c_str(); + c.model_config.moonshine.uncached_decoder = + config.model_config.moonshine.uncached_decoder.c_str(); + c.model_config.moonshine.cached_decoder = + config.model_config.moonshine.cached_decoder.c_str(); + c.lm_config.model = config.lm_config.model.c_str(); c.lm_config.scale = config.lm_config.scale; diff --git a/sherpa-onnx/c-api/cxx-api.h b/sherpa-onnx/c-api/cxx-api.h index 17727f817..b8a46e113 100644 --- a/sherpa-onnx/c-api/cxx-api.h +++ b/sherpa-onnx/c-api/cxx-api.h @@ -225,6 +225,13 @@ struct SHERPA_ONNX_API OfflineSenseVoiceModelConfig { bool use_itn = false; }; +struct SHERPA_ONNX_API OfflineMoonshineModelConfig { + std::string preprocessor; + std::string encoder; + std::string uncached_decoder; + std::string cached_decoder; +}; + struct SHERPA_ONNX_API OfflineModelConfig { OfflineTransducerModelConfig transducer; OfflineParaformerModelConfig paraformer; @@ -241,6 +248,7 @@ struct SHERPA_ONNX_API OfflineModelConfig { std::string bpe_vocab; std::string telespeech_ctc; OfflineSenseVoiceModelConfig sense_voice; + OfflineMoonshineModelConfig moonshine; }; struct SHERPA_ONNX_API OfflineLMConfig { From 4a4659aa4fad632280abdf6e5ae9f7896cb75d09 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sun, 27 Oct 2024 08:19:01 +0800 Subject: [PATCH 044/183] Add Swift API for Moonshine models. (#1477) --- swift-api-examples/SherpaOnnx.swift | 20 ++++++++++++++++-- .../decode-file-non-streaming.swift | 21 +++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/swift-api-examples/SherpaOnnx.swift b/swift-api-examples/SherpaOnnx.swift index 783f59224..661ebba28 100644 --- a/swift-api-examples/SherpaOnnx.swift +++ b/swift-api-examples/SherpaOnnx.swift @@ -357,6 +357,20 @@ func sherpaOnnxOfflineWhisperModelConfig( ) } +func sherpaOnnxOfflineMoonshineModelConfig( + preprocessor: String = "", + encoder: String = "", + uncachedDecoder: String = "", + cachedDecoder: String = "" +) -> SherpaOnnxOfflineMoonshineModelConfig { + return SherpaOnnxOfflineMoonshineModelConfig( + preprocessor: toCPointer(preprocessor), + encoder: toCPointer(encoder), + uncached_decoder: toCPointer(uncachedDecoder), + cached_decoder: toCPointer(cachedDecoder) + ) +} + func sherpaOnnxOfflineTdnnModelConfig( model: String = "" ) -> SherpaOnnxOfflineTdnnModelConfig { @@ -401,7 +415,8 @@ func sherpaOnnxOfflineModelConfig( modelingUnit: String = "cjkchar", bpeVocab: String = "", teleSpeechCtc: String = "", - senseVoice: SherpaOnnxOfflineSenseVoiceModelConfig = sherpaOnnxOfflineSenseVoiceModelConfig() + senseVoice: SherpaOnnxOfflineSenseVoiceModelConfig = sherpaOnnxOfflineSenseVoiceModelConfig(), + moonshine: SherpaOnnxOfflineMoonshineModelConfig = sherpaOnnxOfflineMoonshineModelConfig() ) -> SherpaOnnxOfflineModelConfig { return SherpaOnnxOfflineModelConfig( transducer: transducer, @@ -417,7 +432,8 @@ func sherpaOnnxOfflineModelConfig( modeling_unit: toCPointer(modelingUnit), bpe_vocab: toCPointer(bpeVocab), telespeech_ctc: toCPointer(teleSpeechCtc), - sense_voice: senseVoice + sense_voice: senseVoice, + moonshine: moonshine ) } diff --git a/swift-api-examples/decode-file-non-streaming.swift b/swift-api-examples/decode-file-non-streaming.swift index a60777832..2e5e9f9ac 100644 --- a/swift-api-examples/decode-file-non-streaming.swift +++ b/swift-api-examples/decode-file-non-streaming.swift @@ -18,6 +18,7 @@ func run() { var modelType = "whisper" // modelType = "paraformer" // modelType = "sense_voice" + // modelType = "moonshine" if modelType == "whisper" { let encoder = "./sherpa-onnx-whisper-tiny.en/tiny.en-encoder.int8.onnx" @@ -61,6 +62,24 @@ func run() { debug: 0, senseVoice: senseVoiceConfig ) + } else if modelType == "moonshine" { + let preprocessor = "./sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx" + let encoder = "./sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx" + let uncachedDecoder = "./sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx" + let cachedDecoder = "./sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx" + let tokens = "./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt" + let moonshine = sherpaOnnxOfflineMoonshineModelConfig( + preprocessor: preprocessor, + encoder: encoder, + uncachedDecoder: uncachedDecoder, + cachedDecoder: cachedDecoder + ) + + modelConfig = sherpaOnnxOfflineModelConfig( + tokens: tokens, + debug: 0, + moonshine: moonshine + ) } else { print("Please specify a supported modelType \(modelType)") return @@ -80,6 +99,8 @@ func run() { var filePath = "./sherpa-onnx-whisper-tiny.en/test_wavs/0.wav" if modelType == "sense_voice" { filePath = "./sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/test_wavs/zh.wav" + } else if modelType == "moonshine" { + filePath = "./sherpa-onnx-moonshine-tiny-en-int8/test_wavs/0.wav" } let fileURL: NSURL = NSURL(fileURLWithPath: filePath) let audioFile = try! AVAudioFile(forReading: fileURL as URL) From 052b8645ba49194460bbf12a50d5079f33ba9545 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sun, 27 Oct 2024 09:04:05 +0800 Subject: [PATCH 045/183] Add Go API examples for adding punctuations to text. (#1478) --- .github/workflows/test-go-package.yaml | 7 +++++ .github/workflows/test-go.yaml | 6 ++++ go-api-examples/add-punctuation/go.mod | 3 ++ go-api-examples/add-punctuation/main.go | 31 ++++++++++++++++++++ go-api-examples/add-punctuation/run.sh | 14 +++++++++ scripts/go/_internal/add-punctuation/go.mod | 5 ++++ scripts/go/_internal/add-punctuation/main.go | 1 + scripts/go/_internal/add-punctuation/run.sh | 1 + scripts/go/sherpa_onnx.go | 12 ++++---- 9 files changed, 74 insertions(+), 6 deletions(-) create mode 100644 go-api-examples/add-punctuation/go.mod create mode 100644 go-api-examples/add-punctuation/main.go create mode 100755 go-api-examples/add-punctuation/run.sh create mode 100644 scripts/go/_internal/add-punctuation/go.mod create mode 120000 scripts/go/_internal/add-punctuation/main.go create mode 120000 scripts/go/_internal/add-punctuation/run.sh diff --git a/.github/workflows/test-go-package.yaml b/.github/workflows/test-go-package.yaml index 6839b0bab..9074449b4 100644 --- a/.github/workflows/test-go-package.yaml +++ b/.github/workflows/test-go-package.yaml @@ -68,6 +68,13 @@ jobs: run: | gcc --version + - name: Test adding punctuation + if: matrix.os != 'windows-latest' + shell: bash + run: | + cd go-api-examples/add-punctuation/ + ./run.sh + - name: Test non-streaming speaker diarization if: matrix.os != 'windows-latest' shell: bash diff --git a/.github/workflows/test-go.yaml b/.github/workflows/test-go.yaml index ccb3eb8d9..1e8ad9c83 100644 --- a/.github/workflows/test-go.yaml +++ b/.github/workflows/test-go.yaml @@ -134,6 +134,12 @@ jobs: name: ${{ matrix.os }}-libs path: to-upload/ + - name: Test adding punctuation + shell: bash + run: | + cd scripts/go/_internal/add-punctuation/ + ./run.sh + - name: Test non-streaming speaker diarization shell: bash run: | diff --git a/go-api-examples/add-punctuation/go.mod b/go-api-examples/add-punctuation/go.mod new file mode 100644 index 000000000..ec6d75805 --- /dev/null +++ b/go-api-examples/add-punctuation/go.mod @@ -0,0 +1,3 @@ +module add-punctuation + +go 1.12 diff --git a/go-api-examples/add-punctuation/main.go b/go-api-examples/add-punctuation/main.go new file mode 100644 index 000000000..055748ea8 --- /dev/null +++ b/go-api-examples/add-punctuation/main.go @@ -0,0 +1,31 @@ +package main + +import ( + sherpa "github.com/k2-fsa/sherpa-onnx-go/sherpa_onnx" + "log" +) + +func main() { + log.SetFlags(log.LstdFlags | log.Lmicroseconds) + + config := sherpa.OfflinePunctuationConfig{} + config.Model.CtTransformer = "./sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12/model.onnx" + config.Model.NumThreads = 1 + config.Model.Provider = "cpu" + + punct := sherpa.NewOfflinePunctuation(&config) + defer sherpa.DeleteOfflinePunc(punct) + + textArray := []string{ + "这是一个测试你好吗How are you我很好thank you are you ok谢谢你", + "我们都是木头人不会说话不会动", + "The African blogosphere is rapidly expanding bringing more voices online in the form of commentaries opinions analyses rants and poetry", + } + log.Println("----------") + for _, text := range textArray { + newText := punct.AddPunct(text) + log.Printf("Input text: %v", text) + log.Printf("Output text: %v", newText) + log.Println("----------") + } +} diff --git a/go-api-examples/add-punctuation/run.sh b/go-api-examples/add-punctuation/run.sh new file mode 100755 index 000000000..6d43b84f0 --- /dev/null +++ b/go-api-examples/add-punctuation/run.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -ex + +if [ ! -d ./sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12 ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/punctuation-models/sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2 + tar xvf sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2 + rm sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2 +fi + +go mod tidy +go build + +./add-punctuation diff --git a/scripts/go/_internal/add-punctuation/go.mod b/scripts/go/_internal/add-punctuation/go.mod new file mode 100644 index 000000000..f25ca0a86 --- /dev/null +++ b/scripts/go/_internal/add-punctuation/go.mod @@ -0,0 +1,5 @@ +module add-punctuation + +go 1.12 + +replace github.com/k2-fsa/sherpa-onnx-go/sherpa_onnx => ../ diff --git a/scripts/go/_internal/add-punctuation/main.go b/scripts/go/_internal/add-punctuation/main.go new file mode 120000 index 000000000..84df910cd --- /dev/null +++ b/scripts/go/_internal/add-punctuation/main.go @@ -0,0 +1 @@ +../../../../go-api-examples/add-punctuation/main.go \ No newline at end of file diff --git a/scripts/go/_internal/add-punctuation/run.sh b/scripts/go/_internal/add-punctuation/run.sh new file mode 120000 index 000000000..2b1ee21b4 --- /dev/null +++ b/scripts/go/_internal/add-punctuation/run.sh @@ -0,0 +1 @@ +../../../../go-api-examples/add-punctuation/run.sh \ No newline at end of file diff --git a/scripts/go/sherpa_onnx.go b/scripts/go/sherpa_onnx.go index 45bf714bb..30ca31dce 100644 --- a/scripts/go/sherpa_onnx.go +++ b/scripts/go/sherpa_onnx.go @@ -1322,10 +1322,10 @@ func (sd *OfflineSpeakerDiarization) Process(samples []float32) []OfflineSpeaker // For punctuation // ============================================================ type OfflinePunctuationModelConfig struct { - Ct_transformer string - Num_threads C.int - Debug C.int // true to print debug information of the model - Provider string + CtTransformer string + NumThreads C.int + Debug C.int // true to print debug information of the model + Provider string } type OfflinePunctuationConfig struct { @@ -1338,10 +1338,10 @@ type OfflinePunctuation struct { func NewOfflinePunctuation(config *OfflinePunctuationConfig) *OfflinePunctuation { cfg := C.struct_SherpaOnnxOfflinePunctuationConfig{} - cfg.model.ct_transformer = C.CString(config.Model.Ct_transformer) + cfg.model.ct_transformer = C.CString(config.Model.CtTransformer) defer C.free(unsafe.Pointer(cfg.model.ct_transformer)) - cfg.model.num_threads = config.Model.Num_threads + cfg.model.num_threads = config.Model.NumThreads cfg.model.debug = config.Model.Debug cfg.model.provider = C.CString(config.Model.Provider) defer C.free(unsafe.Pointer(cfg.model.provider)) From 3d3edabb5ff9fbd68f511d2fffea9353e539c2e7 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sun, 27 Oct 2024 09:39:09 +0800 Subject: [PATCH 046/183] Add Go API for Moonshine models (#1479) --- .github/workflows/test-go.yaml | 90 ++++++++++--------- go-api-examples/README.md | 25 ++++-- .../non-streaming-decode-files/main.go | 13 +-- .../run-moonshine.sh | 21 +++++ .../run-moonshine.sh | 1 + scripts/go/sherpa_onnx.go | 20 +++++ 6 files changed, 115 insertions(+), 55 deletions(-) create mode 100755 go-api-examples/non-streaming-decode-files/run-moonshine.sh create mode 120000 scripts/go/_internal/non-streaming-decode-files/run-moonshine.sh diff --git a/.github/workflows/test-go.yaml b/.github/workflows/test-go.yaml index 1e8ad9c83..9c9951111 100644 --- a/.github/workflows/test-go.yaml +++ b/.github/workflows/test-go.yaml @@ -134,6 +134,53 @@ jobs: name: ${{ matrix.os }}-libs path: to-upload/ + - name: Test non-streaming decoding files + shell: bash + run: | + cd scripts/go/_internal/non-streaming-decode-files/ + ls -lh + go mod tidy + cat go.mod + go build + ls -lh + + echo "Test Moonshine" + ./run-moonshine.sh + rm -rf sherpa-onnx-* + + echo "Test SenseVoice ctc" + ./run-sense-voice-small.sh + rm -rf sherpa-onnx-sense-* + + echo "Test telespeech ctc" + ./run-telespeech-ctc.sh + rm -rf sherpa-onnx-telespeech-ctc-* + + echo "Test transducer" + ./run-transducer.sh + rm -rf sherpa-onnx-zipformer-en-2023-06-26 + + echo "Test transducer" + ./run-transducer.sh + rm -rf sherpa-onnx-zipformer-en-2023-06-26 + + echo "Test paraformer" + ./run-paraformer.sh + ./run-paraformer-itn.sh + rm -rf sherpa-onnx-paraformer-zh-2023-09-14 + + echo "Test NeMo CTC" + ./run-nemo-ctc.sh + rm -rf sherpa-onnx-nemo-ctc-en-conformer-medium + + echo "Test Whisper tiny.en" + ./run-whisper.sh + rm -rf sherpa-onnx-whisper-tiny.en + + echo "Test Tdnn yesno" + ./run-tdnn-yesno.sh + rm -rf sherpa-onnx-tdnn-yesno + - name: Test adding punctuation shell: bash run: | @@ -193,49 +240,6 @@ jobs: name: tts-waves-${{ matrix.os }} path: tts-waves - - name: Test non-streaming decoding files - shell: bash - run: | - cd scripts/go/_internal/non-streaming-decode-files/ - ls -lh - go mod tidy - cat go.mod - go build - ls -lh - - echo "Test SenseVoice ctc" - ./run-sense-voice-small.sh - rm -rf sherpa-onnx-sense-* - - echo "Test telespeech ctc" - ./run-telespeech-ctc.sh - rm -rf sherpa-onnx-telespeech-ctc-* - - echo "Test transducer" - ./run-transducer.sh - rm -rf sherpa-onnx-zipformer-en-2023-06-26 - - echo "Test transducer" - ./run-transducer.sh - rm -rf sherpa-onnx-zipformer-en-2023-06-26 - - echo "Test paraformer" - ./run-paraformer.sh - ./run-paraformer-itn.sh - rm -rf sherpa-onnx-paraformer-zh-2023-09-14 - - echo "Test NeMo CTC" - ./run-nemo-ctc.sh - rm -rf sherpa-onnx-nemo-ctc-en-conformer-medium - - echo "Test Whisper tiny.en" - ./run-whisper.sh - rm -rf sherpa-onnx-whisper-tiny.en - - echo "Test Tdnn yesno" - ./run-tdnn-yesno.sh - rm -rf sherpa-onnx-tdnn-yesno - - name: Test streaming decoding files shell: bash run: | diff --git a/go-api-examples/README.md b/go-api-examples/README.md index 91f2c76e1..e16dab690 100644 --- a/go-api-examples/README.md +++ b/go-api-examples/README.md @@ -6,28 +6,41 @@ Please refer to the documentation https://k2-fsa.github.io/sherpa/onnx/go-api/index.html for details. +- [./add-punctuation](./add-punctuation) It shows how to use + a punctuation model to add punctuations to text + - [./non-streaming-decode-files](./non-streaming-decode-files) It shows how to use a non-streaming ASR model to decode files +- [./non-streaming-speaker-diarization](./non-streaming-speaker-diarization) It shows how to use + a speaker segmentation model and a speaker embedding model for speaker diarization. + - [./non-streaming-tts](./non-streaming-tts) It shows how to use a non-streaming TTS model to convert text to speech - [./real-time-speech-recognition-from-microphone](./real-time-speech-recognition-from-microphone) It shows how to use a streaming ASR model to recognize speech from a microphone in real-time +- [./speaker-identification](./speaker-identification) It shows how to use a speaker + embedding model for speaker identification. + +- [./streaming-decode-files](./streaming-decode-files) It shows how to use a streaming + model for streaming speech recognition + +- [./streaming-hlg-decoding](./streaming-hlg-decoding) It shows how to use a streaming + model for streaming speech recognition with HLG decoding + - [./vad](./vad) It shows how to use silero VAD with Golang. -- [./vad-asr-whisper](./vad-asr-whisper) It shows how to use silero VAD + Whisper +- [./vad-asr-paraformer](./vad-asr-paraformer) It shows how to use silero VAD + Paraformer for speech recognition. -- [./vad-asr-paraformer](./vad-asr-paraformer) It shows how to use silero VAD + Paraformer +- [./vad-asr-whisper](./vad-asr-whisper) It shows how to use silero VAD + Whisper + +- [./vad-speaker-identification](./vad-speaker-identification) It shows how to use Go API for VAD + speaker identification. for speech recognition. - [./vad-spoken-language-identification](./vad-spoken-language-identification) It shows how to use silero VAD + Whisper for spoken language identification. -- [./speaker-identification](./speaker-identification) It shows how to use Go API for speaker identification. - -- [./vad-speaker-identification](./vad-speaker-identification) It shows how to use Go API for VAD + speaker identification. - [sherpa-onnx]: https://github.com/k2-fsa/sherpa-onnx diff --git a/go-api-examples/non-streaming-decode-files/main.go b/go-api-examples/non-streaming-decode-files/main.go index 5373dcf29..92b23dc19 100644 --- a/go-api-examples/non-streaming-decode-files/main.go +++ b/go-api-examples/non-streaming-decode-files/main.go @@ -34,6 +34,11 @@ func main() { flag.StringVar(&config.ModelConfig.Whisper.Task, "whisper-task", "transcribe", "transcribe or translate") flag.IntVar(&config.ModelConfig.Whisper.TailPaddings, "whisper-tail-paddings", -1, "tail paddings for whisper") + flag.StringVar(&config.ModelConfig.Moonshine.Preprocessor, "moonshine-preprocessor", "", "Path to the moonshine preprocessor model") + flag.StringVar(&config.ModelConfig.Moonshine.Encoder, "moonshine-encoder", "", "Path to the moonshine encoder model") + flag.StringVar(&config.ModelConfig.Moonshine.UncachedDecoder, "moonshine-uncached-decoder", "", "Path to the moonshine uncached decoder model") + flag.StringVar(&config.ModelConfig.Moonshine.CachedDecoder, "moonshine-cached-decoder", "", "Path to the moonshine cached decoder model") + flag.StringVar(&config.ModelConfig.Tdnn.Model, "tdnn-model", "", "Path to the tdnn model") flag.StringVar(&config.ModelConfig.SenseVoice.Model, "sense-voice-model", "", "Path to the SenseVoice model") @@ -85,12 +90,8 @@ func main() { log.Println("Emotion: " + result.Emotion) log.Println("Lang: " + result.Lang) log.Println("Event: " + result.Event) - for _, v := range result.Timestamps { - log.Printf("Timestamp: %+v\n", v) - } - for _, v := range result.Tokens { - log.Println("Token: " + v) - } + log.Printf("Timestamp: %v\n", result.Timestamps) + log.Printf("Tokens: %v\n", result.Tokens) log.Printf("Wave duration: %v seconds", float32(len(samples))/float32(sampleRate)) } diff --git a/go-api-examples/non-streaming-decode-files/run-moonshine.sh b/go-api-examples/non-streaming-decode-files/run-moonshine.sh new file mode 100755 index 000000000..409101e4e --- /dev/null +++ b/go-api-examples/non-streaming-decode-files/run-moonshine.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -ex + +if [ ! -f ./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +fi + +go mod tidy +go build + +./non-streaming-decode-files \ + --moonshine-preprocessor=./sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx \ + --moonshine-encoder=./sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx \ + --moonshine-uncached-decoder=./sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx \ + --moonshine-cached-decoder=./sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx \ + --tokens=./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt \ + ./sherpa-onnx-moonshine-tiny-en-int8/test_wavs/0.wav + diff --git a/scripts/go/_internal/non-streaming-decode-files/run-moonshine.sh b/scripts/go/_internal/non-streaming-decode-files/run-moonshine.sh new file mode 120000 index 000000000..95064f1a8 --- /dev/null +++ b/scripts/go/_internal/non-streaming-decode-files/run-moonshine.sh @@ -0,0 +1 @@ +../../../../go-api-examples/non-streaming-decode-files/run-moonshine.sh \ No newline at end of file diff --git a/scripts/go/sherpa_onnx.go b/scripts/go/sherpa_onnx.go index 30ca31dce..cde6513da 100644 --- a/scripts/go/sherpa_onnx.go +++ b/scripts/go/sherpa_onnx.go @@ -382,6 +382,13 @@ type OfflineWhisperModelConfig struct { TailPaddings int } +type OfflineMoonshineModelConfig struct { + Preprocessor string + Encoder string + UncachedDecoder string + CachedDecoder string +} + type OfflineTdnnModelConfig struct { Model string } @@ -405,6 +412,7 @@ type OfflineModelConfig struct { Whisper OfflineWhisperModelConfig Tdnn OfflineTdnnModelConfig SenseVoice OfflineSenseVoiceModelConfig + Moonshine OfflineMoonshineModelConfig Tokens string // Path to tokens.txt // Number of threads to use for neural network computation @@ -515,6 +523,18 @@ func NewOfflineRecognizer(config *OfflineRecognizerConfig) *OfflineRecognizer { c.model_config.sense_voice.use_itn = C.int(config.ModelConfig.SenseVoice.UseInverseTextNormalization) + c.model_config.moonshine.preprocessor = C.CString(config.ModelConfig.Moonshine.Preprocessor) + defer C.free(unsafe.Pointer(c.model_config.moonshine.preprocessor)) + + c.model_config.moonshine.encoder = C.CString(config.ModelConfig.Moonshine.Encoder) + defer C.free(unsafe.Pointer(c.model_config.moonshine.encoder)) + + c.model_config.moonshine.uncached_decoder = C.CString(config.ModelConfig.Moonshine.UncachedDecoder) + defer C.free(unsafe.Pointer(c.model_config.moonshine.uncached_decoder)) + + c.model_config.moonshine.cached_decoder = C.CString(config.ModelConfig.Moonshine.CachedDecoder) + defer C.free(unsafe.Pointer(c.model_config.moonshine.cached_decoder)) + c.model_config.tokens = C.CString(config.ModelConfig.Tokens) defer C.free(unsafe.Pointer(c.model_config.tokens)) From 6f261d39f392eb9c8c7a713d4eaecb3f13677854 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sun, 27 Oct 2024 11:31:01 +0800 Subject: [PATCH 047/183] Add JavaScript API for Moonshine models (#1480) --- .github/scripts/test-nodejs-addon-npm.sh | 20 ++- .github/scripts/test-nodejs-npm.sh | 24 ++++ nodejs-addon-examples/README.md | 30 ++++ .../test_asr_non_streaming_moonshine.js | 50 +++++++ ..._asr_non_streaming_moonshine_microphone.js | 113 +++++++++++++++ ...st_vad_with_non_streaming_asr_moonshine.js | 132 ++++++++++++++++++ nodejs-examples/README.md | 51 +++++++ nodejs-examples/test-offline-moonshine.js | 37 +++++ ...st-vad-with-non-streaming-asr-moonshine.js | 128 +++++++++++++++++ scripts/node-addon-api/src/macros.h | 7 + .../node-addon-api/src/non-streaming-asr.cc | 129 +++++++---------- wasm/asr/sherpa-onnx-asr.js | 75 +++++++++- wasm/nodejs/sherpa-onnx-wasm-nodejs.cc | 11 +- 13 files changed, 719 insertions(+), 88 deletions(-) create mode 100644 nodejs-addon-examples/test_asr_non_streaming_moonshine.js create mode 100644 nodejs-addon-examples/test_vad_asr_non_streaming_moonshine_microphone.js create mode 100644 nodejs-addon-examples/test_vad_with_non_streaming_asr_moonshine.js create mode 100644 nodejs-examples/test-offline-moonshine.js create mode 100644 nodejs-examples/test-vad-with-non-streaming-asr-moonshine.js diff --git a/.github/scripts/test-nodejs-addon-npm.sh b/.github/scripts/test-nodejs-addon-npm.sh index 42c753ebf..755cde74b 100755 --- a/.github/scripts/test-nodejs-addon-npm.sh +++ b/.github/scripts/test-nodejs-addon-npm.sh @@ -10,6 +10,19 @@ arch=$(node -p "require('os').arch()") platform=$(node -p "require('os').platform()") node_version=$(node -p "process.versions.node.split('.')[0]") +echo "----------non-streaming asr moonshine + vad----------" +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/Obama.wav +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx + +node ./test_vad_with_non_streaming_asr_moonshine.js +rm -rf sherpa-onnx-* +rm *.wav +rm *.onnx + echo "----------non-streaming speaker diarization----------" curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 @@ -24,7 +37,7 @@ node ./test_offline_speaker_diarization.js rm -rfv *.onnx *.wav sherpa-onnx-pyannote-* -echo "----------non-streaming asr + vad----------" +echo "----------non-streaming asr whisper + vad----------" curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2 tar xvf sherpa-onnx-whisper-tiny.en.tar.bz2 rm sherpa-onnx-whisper-tiny.en.tar.bz2 @@ -218,6 +231,11 @@ rm sherpa-onnx-whisper-tiny.en.tar.bz2 node ./test_asr_non_streaming_whisper.js rm -rf sherpa-onnx-whisper-tiny.en +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +node ./test_asr_non_streaming_moonshine.js +rm -rf sherpa-onnx-* ls -lh diff --git a/.github/scripts/test-nodejs-npm.sh b/.github/scripts/test-nodejs-npm.sh index 03dec04aa..518d173b6 100755 --- a/.github/scripts/test-nodejs-npm.sh +++ b/.github/scripts/test-nodejs-npm.sh @@ -21,6 +21,23 @@ curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segm node ./test-offline-speaker-diarization.js rm -rfv *.wav *.onnx sherpa-onnx-pyannote-* +echo '-----vad+moonshine----------' + +curl -LS -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2 +tar xvf sherpa-onnx-whisper-tiny.en.tar.bz2 +rm sherpa-onnx-whisper-tiny.en.tar.bz2 + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/Obama.wav +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx +node ./test-vad-with-non-streaming-asr-whisper.js +rm Obama.wav +rm silero_vad.onnx +rm -rf sherpa-onnx-moonshine-* + echo '-----vad+whisper----------' curl -LS -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2 @@ -90,6 +107,13 @@ rm sherpa-onnx-whisper-tiny.en.tar.bz2 node ./test-offline-whisper.js rm -rf sherpa-onnx-whisper-tiny.en +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + +node ./test-offline-moonshine.js +rm -rf sherpa-onnx-moonshine-* + # online asr curl -LS -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2 tar xvf sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2 diff --git a/nodejs-addon-examples/README.md b/nodejs-addon-examples/README.md index 8851c6265..f436bcae2 100644 --- a/nodejs-addon-examples/README.md +++ b/nodejs-addon-examples/README.md @@ -112,6 +112,8 @@ The following tables list the examples in this folder. |[./test_asr_non_streaming_transducer.js](./test_asr_non_streaming_transducer.js)|Non-streaming speech recognition from a file with a Zipformer transducer model| |[./test_asr_non_streaming_whisper.js](./test_asr_non_streaming_whisper.js)| Non-streaming speech recognition from a file using [Whisper](https://github.com/openai/whisper)| |[./test_vad_with_non_streaming_asr_whisper.js](./test_vad_with_non_streaming_asr_whisper.js)| Non-streaming speech recognition from a file using [Whisper](https://github.com/openai/whisper) + [Silero VAD](https://github.com/snakers4/silero-vad)| +|[./test_asr_non_streaming_moonshine.js](./test_asr_non_streaming_moonshine.js)|Non-streaming speech recognition from a file using [Moonshine](https://github.com/usefulsensors/moonshine)| +|[./test_vad_with_non_streaming_asr_moonshine.js](./test_vad_with_non_streaming_asr_moonshine.js)| Non-streaming speech recognition from a file using [Moonshine](https://github.com/usefulsensors/moonshine) + [Silero VAD](https://github.com/snakers4/silero-vad)| |[./test_asr_non_streaming_nemo_ctc.js](./test_asr_non_streaming_nemo_ctc.js)|Non-streaming speech recognition from a file using a [NeMo](https://github.com/NVIDIA/NeMo) CTC model with greedy search| |[./test_asr_non_streaming_paraformer.js](./test_asr_non_streaming_paraformer.js)|Non-streaming speech recognition from a file using [Paraformer](https://github.com/alibaba-damo-academy/FunASR)| |[./test_asr_non_streaming_sense_voice.js](./test_asr_non_streaming_sense_voice.js)|Non-streaming speech recognition from a file using [SenseVoice](https://github.com/FunAudioLLM/SenseVoice)| @@ -122,6 +124,7 @@ The following tables list the examples in this folder. |---|---| |[./test_vad_asr_non_streaming_transducer_microphone.js](./test_vad_asr_non_streaming_transducer_microphone.js)|VAD + Non-streaming speech recognition from a microphone using a Zipformer transducer model| |[./test_vad_asr_non_streaming_whisper_microphone.js](./test_vad_asr_non_streaming_whisper_microphone.js)|VAD + Non-streaming speech recognition from a microphone using [Whisper](https://github.com/openai/whisper)| +|[./test_vad_asr_non_streaming_moonshine_microphone.js](./test_vad_asr_non_streaming_moonshine_microphone.js)|VAD + Non-streaming speech recognition from a microphone using [Moonshine](https://github.com/usefulsensors/moonshine)| |[./test_vad_asr_non_streaming_nemo_ctc_microphone.js](./test_vad_asr_non_streaming_nemo_ctc_microphone.js)|VAD + Non-streaming speech recognition from a microphone using a [NeMo](https://github.com/NVIDIA/NeMo) CTC model with greedy search| |[./test_vad_asr_non_streaming_paraformer_microphone.js](./test_vad_asr_non_streaming_paraformer_microphone.js)|VAD + Non-streaming speech recognition from a microphone using [Paraformer](https://github.com/alibaba-damo-academy/FunASR)| |[./test_vad_asr_non_streaming_sense_voice_microphone.js](./test_vad_asr_non_streaming_sense_voice_microphone.js)|VAD + Non-streaming speech recognition from a microphone using [SenseVoice](https://github.com/FunAudioLLM/SenseVoice)| @@ -260,6 +263,33 @@ npm install naudiodon2 node ./test_vad_asr_non_streaming_whisper_microphone.js ``` +### Non-streaming speech recognition with Moonshine + +```bash +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + +node ./test_asr_non_streaming_moonshine.js + +# To run VAD + non-streaming ASR with Moonshine using a microphone +npm install naudiodon2 +node ./test_vad_asr_non_streaming_moonshine_microphone.js +``` + +### Non-streaming speech recognition with Moonshine + VAD + +```bash +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/Obama.wav +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx + +node ./test_vad_with_non_streaming_asr_moonshine.js +``` + ### Non-streaming speech recognition with Whisper + VAD ```bash diff --git a/nodejs-addon-examples/test_asr_non_streaming_moonshine.js b/nodejs-addon-examples/test_asr_non_streaming_moonshine.js new file mode 100644 index 000000000..9e676b8c5 --- /dev/null +++ b/nodejs-addon-examples/test_asr_non_streaming_moonshine.js @@ -0,0 +1,50 @@ +// Copyright (c) 2024 Xiaomi Corporation +const sherpa_onnx = require('sherpa-onnx-node'); + +// Please download test files from +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models +const config = { + 'featConfig': { + 'sampleRate': 16000, + 'featureDim': 80, + }, + 'modelConfig': { + 'moonshine': { + 'preprocessor': './sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx', + 'encoder': './sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx', + 'uncachedDecoder': + './sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx', + 'cachedDecoder': + './sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx', + }, + 'tokens': './sherpa-onnx-moonshine-tiny-en-int8/tokens.txt', + 'numThreads': 2, + 'provider': 'cpu', + 'debug': 1, + } +}; + +const waveFilename = './sherpa-onnx-moonshine-tiny-en-int8/test_wavs/0.wav'; + +const recognizer = new sherpa_onnx.OfflineRecognizer(config); +console.log('Started') +let start = Date.now(); +const stream = recognizer.createStream(); +const wave = sherpa_onnx.readWave(waveFilename); +stream.acceptWaveform({sampleRate: wave.sampleRate, samples: wave.samples}); + +recognizer.decode(stream); +result = recognizer.getResult(stream) +let stop = Date.now(); +console.log('Done') + +const elapsed_seconds = (stop - start) / 1000; +const duration = wave.samples.length / wave.sampleRate; +const real_time_factor = elapsed_seconds / duration; +console.log('Wave duration', duration.toFixed(3), 'secodns') +console.log('Elapsed', elapsed_seconds.toFixed(3), 'secodns') +console.log( + `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, + real_time_factor.toFixed(3)) +console.log(waveFilename) +console.log('result\n', result) diff --git a/nodejs-addon-examples/test_vad_asr_non_streaming_moonshine_microphone.js b/nodejs-addon-examples/test_vad_asr_non_streaming_moonshine_microphone.js new file mode 100644 index 000000000..b3cbcca5d --- /dev/null +++ b/nodejs-addon-examples/test_vad_asr_non_streaming_moonshine_microphone.js @@ -0,0 +1,113 @@ +// Copyright (c) 2023-2024 Xiaomi Corporation (authors: Fangjun Kuang) +// +const portAudio = require('naudiodon2'); +// console.log(portAudio.getDevices()); + +const sherpa_onnx = require('sherpa-onnx-node'); + +function createRecognizer() { + // Please download test files from + // https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models + const config = { + 'featConfig': { + 'sampleRate': 16000, + 'featureDim': 80, + }, + 'modelConfig': { + 'moonshine': { + 'preprocessor': './sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx', + 'encoder': './sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx', + 'uncachedDecoder': + './sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx', + 'cachedDecoder': + './sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx', + }, + 'tokens': './sherpa-onnx-moonshine-tiny-en-int8/tokens.txt', + 'numThreads': 2, + 'provider': 'cpu', + 'debug': 1, + } + }; + + return new sherpa_onnx.OfflineRecognizer(config); +} + +function createVad() { + // please download silero_vad.onnx from + // https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx + const config = { + sileroVad: { + model: './silero_vad.onnx', + threshold: 0.5, + minSpeechDuration: 0.25, + minSilenceDuration: 0.5, + windowSize: 512, + }, + sampleRate: 16000, + debug: true, + numThreads: 1, + }; + + const bufferSizeInSeconds = 60; + + return new sherpa_onnx.Vad(config, bufferSizeInSeconds); +} + +const recognizer = createRecognizer(); +const vad = createVad(); + +const bufferSizeInSeconds = 30; +const buffer = + new sherpa_onnx.CircularBuffer(bufferSizeInSeconds * vad.config.sampleRate); + +const ai = new portAudio.AudioIO({ + inOptions: { + channelCount: 1, + closeOnError: true, // Close the stream if an audio error is detected, if + // set false then just log the error + deviceId: -1, // Use -1 or omit the deviceId to select the default device + sampleFormat: portAudio.SampleFormatFloat32, + sampleRate: vad.config.sampleRate + } +}); + +let printed = false; +let index = 0; +ai.on('data', data => { + const windowSize = vad.config.sileroVad.windowSize; + buffer.push(new Float32Array(data.buffer)); + while (buffer.size() > windowSize) { + const samples = buffer.get(buffer.head(), windowSize); + buffer.pop(windowSize); + vad.acceptWaveform(samples); + } + + while (!vad.isEmpty()) { + const segment = vad.front(); + vad.pop(); + const stream = recognizer.createStream(); + stream.acceptWaveform({ + samples: segment.samples, + sampleRate: recognizer.config.featConfig.sampleRate + }); + recognizer.decode(stream); + const r = recognizer.getResult(stream); + if (r.text.length > 0) { + const text = r.text.toLowerCase().trim(); + console.log(`${index}: ${text}`); + + const filename = `${index}-${text}-${ + new Date() + .toLocaleTimeString('en-US', {hour12: false}) + .split(' ')[0]}.wav`; + sherpa_onnx.writeWave( + filename, + {samples: segment.samples, sampleRate: vad.config.sampleRate}); + + index += 1; + } + } +}); + +ai.start(); +console.log('Started! Please speak') diff --git a/nodejs-addon-examples/test_vad_with_non_streaming_asr_moonshine.js b/nodejs-addon-examples/test_vad_with_non_streaming_asr_moonshine.js new file mode 100644 index 000000000..3b0510604 --- /dev/null +++ b/nodejs-addon-examples/test_vad_with_non_streaming_asr_moonshine.js @@ -0,0 +1,132 @@ +// Copyright (c) 2023-2024 Xiaomi Corporation (authors: Fangjun Kuang) + +const sherpa_onnx = require('sherpa-onnx-node'); + +function createRecognizer() { + // Please download test files from + // https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models + const config = { + 'featConfig': { + 'sampleRate': 16000, + 'featureDim': 80, + }, + 'modelConfig': { + 'moonshine': { + 'preprocessor': './sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx', + 'encoder': './sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx', + 'uncachedDecoder': + './sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx', + 'cachedDecoder': + './sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx', + }, + 'tokens': './sherpa-onnx-moonshine-tiny-en-int8/tokens.txt', + 'numThreads': 2, + 'provider': 'cpu', + 'debug': 1, + } + }; + + return new sherpa_onnx.OfflineRecognizer(config); +} + +function createVad() { + // please download silero_vad.onnx from + // https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx + const config = { + sileroVad: { + model: './silero_vad.onnx', + threshold: 0.5, + minSpeechDuration: 0.25, + minSilenceDuration: 0.5, + maxSpeechDuration: 5, + windowSize: 512, + }, + sampleRate: 16000, + debug: true, + numThreads: 1, + }; + + const bufferSizeInSeconds = 60; + + return new sherpa_onnx.Vad(config, bufferSizeInSeconds); +} + +const recognizer = createRecognizer(); +const vad = createVad(); + +// please download ./Obama.wav from +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models +const waveFilename = './Obama.wav'; +const wave = sherpa_onnx.readWave(waveFilename); + +if (wave.sampleRate != recognizer.config.featConfig.sampleRate) { + throw new Error( + 'Expected sample rate: ${recognizer.config.featConfig.sampleRate}. Given: ${wave.sampleRate}'); +} + +console.log('Started') +let start = Date.now(); + +const windowSize = vad.config.sileroVad.windowSize; +for (let i = 0; i < wave.samples.length; i += windowSize) { + const thisWindow = wave.samples.subarray(i, i + windowSize); + vad.acceptWaveform(thisWindow); + + while (!vad.isEmpty()) { + const segment = vad.front(); + vad.pop(); + + let start_time = segment.start / wave.sampleRate; + let end_time = start_time + segment.samples.length / wave.sampleRate; + + start_time = start_time.toFixed(2); + end_time = end_time.toFixed(2); + + const stream = recognizer.createStream(); + stream.acceptWaveform( + {samples: segment.samples, sampleRate: wave.sampleRate}); + + recognizer.decode(stream); + const r = recognizer.getResult(stream); + if (r.text.length > 0) { + const text = r.text.toLowerCase().trim(); + console.log(`${start_time} -- ${end_time}: ${text}`); + } + } +} + +vad.flush(); + +while (!vad.isEmpty()) { + const segment = vad.front(); + vad.pop(); + + let start_time = segment.start / wave.sampleRate; + let end_time = start_time + segment.samples.length / wave.sampleRate; + + start_time = start_time.toFixed(2); + end_time = end_time.toFixed(2); + + const stream = recognizer.createStream(); + stream.acceptWaveform( + {samples: segment.samples, sampleRate: wave.sampleRate}); + + recognizer.decode(stream); + const r = recognizer.getResult(stream); + if (r.text.length > 0) { + const text = r.text.toLowerCase().trim(); + console.log(`${start_time} -- ${end_time}: ${text}`); + } +} + +let stop = Date.now(); +console.log('Done') + +const elapsed_seconds = (stop - start) / 1000; +const duration = wave.samples.length / wave.sampleRate; +const real_time_factor = elapsed_seconds / duration; +console.log('Wave duration', duration.toFixed(3), 'seconds') +console.log('Elapsed', elapsed_seconds.toFixed(3), 'seconds') +console.log( + `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, + real_time_factor.toFixed(3)) diff --git a/nodejs-examples/README.md b/nodejs-examples/README.md index 496a0062b..a953573b6 100644 --- a/nodejs-examples/README.md +++ b/nodejs-examples/README.md @@ -133,7 +133,25 @@ tar xvf sherpa-onnx-zipformer-en-2023-06-26.tar.bz2 node ./test-offline-transducer.js ``` +## ./test-vad-with-non-streaming-asr-whisper.js + +[./test-vad-with-non-streaming-asr-whisper.js](./test-vad-with-non-streaming-asr-whisper.js) +shows how to use VAD + whisper to decode a very long file. + +You can use the following command to run it: + +```bash +wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2 +tar xvf sherpa-onnx-whisper-tiny.en.tar.bz2 + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/Obama.wav +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx + +node ./test-vad-with-non-streaming-asr-whisper.js +``` + ## ./test-offline-whisper.js + [./test-offline-whisper.js](./test-offline-whisper.js) demonstrates how to decode a file with a Whisper model. In the code we use [sherpa-onnx-whisper-tiny.en](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/whisper/tiny.en.html). @@ -146,7 +164,40 @@ tar xvf sherpa-onnx-whisper-tiny.en.tar.bz2 node ./test-offline-whisper.js ``` +## ./test-offline-moonshine.js + +[./test-offline-moonshine.js](./test-offline-moonshine.js) demonstrates +how to decode a file with a Moonshine model. In the code we use +[sherpa-onnx-moonshine-tiny-en-int8](https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2). + +You can use the following command to run it: + +```bash +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + +node ./test-offline-moonshine.js +``` + +## ./test-vad-with-non-streaming-asr-moonshine.js + +[./test-vad-with-non-streaming-asr-moonshine.js](./test-vad-with-non-streaming-asr-moonshine.js) +shows how to use VAD + whisper to decode a very long file. + +You can use the following command to run it: + +```bash +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/Obama.wav +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx + +node ./test-vad-with-non-streaming-asr-moonshine.js +``` + ## ./test-online-paraformer-microphone.js + [./test-online-paraformer-microphone.js](./test-online-paraformer-microphone.js) demonstrates how to do real-time speech recognition from microphone with a streaming Paraformer model. In the code we use diff --git a/nodejs-examples/test-offline-moonshine.js b/nodejs-examples/test-offline-moonshine.js new file mode 100644 index 000000000..8f5d2f00b --- /dev/null +++ b/nodejs-examples/test-offline-moonshine.js @@ -0,0 +1,37 @@ +// Copyright (c) 2023 Xiaomi Corporation (authors: Fangjun Kuang) +// +const sherpa_onnx = require('sherpa-onnx'); + +function createOfflineRecognizer() { + let modelConfig = { + moonshine: { + preprocessor: './sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx', + encoder: './sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx', + uncachedDecoder: + './sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx', + cachedDecoder: + './sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx', + }, + tokens: './sherpa-onnx-moonshine-tiny-en-int8/tokens.txt', + }; + + let config = { + modelConfig: modelConfig, + }; + + return sherpa_onnx.createOfflineRecognizer(config); +} + +recognizer = createOfflineRecognizer(); +stream = recognizer.createStream(); + +const waveFilename = './sherpa-onnx-moonshine-tiny-en-int8/test_wavs/0.wav'; +const wave = sherpa_onnx.readWave(waveFilename); +stream.acceptWaveform(wave.sampleRate, wave.samples); + +recognizer.decode(stream); +const text = recognizer.getResult(stream).text; +console.log(text); + +stream.free(); +recognizer.free(); diff --git a/nodejs-examples/test-vad-with-non-streaming-asr-moonshine.js b/nodejs-examples/test-vad-with-non-streaming-asr-moonshine.js new file mode 100644 index 000000000..0d6bd7648 --- /dev/null +++ b/nodejs-examples/test-vad-with-non-streaming-asr-moonshine.js @@ -0,0 +1,128 @@ +// Copyright (c) 2023-2024 Xiaomi Corporation (authors: Fangjun Kuang) + +const sherpa_onnx = require('sherpa-onnx'); + +function createRecognizer() { + // Please download test files from + // https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models + const config = { + 'modelConfig': { + 'moonshine': { + 'preprocessor': './sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx', + 'encoder': './sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx', + 'uncachedDecoder': + './sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx', + 'cachedDecoder': + './sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx', + }, + 'tokens': './sherpa-onnx-moonshine-tiny-en-int8/tokens.txt', + 'debug': 0, + } + }; + + return sherpa_onnx.createOfflineRecognizer(config); +} + +function createVad() { + // please download silero_vad.onnx from + // https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx + const config = { + sileroVad: { + model: './silero_vad.onnx', + threshold: 0.5, + minSpeechDuration: 0.25, + minSilenceDuration: 0.5, + maxSpeechDuration: 5, + windowSize: 512, + }, + sampleRate: 16000, + debug: true, + numThreads: 1, + bufferSizeInSeconds: 60, + }; + + return sherpa_onnx.createVad(config); +} + +const recognizer = createRecognizer(); +const vad = createVad(); + +// please download ./Obama.wav from +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models +const waveFilename = './Obama.wav'; +const wave = sherpa_onnx.readWave(waveFilename); + +if (wave.sampleRate != recognizer.config.featConfig.sampleRate) { + throw new Error( + 'Expected sample rate: ${recognizer.config.featConfig.sampleRate}. Given: ${wave.sampleRate}'); +} + +console.log('Started') +let start = Date.now(); + +const windowSize = vad.config.sileroVad.windowSize; +for (let i = 0; i < wave.samples.length; i += windowSize) { + const thisWindow = wave.samples.subarray(i, i + windowSize); + vad.acceptWaveform(thisWindow); + + while (!vad.isEmpty()) { + const segment = vad.front(); + vad.pop(); + + let start_time = segment.start / wave.sampleRate; + let end_time = start_time + segment.samples.length / wave.sampleRate; + + start_time = start_time.toFixed(2); + end_time = end_time.toFixed(2); + + const stream = recognizer.createStream(); + stream.acceptWaveform(wave.sampleRate, segment.samples); + + recognizer.decode(stream); + const r = recognizer.getResult(stream); + if (r.text.length > 0) { + const text = r.text.toLowerCase().trim(); + console.log(`${start_time} -- ${end_time}: ${text}`); + } + + stream.free(); + } +} + +vad.flush(); + +while (!vad.isEmpty()) { + const segment = vad.front(); + vad.pop(); + + let start_time = segment.start / wave.sampleRate; + let end_time = start_time + segment.samples.length / wave.sampleRate; + + start_time = start_time.toFixed(2); + end_time = end_time.toFixed(2); + + const stream = recognizer.createStream(); + stream.acceptWaveform(wave.sampleRate, segment.samples); + + recognizer.decode(stream); + const r = recognizer.getResult(stream); + if (r.text.length > 0) { + const text = r.text.toLowerCase().trim(); + console.log(`${start_time} -- ${end_time}: ${text}`); + } +} + +let stop = Date.now(); +console.log('Done') + +const elapsed_seconds = (stop - start) / 1000; +const duration = wave.samples.length / wave.sampleRate; +const real_time_factor = elapsed_seconds / duration; +console.log('Wave duration', duration.toFixed(3), 'seconds') +console.log('Elapsed', elapsed_seconds.toFixed(3), 'seconds') +console.log( + `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, + real_time_factor.toFixed(3)) + +vad.free(); +recognizer.free(); diff --git a/scripts/node-addon-api/src/macros.h b/scripts/node-addon-api/src/macros.h index ac0dbd567..1f4f401e3 100644 --- a/scripts/node-addon-api/src/macros.h +++ b/scripts/node-addon-api/src/macros.h @@ -41,4 +41,11 @@ } \ } while (0) +#define SHERPA_ONNX_DELETE_C_STR(p) \ + do { \ + if (p) { \ + delete[] p; \ + } \ + } while (0) + #endif // SCRIPTS_NODE_ADDON_API_SRC_MACROS_H_ diff --git a/scripts/node-addon-api/src/non-streaming-asr.cc b/scripts/node-addon-api/src/non-streaming-asr.cc index 28c0b31ed..a95c892a8 100644 --- a/scripts/node-addon-api/src/non-streaming-asr.cc +++ b/scripts/node-addon-api/src/non-streaming-asr.cc @@ -80,6 +80,25 @@ static SherpaOnnxOfflineWhisperModelConfig GetOfflineWhisperModelConfig( return c; } +static SherpaOnnxOfflineMoonshineModelConfig GetOfflineMoonshineModelConfig( + Napi::Object obj) { + SherpaOnnxOfflineMoonshineModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("moonshine") || !obj.Get("moonshine").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("moonshine").As(); + + SHERPA_ONNX_ASSIGN_ATTR_STR(preprocessor, preprocessor); + SHERPA_ONNX_ASSIGN_ATTR_STR(encoder, encoder); + SHERPA_ONNX_ASSIGN_ATTR_STR(uncached_decoder, uncachedDecoder); + SHERPA_ONNX_ASSIGN_ATTR_STR(cached_decoder, cachedDecoder); + + return c; +} + static SherpaOnnxOfflineTdnnModelConfig GetOfflineTdnnModelConfig( Napi::Object obj) { SherpaOnnxOfflineTdnnModelConfig c; @@ -130,6 +149,7 @@ static SherpaOnnxOfflineModelConfig GetOfflineModelConfig(Napi::Object obj) { c.whisper = GetOfflineWhisperModelConfig(o); c.tdnn = GetOfflineTdnnModelConfig(o); c.sense_voice = GetOfflineSenseVoiceModelConfig(o); + c.moonshine = GetOfflineMoonshineModelConfig(o); SHERPA_ONNX_ASSIGN_ATTR_STR(tokens, tokens); SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); @@ -206,97 +226,42 @@ CreateOfflineRecognizerWrapper(const Napi::CallbackInfo &info) { const SherpaOnnxOfflineRecognizer *recognizer = SherpaOnnxCreateOfflineRecognizer(&c); - if (c.model_config.transducer.encoder) { - delete[] c.model_config.transducer.encoder; - } - - if (c.model_config.transducer.decoder) { - delete[] c.model_config.transducer.decoder; - } - - if (c.model_config.transducer.joiner) { - delete[] c.model_config.transducer.joiner; - } - - if (c.model_config.paraformer.model) { - delete[] c.model_config.paraformer.model; - } - - if (c.model_config.nemo_ctc.model) { - delete[] c.model_config.nemo_ctc.model; - } - - if (c.model_config.whisper.encoder) { - delete[] c.model_config.whisper.encoder; - } - - if (c.model_config.whisper.decoder) { - delete[] c.model_config.whisper.decoder; - } - - if (c.model_config.whisper.language) { - delete[] c.model_config.whisper.language; - } - - if (c.model_config.whisper.task) { - delete[] c.model_config.whisper.task; - } - - if (c.model_config.tdnn.model) { - delete[] c.model_config.tdnn.model; - } - - if (c.model_config.sense_voice.model) { - delete[] c.model_config.sense_voice.model; - } - - if (c.model_config.sense_voice.language) { - delete[] c.model_config.sense_voice.language; - } - - if (c.model_config.tokens) { - delete[] c.model_config.tokens; - } - - if (c.model_config.provider) { - delete[] c.model_config.provider; - } + SHERPA_ONNX_DELETE_C_STR(c.model_config.transducer.encoder); + SHERPA_ONNX_DELETE_C_STR(c.model_config.transducer.decoder); + SHERPA_ONNX_DELETE_C_STR(c.model_config.transducer.joiner); - if (c.model_config.model_type) { - delete[] c.model_config.model_type; - } + SHERPA_ONNX_DELETE_C_STR(c.model_config.paraformer.model); - if (c.model_config.modeling_unit) { - delete[] c.model_config.modeling_unit; - } + SHERPA_ONNX_DELETE_C_STR(c.model_config.nemo_ctc.model); - if (c.model_config.bpe_vocab) { - delete[] c.model_config.bpe_vocab; - } + SHERPA_ONNX_DELETE_C_STR(c.model_config.whisper.encoder); + SHERPA_ONNX_DELETE_C_STR(c.model_config.whisper.decoder); + SHERPA_ONNX_DELETE_C_STR(c.model_config.whisper.language); + SHERPA_ONNX_DELETE_C_STR(c.model_config.whisper.task); - if (c.model_config.telespeech_ctc) { - delete[] c.model_config.telespeech_ctc; - } + SHERPA_ONNX_DELETE_C_STR(c.model_config.tdnn.model); - if (c.lm_config.model) { - delete[] c.lm_config.model; - } + SHERPA_ONNX_DELETE_C_STR(c.model_config.sense_voice.model); + SHERPA_ONNX_DELETE_C_STR(c.model_config.sense_voice.language); - if (c.decoding_method) { - delete[] c.decoding_method; - } + SHERPA_ONNX_DELETE_C_STR(c.model_config.moonshine.preprocessor); + SHERPA_ONNX_DELETE_C_STR(c.model_config.moonshine.encoder); + SHERPA_ONNX_DELETE_C_STR(c.model_config.moonshine.uncached_decoder); + SHERPA_ONNX_DELETE_C_STR(c.model_config.moonshine.cached_decoder); - if (c.hotwords_file) { - delete[] c.hotwords_file; - } + SHERPA_ONNX_DELETE_C_STR(c.model_config.tokens); + SHERPA_ONNX_DELETE_C_STR(c.model_config.provider); + SHERPA_ONNX_DELETE_C_STR(c.model_config.model_type); + SHERPA_ONNX_DELETE_C_STR(c.model_config.modeling_unit); + SHERPA_ONNX_DELETE_C_STR(c.model_config.bpe_vocab); + SHERPA_ONNX_DELETE_C_STR(c.model_config.telespeech_ctc); - if (c.rule_fsts) { - delete[] c.rule_fsts; - } + SHERPA_ONNX_DELETE_C_STR(c.lm_config.model); - if (c.rule_fars) { - delete[] c.rule_fars; - } + SHERPA_ONNX_DELETE_C_STR(c.decoding_method); + SHERPA_ONNX_DELETE_C_STR(c.hotwords_file); + SHERPA_ONNX_DELETE_C_STR(c.rule_fsts); + SHERPA_ONNX_DELETE_C_STR(c.rule_fars); if (!recognizer) { Napi::TypeError::New(env, "Please check your config!") diff --git a/wasm/asr/sherpa-onnx-asr.js b/wasm/asr/sherpa-onnx-asr.js index 9b966090c..f0c34c97f 100644 --- a/wasm/asr/sherpa-onnx-asr.js +++ b/wasm/asr/sherpa-onnx-asr.js @@ -35,6 +35,10 @@ function freeConfig(config, Module) { freeConfig(config.whisper, Module) } + if ('moonshine' in config) { + freeConfig(config.moonshine, Module) + } + if ('tdnn' in config) { freeConfig(config.tdnn, Module) } @@ -563,7 +567,7 @@ function initSherpaOnnxOfflineWhisperModelConfig(config, Module) { const n = encoderLen + decoderLen + languageLen + taskLen; const buffer = Module._malloc(n); - const len = 5 * 4; // 4 pointers + const len = 5 * 4; // 4 pointers + 1 int32 const ptr = Module._malloc(len); let offset = 0; @@ -598,6 +602,55 @@ function initSherpaOnnxOfflineWhisperModelConfig(config, Module) { } } +function initSherpaOnnxOfflineMoonshineModelConfig(config, Module) { + const preprocessorLen = Module.lengthBytesUTF8(config.preprocessor || '') + 1; + const encoderLen = Module.lengthBytesUTF8(config.encoder || '') + 1; + const uncachedDecoderLen = + Module.lengthBytesUTF8(config.uncachedDecoder || '') + 1; + const cachedDecoderLen = + Module.lengthBytesUTF8(config.cachedDecoder || '') + 1; + + const n = + preprocessorLen + encoderLen + uncachedDecoderLen + cachedDecoderLen; + const buffer = Module._malloc(n); + + const len = 4 * 4; // 4 pointers + const ptr = Module._malloc(len); + + let offset = 0; + Module.stringToUTF8( + config.preprocessor || '', buffer + offset, preprocessorLen); + offset += preprocessorLen; + + Module.stringToUTF8(config.encoder || '', buffer + offset, encoderLen); + offset += encoderLen; + + Module.stringToUTF8( + config.uncachedDecoder || '', buffer + offset, uncachedDecoderLen); + offset += uncachedDecoderLen; + + Module.stringToUTF8( + config.cachedDecoder || '', buffer + offset, cachedDecoderLen); + offset += cachedDecoderLen; + + offset = 0; + Module.setValue(ptr, buffer + offset, 'i8*'); + offset += preprocessorLen; + + Module.setValue(ptr + 4, buffer + offset, 'i8*'); + offset += encoderLen; + + Module.setValue(ptr + 8, buffer + offset, 'i8*'); + offset += uncachedDecoderLen; + + Module.setValue(ptr + 12, buffer + offset, 'i8*'); + offset += cachedDecoderLen; + + return { + buffer: buffer, ptr: ptr, len: len, + } +} + function initSherpaOnnxOfflineTdnnModelConfig(config, Module) { const n = Module.lengthBytesUTF8(config.model || '') + 1; const buffer = Module._malloc(n); @@ -693,6 +746,15 @@ function initSherpaOnnxOfflineModelConfig(config, Module) { }; } + if (!('moonshine' in config)) { + config.moonshine = { + preprocessor: '', + encoder: '', + uncachedDecoder: '', + cachedDecoder: '', + }; + } + if (!('tdnn' in config)) { config.tdnn = { model: '', @@ -724,8 +786,11 @@ function initSherpaOnnxOfflineModelConfig(config, Module) { const senseVoice = initSherpaOnnxOfflineSenseVoiceModelConfig(config.senseVoice, Module); + const moonshine = + initSherpaOnnxOfflineMoonshineModelConfig(config.moonshine, Module); + const len = transducer.len + paraformer.len + nemoCtc.len + whisper.len + - tdnn.len + 8 * 4 + senseVoice.len; + tdnn.len + 8 * 4 + senseVoice.len + moonshine.len; const ptr = Module._malloc(len); @@ -745,7 +810,6 @@ function initSherpaOnnxOfflineModelConfig(config, Module) { Module._CopyHeap(tdnn.ptr, tdnn.len, ptr + offset); offset += tdnn.len; - const tokensLen = Module.lengthBytesUTF8(config.tokens || '') + 1; const providerLen = Module.lengthBytesUTF8(config.provider || 'cpu') + 1; const modelTypeLen = Module.lengthBytesUTF8(config.modelType || '') + 1; @@ -817,11 +881,14 @@ function initSherpaOnnxOfflineModelConfig(config, Module) { offset += 4; Module._CopyHeap(senseVoice.ptr, senseVoice.len, ptr + offset); + offset += senseVoice.len; + + Module._CopyHeap(moonshine.ptr, moonshine.len, ptr + offset); return { buffer: buffer, ptr: ptr, len: len, transducer: transducer, paraformer: paraformer, nemoCtc: nemoCtc, whisper: whisper, tdnn: tdnn, - senseVoice: senseVoice, + senseVoice: senseVoice, moonshine: moonshine, } } diff --git a/wasm/nodejs/sherpa-onnx-wasm-nodejs.cc b/wasm/nodejs/sherpa-onnx-wasm-nodejs.cc index 43c9d42cb..ff8cd4939 100644 --- a/wasm/nodejs/sherpa-onnx-wasm-nodejs.cc +++ b/wasm/nodejs/sherpa-onnx-wasm-nodejs.cc @@ -15,6 +15,7 @@ static_assert(sizeof(SherpaOnnxOfflineParaformerModelConfig) == 4, ""); static_assert(sizeof(SherpaOnnxOfflineNemoEncDecCtcModelConfig) == 4, ""); static_assert(sizeof(SherpaOnnxOfflineWhisperModelConfig) == 5 * 4, ""); +static_assert(sizeof(SherpaOnnxOfflineMoonshineModelConfig) == 4 * 4, ""); static_assert(sizeof(SherpaOnnxOfflineTdnnModelConfig) == 4, ""); static_assert(sizeof(SherpaOnnxOfflineSenseVoiceModelConfig) == 3 * 4, ""); static_assert(sizeof(SherpaOnnxOfflineLMConfig) == 2 * 4, ""); @@ -25,7 +26,8 @@ static_assert(sizeof(SherpaOnnxOfflineModelConfig) == sizeof(SherpaOnnxOfflineNemoEncDecCtcModelConfig) + sizeof(SherpaOnnxOfflineWhisperModelConfig) + sizeof(SherpaOnnxOfflineTdnnModelConfig) + 8 * 4 + - sizeof(SherpaOnnxOfflineSenseVoiceModelConfig), + sizeof(SherpaOnnxOfflineSenseVoiceModelConfig) + + sizeof(SherpaOnnxOfflineMoonshineModelConfig), ""); static_assert(sizeof(SherpaOnnxFeatureConfig) == 2 * 4, ""); static_assert(sizeof(SherpaOnnxOfflineRecognizerConfig) == @@ -66,6 +68,7 @@ void PrintOfflineRecognizerConfig(SherpaOnnxOfflineRecognizerConfig *config) { auto whisper = &model_config->whisper; auto tdnn = &model_config->tdnn; auto sense_voice = &model_config->sense_voice; + auto moonshine = &model_config->moonshine; fprintf(stdout, "----------offline transducer model config----------\n"); fprintf(stdout, "encoder: %s\n", transducer->encoder); @@ -93,6 +96,12 @@ void PrintOfflineRecognizerConfig(SherpaOnnxOfflineRecognizerConfig *config) { fprintf(stdout, "language: %s\n", sense_voice->language); fprintf(stdout, "use_itn: %d\n", sense_voice->use_itn); + fprintf(stdout, "----------offline moonshine model config----------\n"); + fprintf(stdout, "preprocessor: %s\n", moonshine->preprocessor); + fprintf(stdout, "encoder: %s\n", moonshine->encoder); + fprintf(stdout, "uncached_decoder: %s\n", moonshine->uncached_decoder); + fprintf(stdout, "cached_decoder: %s\n", moonshine->cached_decoder); + fprintf(stdout, "tokens: %s\n", model_config->tokens); fprintf(stdout, "num_threads: %d\n", model_config->num_threads); fprintf(stdout, "provider: %s\n", model_config->provider); From 54468a737071c89f2a323c934980290fa0593743 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sun, 27 Oct 2024 12:04:12 +0800 Subject: [PATCH 048/183] Add Dart API for Moonshine models. (#1481) --- .github/scripts/test-dart.sh | 8 ++ .../non-streaming-asr/bin/moonshine.dart | 69 +++++++++ .../non-streaming-asr/run-moonshine.sh | 20 +++ .../bin/moonshine.dart | 134 ++++++++++++++++++ .../run-moonshine.sh | 29 ++++ .../lib/src/offline_recognizer.dart | 35 ++++- .../lib/src/sherpa_onnx_bindings.dart | 8 ++ 7 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 dart-api-examples/non-streaming-asr/bin/moonshine.dart create mode 100755 dart-api-examples/non-streaming-asr/run-moonshine.sh create mode 100644 dart-api-examples/vad-with-non-streaming-asr/bin/moonshine.dart create mode 100755 dart-api-examples/vad-with-non-streaming-asr/run-moonshine.sh diff --git a/.github/scripts/test-dart.sh b/.github/scripts/test-dart.sh index 27c21573a..881a4765b 100755 --- a/.github/scripts/test-dart.sh +++ b/.github/scripts/test-dart.sh @@ -36,6 +36,10 @@ echo "----zipformer transducer----" ./run-zipformer-transducer.sh rm -rf sherpa-onnx-* +echo "----moonshine----" +./run-moonshine.sh +rm -rf sherpa-onnx-* + echo "----whisper----" ./run-whisper.sh rm -rf sherpa-onnx-* @@ -77,6 +81,10 @@ echo '----------TeleSpeech CTC----------' ./run-telespeech-ctc.sh rm -rf sherpa-onnx-* +echo '----------moonshine----------' +./run-moonshine.sh +rm -rf sherpa-onnx-* + echo '----------whisper----------' ./run-whisper.sh rm -rf sherpa-onnx-* diff --git a/dart-api-examples/non-streaming-asr/bin/moonshine.dart b/dart-api-examples/non-streaming-asr/bin/moonshine.dart new file mode 100644 index 000000000..68b653648 --- /dev/null +++ b/dart-api-examples/non-streaming-asr/bin/moonshine.dart @@ -0,0 +1,69 @@ +// Copyright (c) 2024 Xiaomi Corporation +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; + +import './init.dart'; + +void main(List arguments) async { + await initSherpaOnnx(); + + final parser = ArgParser() + ..addOption('preprocessor', + help: 'Path to the moonshine preprocessor model') + ..addOption('encoder', help: 'Path to the moonshine encoder model') + ..addOption('uncached-decoder', + help: 'Path to moonshine uncached decoder model') + ..addOption('cached-decoder', + help: 'Path to moonshine cached decoder model') + ..addOption('tokens', help: 'Path to tokens.txt') + ..addOption('input-wav', help: 'Path to input.wav to transcribe'); + + final res = parser.parse(arguments); + if (res['preprocessor'] == null || + res['encoder'] == null || + res['uncached-decoder'] == null || + res['cached-decoder'] == null || + res['tokens'] == null || + res['input-wav'] == null) { + print(parser.usage); + exit(1); + } + + final preprocessor = res['preprocessor'] as String; + final encoder = res['encoder'] as String; + final uncachedDecoder = res['uncached-decoder'] as String; + final cachedDecoder = res['cached-decoder'] as String; + final tokens = res['tokens'] as String; + final inputWav = res['input-wav'] as String; + + final moonshine = sherpa_onnx.OfflineMoonshineModelConfig( + preprocessor: preprocessor, + encoder: encoder, + uncachedDecoder: uncachedDecoder, + cachedDecoder: cachedDecoder, + ); + + final modelConfig = sherpa_onnx.OfflineModelConfig( + moonshine: moonshine, + tokens: tokens, + debug: false, + numThreads: 1, + ); + final config = sherpa_onnx.OfflineRecognizerConfig(model: modelConfig); + final recognizer = sherpa_onnx.OfflineRecognizer(config); + + final waveData = sherpa_onnx.readWave(inputWav); + final stream = recognizer.createStream(); + + stream.acceptWaveform( + samples: waveData.samples, sampleRate: waveData.sampleRate); + recognizer.decode(stream); + + final result = recognizer.getResult(stream); + print(result.text); + + stream.free(); + recognizer.free(); +} diff --git a/dart-api-examples/non-streaming-asr/run-moonshine.sh b/dart-api-examples/non-streaming-asr/run-moonshine.sh new file mode 100755 index 000000000..213a230d0 --- /dev/null +++ b/dart-api-examples/non-streaming-asr/run-moonshine.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -ex + +dart pub get + +if [ ! -f ./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +fi + +dart run \ + ./bin/moonshine.dart \ + --preprocessor ./sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx \ + --encoder ./sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx \ + --uncached-decoder ./sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx \ + --cached-decoder ./sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx \ + --tokens ./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt \ + --input-wav ./sherpa-onnx-moonshine-tiny-en-int8/test_wavs/0.wav diff --git a/dart-api-examples/vad-with-non-streaming-asr/bin/moonshine.dart b/dart-api-examples/vad-with-non-streaming-asr/bin/moonshine.dart new file mode 100644 index 000000000..f9d96e694 --- /dev/null +++ b/dart-api-examples/vad-with-non-streaming-asr/bin/moonshine.dart @@ -0,0 +1,134 @@ +// Copyright (c) 2024 Xiaomi Corporation +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:args/args.dart'; +import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; + +import './init.dart'; + +void main(List arguments) async { + await initSherpaOnnx(); + + final parser = ArgParser() + ..addOption('silero-vad', help: 'Path to silero_vad.onnx') + ..addOption('preprocessor', + help: 'Path to the moonshine preprocessor model') + ..addOption('encoder', help: 'Path to the moonshine encoder model') + ..addOption('uncached-decoder', + help: 'Path to moonshine uncached decoder model') + ..addOption('cached-decoder', + help: 'Path to moonshine cached decoder model') + ..addOption('tokens', help: 'Path to tokens.txt') + ..addOption('input-wav', help: 'Path to input.wav to transcribe'); + + final res = parser.parse(arguments); + if (res['silero-vad'] == null || + res['preprocessor'] == null || + res['encoder'] == null || + res['uncached-decoder'] == null || + res['cached-decoder'] == null || + res['tokens'] == null || + res['input-wav'] == null) { + print(parser.usage); + exit(1); + } + + // create VAD + final sileroVad = res['silero-vad'] as String; + + final sileroVadConfig = sherpa_onnx.SileroVadModelConfig( + model: sileroVad, + minSilenceDuration: 0.25, + minSpeechDuration: 0.5, + maxSpeechDuration: 5.0, + ); + + final vadConfig = sherpa_onnx.VadModelConfig( + sileroVad: sileroVadConfig, + numThreads: 1, + debug: true, + ); + + final vad = sherpa_onnx.VoiceActivityDetector( + config: vadConfig, bufferSizeInSeconds: 10); + + // create whisper recognizer + final preprocessor = res['preprocessor'] as String; + final encoder = res['encoder'] as String; + final uncachedDecoder = res['uncached-decoder'] as String; + final cachedDecoder = res['cached-decoder'] as String; + final tokens = res['tokens'] as String; + final inputWav = res['input-wav'] as String; + + final moonshine = sherpa_onnx.OfflineMoonshineModelConfig( + preprocessor: preprocessor, + encoder: encoder, + uncachedDecoder: uncachedDecoder, + cachedDecoder: cachedDecoder, + ); + final modelConfig = sherpa_onnx.OfflineModelConfig( + moonshine: moonshine, + tokens: tokens, + debug: false, + numThreads: 1, + ); + final config = sherpa_onnx.OfflineRecognizerConfig(model: modelConfig); + final recognizer = sherpa_onnx.OfflineRecognizer(config); + + final waveData = sherpa_onnx.readWave(inputWav); + if (waveData.sampleRate != 16000) { + print('Only 16000 Hz is supported. Given: ${waveData.sampleRate}'); + exit(1); + } + + int numSamples = waveData.samples.length; + int numIter = numSamples ~/ vadConfig.sileroVad.windowSize; + + for (int i = 0; i != numIter; ++i) { + int start = i * vadConfig.sileroVad.windowSize; + vad.acceptWaveform(Float32List.sublistView( + waveData.samples, start, start + vadConfig.sileroVad.windowSize)); + + while (!vad.isEmpty()) { + final samples = vad.front().samples; + final startTime = vad.front().start.toDouble() / waveData.sampleRate; + final endTime = + startTime + samples.length.toDouble() / waveData.sampleRate; + + final stream = recognizer.createStream(); + stream.acceptWaveform(samples: samples, sampleRate: waveData.sampleRate); + recognizer.decode(stream); + + final result = recognizer.getResult(stream); + stream.free(); + print( + '${startTime.toStringAsPrecision(5)} -- ${endTime.toStringAsPrecision(5)} : ${result.text}'); + + vad.pop(); + } + } + + vad.flush(); + + while (!vad.isEmpty()) { + final samples = vad.front().samples; + final startTime = vad.front().start.toDouble() / waveData.sampleRate; + final endTime = startTime + samples.length.toDouble() / waveData.sampleRate; + + final stream = recognizer.createStream(); + stream.acceptWaveform(samples: samples, sampleRate: waveData.sampleRate); + recognizer.decode(stream); + + final result = recognizer.getResult(stream); + stream.free(); + print( + '${startTime.toStringAsPrecision(5)} -- ${endTime.toStringAsPrecision(5)} : ${result.text}'); + + vad.pop(); + } + + vad.free(); + + recognizer.free(); +} diff --git a/dart-api-examples/vad-with-non-streaming-asr/run-moonshine.sh b/dart-api-examples/vad-with-non-streaming-asr/run-moonshine.sh new file mode 100755 index 000000000..cd531fec5 --- /dev/null +++ b/dart-api-examples/vad-with-non-streaming-asr/run-moonshine.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -ex + +dart pub get + +if [ ! -f ./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +fi + +if [ ! -f ./Obama.wav ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/Obama.wav +fi + +if [[ ! -f ./silero_vad.onnx ]]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx +fi + +dart run \ + ./bin/moonshine.dart \ + --silero-vad ./silero_vad.onnx \ + --preprocessor ./sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx \ + --encoder ./sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx \ + --uncached-decoder ./sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx \ + --cached-decoder ./sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx \ + --tokens ./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt \ + --input-wav ./Obama.wav diff --git a/flutter/sherpa_onnx/lib/src/offline_recognizer.dart b/flutter/sherpa_onnx/lib/src/offline_recognizer.dart index 749ffb316..2f6f167d4 100644 --- a/flutter/sherpa_onnx/lib/src/offline_recognizer.dart +++ b/flutter/sherpa_onnx/lib/src/offline_recognizer.dart @@ -68,6 +68,24 @@ class OfflineWhisperModelConfig { final int tailPaddings; } +class OfflineMoonshineModelConfig { + const OfflineMoonshineModelConfig( + {this.preprocessor = '', + this.encoder = '', + this.uncachedDecoder = '', + this.cachedDecoder = ''}); + + @override + String toString() { + return 'OfflineMoonshineModelConfig(preprocessor: $preprocessor, encoder: $encoder, uncachedDecoder: $uncachedDecoder, cachedDecoder: $cachedDecoder)'; + } + + final String preprocessor; + final String encoder; + final String uncachedDecoder; + final String cachedDecoder; +} + class OfflineTdnnModelConfig { const OfflineTdnnModelConfig({this.model = ''}); @@ -116,6 +134,7 @@ class OfflineModelConfig { this.whisper = const OfflineWhisperModelConfig(), this.tdnn = const OfflineTdnnModelConfig(), this.senseVoice = const OfflineSenseVoiceModelConfig(), + this.moonshine = const OfflineMoonshineModelConfig(), required this.tokens, this.numThreads = 1, this.debug = true, @@ -128,7 +147,7 @@ class OfflineModelConfig { @override String toString() { - return 'OfflineModelConfig(transducer: $transducer, paraformer: $paraformer, nemoCtc: $nemoCtc, whisper: $whisper, tdnn: $tdnn, senseVoice: $senseVoice, tokens: $tokens, numThreads: $numThreads, debug: $debug, provider: $provider, modelType: $modelType, modelingUnit: $modelingUnit, bpeVocab: $bpeVocab, telespeechCtc: $telespeechCtc)'; + return 'OfflineModelConfig(transducer: $transducer, paraformer: $paraformer, nemoCtc: $nemoCtc, whisper: $whisper, tdnn: $tdnn, senseVoice: $senseVoice, moonshine: $moonshine, tokens: $tokens, numThreads: $numThreads, debug: $debug, provider: $provider, modelType: $modelType, modelingUnit: $modelingUnit, bpeVocab: $bpeVocab, telespeechCtc: $telespeechCtc)'; } final OfflineTransducerModelConfig transducer; @@ -137,6 +156,7 @@ class OfflineModelConfig { final OfflineWhisperModelConfig whisper; final OfflineTdnnModelConfig tdnn; final OfflineSenseVoiceModelConfig senseVoice; + final OfflineMoonshineModelConfig moonshine; final String tokens; final int numThreads; @@ -257,6 +277,15 @@ class OfflineRecognizer { c.ref.model.senseVoice.useInverseTextNormalization = config.model.senseVoice.useInverseTextNormalization ? 1 : 0; + c.ref.model.moonshine.preprocessor = + config.model.moonshine.preprocessor.toNativeUtf8(); + c.ref.model.moonshine.encoder = + config.model.moonshine.encoder.toNativeUtf8(); + c.ref.model.moonshine.uncachedDecoder = + config.model.moonshine.uncachedDecoder.toNativeUtf8(); + c.ref.model.moonshine.cachedDecoder = + config.model.moonshine.cachedDecoder.toNativeUtf8(); + c.ref.model.tokens = config.model.tokens.toNativeUtf8(); c.ref.model.numThreads = config.model.numThreads; @@ -294,6 +323,10 @@ class OfflineRecognizer { calloc.free(c.ref.model.modelType); calloc.free(c.ref.model.provider); calloc.free(c.ref.model.tokens); + calloc.free(c.ref.model.moonshine.cachedDecoder); + calloc.free(c.ref.model.moonshine.uncachedDecoder); + calloc.free(c.ref.model.moonshine.encoder); + calloc.free(c.ref.model.moonshine.preprocessor); calloc.free(c.ref.model.senseVoice.language); calloc.free(c.ref.model.senseVoice.model); calloc.free(c.ref.model.tdnn.model); diff --git a/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart b/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart index 8a8817d63..0d463cf6d 100644 --- a/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart +++ b/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart @@ -194,6 +194,13 @@ final class SherpaOnnxOfflineWhisperModelConfig extends Struct { external int tailPaddings; } +final class SherpaOnnxOfflineMoonshineModelConfig extends Struct { + external Pointer preprocessor; + external Pointer encoder; + external Pointer uncachedDecoder; + external Pointer cachedDecoder; +} + final class SherpaOnnxOfflineTdnnModelConfig extends Struct { external Pointer model; } @@ -236,6 +243,7 @@ final class SherpaOnnxOfflineModelConfig extends Struct { external Pointer telespeechCtc; external SherpaOnnxOfflineSenseVoiceModelConfig senseVoice; + external SherpaOnnxOfflineMoonshineModelConfig moonshine; } final class SherpaOnnxOfflineRecognizerConfig extends Struct { From cdd8e1bbcb51aeb0b1bd73c4138f0e4ad7061bbd Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sun, 27 Oct 2024 12:21:16 +0800 Subject: [PATCH 049/183] Add Pascal API for Moonshine models (#1482) --- .github/workflows/pascal.yaml | 8 + .../non-streaming-asr/.gitignore | 1 + .../non-streaming-asr/moonshine.pas | 80 ++++++++++ .../non-streaming-asr/run-moonshine.sh | 42 ++++++ .../vad-with-non-streaming-asr/.gitignore | 1 + .../run-vad-with-moonshine.sh | 49 ++++++ .../vad_with_moonshine.pas | 139 ++++++++++++++++++ sherpa-onnx/pascal-api/sherpa_onnx.pas | 37 ++++- 8 files changed, 354 insertions(+), 3 deletions(-) create mode 100644 pascal-api-examples/non-streaming-asr/moonshine.pas create mode 100755 pascal-api-examples/non-streaming-asr/run-moonshine.sh create mode 100755 pascal-api-examples/vad-with-non-streaming-asr/run-vad-with-moonshine.sh create mode 100644 pascal-api-examples/vad-with-non-streaming-asr/vad_with_moonshine.pas diff --git a/.github/workflows/pascal.yaml b/.github/workflows/pascal.yaml index ba9a73163..306ae6480 100644 --- a/.github/workflows/pascal.yaml +++ b/.github/workflows/pascal.yaml @@ -165,6 +165,10 @@ jobs: cd ./pascal-api-examples pushd vad-with-non-streaming-asr + time ./run-vad-with-moonshine.sh + rm -rf sherpa-onnx-* + echo "---" + time ./run-vad-with-whisper.sh rm -rf sherpa-onnx-* echo "---" @@ -220,6 +224,10 @@ jobs: rm -rf sherpa-onnx-* echo "---" + ./run-moonshine.sh + rm -rf sherpa-onnx-* + echo "---" + ./run-whisper.sh rm -rf sherpa-onnx-* echo "---" diff --git a/pascal-api-examples/non-streaming-asr/.gitignore b/pascal-api-examples/non-streaming-asr/.gitignore index fbcf1c968..aba0585a3 100644 --- a/pascal-api-examples/non-streaming-asr/.gitignore +++ b/pascal-api-examples/non-streaming-asr/.gitignore @@ -7,3 +7,4 @@ paraformer paraformer_itn sense_voice telespeech_ctc +moonshine diff --git a/pascal-api-examples/non-streaming-asr/moonshine.pas b/pascal-api-examples/non-streaming-asr/moonshine.pas new file mode 100644 index 000000000..04597ad64 --- /dev/null +++ b/pascal-api-examples/non-streaming-asr/moonshine.pas @@ -0,0 +1,80 @@ +{ Copyright (c) 2024 Xiaomi Corporation } + +{ +This file shows how to use a non-streaming Moonshine model +to decode files. + +You can download the model files from +https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models +} + +program moonshine; + +{$mode objfpc} + +uses + sherpa_onnx, + DateUtils, + SysUtils; + +var + Wave: TSherpaOnnxWave; + WaveFilename: AnsiString; + + Config: TSherpaOnnxOfflineRecognizerConfig; + Recognizer: TSherpaOnnxOfflineRecognizer; + Stream: TSherpaOnnxOfflineStream; + RecognitionResult: TSherpaOnnxOfflineRecognizerResult; + + Start: TDateTime; + Stop: TDateTime; + + Elapsed: Single; + Duration: Single; + RealTimeFactor: Single; +begin + Initialize(Config); + + Config.ModelConfig.Moonshine.Preprocessor := './sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx'; + Config.ModelConfig.Moonshine.Encoder := './sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx'; + Config.ModelConfig.Moonshine.UncachedDecoder := './sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx'; + Config.ModelConfig.Moonshine.CachedDecoder := './sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx'; + + Config.ModelConfig.Tokens := './sherpa-onnx-moonshine-tiny-en-int8/tokens.txt'; + Config.ModelConfig.Provider := 'cpu'; + Config.ModelConfig.NumThreads := 1; + Config.ModelConfig.Debug := False; + + WaveFilename := './sherpa-onnx-moonshine-tiny-en-int8/test_wavs/0.wav'; + + Wave := SherpaOnnxReadWave(WaveFilename); + + Recognizer := TSherpaOnnxOfflineRecognizer.Create(Config); + Stream := Recognizer.CreateStream(); + Start := Now; + + Stream.AcceptWaveform(Wave.Samples, Wave.SampleRate); + Recognizer.Decode(Stream); + + RecognitionResult := Recognizer.GetResult(Stream); + + Stop := Now; + + Elapsed := MilliSecondsBetween(Stop, Start) / 1000; + Duration := Length(Wave.Samples) / Wave.SampleRate; + RealTimeFactor := Elapsed / Duration; + + WriteLn(RecognitionResult.ToString); + WriteLn(Format('NumThreads %d', [Config.ModelConfig.NumThreads])); + WriteLn(Format('Elapsed %.3f s', [Elapsed])); + WriteLn(Format('Wave duration %.3f s', [Duration])); + WriteLn(Format('RTF = %.3f/%.3f = %.3f', [Elapsed, Duration, RealTimeFactor])); + + {Free resources to avoid memory leak. + + Note: You don't need to invoke them for this simple script. + However, you have to invoke them in your own large/complex project. + } + FreeAndNil(Stream); + FreeAndNil(Recognizer); +end. diff --git a/pascal-api-examples/non-streaming-asr/run-moonshine.sh b/pascal-api-examples/non-streaming-asr/run-moonshine.sh new file mode 100755 index 000000000..9486b06e9 --- /dev/null +++ b/pascal-api-examples/non-streaming-asr/run-moonshine.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -ex + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +SHERPA_ONNX_DIR=$(cd $SCRIPT_DIR/../.. && pwd) + +echo "SHERPA_ONNX_DIR: $SHERPA_ONNX_DIR" + +if [[ ! -f ../../build/install/lib/libsherpa-onnx-c-api.dylib && ! -f ../../build/install/lib/libsherpa-onnx-c-api.so && ! -f ../../build/install/lib/sherpa-onnx-c-api.dll ]]; then + mkdir -p ../../build + pushd ../../build + cmake \ + -DCMAKE_INSTALL_PREFIX=./install \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + .. + + cmake --build . --target install --config Release + ls -lh lib + popd +fi + +if [ ! -f ./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +fi + +fpc \ + -dSHERPA_ONNX_USE_SHARED_LIBS \ + -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \ + -Fl$SHERPA_ONNX_DIR/build/install/lib \ + ./moonshine.pas + +export LD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$LD_LIBRARY_PATH +export DYLD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$DYLD_LIBRARY_PATH + +./moonshine diff --git a/pascal-api-examples/vad-with-non-streaming-asr/.gitignore b/pascal-api-examples/vad-with-non-streaming-asr/.gitignore index 4718ed421..d499ad3b3 100644 --- a/pascal-api-examples/vad-with-non-streaming-asr/.gitignore +++ b/pascal-api-examples/vad-with-non-streaming-asr/.gitignore @@ -1,3 +1,4 @@ !run-*.sh vad_with_whisper vad_with_sense_voice +vad_with_moonshine diff --git a/pascal-api-examples/vad-with-non-streaming-asr/run-vad-with-moonshine.sh b/pascal-api-examples/vad-with-non-streaming-asr/run-vad-with-moonshine.sh new file mode 100755 index 000000000..fdf04b639 --- /dev/null +++ b/pascal-api-examples/vad-with-non-streaming-asr/run-vad-with-moonshine.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -ex + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +SHERPA_ONNX_DIR=$(cd $SCRIPT_DIR/../.. && pwd) + +echo "SHERPA_ONNX_DIR: $SHERPA_ONNX_DIR" + +if [[ ! -f ../../build/install/lib/libsherpa-onnx-c-api.dylib && ! -f ../../build/install/lib/libsherpa-onnx-c-api.so && ! -f ../../build/install/lib/sherpa-onnx-c-api.dll ]]; then + mkdir -p ../../build + pushd ../../build + cmake \ + -DCMAKE_INSTALL_PREFIX=./install \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + .. + + cmake --build . --target install --config Release + popd +fi + +if [[ ! -f ./silero_vad.onnx ]]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx +fi + +if [ ! -f ./Obama.wav ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/Obama.wav +fi + +if [ ! -f ./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +fi + +fpc \ + -dSHERPA_ONNX_USE_SHARED_LIBS \ + -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \ + -Fl$SHERPA_ONNX_DIR/build/install/lib \ + ./vad_with_moonshine.pas + +export LD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$LD_LIBRARY_PATH +export DYLD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$DYLD_LIBRARY_PATH + +./vad_with_moonshine diff --git a/pascal-api-examples/vad-with-non-streaming-asr/vad_with_moonshine.pas b/pascal-api-examples/vad-with-non-streaming-asr/vad_with_moonshine.pas new file mode 100644 index 000000000..50a2e95d2 --- /dev/null +++ b/pascal-api-examples/vad-with-non-streaming-asr/vad_with_moonshine.pas @@ -0,0 +1,139 @@ +{ Copyright (c) 2024 Xiaomi Corporation } + +{ +This file shows how to use a non-streaming Moonshine model +with silero VAD to decode files. + +You can download the model files from +https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models +} + +program vad_with_moonshine; + +{$mode objfpc} + +uses + sherpa_onnx, + SysUtils; + +function CreateVad(): TSherpaOnnxVoiceActivityDetector; +var + Config: TSherpaOnnxVadModelConfig; + + SampleRate: Integer; + WindowSize: Integer; +begin + Initialize(Config); + + SampleRate := 16000; {Please don't change it unless you know the details} + WindowSize := 512; {Please don't change it unless you know the details} + + Config.SileroVad.Model := './silero_vad.onnx'; + Config.SileroVad.MinSpeechDuration := 0.5; + Config.SileroVad.MinSilenceDuration := 0.5; + Config.SileroVad.Threshold := 0.5; + Config.SileroVad.WindowSize := WindowSize; + Config.NumThreads:= 1; + Config.Debug:= True; + Config.Provider:= 'cpu'; + Config.SampleRate := SampleRate; + + Result := TSherpaOnnxVoiceActivityDetector.Create(Config, 30); +end; + +function CreateOfflineRecognizer(): TSherpaOnnxOfflineRecognizer; +var + Config: TSherpaOnnxOfflineRecognizerConfig; +begin + Initialize(Config); + + Config.ModelConfig.Moonshine.Preprocessor := './sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx'; + Config.ModelConfig.Moonshine.Encoder := './sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx'; + Config.ModelConfig.Moonshine.UncachedDecoder := './sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx'; + Config.ModelConfig.Moonshine.CachedDecoder := './sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx'; + + Config.ModelConfig.Tokens := './sherpa-onnx-moonshine-tiny-en-int8/tokens.txt'; + Config.ModelConfig.Provider := 'cpu'; + Config.ModelConfig.NumThreads := 1; + Config.ModelConfig.Debug := False; + + Result := TSherpaOnnxOfflineRecognizer.Create(Config); +end; + +var + Wave: TSherpaOnnxWave; + + Recognizer: TSherpaOnnxOfflineRecognizer; + Vad: TSherpaOnnxVoiceActivityDetector; + + Offset: Integer; + WindowSize: Integer; + SpeechSegment: TSherpaOnnxSpeechSegment; + + Start: Single; + Duration: Single; + + Stream: TSherpaOnnxOfflineStream; + RecognitionResult: TSherpaOnnxOfflineRecognizerResult; +begin + Vad := CreateVad(); + Recognizer := CreateOfflineRecognizer(); + + Wave := SherpaOnnxReadWave('./Obama.wav'); + if Wave.SampleRate <> Vad.Config.SampleRate then + begin + WriteLn(Format('Expected sample rate: %d. Given: %d', + [Vad.Config.SampleRate, Wave.SampleRate])); + + Exit; + end; + + WindowSize := Vad.Config.SileroVad.WindowSize; + Offset := 0; + while Offset + WindowSize <= Length(Wave.Samples) do + begin + Vad.AcceptWaveform(Wave.Samples, Offset, WindowSize); + Offset += WindowSize; + + while not Vad.IsEmpty do + begin + SpeechSegment := Vad.Front(); + Vad.Pop(); + Stream := Recognizer.CreateStream(); + + Stream.AcceptWaveform(SpeechSegment.Samples, Wave.SampleRate); + Recognizer.Decode(Stream); + RecognitionResult := Recognizer.GetResult(Stream); + + Start := SpeechSegment.Start / Wave.SampleRate; + Duration := Length(SpeechSegment.Samples) / Wave.SampleRate; + WriteLn(Format('%.3f -- %.3f %s', + [Start, Start + Duration, RecognitionResult.Text])); + + FreeAndNil(Stream); + end; + end; + + Vad.Flush; + + while not Vad.IsEmpty do + begin + SpeechSegment := Vad.Front(); + Vad.Pop(); + Stream := Recognizer.CreateStream(); + + Stream.AcceptWaveform(SpeechSegment.Samples, Wave.SampleRate); + Recognizer.Decode(Stream); + RecognitionResult := Recognizer.GetResult(Stream); + + Start := SpeechSegment.Start / Wave.SampleRate; + Duration := Length(SpeechSegment.Samples) / Wave.SampleRate; + WriteLn(Format('%.3f -- %.3f %s', + [Start, Start + Duration, RecognitionResult.Text])); + + FreeAndNil(Stream); + end; + + FreeAndNil(Recognizer); + FreeAndNil(Vad); +end. diff --git a/sherpa-onnx/pascal-api/sherpa_onnx.pas b/sherpa-onnx/pascal-api/sherpa_onnx.pas index 1b24dec80..cff215759 100644 --- a/sherpa-onnx/pascal-api/sherpa_onnx.pas +++ b/sherpa-onnx/pascal-api/sherpa_onnx.pas @@ -250,6 +250,14 @@ TSherpaOnnxOfflineWhisperModelConfig = record class operator Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineWhisperModelConfig); end; + TSherpaOnnxOfflineMoonshineModelConfig = record + Preprocessor: AnsiString; + Encoder: AnsiString; + UncachedDecoder: AnsiString; + CachedDecoder: AnsiString; + function ToString: AnsiString; + end; + TSherpaOnnxOfflineTdnnModelConfig = record Model: AnsiString; function ToString: AnsiString; @@ -285,6 +293,7 @@ TSherpaOnnxOfflineModelConfig = record BpeVocab: AnsiString; TeleSpeechCtc: AnsiString; SenseVoice: TSherpaOnnxOfflineSenseVoiceModelConfig; + Moonshine: TSherpaOnnxOfflineMoonshineModelConfig; class operator Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineModelConfig); function ToString: AnsiString; end; @@ -617,6 +626,12 @@ SherpaOnnxOfflineWhisperModelConfig = record Task: PAnsiChar; TailPaddings: cint32; end; + SherpaOnnxOfflineMoonshineModelConfig = record + Preprocessor: PAnsiChar; + Encoder: PAnsiChar; + UncachedDecoder: PAnsiChar; + CachedDecoder: PAnsiChar; + end; SherpaOnnxOfflineTdnnModelConfig = record Model: PAnsiChar; end; @@ -644,6 +659,7 @@ SherpaOnnxOfflineModelConfig = record BpeVocab: PAnsiChar; TeleSpeechCtc: PAnsiChar; SenseVoice: SherpaOnnxOfflineSenseVoiceModelConfig; + Moonshine: SherpaOnnxOfflineMoonshineModelConfig; end; SherpaOnnxOfflineRecognizerConfig = record @@ -1312,6 +1328,16 @@ function TSherpaOnnxOfflineWhisperModelConfig.ToString: AnsiString; [Self.Encoder, Self.Decoder, Self.Language, Self.Task, Self.TailPaddings]); end; +function TSherpaOnnxOfflineMoonshineModelConfig.ToString: AnsiString; +begin + Result := Format('TSherpaOnnxOfflineMoonshineModelConfig(' + + 'Preprocessor := %s, ' + + 'Encoder := %s, ' + + 'UncachedDecoder := %s, ' + + 'CachedDecoder := %s)', + [Self.Preprocessor, Self.Encoder, Self.UncachedDecoder, Self.CachedDecoder]); +end; + function TSherpaOnnxOfflineTdnnModelConfig.ToString: AnsiString; begin Result := Format('TSherpaOnnxOfflineTdnnModelConfig(Model := %s)', @@ -1353,13 +1379,14 @@ function TSherpaOnnxOfflineModelConfig.ToString: AnsiString; 'ModelingUnit := %s, ' + 'BpeVocab := %s, ' + 'TeleSpeechCtc := %s, ' + - 'SenseVoice := %s' + + 'SenseVoice := %s, ' + + 'Moonshine := %s' + ')', [Self.Transducer.ToString, Self.Paraformer.ToString, Self.NeMoCtc.ToString, Self.Whisper.ToString, Self.Tdnn.ToString, Self.Tokens, Self.NumThreads, Self.Debug.ToString, Self.Provider, Self.ModelType, Self.ModelingUnit, Self.BpeVocab, - Self.TeleSpeechCtc, Self.SenseVoice.ToString + Self.TeleSpeechCtc, Self.SenseVoice.ToString, Self.Moonshine.ToString ]); end; @@ -1407,7 +1434,6 @@ constructor TSherpaOnnxOfflineRecognizer.Create(Config: TSherpaOnnxOfflineRecogn C.ModelConfig.Tdnn.Model := PAnsiChar(Config.ModelConfig.Tdnn.Model); - C.ModelConfig.Tokens := PAnsiChar(Config.ModelConfig.Tokens); C.ModelConfig.NumThreads := Config.ModelConfig.NumThreads; C.ModelConfig.Debug := Ord(Config.ModelConfig.Debug); @@ -1421,6 +1447,11 @@ constructor TSherpaOnnxOfflineRecognizer.Create(Config: TSherpaOnnxOfflineRecogn C.ModelConfig.SenseVoice.Language := PAnsiChar(Config.ModelConfig.SenseVoice.Language); C.ModelConfig.SenseVoice.UseItn := Ord(Config.ModelConfig.SenseVoice.UseItn); + C.ModelConfig.Moonshine.Preprocessor := PAnsiChar(Config.ModelConfig.Moonshine.Preprocessor); + C.ModelConfig.Moonshine.Encoder := PAnsiChar(Config.ModelConfig.Moonshine.Encoder); + C.ModelConfig.Moonshine.UncachedDecoder := PAnsiChar(Config.ModelConfig.Moonshine.UncachedDecoder); + C.ModelConfig.Moonshine.CachedDecoder := PAnsiChar(Config.ModelConfig.Moonshine.CachedDecoder); + C.LMConfig.Model := PAnsiChar(Config.LMConfig.Model); C.LMConfig.Scale := Config.LMConfig.Scale; From 3622104133701514f30d555d8c4171bf3bac85f6 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sun, 27 Oct 2024 13:14:25 +0800 Subject: [PATCH 050/183] Add C# API for Moonshine models. (#1483) * Also, return timestamps for non-streaming ASR. --- .github/scripts/test-dot-net.sh | 3 + .../offline-decode-files/Program.cs | 44 ++++++++++++--- .../offline-decode-files/run-moonshine.sh | 18 ++++++ scripts/dotnet/OfflineModelConfig.cs | 2 + scripts/dotnet/OfflineMoonshineModelConfig.cs | 29 ++++++++++ scripts/dotnet/OfflineRecognizerResult.cs | 55 ++++++++++++++++++- 6 files changed, 143 insertions(+), 8 deletions(-) create mode 100755 dotnet-examples/offline-decode-files/run-moonshine.sh create mode 100644 scripts/dotnet/OfflineMoonshineModelConfig.cs diff --git a/.github/scripts/test-dot-net.sh b/.github/scripts/test-dot-net.sh index eec3b6bb4..f4bfc66c5 100755 --- a/.github/scripts/test-dot-net.sh +++ b/.github/scripts/test-dot-net.sh @@ -9,6 +9,9 @@ rm -fv *.wav rm -rfv sherpa-onnx-pyannote-* cd ../offline-decode-files +./run-moonshine.sh +rm -rf sherpa-onnx-* + ./run-sense-voice-ctc.sh rm -rf sherpa-onnx-* diff --git a/dotnet-examples/offline-decode-files/Program.cs b/dotnet-examples/offline-decode-files/Program.cs index d971becd3..d855da6f8 100644 --- a/dotnet-examples/offline-decode-files/Program.cs +++ b/dotnet-examples/offline-decode-files/Program.cs @@ -17,7 +17,7 @@ class Options { [Option("sample-rate", Required = false, Default = 16000, HelpText = "Sample rate of the data used to train the model")] - public int SampleRate { get; set; } = 16000; + public int SampleRate { get; set; } = 16000; [Option("feat-dim", Required = false, Default = 80, HelpText = "Dimension of the features used to train the model")] public int FeatureDim { get; set; } = 80; @@ -31,7 +31,7 @@ class Options [Option(Required = false, Default = "", HelpText = "Path to transducer decoder.onnx. Used only for transducer models")] public string Decoder { get; set; } = ""; - [Option(Required = false, Default = "",HelpText = "Path to transducer joiner.onnx. Used only for transducer models")] + [Option(Required = false, Default = "", HelpText = "Path to transducer joiner.onnx. Used only for transducer models")] public string Joiner { get; set; } = ""; [Option("model-type", Required = false, Default = "", HelpText = "model type")] @@ -44,10 +44,22 @@ class Options public string WhisperDecoder { get; set; } = ""; [Option("whisper-language", Required = false, Default = "", HelpText = "Language of the input file. Can be empty")] - public string WhisperLanguage{ get; set; } = ""; + public string WhisperLanguage { get; set; } = ""; [Option("whisper-task", Required = false, Default = "transcribe", HelpText = "transcribe or translate")] - public string WhisperTask{ get; set; } = "transcribe"; + public string WhisperTask { get; set; } = "transcribe"; + + [Option("moonshine-preprocessor", Required = false, Default = "", HelpText = "Path to preprocess.onnx. Used only for Moonshine models")] + public string MoonshinePreprocessor { get; set; } = ""; + + [Option("moonshine-encoder", Required = false, Default = "", HelpText = "Path to encode.onnx. Used only for Moonshine models")] + public string MoonshineEncoder { get; set; } = ""; + + [Option("moonshine-uncached-decoder", Required = false, Default = "", HelpText = "Path to uncached_decode.onnx. Used only for Moonshine models")] + public string MoonshineUncachedDecoder { get; set; } = ""; + + [Option("moonshine-cached-decoder", Required = false, Default = "", HelpText = "Path to cached_decode.onnx. Used only for Moonshine models")] + public string MoonshineCachedDecoder { get; set; } = ""; [Option("tdnn-model", Required = false, Default = "", HelpText = "Path to tdnn yesno model")] public string TdnnModel { get; set; } = ""; @@ -90,7 +102,7 @@ class Options public float HotwordsScore { get; set; } = 1.5F; [Option("files", Required = true, HelpText = "Audio files for decoding")] - public IEnumerable Files { get; set; } = new string[] {}; + public IEnumerable Files { get; set; } = new string[] { }; } static void Main(string[] args) @@ -236,6 +248,13 @@ private static void Run(Options options) config.ModelConfig.SenseVoice.Model = options.SenseVoiceModel; config.ModelConfig.SenseVoice.UseInverseTextNormalization = options.SenseVoiceUseItn; } + else if (!String.IsNullOrEmpty(options.MoonshinePreprocessor)) + { + config.ModelConfig.Moonshine.Preprocessor = options.MoonshinePreprocessor; + config.ModelConfig.Moonshine.Encoder = options.MoonshineEncoder; + config.ModelConfig.Moonshine.UncachedDecoder = options.MoonshineUncachedDecoder; + config.ModelConfig.Moonshine.CachedDecoder = options.MoonshineCachedDecoder; + } else { Console.WriteLine("Please provide a model"); @@ -273,10 +292,21 @@ private static void Run(Options options) // display results for (int i = 0; i != files.Length; ++i) { - var text = streams[i].Result.Text; + var r = streams[i].Result; Console.WriteLine("--------------------"); Console.WriteLine(files[i]); - Console.WriteLine(text); + Console.WriteLine("Text: {0}", r.Text); + Console.WriteLine("Tokens: [{0}]", string.Join(", ", r.Tokens)); + if (r.Timestamps != null && r.Timestamps.Length > 0) { + Console.Write("Timestamps: ["); + var sep = ""; + for (int k = 0; k != r.Timestamps.Length; ++k) + { + Console.Write("{0}{1}", sep, r.Timestamps[k].ToString("0.00")); + sep = ", "; + } + Console.WriteLine("]"); + } } Console.WriteLine("--------------------"); } diff --git a/dotnet-examples/offline-decode-files/run-moonshine.sh b/dotnet-examples/offline-decode-files/run-moonshine.sh new file mode 100755 index 000000000..025e0902d --- /dev/null +++ b/dotnet-examples/offline-decode-files/run-moonshine.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -ex + +if [ ! -f ./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +fi + +dotnet run \ + --num-threads=2 \ + --moonshine-preprocessor=./sherpa-onnx-moonshine-tiny-en-int8/preprocess.onnx \ + --moonshine-encoder=./sherpa-onnx-moonshine-tiny-en-int8/encode.int8.onnx \ + --moonshine-uncached-decoder=./sherpa-onnx-moonshine-tiny-en-int8/uncached_decode.int8.onnx \ + --moonshine-cached-decoder=./sherpa-onnx-moonshine-tiny-en-int8/cached_decode.int8.onnx \ + --tokens=./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt \ + --files ./sherpa-onnx-moonshine-tiny-en-int8/test_wavs/0.wav diff --git a/scripts/dotnet/OfflineModelConfig.cs b/scripts/dotnet/OfflineModelConfig.cs index b24aeaf89..b7433c277 100644 --- a/scripts/dotnet/OfflineModelConfig.cs +++ b/scripts/dotnet/OfflineModelConfig.cs @@ -24,6 +24,7 @@ public OfflineModelConfig() BpeVocab = ""; TeleSpeechCtc = ""; SenseVoice = new OfflineSenseVoiceModelConfig(); + Moonshine = new OfflineMoonshineModelConfig(); } public OfflineTransducerModelConfig Transducer; public OfflineParaformerModelConfig Paraformer; @@ -54,5 +55,6 @@ public OfflineModelConfig() public string TeleSpeechCtc; public OfflineSenseVoiceModelConfig SenseVoice; + public OfflineMoonshineModelConfig Moonshine; } } diff --git a/scripts/dotnet/OfflineMoonshineModelConfig.cs b/scripts/dotnet/OfflineMoonshineModelConfig.cs new file mode 100644 index 000000000..53c205173 --- /dev/null +++ b/scripts/dotnet/OfflineMoonshineModelConfig.cs @@ -0,0 +1,29 @@ +/// Copyright (c) 2024 Xiaomi Corporation (authors: Fangjun Kuang) + +using System.Runtime.InteropServices; + +namespace SherpaOnnx +{ + [StructLayout(LayoutKind.Sequential)] + public struct OfflineMoonshineModelConfig + { + public OfflineMoonshineModelConfig() + { + Preprocessor = ""; + Encoder = ""; + UncachedDecoder = ""; + CachedDecoder = ""; + } + [MarshalAs(UnmanagedType.LPStr)] + public string Preprocessor; + + [MarshalAs(UnmanagedType.LPStr)] + public string Encoder; + + [MarshalAs(UnmanagedType.LPStr)] + public string UncachedDecoder; + + [MarshalAs(UnmanagedType.LPStr)] + public string CachedDecoder; + } +} diff --git a/scripts/dotnet/OfflineRecognizerResult.cs b/scripts/dotnet/OfflineRecognizerResult.cs index 8c10b6f88..ecf682b19 100644 --- a/scripts/dotnet/OfflineRecognizerResult.cs +++ b/scripts/dotnet/OfflineRecognizerResult.cs @@ -31,17 +31,70 @@ public OfflineRecognizerResult(IntPtr handle) byte[] stringBuffer = new byte[length]; Marshal.Copy(impl.Text, stringBuffer, 0, length); _text = Encoding.UTF8.GetString(stringBuffer); + + _tokens = new String[impl.Count]; + + unsafe + { + byte* buf = (byte*)impl.Tokens; + for (int i = 0; i < impl.Count; i++) + { + length = 0; + byte* start = buf; + while (*buf != 0) + { + ++buf; + length += 1; + } + ++buf; + + stringBuffer = new byte[length]; + fixed (byte* pTarget = stringBuffer) + { + for (int k = 0; k < length; k++) + { + pTarget[k] = start[k]; + } + } + + _tokens[i] = Encoding.UTF8.GetString(stringBuffer); + } + } + + unsafe + { + if (impl.Timestamps != IntPtr.Zero) + { + float *t = (float*)impl.Timestamps; + _timestamps = new float[impl.Count]; + fixed (float* f = _timestamps) + { + for (int k = 0; k < impl.Count; k++) + { + f[k] = t[k]; + } + } + } + } + } [StructLayout(LayoutKind.Sequential)] struct Impl { public IntPtr Text; + public IntPtr Timestamps; + public int Count; + public IntPtr Tokens; } private String _text; public String Text => _text; - } + private String[] _tokens; + public String[] Tokens => _tokens; + private float[] _timestamps; + public float[] Timestamps => _timestamps; + } } From 91e090ff86f0773556059cb55183837d5687450b Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sun, 27 Oct 2024 13:45:13 +0800 Subject: [PATCH 051/183] Release v1.10.30 (#1484) --- CHANGELOG.md | 16 ++++++++++++++++ CMakeLists.txt | 2 +- dart-api-examples/add-punctuations/pubspec.yaml | 2 +- dart-api-examples/audio-tagging/pubspec.yaml | 2 +- dart-api-examples/keyword-spotter/pubspec.yaml | 2 +- dart-api-examples/non-streaming-asr/pubspec.yaml | 2 +- .../speaker-diarization/pubspec.yaml | 2 +- .../speaker-identification/pubspec.yaml | 2 +- dart-api-examples/streaming-asr/pubspec.yaml | 2 +- dart-api-examples/tts/pubspec.yaml | 2 +- .../vad-with-non-streaming-asr/pubspec.yaml | 2 +- dart-api-examples/vad/pubspec.yaml | 2 +- flutter-examples/streaming_asr/pubspec.yaml | 4 ++-- flutter-examples/tts/pubspec.yaml | 4 ++-- flutter/sherpa_onnx/pubspec.yaml | 12 ++++++------ .../sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec | 2 +- .../macos/sherpa_onnx_macos.podspec | 2 +- new-release.sh | 10 +++++----- nodejs-addon-examples/package.json | 2 +- 19 files changed, 45 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 176d8ee23..8787fda3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,19 @@ +## 1.10.30 + +* Fix building node-addon for Windows x86. (#1469) +* Begin to support https://github.com/usefulsensors/moonshine (#1470) +* Publish pre-built JNI libs for Linux aarch64 (#1472) +* Add C++ runtime and Python APIs for Moonshine models (#1473) +* Add Kotlin and Java API for Moonshine models (#1474) +* Add C and C++ API for Moonshine models (#1476) +* Add Swift API for Moonshine models. (#1477) +* Add Go API examples for adding punctuations to text. (#1478) +* Add Go API for Moonshine models (#1479) +* Add JavaScript API for Moonshine models (#1480) +* Add Dart API for Moonshine models. (#1481) +* Add Pascal API for Moonshine models (#1482) +* Add C# API for Moonshine models. (#1483) + ## 1.10.29 * Add Go API for offline punctuation models (#1434) diff --git a/CMakeLists.txt b/CMakeLists.txt index c9edbf43d..ac357b746 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,7 @@ project(sherpa-onnx) # ./nodejs-addon-examples # ./dart-api-examples/ # ./CHANGELOG.md -set(SHERPA_ONNX_VERSION "1.10.29") +set(SHERPA_ONNX_VERSION "1.10.30") # Disable warning about # diff --git a/dart-api-examples/add-punctuations/pubspec.yaml b/dart-api-examples/add-punctuations/pubspec.yaml index 892535c85..9cf4cf28e 100644 --- a/dart-api-examples/add-punctuations/pubspec.yaml +++ b/dart-api-examples/add-punctuations/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.29 + sherpa_onnx: ^1.10.30 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/audio-tagging/pubspec.yaml b/dart-api-examples/audio-tagging/pubspec.yaml index ddd5ac256..06aff0384 100644 --- a/dart-api-examples/audio-tagging/pubspec.yaml +++ b/dart-api-examples/audio-tagging/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.29 + sherpa_onnx: ^1.10.30 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/keyword-spotter/pubspec.yaml b/dart-api-examples/keyword-spotter/pubspec.yaml index 61da20d70..28d010ac8 100644 --- a/dart-api-examples/keyword-spotter/pubspec.yaml +++ b/dart-api-examples/keyword-spotter/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.29 + sherpa_onnx: ^1.10.30 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/non-streaming-asr/pubspec.yaml b/dart-api-examples/non-streaming-asr/pubspec.yaml index be17050f6..71a1f05ed 100644 --- a/dart-api-examples/non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.29 + sherpa_onnx: ^1.10.30 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/speaker-diarization/pubspec.yaml b/dart-api-examples/speaker-diarization/pubspec.yaml index 17ce94336..28e0c39d6 100644 --- a/dart-api-examples/speaker-diarization/pubspec.yaml +++ b/dart-api-examples/speaker-diarization/pubspec.yaml @@ -8,7 +8,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.29 + sherpa_onnx: ^1.10.30 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/speaker-identification/pubspec.yaml b/dart-api-examples/speaker-identification/pubspec.yaml index 490c80732..21cbea554 100644 --- a/dart-api-examples/speaker-identification/pubspec.yaml +++ b/dart-api-examples/speaker-identification/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.29 + sherpa_onnx: ^1.10.30 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/streaming-asr/pubspec.yaml b/dart-api-examples/streaming-asr/pubspec.yaml index 1dc07924d..9f837d92e 100644 --- a/dart-api-examples/streaming-asr/pubspec.yaml +++ b/dart-api-examples/streaming-asr/pubspec.yaml @@ -11,7 +11,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.29 + sherpa_onnx: ^1.10.30 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/tts/pubspec.yaml b/dart-api-examples/tts/pubspec.yaml index 0a36a2324..607db08ee 100644 --- a/dart-api-examples/tts/pubspec.yaml +++ b/dart-api-examples/tts/pubspec.yaml @@ -8,7 +8,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.29 + sherpa_onnx: ^1.10.30 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml index 76230d26e..9328dfca2 100644 --- a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.29 + sherpa_onnx: ^1.10.30 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad/pubspec.yaml b/dart-api-examples/vad/pubspec.yaml index f0ddcbbc0..45ee33487 100644 --- a/dart-api-examples/vad/pubspec.yaml +++ b/dart-api-examples/vad/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.29 + sherpa_onnx: ^1.10.30 path: ^1.9.0 args: ^2.5.0 diff --git a/flutter-examples/streaming_asr/pubspec.yaml b/flutter-examples/streaming_asr/pubspec.yaml index e080857c7..68b000976 100644 --- a/flutter-examples/streaming_asr/pubspec.yaml +++ b/flutter-examples/streaming_asr/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' -version: 1.10.29 +version: 1.10.30 topics: - speech-recognition @@ -31,7 +31,7 @@ dependencies: record: ^5.1.0 url_launcher: ^6.2.6 - sherpa_onnx: ^1.10.29 + sherpa_onnx: ^1.10.30 # sherpa_onnx: # path: ../../flutter/sherpa_onnx diff --git a/flutter-examples/tts/pubspec.yaml b/flutter-examples/tts/pubspec.yaml index 211685b01..d3b1e26f6 100644 --- a/flutter-examples/tts/pubspec.yaml +++ b/flutter-examples/tts/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' # Remove this line if you wish to publish to pub.dev -version: 1.10.29 +version: 1.10.30 environment: sdk: ">=2.17.0 <4.0.0" @@ -18,7 +18,7 @@ dependencies: cupertino_icons: ^1.0.6 path_provider: ^2.1.3 path: ^1.9.0 - sherpa_onnx: ^1.10.29 + sherpa_onnx: ^1.10.30 # sherpa_onnx: # path: ../../flutter/sherpa_onnx url_launcher: 6.2.6 diff --git a/flutter/sherpa_onnx/pubspec.yaml b/flutter/sherpa_onnx/pubspec.yaml index 221cc7c3e..9e367fa64 100644 --- a/flutter/sherpa_onnx/pubspec.yaml +++ b/flutter/sherpa_onnx/pubspec.yaml @@ -17,7 +17,7 @@ topics: - voice-activity-detection # remember to change the version in ../sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec -version: 1.10.29 +version: 1.10.30 homepage: https://github.com/k2-fsa/sherpa-onnx @@ -30,23 +30,23 @@ dependencies: flutter: sdk: flutter - sherpa_onnx_android: ^1.10.29 + sherpa_onnx_android: ^1.10.30 # sherpa_onnx_android: # path: ../sherpa_onnx_android - sherpa_onnx_macos: ^1.10.29 + sherpa_onnx_macos: ^1.10.30 # sherpa_onnx_macos: # path: ../sherpa_onnx_macos - sherpa_onnx_linux: ^1.10.29 + sherpa_onnx_linux: ^1.10.30 # sherpa_onnx_linux: # path: ../sherpa_onnx_linux - sherpa_onnx_windows: ^1.10.29 + sherpa_onnx_windows: ^1.10.30 # sherpa_onnx_windows: # path: ../sherpa_onnx_windows - sherpa_onnx_ios: ^1.10.29 + sherpa_onnx_ios: ^1.10.30 # sherpa_onnx_ios: # path: ../sherpa_onnx_ios diff --git a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec index 9c5d4bd34..e0e0a9769 100644 --- a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec +++ b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec @@ -7,7 +7,7 @@ # https://groups.google.com/g/dart-ffi/c/nUATMBy7r0c Pod::Spec.new do |s| s.name = 'sherpa_onnx_ios' - s.version = '1.10.29' + s.version = '1.10.30' s.summary = 'A new Flutter FFI plugin project.' s.description = <<-DESC A new Flutter FFI plugin project. diff --git a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec index fbf3b5e20..f74dddad9 100644 --- a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec +++ b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec @@ -4,7 +4,7 @@ # Pod::Spec.new do |s| s.name = 'sherpa_onnx_macos' - s.version = '1.10.29' + s.version = '1.10.30' s.summary = 'sherpa-onnx Flutter FFI plugin project.' s.description = <<-DESC sherpa-onnx Flutter FFI plugin project. diff --git a/new-release.sh b/new-release.sh index 1096e82e2..48095178e 100755 --- a/new-release.sh +++ b/new-release.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.28/1\.10\.29/g' {} \; -find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.28/1\.10\.29/g' {} \; -find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.28/1\.10\.29/g' {} \; -find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.28/1\.10\.29/g' {} \; -find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.28/1\.10\.29/g' {} \; +find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.29/1\.10\.30/g' {} \; +find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.29/1\.10\.30/g' {} \; +find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.29/1\.10\.30/g' {} \; +find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.29/1\.10\.30/g' {} \; +find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.29/1\.10\.30/g' {} \; diff --git a/nodejs-addon-examples/package.json b/nodejs-addon-examples/package.json index 88bee9b32..3afbc2f3b 100644 --- a/nodejs-addon-examples/package.json +++ b/nodejs-addon-examples/package.json @@ -1,5 +1,5 @@ { "dependencies": { - "sherpa-onnx-node": "^1.10.29" + "sherpa-onnx-node": "^1.10.30" } } From 9eb493f6bfbe776dcbfbc64cceaae358ee530714 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 28 Oct 2024 10:37:50 +0800 Subject: [PATCH 052/183] Publish pre-built wheels for Python 3.13 (#1485) --- .github/workflows/build-wheels-aarch64.yaml | 4 ++-- .github/workflows/build-wheels-linux-cuda.yaml | 2 +- .github/workflows/build-wheels-linux.yaml | 4 ++-- .github/workflows/build-wheels-macos-arm64.yaml | 4 ++-- .github/workflows/build-wheels-macos-universal2.yaml | 4 ++-- .github/workflows/build-wheels-macos-x64.yaml | 4 ++-- .github/workflows/build-wheels-win32.yaml | 4 ++-- .github/workflows/build-wheels-win64-cuda.yaml | 2 +- .github/workflows/build-wheels-win64.yaml | 2 +- .github/workflows/run-python-test-macos.yaml | 3 +++ .github/workflows/run-python-test.yaml | 2 ++ .github/workflows/test-build-wheel.yaml | 1 + .github/workflows/test-pip-install.yaml | 7 +++++++ 13 files changed, 28 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-wheels-aarch64.yaml b/.github/workflows/build-wheels-aarch64.yaml index 860ccdcda..6b56293bd 100644 --- a/.github/workflows/build-wheels-aarch64.yaml +++ b/.github/workflows/build-wheels-aarch64.yaml @@ -21,7 +21,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - python-version: ["cp37", "cp38", "cp39", "cp310", "cp311", "cp312"] + python-version: ["cp37", "cp38", "cp39", "cp310", "cp311", "cp312", "cp313"] manylinux: [manylinux2014] #, manylinux_2_28] steps: @@ -35,7 +35,7 @@ jobs: # see https://cibuildwheel.readthedocs.io/en/stable/changelog/ # for a list of versions - name: Build wheels - uses: pypa/cibuildwheel@v2.16.5 + uses: pypa/cibuildwheel@v2.21.3 env: CIBW_BEFORE_ALL: | git clone --depth 1 --branch v1.2.12 https://github.com/alsa-project/alsa-lib diff --git a/.github/workflows/build-wheels-linux-cuda.yaml b/.github/workflows/build-wheels-linux-cuda.yaml index 3f3f66dda..28195fb24 100644 --- a/.github/workflows/build-wheels-linux-cuda.yaml +++ b/.github/workflows/build-wheels-linux-cuda.yaml @@ -21,7 +21,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-20.04] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/build-wheels-linux.yaml b/.github/workflows/build-wheels-linux.yaml index 5ca10639a..2abd236f9 100644 --- a/.github/workflows/build-wheels-linux.yaml +++ b/.github/workflows/build-wheels-linux.yaml @@ -21,7 +21,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - python-version: ["cp37", "cp38", "cp39", "cp310", "cp311", "cp312"] + python-version: ["cp37", "cp38", "cp39", "cp310", "cp311", "cp312", "cp313"] manylinux: [manylinux2014] #, manylinux_2_28] @@ -31,7 +31,7 @@ jobs: # see https://cibuildwheel.readthedocs.io/en/stable/changelog/ # for a list of versions - name: Build wheels - uses: pypa/cibuildwheel@v2.16.5 + uses: pypa/cibuildwheel@v2.21.3 env: CIBW_BEFORE_ALL: | git clone --depth 1 --branch v1.2.12 https://github.com/alsa-project/alsa-lib diff --git a/.github/workflows/build-wheels-macos-arm64.yaml b/.github/workflows/build-wheels-macos-arm64.yaml index 181246e8b..ba8739f5b 100644 --- a/.github/workflows/build-wheels-macos-arm64.yaml +++ b/.github/workflows/build-wheels-macos-arm64.yaml @@ -21,13 +21,13 @@ jobs: fail-fast: false matrix: os: [macos-13] - python-version: ["cp38", "cp39", "cp310", "cp311", "cp312"] + python-version: ["cp38", "cp39", "cp310", "cp311", "cp312", "cp313"] steps: - uses: actions/checkout@v4 - name: Build wheels - uses: pypa/cibuildwheel@v2.15.0 + uses: pypa/cibuildwheel@v2.21.3 env: CIBW_BUILD: "${{ matrix.python-version}}-* " CIBW_ENVIRONMENT: SHERPA_ONNX_CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES='arm64'" diff --git a/.github/workflows/build-wheels-macos-universal2.yaml b/.github/workflows/build-wheels-macos-universal2.yaml index 88c51b3a0..1d25a2314 100644 --- a/.github/workflows/build-wheels-macos-universal2.yaml +++ b/.github/workflows/build-wheels-macos-universal2.yaml @@ -21,13 +21,13 @@ jobs: fail-fast: false matrix: os: [macos-latest] - python-version: ["cp38", "cp39", "cp310", "cp311", "cp312"] + python-version: ["cp38", "cp39", "cp310", "cp311", "cp312", "cp313"] steps: - uses: actions/checkout@v4 - name: Build wheels - uses: pypa/cibuildwheel@v2.15.0 + uses: pypa/cibuildwheel@v2.21.3 env: CIBW_BUILD: "${{ matrix.python-version}}-* " CIBW_ENVIRONMENT: SHERPA_ONNX_CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES='arm64;x86_64'" diff --git a/.github/workflows/build-wheels-macos-x64.yaml b/.github/workflows/build-wheels-macos-x64.yaml index 695186958..7d0f59062 100644 --- a/.github/workflows/build-wheels-macos-x64.yaml +++ b/.github/workflows/build-wheels-macos-x64.yaml @@ -21,7 +21,7 @@ jobs: fail-fast: false matrix: os: [macos-13] - python-version: ["cp37", "cp38", "cp39", "cp310", "cp311", "cp312"] + python-version: ["cp37", "cp38", "cp39", "cp310", "cp311", "cp312", "cp313"] steps: - uses: actions/checkout@v4 @@ -42,7 +42,7 @@ jobs: - name: Build wheels if: matrix.python-version != 'cp37' - uses: pypa/cibuildwheel@v2.15.0 + uses: pypa/cibuildwheel@v2.21.3 env: CIBW_BUILD: "${{ matrix.python-version}}-* " CIBW_ENVIRONMENT: SHERPA_ONNX_CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES='x86_64'" diff --git a/.github/workflows/build-wheels-win32.yaml b/.github/workflows/build-wheels-win32.yaml index 04a02d45f..8138b37cf 100644 --- a/.github/workflows/build-wheels-win32.yaml +++ b/.github/workflows/build-wheels-win32.yaml @@ -21,7 +21,7 @@ jobs: fail-fast: false matrix: os: [windows-latest] - python-version: ["cp37", "cp38", "cp39", "cp310", "cp311", "cp312"] + python-version: ["cp37", "cp38", "cp39", "cp310", "cp311", "cp312", "cp313"] steps: - uses: actions/checkout@v4 @@ -29,7 +29,7 @@ jobs: # see https://cibuildwheel.readthedocs.io/en/stable/changelog/ # for a list of versions - name: Build wheels - uses: pypa/cibuildwheel@v2.16.5 + uses: pypa/cibuildwheel@v2.21.3 env: CIBW_ENVIRONMENT: SHERPA_ONNX_CMAKE_ARGS="-A Win32" CIBW_BUILD: "${{ matrix.python-version}}-* " diff --git a/.github/workflows/build-wheels-win64-cuda.yaml b/.github/workflows/build-wheels-win64-cuda.yaml index 5224f20f0..27b4fb87e 100644 --- a/.github/workflows/build-wheels-win64-cuda.yaml +++ b/.github/workflows/build-wheels-win64-cuda.yaml @@ -21,7 +21,7 @@ jobs: fail-fast: false matrix: os: [windows-2019] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/build-wheels-win64.yaml b/.github/workflows/build-wheels-win64.yaml index 0bae16c4a..a87e63537 100644 --- a/.github/workflows/build-wheels-win64.yaml +++ b/.github/workflows/build-wheels-win64.yaml @@ -21,7 +21,7 @@ jobs: fail-fast: false matrix: os: [windows-2019] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/run-python-test-macos.yaml b/.github/workflows/run-python-test-macos.yaml index ed51379d2..a81d6f249 100644 --- a/.github/workflows/run-python-test-macos.yaml +++ b/.github/workflows/run-python-test-macos.yaml @@ -54,6 +54,9 @@ jobs: - os: macos-latest python-version: "3.12" + - os: macos-latest + python-version: "3.13" + steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/run-python-test.yaml b/.github/workflows/run-python-test.yaml index 80fa86a74..9260775cf 100644 --- a/.github/workflows/run-python-test.yaml +++ b/.github/workflows/run-python-test.yaml @@ -53,6 +53,8 @@ jobs: python-version: "3.11" - os: ubuntu-22.04 python-version: "3.12" + - os: ubuntu-22.04 + python-version: "3.13" steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/test-build-wheel.yaml b/.github/workflows/test-build-wheel.yaml index 8b7472b84..ff6ad0a2d 100644 --- a/.github/workflows/test-build-wheel.yaml +++ b/.github/workflows/test-build-wheel.yaml @@ -140,6 +140,7 @@ jobs: export PATH=/c/hostedtoolcache/windows/Python/3.10.11/x64/bin:$PATH export PATH=/c/hostedtoolcache/windows/Python/3.11.9/x64/bin:$PATH export PATH=/c/hostedtoolcache/windows/Python/3.12.7/x64/bin:$PATH + export PATH=/c/hostedtoolcache/windows/Python/3.13.0/x64/bin:$PATH which sherpa-onnx sherpa-onnx --help diff --git a/.github/workflows/test-pip-install.yaml b/.github/workflows/test-pip-install.yaml index b59b66b53..6d2faa512 100644 --- a/.github/workflows/test-pip-install.yaml +++ b/.github/workflows/test-pip-install.yaml @@ -42,6 +42,8 @@ jobs: python-version: "3.11" - os: ubuntu-22.04 python-version: "3.12" + - os: ubuntu-22.04 + python-version: "3.13" - os: macos-12 python-version: "3.8" @@ -55,6 +57,8 @@ jobs: - os: macos-14 python-version: "3.12" + - os: macos-14 + python-version: "3.13" - os: windows-2019 python-version: "3.7" @@ -69,6 +73,8 @@ jobs: python-version: "3.11" - os: windows-2022 python-version: "3.12" + - os: windows-2022 + python-version: "3.13" steps: - uses: actions/checkout@v4 @@ -105,6 +111,7 @@ jobs: export PATH=/c/hostedtoolcache/windows/Python/3.10.11/x64/bin:$PATH export PATH=/c/hostedtoolcache/windows/Python/3.11.9/x64/bin:$PATH export PATH=/c/hostedtoolcache/windows/Python/3.12.7/x64/bin:$PATH + export PATH=/c/hostedtoolcache/windows/Python/3.13.0/x64/bin:$PATH sherpa-onnx --help sherpa-onnx-keyword-spotter --help From 36a0e788845c92eaf7cb13624444d9a16595e2f8 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 28 Oct 2024 12:49:47 +0800 Subject: [PATCH 053/183] Add some commonly used models to README.md (#1486) --- README.md | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/README.md b/README.md index b1385f298..ebc90bca8 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,61 @@ We also have spaces built using WebAssembly. They are listed below: +#### Some pre-trained ASR models (Streaming) + +

+ +Please see + + - + - + - + +for more models. The following table lists only **SOME** of them. + + +|Name | Supported Languages| Description| +|-----|-----|----| +|[sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20][sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20]| Chinese, English| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#csukuangfj-sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-bilingual-chinese-english)| +|[sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16][sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16]| Chinese, English| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16-bilingual-chinese-english)| +|[sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23][sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23]|Chinese| Suitable for Cortex A7 CPU. See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#sherpa-onnx-streaming-zipformer-zh-14m-2023-02-23)| +|[sherpa-onnx-streaming-zipformer-en-20M-2023-02-17][sherpa-onnx-streaming-zipformer-en-20M-2023-02-17]|English|Suitable for Cortex A7 CPU. See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#sherpa-onnx-streaming-zipformer-en-20m-2023-02-17)| +|[sherpa-onnx-streaming-zipformer-korean-2024-06-16][sherpa-onnx-streaming-zipformer-korean-2024-06-16]|Korean| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#sherpa-onnx-streaming-zipformer-korean-2024-06-16-korean)| +|[sherpa-onnx-streaming-zipformer-fr-2023-04-14][sherpa-onnx-streaming-zipformer-fr-2023-04-14]|French| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#shaojieli-sherpa-onnx-streaming-zipformer-fr-2023-04-14-french)| + +
+ + +#### Some pre-trained ASR models (Non-Streaming) + +
+ +Please see + + - + - + - + - + - + +for more models. The following table lists only **SOME** of them. + +|Name | Supported Languages| Description| +|-----|-----|----| +|[Whisper tiny.en](https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2)|English| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/whisper/tiny.en.html)| +|[Moonshine tiny](https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2)|English|See [also](https://github.com/usefulsensors/moonshine)| +|[sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17][sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17]|Chinese, Cantonese, English, Korean, Japanese| 支持多种中文方言. See [also](https://k2-fsa.github.io/sherpa/onnx/sense-voice/index.html)| +|[sherpa-onnx-paraformer-zh-2024-03-09][sherpa-onnx-paraformer-zh-2024-03-09]|Chinese, English| 也支持多种中文方言. See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-paraformer/paraformer-models.html#csukuangfj-sherpa-onnx-paraformer-zh-2024-03-09-chinese-english)| +|[sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01][sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01]|Japanese|See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01-japanese)| +|[sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24][sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24]|Russian|See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/nemo-transducer-models.html#sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24-russian)| +|[sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24][sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24]|Russian| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-ctc/nemo/russian.html#sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24)| +|[sherpa-onnx-zipformer-ru-2024-09-18][sherpa-onnx-zipformer-ru-2024-09-18]|Russian|See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#sherpa-onnx-zipformer-ru-2024-09-18-russian)| +|[sherpa-onnx-zipformer-korean-2024-06-24][sherpa-onnx-zipformer-korean-2024-06-24]|Korean|See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#sherpa-onnx-zipformer-korean-2024-06-24-korean)| +|[sherpa-onnx-zipformer-thai-2024-06-20][sherpa-onnx-zipformer-thai-2024-06-20]|Thai| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#sherpa-onnx-zipformer-thai-2024-06-20-thai)| +|[sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04][sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04]|Chinese| 支持多种方言. See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/telespeech/models.html#sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04)| + +
+ ### Useful links - Documentation: https://k2-fsa.github.io/sherpa/onnx/ @@ -335,3 +390,18 @@ Video demo in Chinese: [爆了！炫神教你开打字挂！真正影响胜率 [speaker-segmentation-models]: https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models [GigaSpeech]: https://github.com/SpeechColab/GigaSpeech [WenetSpeech]: https://github.com/wenet-e2e/WenetSpeech +[sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 +[sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16.tar.bz2 +[sherpa-onnx-streaming-zipformer-korean-2024-06-16]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-korean-2024-06-16.tar.bz2 +[sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23.tar.bz2 +[sherpa-onnx-streaming-zipformer-en-20M-2023-02-17]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17.tar.bz2 +[sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01.tar.bz2 +[sherpa-onnx-zipformer-ru-2024-09-18]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ru-2024-09-18.tar.bz2 +[sherpa-onnx-zipformer-korean-2024-06-24]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-korean-2024-06-24.tar.bz2 +[sherpa-onnx-zipformer-thai-2024-06-20]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-thai-2024-06-20.tar.bz2 +[sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24.tar.bz2 +[sherpa-onnx-paraformer-zh-2024-03-09]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-paraformer-zh-2024-03-09.tar.bz2 +[sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24.tar.bz2 +[sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04.tar.bz2 +[sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 +[sherpa-onnx-streaming-zipformer-fr-2023-04-14]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-fr-2023-04-14.tar.bz2 From 72dc68c8fa19d0ecbb5bfc4326b1ad235046de2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BD=AD=E9=9C=87=E4=B8=9C?= <275331498@qq.com> Date: Mon, 28 Oct 2024 21:30:18 +0800 Subject: [PATCH 054/183] fix typo (#1488) --- python-api-examples/offline-speaker-diarization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python-api-examples/offline-speaker-diarization.py b/python-api-examples/offline-speaker-diarization.py index 3e3ff1618..01627c7f5 100755 --- a/python-api-examples/offline-speaker-diarization.py +++ b/python-api-examples/offline-speaker-diarization.py @@ -102,9 +102,9 @@ def main(): f"Expected samples rate: {sd.sample_rate}, given: {sample_rate}" ) - show_porgress = True + show_progress = True - if show_porgress: + if show_progress: result = sd.process(audio, callback=progress_callback).sort_by_start_time() else: result = sd.process(audio).sort_by_start_time() From 356da3b54cc06d605fba7bc07a2705335ffde36e Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 29 Oct 2024 12:26:26 +0800 Subject: [PATCH 055/183] Publish pre-built macos xcframework (#1490) --- .github/workflows/swift.yaml | 26 ++++++++++++++++++++++++++ build-swift-macos.sh | 9 +++++++++ sherpa-onnx/c-api/cxx-api.h | 1 - 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/.github/workflows/swift.yaml b/.github/workflows/swift.yaml index 3176c9b31..2143dccde 100644 --- a/.github/workflows/swift.yaml +++ b/.github/workflows/swift.yaml @@ -4,6 +4,8 @@ on: push: branches: - master + tags: + - 'v[0-9]+.[0-9]+.[0-9]+*' paths: - './build-swift-macos.sh' - '.github/workflows/swift.yaml' @@ -65,6 +67,30 @@ jobs: ./build-swift-macos.sh + - name: Copy files + if: matrix.os == 'macos-13' && (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') + shell: bash + run: | + SHERPA_ONNX_VERSION=v$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + + dst=sherpa-onnx-${SHERPA_ONNX_VERSION}-macos-xcframework-static + mkdir $dst + + mv -v build-swift-macos/sherpa-onnx.xcframework $dst + + brew install tree + tree $dst + + tar cjvf ${dst}.tar.bz2 $dst + + - name: Release pre-compiled binaries and libs for macOS + if: matrix.os == 'macos-13' && (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + overwrite: true + file: sherpa-onnx-*macos-xcframework-static.tar.bz2 + - name: test shell: bash run: | diff --git a/build-swift-macos.sh b/build-swift-macos.sh index f41dd7d5c..359ea9371 100755 --- a/build-swift-macos.sh +++ b/build-swift-macos.sh @@ -7,6 +7,9 @@ mkdir -p $dir cd $dir cmake \ + -DSHERPA_ONNX_ENABLE_BINARY=OFF \ + -DSHERPA_ONNX_BUILD_C_API_EXAMPLES=OFF \ + -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \ -DCMAKE_INSTALL_PREFIX=./install \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_SHARED_LIBS=OFF \ @@ -21,6 +24,7 @@ cmake \ make VERBOSE=1 -j4 make install +rm -fv ./install/include/cargs.h libtool -static -o ./install/lib/libsherpa-onnx.a \ ./install/lib/libsherpa-onnx-c-api.a \ @@ -34,3 +38,8 @@ libtool -static -o ./install/lib/libsherpa-onnx.a \ ./install/lib/libpiper_phonemize.a \ ./install/lib/libespeak-ng.a \ ./install/lib/libssentencepiece_core.a + +xcodebuild -create-xcframework \ + -library install/lib/libsherpa-onnx.a \ + -headers install/include \ + -output sherpa-onnx.xcframework diff --git a/sherpa-onnx/c-api/cxx-api.h b/sherpa-onnx/c-api/cxx-api.h index b8a46e113..ade52b9aa 100644 --- a/sherpa-onnx/c-api/cxx-api.h +++ b/sherpa-onnx/c-api/cxx-api.h @@ -315,4 +315,3 @@ class SHERPA_ONNX_API OfflineRecognizer } // namespace sherpa_onnx::cxx #endif // SHERPA_ONNX_C_API_CXX_API_H_ - // From d9c586ccf2e5232067a21c792bca67ef4feecffd Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 29 Oct 2024 14:59:12 +0800 Subject: [PATCH 056/183] Removed unused TTS example code in .Net examples (#1492) --- build-wasm-simd-asr.sh | 1 + build-wasm-simd-kws.sh | 1 + build-wasm-simd-nodejs.sh | 1 + build-wasm-simd-speaker-diarization.sh | 1 + build-wasm-simd-tts.sh | 1 + build-wasm-simd-vad-asr.sh | 1 + build-wasm-simd-vad.sh | 1 + ...SherpaOnnxGeneratedAudioResultPlayAudio.cs | 44 ---- dotnet-examples/TTS/Program.cs | 66 ------ .../TTS/Struct/SherpaOnnxGeneratedAudio.cs | 198 ------------------ .../TTS/Struct/SherpaOnnxOfflineTtsConfig.cs | 18 -- .../Struct/SherpaOnnxOfflineTtsModelConfig.cs | 23 -- .../SherpaOnnxOfflineTtsVitsModelConfig.cs | 56 ----- dotnet-examples/TTS/TTS.csproj | 15 -- dotnet-examples/TTS/TTSCore.cs | 75 ------- 15 files changed, 7 insertions(+), 495 deletions(-) delete mode 100644 dotnet-examples/TTS/PlayAudioPartial/SherpaOnnxGeneratedAudioResultPlayAudio.cs delete mode 100644 dotnet-examples/TTS/Program.cs delete mode 100644 dotnet-examples/TTS/Struct/SherpaOnnxGeneratedAudio.cs delete mode 100644 dotnet-examples/TTS/Struct/SherpaOnnxOfflineTtsConfig.cs delete mode 100644 dotnet-examples/TTS/Struct/SherpaOnnxOfflineTtsModelConfig.cs delete mode 100644 dotnet-examples/TTS/Struct/SherpaOnnxOfflineTtsVitsModelConfig.cs delete mode 100644 dotnet-examples/TTS/TTS.csproj delete mode 100644 dotnet-examples/TTS/TTSCore.cs diff --git a/build-wasm-simd-asr.sh b/build-wasm-simd-asr.sh index f5e755047..c19539332 100755 --- a/build-wasm-simd-asr.sh +++ b/build-wasm-simd-asr.sh @@ -20,6 +20,7 @@ if [ x"$EMSCRIPTEN" == x"" ]; then exit 1 else EMSCRIPTEN=$(dirname $(realpath $(which emcc))) + emcc --version fi fi diff --git a/build-wasm-simd-kws.sh b/build-wasm-simd-kws.sh index 301bd8711..408fd75a8 100755 --- a/build-wasm-simd-kws.sh +++ b/build-wasm-simd-kws.sh @@ -15,6 +15,7 @@ if [ x"$EMSCRIPTEN" == x"" ]; then exit 1 else EMSCRIPTEN=$(dirname $(realpath $(which emcc))) + emcc --version fi fi diff --git a/build-wasm-simd-nodejs.sh b/build-wasm-simd-nodejs.sh index 13bf3c854..43023cbed 100755 --- a/build-wasm-simd-nodejs.sh +++ b/build-wasm-simd-nodejs.sh @@ -22,6 +22,7 @@ if [ x"$EMSCRIPTEN" == x"" ]; then exit 1 else EMSCRIPTEN=$(dirname $(realpath $(which emcc))) + emcc --version fi fi diff --git a/build-wasm-simd-speaker-diarization.sh b/build-wasm-simd-speaker-diarization.sh index da4be0381..888abb566 100755 --- a/build-wasm-simd-speaker-diarization.sh +++ b/build-wasm-simd-speaker-diarization.sh @@ -20,6 +20,7 @@ if [ x"$EMSCRIPTEN" == x"" ]; then exit 1 else EMSCRIPTEN=$(dirname $(realpath $(which emcc))) + emcc --version fi fi diff --git a/build-wasm-simd-tts.sh b/build-wasm-simd-tts.sh index 4e37d2047..c707bef6e 100755 --- a/build-wasm-simd-tts.sh +++ b/build-wasm-simd-tts.sh @@ -20,6 +20,7 @@ if [ x"$EMSCRIPTEN" == x"" ]; then exit 1 else EMSCRIPTEN=$(dirname $(realpath $(which emcc))) + emcc --version fi fi diff --git a/build-wasm-simd-vad-asr.sh b/build-wasm-simd-vad-asr.sh index 4bf899da3..621931550 100755 --- a/build-wasm-simd-vad-asr.sh +++ b/build-wasm-simd-vad-asr.sh @@ -21,6 +21,7 @@ if [ x"$EMSCRIPTEN" == x"" ]; then exit 1 else EMSCRIPTEN=$(dirname $(realpath $(which emcc))) + emcc --version fi fi diff --git a/build-wasm-simd-vad.sh b/build-wasm-simd-vad.sh index b73f7f156..2ab11249d 100755 --- a/build-wasm-simd-vad.sh +++ b/build-wasm-simd-vad.sh @@ -20,6 +20,7 @@ if [ x"$EMSCRIPTEN" == x"" ]; then exit 1 else EMSCRIPTEN=$(dirname $(realpath $(which emcc))) + emcc --version fi fi diff --git a/dotnet-examples/TTS/PlayAudioPartial/SherpaOnnxGeneratedAudioResultPlayAudio.cs b/dotnet-examples/TTS/PlayAudioPartial/SherpaOnnxGeneratedAudioResultPlayAudio.cs deleted file mode 100644 index 1eb1e3568..000000000 --- a/dotnet-examples/TTS/PlayAudioPartial/SherpaOnnxGeneratedAudioResultPlayAudio.cs +++ /dev/null @@ -1,44 +0,0 @@ -using NAudio.Wave; - -namespace TTS.Struct -{ - public sealed partial class SherpaOnnxGeneratedAudioResult - { - private WaveOutEvent waveOut; - private WaveFormat waveFormat; - private BufferedWaveProvider bufferedWaveProvider; - - private int bufferLength = 1; - - public TimeSpan? AudioDuration => bufferedWaveProvider?.BufferedDuration; - - public float PlayProgress => (waveOut?.GetPosition() * 1.0f / bufferLength).Value; - - public void Play() - { - waveOut ??= new WaveOutEvent(); - - waveFormat ??= new WaveFormat(sample_rate, AudioDataBit, Channels); // 32-bit 浮点，单声道 - - if (bufferedWaveProvider == null) - { - bufferedWaveProvider ??= new BufferedWaveProvider(waveFormat); - - var buffer = AudioByteData; - - bufferLength = buffer.Length; - - bufferedWaveProvider.AddSamples(buffer, 0, bufferLength); - bufferedWaveProvider.BufferLength = bufferLength; - waveOut.Init(bufferedWaveProvider); - } - waveOut.Play(); - } - - public void Stop() - { - waveOut?.Stop(); - } - - } -} diff --git a/dotnet-examples/TTS/Program.cs b/dotnet-examples/TTS/Program.cs deleted file mode 100644 index 07bb1325f..000000000 --- a/dotnet-examples/TTS/Program.cs +++ /dev/null @@ -1,66 +0,0 @@ -using System.Text; -using TTS; -using TTS.Struct; - -internal class Program -{ - private static void Main(string[] args) - { - SherpaOnnxOfflineTtsConfig sherpaOnnxOfflineTtsConfig = new SherpaOnnxOfflineTtsConfig(); - sherpaOnnxOfflineTtsConfig.model = new SherpaOnnxOfflineTtsModelConfig - { - debug = 0, - num_threads = 4, - provider = "cpu", - vits = new SherpaOnnxOfflineTtsVitsModelConfig - { - //lexicon = "vits-zh-aishell3/lexicon.txt", - //model = "vits-zh-aishell3/vits-aishell3.onnx", - //tokens = "vits-zh-aishell3/tokens.txt", - model = @"C:\Services\Sherpa\model.onnx", - lexicon = "", - tokens = @"C:\Services\Sherpa\tokens.txt", - data_dir = @"C:\Services\Sherpa\espeak-ng-data", - - noise_scale = 0.667f, - noise_scale_w = 0.8f, - length_scale = 1, - }, - - }; - - TTSCore i = new TTSCore(sherpaOnnxOfflineTtsConfig); - - Console.InputEncoding = Encoding.Unicode; - Console.OutputEncoding = Encoding.UTF8; - - while (true) - { - var str = Console.ReadLine(); - var audioResult = i.ToSpeech(str, 40, 1f); - - // audioResult.WriteWAVFile("123.wav");保存本地 - - audioResult.Play(); - - int lastIndex = -1; - while (audioResult.PlayProgress <= 1f) - { - int index = (int)(audioResult.PlayProgress * (str.Length - 1)); - if (lastIndex != index) - { - Console.Write(str[index]); - lastIndex = index; - } - Thread.Sleep(100); - } - - if (++lastIndex < str.Length) - Console.Write(str[lastIndex]); - - Console.WriteLine(); - - } - - } -} diff --git a/dotnet-examples/TTS/Struct/SherpaOnnxGeneratedAudio.cs b/dotnet-examples/TTS/Struct/SherpaOnnxGeneratedAudio.cs deleted file mode 100644 index affc3a034..000000000 --- a/dotnet-examples/TTS/Struct/SherpaOnnxGeneratedAudio.cs +++ /dev/null @@ -1,198 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.InteropServices; -using System.Text; -using System.Threading.Tasks; - -namespace TTS.Struct -{ - /// - /// 生成语音结果 - /// - public sealed partial class SherpaOnnxGeneratedAudioResult : IDisposable - { - public const string Filename = "sherpa-onnx-c-api"; - - /// - /// 销毁非托管内存 - /// - /// - [DllImport(Filename)] - private static extern void SherpaOnnxDestroyOfflineTtsGeneratedAudio(IntPtr ttsGenerateIntptr); - - [DllImport(Filename)] - private static extern int SherpaOnnxWriteWave(IntPtr q, int n, int sample_rate, string filename); - - /// - /// 音频数据比特 - /// - public const int AudioDataBit = 16; - /// - /// 单通道 - /// - public const int Channels = 1; - - /// - /// 原生句柄 - /// - internal IntPtr thisHandle; - - internal readonly IntPtr audioData; - internal readonly int dataSize; - - /// - /// 采样率 - /// - public readonly int sample_rate; - - /// - /// 音频数据指针 - /// - public IntPtr AudioDataIntPtr => audioData; - - /// - /// 数据的大小 - /// - public unsafe int AudioDataLength - { - get - { - return dataSize; - - //float* buffer = (float*)audioData; - //while (*buffer != 0) - // ++buffer; - //return (int)(buffer - (float*)audioData); - } - } - - /// - /// 获得音频数据 float[] - /// 这个内部创建一个数组 - /// - public unsafe float[] AudioFloatData - { - get - { - int length = AudioDataLength; - - float[] floatAudioData = new float[length]; - Marshal.Copy(audioData, floatAudioData, 0, floatAudioData.Length); - return floatAudioData; - } - } - - - /// - /// 获得音频数据 byte[] - /// 这个内部创建一个数组 - /// - public byte[] AudioByteData - { - get - { - byte[] bytes = new byte[AudioDataLength * 2]; - ReadData(bytes, 0); - return bytes; - } - } - - internal SherpaOnnxGeneratedAudioResult(IntPtr intPtr, SherpaOnnxGeneratedAudio sherpaOnnx) - { - this.thisHandle = intPtr; - this.audioData = sherpaOnnx.audioData; - this.dataSize = sherpaOnnx.dataSize; - this.sample_rate = sherpaOnnx.sample_rate; - } - - ~SherpaOnnxGeneratedAudioResult() - { - Dispose(); - } - - /// - /// 读取数据 - /// 没有垃圾产生，自己传递数组进来 - /// - /// 数组 - /// 数组那个位置写入 - /// 写入了多少个 - public int ReadData(float[] audioFloats, int offset) - { - int length = AudioDataLength; - - int c = audioFloats.Length - offset; - length = c >= length ? length : c; - - Marshal.Copy(audioData, audioFloats, offset, length); - return length; - } - - /// - /// 读取数据 - /// 这个内部转换成byte[] 音频数组 - /// 没有垃圾产生，自己传递数组进来 - /// - /// 数组，这个长度需要是AudioDataLength*2大小 - /// 数组那个位置写入 - /// 写入了多少个 - public int ReadData(byte[] audioFloats, int offset) - { - //因为是16bit存储音频数据，所以float会转换成两个字节存储 - var audiodata = AudioFloatData; - - int length = audiodata.Length * 2; - - int c = audioFloats.Length - offset; - c = c % 2 == 0 ? c : c - 1; - - length = c >= length ? length : c; - - int p = length / 2; - - for (int i = 0; i < p; i++) - { - short value = (short)(audiodata[i] * short.MaxValue); - - audioFloats[offset++] = (byte)value; - audioFloats[offset++] = (byte)(value >> 8); - } - - return length; - - } - - /// - /// 写入WAV音频数据 - /// - /// - /// - public bool WriteWAVFile(string filename) - { - return 1 == SherpaOnnxWriteWave(audioData, this.dataSize, this.sample_rate, filename); - } - - public void Dispose() - { - if (this.thisHandle != IntPtr.Zero) - { - SherpaOnnxDestroyOfflineTtsGeneratedAudio(this.thisHandle); - GC.SuppressFinalize(this); - this.thisHandle = IntPtr.Zero; - } - } - } - - [StructLayout(LayoutKind.Sequential)] - internal struct SherpaOnnxGeneratedAudio - { - internal readonly IntPtr audioData; - internal readonly int dataSize; - - /// - /// 采样率 - /// - public readonly int sample_rate; - } -} diff --git a/dotnet-examples/TTS/Struct/SherpaOnnxOfflineTtsConfig.cs b/dotnet-examples/TTS/Struct/SherpaOnnxOfflineTtsConfig.cs deleted file mode 100644 index f33e37dcd..000000000 --- a/dotnet-examples/TTS/Struct/SherpaOnnxOfflineTtsConfig.cs +++ /dev/null @@ -1,18 +0,0 @@ -using System.Runtime.InteropServices; - -namespace TTS.Struct -{ - [StructLayout(LayoutKind.Sequential)] - public struct SherpaOnnxOfflineTtsConfig - { - public SherpaOnnxOfflineTtsModelConfig model; - - [MarshalAs(UnmanagedType.LPStr)] - public string rule_fsts; - - public int max_num_sentences; - - [MarshalAs(UnmanagedType.LPStr)] - public string rule_fars; - } -} diff --git a/dotnet-examples/TTS/Struct/SherpaOnnxOfflineTtsModelConfig.cs b/dotnet-examples/TTS/Struct/SherpaOnnxOfflineTtsModelConfig.cs deleted file mode 100644 index 46dd55859..000000000 --- a/dotnet-examples/TTS/Struct/SherpaOnnxOfflineTtsModelConfig.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System.Runtime.InteropServices; - -namespace TTS.Struct -{ - [StructLayout(LayoutKind.Sequential)] - public struct SherpaOnnxOfflineTtsModelConfig - { - /// - /// 模型配置 - /// - public SherpaOnnxOfflineTtsVitsModelConfig vits; - /// - /// 线程数 - /// - public int num_threads; - public int debug; - /// - /// 使用cpu - /// - [MarshalAs(UnmanagedType.LPStr)] - public string provider; - } -} diff --git a/dotnet-examples/TTS/Struct/SherpaOnnxOfflineTtsVitsModelConfig.cs b/dotnet-examples/TTS/Struct/SherpaOnnxOfflineTtsVitsModelConfig.cs deleted file mode 100644 index 266df5ae7..000000000 --- a/dotnet-examples/TTS/Struct/SherpaOnnxOfflineTtsVitsModelConfig.cs +++ /dev/null @@ -1,56 +0,0 @@ -using System.Runtime.InteropServices; - -namespace TTS.Struct -{ - [StructLayout(LayoutKind.Sequential)] - public struct SherpaOnnxOfflineTtsVitsModelConfig - { - /// - /// 模型 - /// "vits-zh-aishell3/vits-aishell3.onnx" - /// - [MarshalAs(UnmanagedType.LPStr)] - public string model; - /// - /// 词典文件 - /// "vits-zh-aishell3/lexicon.txt" - /// - [MarshalAs(UnmanagedType.LPStr)] - public string lexicon; - - [MarshalAs(UnmanagedType.LPStr)] - public string tokens; - - [MarshalAs(UnmanagedType.LPStr)] - public string data_dir; - - /// - /// VITS模型的noise_scale (float，默认值= 0.667) - /// - public float noise_scale = 0.667f; - /// - /// VITS模型的noise_scale_w (float，默认值= 0.8) - /// - public float noise_scale_w = 0.8f; - /// - /// 演讲的速度。大→慢;小→更快。(float, default = 1) - /// - public float length_scale = 1f; - - [MarshalAs(UnmanagedType.LPStr)] - public string dict_dir; - - public SherpaOnnxOfflineTtsVitsModelConfig() - { - noise_scale = 0.667f; - noise_scale_w = 0.8f; - length_scale = 1f; - - model = "vits-zh-aishell3/vits-aishell3.onnx"; - lexicon = "vits-zh-aishell3/lexicon.txt"; - tokens = "vits-zh-aishell3/tokens.txt"; - data_dir = ""; - dict_dir = ""; - } - } -} diff --git a/dotnet-examples/TTS/TTS.csproj b/dotnet-examples/TTS/TTS.csproj deleted file mode 100644 index cb1a419ea..000000000 --- a/dotnet-examples/TTS/TTS.csproj +++ /dev/null @@ -1,15 +0,0 @@ - - - - Exe - net6.0 - enable - enable - true - - - - - - - diff --git a/dotnet-examples/TTS/TTSCore.cs b/dotnet-examples/TTS/TTSCore.cs deleted file mode 100644 index a15cb19e6..000000000 --- a/dotnet-examples/TTS/TTSCore.cs +++ /dev/null @@ -1,75 +0,0 @@ -using System.Runtime.InteropServices; -using TTS.Struct; - -namespace TTS -{ - internal sealed class TTSCore : IDisposable - { - public const string Filename = "sherpa-onnx-c-api"; - - [DllImport(Filename, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr SherpaOnnxCreateOfflineTts(SherpaOnnxOfflineTtsConfig handle); - - [DllImport(Filename)] - private static extern IntPtr SherpaOnnxOfflineTtsGenerate(IntPtr createOfflineTtsIntptr, IntPtr text, int sid, float speed); - - [DllImport(Filename)] - private static extern void SherpaOnnxDestroyOfflineTts(IntPtr intPtr); - - /// - /// 原生句柄 - /// - private IntPtr thisHandle; - - public TTSCore(SherpaOnnxOfflineTtsConfig modelConfig) - { - IntPtr ttsHandle = SherpaOnnxCreateOfflineTts(modelConfig); - if (ttsHandle == IntPtr.Zero) - { - throw new InvalidOperationException("Failed to create SherpaOnnx TTS engine."); - } - thisHandle = ttsHandle; - } - - /// - /// 文字转语音 - /// - /// 文字 - /// 音色 - /// 速度 - /// - public SherpaOnnxGeneratedAudioResult ToSpeech(string text, int sid, float speed = 1f) - { - var result = SherpaOnnxOfflineTtsGenerate(thisHandle, Marshal.StringToCoTaskMemUTF8(text), sid, speed); - SherpaOnnxGeneratedAudio impl = (SherpaOnnxGeneratedAudio)Marshal.PtrToStructure(result, typeof(SherpaOnnxGeneratedAudio)); - return new SherpaOnnxGeneratedAudioResult(result, impl); - } - - /// - /// 文字转语音 - /// - /// 文字 - /// 音色 - /// 速度 - /// - public Task ToSpeechAsync(string text, int sid, float speed = 1f) - { - return Task.Run(() => ToSpeech(text, sid, speed)); - } - - ~TTSCore() - { - Dispose(); - } - - public void Dispose() - { - if (this.thisHandle != IntPtr.Zero) - { - SherpaOnnxDestroyOfflineTts(this.thisHandle); - GC.SuppressFinalize(this); - this.thisHandle = IntPtr.Zero; - } - } - } -} From d9f65c984009faa3c958cd750d6d3f1d9208251b Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 29 Oct 2024 17:00:39 +0800 Subject: [PATCH 057/183] Update pybind11 to support numpy 2.0 (#1493) --- cmake/pybind11.cmake | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmake/pybind11.cmake b/cmake/pybind11.cmake index fe69ccc5a..bc06a3d1c 100644 --- a/cmake/pybind11.cmake +++ b/cmake/pybind11.cmake @@ -1,18 +1,18 @@ function(download_pybind11) include(FetchContent) - set(pybind11_URL "https://github.com/pybind/pybind11/archive/refs/tags/v2.10.2.tar.gz") - set(pybind11_URL2 "https://hf-mirror.com/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/pybind11-2.10.2.tar.gz") - set(pybind11_HASH "SHA256=93bd1e625e43e03028a3ea7389bba5d3f9f2596abc074b068e70f4ef9b1314ae") + set(pybind11_URL "https://github.com/pybind/pybind11/archive/refs/tags/v2.12.0.tar.gz") + set(pybind11_URL2 "https://hf-mirror.com/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/pybind11-2.12.0.tar.gz") + set(pybind11_HASH "SHA256=bf8f242abd1abcd375d516a7067490fb71abd79519a282d22b6e4d19282185a7") # If you don't have access to the Internet, # please pre-download pybind11 set(possible_file_locations - $ENV{HOME}/Downloads/pybind11-2.10.2.tar.gz - ${CMAKE_SOURCE_DIR}/pybind11-2.10.2.tar.gz - ${CMAKE_BINARY_DIR}/pybind11-2.10.2.tar.gz - /tmp/pybind11-2.10.2.tar.gz - /star-fj/fangjun/download/github/pybind11-2.10.2.tar.gz + $ENV{HOME}/Downloads/pybind11-2.12.0.tar.gz + ${CMAKE_SOURCE_DIR}/pybind11-2.12.0.tar.gz + ${CMAKE_BINARY_DIR}/pybind11-2.12.0.tar.gz + /tmp/pybind11-2.12.0.tar.gz + /star-fj/fangjun/download/github/pybind11-2.12.0.tar.gz ) foreach(f IN LISTS possible_file_locations) From 9fa3bc40d7c0f6a5adce79824ae11a28279b19e3 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 30 Oct 2024 12:13:11 +0800 Subject: [PATCH 058/183] Fix reading tokens.txt on Windows. (#1497) --- sherpa-onnx/csrc/symbol-table.cc | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/sherpa-onnx/csrc/symbol-table.cc b/sherpa-onnx/csrc/symbol-table.cc index eed7a1e53..173b060b4 100644 --- a/sherpa-onnx/csrc/symbol-table.cc +++ b/sherpa-onnx/csrc/symbol-table.cc @@ -23,6 +23,29 @@ namespace sherpa_onnx { +namespace { +// copied from +// https://stackoverflow.com/questions/216823/how-to-trim-a-stdstring +const char *ws = " \t\n\r\f\v"; + +// trim from end of string (right) +inline std::string &TrimRight(std::string &s, const char *t = ws) { + s.erase(s.find_last_not_of(t) + 1); + return s; +} + +// trim from beginning of string (left) +inline std::string &TrimLeft(std::string &s, const char *t = ws) { + s.erase(0, s.find_first_not_of(t)); + return s; +} + +// trim from both ends of string (right then left) +inline std::string &Trim(std::string &s, const char *t = ws) { + return TrimLeft(TrimRight(s, t), t); +} +} // namespace + std::unordered_map ReadTokens( std::istream &is, std::unordered_map *id2token /*= nullptr*/) { @@ -33,6 +56,7 @@ std::unordered_map ReadTokens( std::string sym; int32_t id = -1; while (std::getline(is, line)) { + Trim(line); std::istringstream iss(line); iss >> sym; if (iss.eof()) { From a3c89aa0d8470d8eab30d7781b3dd7b9ed448046 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 31 Oct 2024 17:54:16 +0800 Subject: [PATCH 059/183] Add two-pass ASR Android APKs for Moonshine models. (#1499) --- .github/workflows/apk-asr-2pass.yaml | 4 +-- scripts/apk/generate-asr-2pass-apk-script.py | 38 ++++++++++++++++++++ scripts/apk/generate-vad-asr-apk-script.py | 15 ++++++++ sherpa-onnx/kotlin-api/OfflineRecognizer.kt | 13 +++++++ 4 files changed, 68 insertions(+), 2 deletions(-) diff --git a/.github/workflows/apk-asr-2pass.yaml b/.github/workflows/apk-asr-2pass.yaml index bfbc2f170..72885db45 100644 --- a/.github/workflows/apk-asr-2pass.yaml +++ b/.github/workflows/apk-asr-2pass.yaml @@ -23,8 +23,8 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - total: ["2"] - index: ["0", "1"] + total: ["4"] + index: ["0", "1", "2", "3"] steps: - uses: actions/checkout@v4 diff --git a/scripts/apk/generate-asr-2pass-apk-script.py b/scripts/apk/generate-asr-2pass-apk-script.py index e35a34fb1..87cb0a0b3 100755 --- a/scripts/apk/generate-asr-2pass-apk-script.py +++ b/scripts/apk/generate-asr-2pass-apk-script.py @@ -127,6 +127,36 @@ def get_2nd_models(): ls -lh + popd + """, + ), + Model( + model_name="sherpa-onnx-moonshine-tiny-en-int8", + idx=21, + lang="en", + short_name="moonshine_tiny_int8", + cmd=""" + pushd $model_name + + rm -rfv test_wavs + + ls -lh + + popd + """, + ), + Model( + model_name="sherpa-onnx-moonshine-base-en-int8", + idx=22, + lang="en", + short_name="moonshine_base_int8", + cmd=""" + pushd $model_name + + rm -rfv test_wavs + + ls -lh + popd """, ), @@ -300,6 +330,14 @@ def get_models(): "sherpa-onnx-streaming-zipformer-en-20M-2023-02-17", "sherpa-onnx-whisper-tiny.en", ), + ( + "sherpa-onnx-streaming-zipformer-en-20M-2023-02-17", + "sherpa-onnx-moonshine-tiny-en-int8", + ), + ( + "sherpa-onnx-streaming-zipformer-en-20M-2023-02-17", + "sherpa-onnx-moonshine-base-en-int8", + ), ( "sherpa-onnx-streaming-zipformer-en-20M-2023-02-17", "sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17", diff --git a/scripts/apk/generate-vad-asr-apk-script.py b/scripts/apk/generate-vad-asr-apk-script.py index 5c3694d69..a111ff610 100755 --- a/scripts/apk/generate-vad-asr-apk-script.py +++ b/scripts/apk/generate-vad-asr-apk-script.py @@ -384,6 +384,21 @@ def get_models(): ls -lh + popd + """, + ), + Model( + model_name="sherpa-onnx-moonshine-base-en-int8", + idx=22, + lang="en", + short_name="moonshine_base_int8", + cmd=""" + pushd $model_name + + rm -rfv test_wavs + + ls -lh + popd """, ), diff --git a/sherpa-onnx/kotlin-api/OfflineRecognizer.kt b/sherpa-onnx/kotlin-api/OfflineRecognizer.kt index 76dc82149..647f2447c 100644 --- a/sherpa-onnx/kotlin-api/OfflineRecognizer.kt +++ b/sherpa-onnx/kotlin-api/OfflineRecognizer.kt @@ -438,6 +438,19 @@ fun getOfflineModelConfig(type: Int): OfflineModelConfig? { tokens = "$modelDir/tokens.txt", ) } + + 22 -> { + val modelDir = "sherpa-onnx-moonshine-base-en-int8" + return OfflineModelConfig( + moonshine = OfflineMoonshineModelConfig( + preprocessor = "$modelDir/preprocess.onnx", + encoder = "$modelDir/encode.int8.onnx", + uncachedDecoder = "$modelDir/uncached_decode.int8.onnx", + cachedDecoder = "$modelDir/cached_decode.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } } return null } From 9ab89c33bccbe0378f7969316e50232668846416 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 1 Nov 2024 11:16:28 +0800 Subject: [PATCH 060/183] Support building GPU-capable sherpa-onnx on Linux aarch64. (#1500) Thanks to @Peakyxh for providing pre-built onnxruntime libraries with CUDA support for Linux aarch64. Tested on Jetson nano b01 --- .../workflows/aarch64-linux-gnu-shared.yaml | 77 ++++-- build-aarch64-linux-gnu.sh | 16 ++ cmake/onnxruntime-linux-aarch64-gpu.cmake | 101 ++++++++ cmake/onnxruntime.cmake | 4 +- cmake/piper-phonemize.cmake | 16 +- sherpa-onnx/csrc/macros.h | 221 +++++++++--------- sherpa-onnx/csrc/offline-ced-model.cc | 2 +- .../csrc/offline-ct-transformer-model.cc | 2 +- sherpa-onnx/csrc/offline-ctc-model.cc | 20 +- sherpa-onnx/csrc/offline-moonshine-model.cc | 2 +- .../csrc/offline-nemo-enc-dec-ctc-model.cc | 2 +- sherpa-onnx/csrc/offline-paraformer-model.cc | 2 +- sherpa-onnx/csrc/offline-recognizer-impl.cc | 14 +- sherpa-onnx/csrc/offline-sense-voice-model.cc | 2 +- sherpa-onnx/csrc/offline-tdnn-ctc-model.cc | 2 +- .../csrc/offline-telespeech-ctc-model.cc | 2 +- sherpa-onnx/csrc/offline-transducer-model.cc | 6 +- .../csrc/offline-transducer-nemo-model.cc | 4 +- sherpa-onnx/csrc/offline-wenet-ctc-model.cc | 2 +- sherpa-onnx/csrc/offline-whisper-model.cc | 2 +- .../offline-zipformer-audio-tagging-model.cc | 2 +- .../csrc/offline-zipformer-ctc-model.cc | 2 +- sherpa-onnx/csrc/online-cnn-bilstm-model.cc | 2 +- .../csrc/online-conformer-transducer-model.cc | 14 +- .../csrc/online-lstm-transducer-model.cc | 11 +- sherpa-onnx/csrc/online-nemo-ctc-model.cc | 10 +- sherpa-onnx/csrc/online-paraformer-model.cc | 2 +- sherpa-onnx/csrc/online-rnn-lm.cc | 87 ++++--- sherpa-onnx/csrc/online-transducer-model.cc | 14 +- .../csrc/online-transducer-nemo-model.cc | 10 +- sherpa-onnx/csrc/online-wenet-ctc-model.cc | 2 +- .../csrc/online-zipformer-transducer-model.cc | 34 +-- .../csrc/online-zipformer2-ctc-model.cc | 6 +- .../online-zipformer2-transducer-model.cc | 38 +-- sherpa-onnx/csrc/onnx-utils.cc | 64 ++++- sherpa-onnx/csrc/onnx-utils.h | 3 + sherpa-onnx/csrc/session.cc | 6 + .../csrc/speaker-embedding-extractor-impl.cc | 12 +- .../speaker-embedding-extractor-nemo-model.cc | 2 +- .../spoken-language-identification-impl.cc | 9 +- sherpa-onnx/csrc/symbol-table.cc | 17 +- 41 files changed, 546 insertions(+), 300 deletions(-) create mode 100644 cmake/onnxruntime-linux-aarch64-gpu.cmake diff --git a/.github/workflows/aarch64-linux-gnu-shared.yaml b/.github/workflows/aarch64-linux-gnu-shared.yaml index dbba7c132..1f548e237 100644 --- a/.github/workflows/aarch64-linux-gnu-shared.yaml +++ b/.github/workflows/aarch64-linux-gnu-shared.yaml @@ -34,11 +34,12 @@ concurrency: jobs: aarch64_linux_gnu_shared: runs-on: ${{ matrix.os }} - name: aarch64 shared lib test + name: aarch64 shared GPU ${{ matrix.gpu }} strategy: fail-fast: false matrix: os: [ubuntu-latest] + gpu: [ON, OFF] steps: - uses: actions/checkout@v4 @@ -79,15 +80,24 @@ jobs: make -j2 make install - - name: cache-toolchain - id: cache-toolchain + - name: cache-toolchain (CPU) + if: matrix.gpu == 'OFF' + id: cache-toolchain-cpu uses: actions/cache@v4 with: path: toolchain key: gcc-linaro-7.5.0-2019.12-x86_64_aarch64-linux-gnu.tar.xz - - name: Download toolchain - if: steps.cache-toolchain.outputs.cache-hit != 'true' + - name: cache-toolchain (GPU) + if: matrix.gpu == 'ON' + id: cache-toolchain-gpu + uses: actions/cache@v4 + with: + path: toolchain + key: gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz + + - name: Download toolchain (CPU, gcc 7.5) + if: steps.cache-toolchain-cpu.outputs.cache-hit != 'true' && matrix.gpu == 'OFF' shell: bash run: | wget -qq https://huggingface.co/csukuangfj/sherpa-ncnn-toolchains/resolve/main/gcc-linaro-7.5.0-2019.12-x86_64_aarch64-linux-gnu.tar.xz @@ -95,6 +105,15 @@ jobs: mkdir $GITHUB_WORKSPACE/toolchain tar xf ./gcc-linaro-7.5.0-2019.12-x86_64_aarch64-linux-gnu.tar.xz --strip-components 1 -C $GITHUB_WORKSPACE/toolchain + - name: Download toolchain (GPU, gcc 10.3) + if: steps.cache-toolchain-gpu.outputs.cache-hit != 'true' && matrix.gpu == 'ON' + shell: bash + run: | + wget -qq https://huggingface.co/csukuangfj/sherpa-ncnn-toolchains/resolve/main/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz + + mkdir $GITHUB_WORKSPACE/toolchain + tar xf ./gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz --strip-components 1 -C $GITHUB_WORKSPACE/toolchain + - name: Set environment variable if: steps.cache-build-result.outputs.cache-hit != 'true' shell: bash @@ -103,19 +122,31 @@ jobs: echo "$GITHUB_WORKSPACE/bin" >> "$GITHUB_PATH" ls -lh "$GITHUB_WORKSPACE/toolchain/bin" - echo "CC=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV" - echo "CXX=aarch64-linux-gnu-g++" >> "$GITHUB_ENV" + if [[ ${{ matrix.gpu }} == OFF ]]; then + echo "CC=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV" + echo "CXX=aarch64-linux-gnu-g++" >> "$GITHUB_ENV" + else + echo "CC=aarch64-none-linux-gnu-gcc" >> "$GITHUB_ENV" + echo "CXX=aarch64-none-linux-gnu-g++" >> "$GITHUB_ENV" + fi - name: Display toolchain info shell: bash run: | - aarch64-linux-gnu-gcc --version + if [[ ${{ matrix.gpu }} == OFF ]]; then + which aarch64-linux-gnu-gcc + aarch64-linux-gnu-gcc --version + else + which aarch64-none-linux-gnu-gcc + aarch64-none-linux-gnu-gcc --version + fi - name: Display qemu-aarch64 -h shell: bash run: | export PATH=$GITHUB_WORKSPACE/qemu-install/bin:$PATH export QEMU_LD_PREFIX=$GITHUB_WORKSPACE/toolchain/aarch64-linux-gnu/libc + export QEMU_LD_PREFIX=$GITHUB_WORKSPACE/toolchain/aarch64-none-linux-gnu/libc qemu-aarch64 -h - name: build aarch64-linux-gnu @@ -127,6 +158,7 @@ jobs: cmake --version export BUILD_SHARED_LIBS=ON + export SHERPA_ONNX_ENABLE_GPU=${{ matrix.gpu }} ./build-aarch64-linux-gnu.sh @@ -140,7 +172,11 @@ jobs: run: | export PATH=$GITHUB_WORKSPACE/toolchain/bin:$PATH export PATH=$GITHUB_WORKSPACE/qemu-install/bin:$PATH - export QEMU_LD_PREFIX=$GITHUB_WORKSPACE/toolchain/aarch64-linux-gnu/libc + if [[ ${{ matrix.gpu }} == OFF ]]; then + export QEMU_LD_PREFIX=$GITHUB_WORKSPACE/toolchain/aarch64-linux-gnu/libc + else + export QEMU_LD_PREFIX=$GITHUB_WORKSPACE/toolchain/aarch64-none-linux-gnu/libc + fi ls -lh ./build-aarch64-linux-gnu/bin @@ -151,11 +187,20 @@ jobs: - name: Copy files shell: bash run: | - aarch64-linux-gnu-strip --version + if [[ ${{ matrix.gpu }} == OFF ]]; then + aarch64-linux-gnu-strip --version + else + aarch64-none-linux-gnu-strip --version + fi SHERPA_ONNX_VERSION=v$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) dst=sherpa-onnx-${SHERPA_ONNX_VERSION}-linux-aarch64-shared + if [[ ${{ matrix.gpu }} == OFF ]]; then + dst=${dst}-cpu + else + dst=${dst}-gpu + fi mkdir $dst cp -a build-aarch64-linux-gnu/install/bin $dst/ @@ -166,7 +211,11 @@ jobs: ls -lh $dst/bin/ echo "strip" - aarch64-linux-gnu-strip $dst/bin/* + if [[ ${{ matrix.gpu }} == OFF ]]; then + aarch64-linux-gnu-strip $dst/bin/* + else + aarch64-none-linux-gnu-strip $dst/bin/* + fi tree $dst @@ -174,8 +223,8 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: sherpa-onnx-linux-aarch64-shared - path: sherpa-onnx-*linux-aarch64-shared.tar.bz2 + name: sherpa-onnx-linux-aarch64-shared-gpu-${{ matrix.gpu }} + path: sherpa-onnx-*linux-aarch64-shared*.tar.bz2 # https://huggingface.co/docs/hub/spaces-github-actions - name: Publish to huggingface @@ -198,7 +247,7 @@ jobs: cd huggingface mkdir -p aarch64 - cp -v ../sherpa-onnx-*-shared.tar.bz2 ./aarch64 + cp -v ../sherpa-onnx-*-shared*.tar.bz2 ./aarch64 git status git lfs track "*.bz2" diff --git a/build-aarch64-linux-gnu.sh b/build-aarch64-linux-gnu.sh index d9851fbe1..62b359c17 100755 --- a/build-aarch64-linux-gnu.sh +++ b/build-aarch64-linux-gnu.sh @@ -44,6 +44,21 @@ if [[ x"$BUILD_SHARED_LIBS" == x"" ]]; then BUILD_SHARED_LIBS=OFF fi +if [[ x"$SHERPA_ONNX_ENABLE_GPU" == x"" ]]; then + # By default, use CPU + SHERPA_ONNX_ENABLE_GPU=OFF + + # If you use GPU, then please make sure you have NVIDIA GPUs on your board. + # It uses onnxruntime 1.11.0. + # + # Tested on Jetson Nano B01 +fi + +if [[ x"$SHERPA_ONNX_ENABLE_GPU" == x"ON" ]]; then + # Build shared libs if building GPU is enabled. + BUILD_SHARED_LIBS=ON +fi + cmake \ -DBUILD_PIPER_PHONMIZE_EXE=OFF \ -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ @@ -51,6 +66,7 @@ cmake \ -DBUILD_ESPEAK_NG_TESTS=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ -DCMAKE_BUILD_TYPE=Release \ + -DSHERPA_ONNX_ENABLE_GPU=$SHERPA_ONNX_ENABLE_GPU \ -DBUILD_SHARED_LIBS=$BUILD_SHARED_LIBS \ -DSHERPA_ONNX_ENABLE_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ diff --git a/cmake/onnxruntime-linux-aarch64-gpu.cmake b/cmake/onnxruntime-linux-aarch64-gpu.cmake new file mode 100644 index 000000000..64db9c22b --- /dev/null +++ b/cmake/onnxruntime-linux-aarch64-gpu.cmake @@ -0,0 +1,101 @@ +# Copyright (c) 2022-2024 Xiaomi Corporation +message(STATUS "CMAKE_SYSTEM_NAME: ${CMAKE_SYSTEM_NAME}") +message(STATUS "CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}") + +if(NOT CMAKE_SYSTEM_NAME STREQUAL Linux) + message(FATAL_ERROR "This file is for Linux only. Given: ${CMAKE_SYSTEM_NAME}") +endif() + +if(NOT CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64) + message(FATAL_ERROR "This file is for aarch64 only. Given: ${CMAKE_SYSTEM_PROCESSOR}") +endif() + +if(NOT BUILD_SHARED_LIBS) + message(FATAL_ERROR "This file is for building shared libraries. BUILD_SHARED_LIBS: ${BUILD_SHARED_LIBS}") +endif() + +if(NOT SHERPA_ONNX_ENABLE_GPU) + message(FATAL_ERROR "This file is for NVIDIA GPU only. Given SHERPA_ONNX_ENABLE_GPU: ${SHERPA_ONNX_ENABLE_GPU}") +endif() + +set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.11.0/onnxruntime-linux-aarch64-gpu-1.11.0.tar.bz2") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-linux-aarch64-gpu-1.11.0.tar.bz2") +set(onnxruntime_HASH "SHA256=36eded935551e23aead09d4173bdf0bd1e7b01fdec15d77f97d6e34029aa60d7") + +# If you don't have access to the Internet, +# please download onnxruntime to one of the following locations. +# You can add more if you want. +set(possible_file_locations + $ENV{HOME}/Downloads/onnxruntime-linux-aarch64-gpu-1.11.0.tar.bz2 + ${CMAKE_SOURCE_DIR}/onnxruntime-linux-aarch64-gpu-1.11.0.tar.bz2 + ${CMAKE_BINARY_DIR}/onnxruntime-linux-aarch64-gpu-1.11.0.tar.bz2 + /tmp/onnxruntime-linux-aarch64-gpu-1.11.0.tar.bz2 + /star-fj/fangjun/download/github/onnxruntime-linux-aarch64-gpu-1.11.0.tar.bz2 +) + +foreach(f IN LISTS possible_file_locations) + if(EXISTS ${f}) + set(onnxruntime_URL "${f}") + file(TO_CMAKE_PATH "${onnxruntime_URL}" onnxruntime_URL) + message(STATUS "Found local downloaded onnxruntime: ${onnxruntime_URL}") + set(onnxruntime_URL2) + break() + endif() +endforeach() + +FetchContent_Declare(onnxruntime + URL + ${onnxruntime_URL} + ${onnxruntime_URL2} + URL_HASH ${onnxruntime_HASH} +) + +FetchContent_GetProperties(onnxruntime) +if(NOT onnxruntime_POPULATED) + message(STATUS "Downloading onnxruntime from ${onnxruntime_URL}") + FetchContent_Populate(onnxruntime) +endif() +message(STATUS "onnxruntime is downloaded to ${onnxruntime_SOURCE_DIR}") + +find_library(location_onnxruntime onnxruntime + PATHS + "${onnxruntime_SOURCE_DIR}/lib" + NO_CMAKE_SYSTEM_PATH +) + +message(STATUS "location_onnxruntime: ${location_onnxruntime}") + +add_library(onnxruntime SHARED IMPORTED) + +set_target_properties(onnxruntime PROPERTIES + IMPORTED_LOCATION ${location_onnxruntime} + INTERFACE_INCLUDE_DIRECTORIES "${onnxruntime_SOURCE_DIR}/include" +) + +find_library(location_onnxruntime_cuda_lib onnxruntime_providers_cuda + PATHS + "${onnxruntime_SOURCE_DIR}/lib" + NO_CMAKE_SYSTEM_PATH +) + +add_library(onnxruntime_providers_cuda SHARED IMPORTED) +set_target_properties(onnxruntime_providers_cuda PROPERTIES + IMPORTED_LOCATION ${location_onnxruntime_cuda_lib} +) +message(STATUS "location_onnxruntime_cuda_lib: ${location_onnxruntime_cuda_lib}") + +# for libonnxruntime_providers_shared.so +find_library(location_onnxruntime_providers_shared_lib onnxruntime_providers_shared + PATHS + "${onnxruntime_SOURCE_DIR}/lib" + NO_CMAKE_SYSTEM_PATH +) +add_library(onnxruntime_providers_shared SHARED IMPORTED) +set_target_properties(onnxruntime_providers_shared PROPERTIES + IMPORTED_LOCATION ${location_onnxruntime_providers_shared_lib} +) +message(STATUS "location_onnxruntime_providers_shared_lib: ${location_onnxruntime_providers_shared_lib}") + +file(GLOB onnxruntime_lib_files "${onnxruntime_SOURCE_DIR}/lib/libonnxruntime*") +message(STATUS "onnxruntime lib files: ${onnxruntime_lib_files}") +install(FILES ${onnxruntime_lib_files} DESTINATION lib) diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake index 6655b45cd..8453b96bd 100644 --- a/cmake/onnxruntime.cmake +++ b/cmake/onnxruntime.cmake @@ -13,7 +13,9 @@ function(download_onnxruntime) include(onnxruntime-linux-riscv64-static) endif() elseif(CMAKE_SYSTEM_NAME STREQUAL Linux AND CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64) - if(BUILD_SHARED_LIBS) + if(SHERPA_ONNX_ENABLE_GPU) + include(onnxruntime-linux-aarch64-gpu) + elseif(BUILD_SHARED_LIBS) include(onnxruntime-linux-aarch64) else() include(onnxruntime-linux-aarch64-static) diff --git a/cmake/piper-phonemize.cmake b/cmake/piper-phonemize.cmake index bcea4e8ac..9c9c71f5a 100644 --- a/cmake/piper-phonemize.cmake +++ b/cmake/piper-phonemize.cmake @@ -1,18 +1,18 @@ function(download_piper_phonemize) include(FetchContent) - set(piper_phonemize_URL "https://github.com/csukuangfj/piper-phonemize/archive/dc6b5f4441bffe521047086930b0fc12686acd56.zip") - set(piper_phonemize_URL2 "https://hf-mirror.com/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/piper-phonemize-dc6b5f4441bffe521047086930b0fc12686acd56.zip") - set(piper_phonemize_HASH "SHA256=b9faa04204b1756fa455a962abb1f037041c040133d55be58d11f11ab9b3ce14") + set(piper_phonemize_URL "https://github.com/csukuangfj/piper-phonemize/archive/38ee199dcc49c7b6de89f7ebfb32ed682763fa1b.zip") + set(piper_phonemize_URL2 "https://hf-mirror.com/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/piper-phonemize-38ee199dcc49c7b6de89f7ebfb32ed682763fa1b.zip") + set(piper_phonemize_HASH "SHA256=ab4d06ca76047e1585c63c482f39ffead5315785345055360703cc9382c5e74b") # If you don't have access to the Internet, # please pre-download kaldi-decoder set(possible_file_locations - $ENV{HOME}/Downloads/piper-phonemize-dc6b5f4441bffe521047086930b0fc12686acd56.zip - ${CMAKE_SOURCE_DIR}/piper-phonemize-dc6b5f4441bffe521047086930b0fc12686acd56.zip - ${CMAKE_BINARY_DIR}/piper-phonemize-dc6b5f4441bffe521047086930b0fc12686acd56.zip - /tmp/piper-phonemize-dc6b5f4441bffe521047086930b0fc12686acd56.zip - /star-fj/fangjun/download/github/piper-phonemize-dc6b5f4441bffe521047086930b0fc12686acd56.zip + $ENV{HOME}/Downloads/piper-phonemize-38ee199dcc49c7b6de89f7ebfb32ed682763fa1b.zip + ${CMAKE_SOURCE_DIR}/piper-phonemize-38ee199dcc49c7b6de89f7ebfb32ed682763fa1b.zip + ${CMAKE_BINARY_DIR}/piper-phonemize-38ee199dcc49c7b6de89f7ebfb32ed682763fa1b.zip + /tmp/piper-phonemize-38ee199dcc49c7b6de89f7ebfb32ed682763fa1b.zip + /star-fj/fangjun/download/github/piper-phonemize-38ee199dcc49c7b6de89f7ebfb32ed682763fa1b.zip ) foreach(f IN LISTS possible_file_locations) diff --git a/sherpa-onnx/csrc/macros.h b/sherpa-onnx/csrc/macros.h index 521739a89..506e63e13 100644 --- a/sherpa-onnx/csrc/macros.h +++ b/sherpa-onnx/csrc/macros.h @@ -7,6 +7,8 @@ #include #include +#include + #if __ANDROID_API__ >= 8 #include "android/log.h" #define SHERPA_ONNX_LOGE(...) \ @@ -36,30 +38,28 @@ #endif // Read an integer -#define SHERPA_ONNX_READ_META_DATA(dst, src_key) \ - do { \ - auto value = \ - meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \ - if (!value) { \ - SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ - exit(-1); \ - } \ - \ - dst = atoi(value.get()); \ - if (dst < 0) { \ - SHERPA_ONNX_LOGE("Invalid value %d for '%s'", dst, src_key); \ - exit(-1); \ - } \ +#define SHERPA_ONNX_READ_META_DATA(dst, src_key) \ + do { \ + auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \ + if (value.empty()) { \ + SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ + exit(-1); \ + } \ + \ + dst = atoi(value.c_str()); \ + if (dst < 0) { \ + SHERPA_ONNX_LOGE("Invalid value %d for '%s'", dst, src_key); \ + exit(-1); \ + } \ } while (0) #define SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(dst, src_key, default_value) \ do { \ - auto value = \ - meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \ - if (!value) { \ + auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \ + if (value.empty()) { \ dst = default_value; \ } else { \ - dst = atoi(value.get()); \ + dst = atoi(value.c_str()); \ if (dst < 0) { \ SHERPA_ONNX_LOGE("Invalid value %d for '%s'", dst, src_key); \ exit(-1); \ @@ -68,118 +68,111 @@ } while (0) // read a vector of integers -#define SHERPA_ONNX_READ_META_DATA_VEC(dst, src_key) \ - do { \ - auto value = \ - meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \ - if (!value) { \ - SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ - exit(-1); \ - } \ - \ - bool ret = SplitStringToIntegers(value.get(), ",", true, &dst); \ - if (!ret) { \ - SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'", value.get(), src_key); \ - exit(-1); \ - } \ +#define SHERPA_ONNX_READ_META_DATA_VEC(dst, src_key) \ + do { \ + auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \ + if (value.empty()) { \ + SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ + exit(-1); \ + } \ + \ + bool ret = SplitStringToIntegers(value.c_str(), ",", true, &dst); \ + if (!ret) { \ + SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'", value.c_str(), src_key); \ + exit(-1); \ + } \ } while (0) // read a vector of floats -#define SHERPA_ONNX_READ_META_DATA_VEC_FLOAT(dst, src_key) \ - do { \ - auto value = \ - meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \ - if (!value) { \ - SHERPA_ONNX_LOGE("%s does not exist in the metadata", src_key); \ - exit(-1); \ - } \ - \ - bool ret = SplitStringToFloats(value.get(), ",", true, &dst); \ - if (!ret) { \ - SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'", value.get(), src_key); \ - exit(-1); \ - } \ +#define SHERPA_ONNX_READ_META_DATA_VEC_FLOAT(dst, src_key) \ + do { \ + auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \ + if (value.empty()) { \ + SHERPA_ONNX_LOGE("%s does not exist in the metadata", src_key); \ + exit(-1); \ + } \ + \ + bool ret = SplitStringToFloats(value.c_str(), ",", true, &dst); \ + if (!ret) { \ + SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'", value.c_str(), src_key); \ + exit(-1); \ + } \ } while (0) // read a vector of strings -#define SHERPA_ONNX_READ_META_DATA_VEC_STRING(dst, src_key) \ - do { \ - auto value = \ - meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \ - if (!value) { \ - SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ - exit(-1); \ - } \ - SplitStringToVector(value.get(), ",", false, &dst); \ - \ - if (dst.empty()) { \ - SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'. Empty vector!", \ - value.get(), src_key); \ - exit(-1); \ - } \ +#define SHERPA_ONNX_READ_META_DATA_VEC_STRING(dst, src_key) \ + do { \ + auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \ + if (value.empty()) { \ + SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ + exit(-1); \ + } \ + SplitStringToVector(value.c_str(), ",", false, &dst); \ + \ + if (dst.empty()) { \ + SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'. Empty vector!", \ + value.c_str(), src_key); \ + exit(-1); \ + } \ } while (0) // read a vector of strings separated by sep -#define SHERPA_ONNX_READ_META_DATA_VEC_STRING_SEP(dst, src_key, sep) \ - do { \ - auto value = \ - meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \ - if (!value) { \ - SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ - exit(-1); \ - } \ - SplitStringToVector(value.get(), sep, false, &dst); \ - \ - if (dst.empty()) { \ - SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'. Empty vector!", \ - value.get(), src_key); \ - exit(-1); \ - } \ +#define SHERPA_ONNX_READ_META_DATA_VEC_STRING_SEP(dst, src_key, sep) \ + do { \ + auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \ + if (value.empty()) { \ + SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ + exit(-1); \ + } \ + SplitStringToVector(value.c_str(), sep, false, &dst); \ + \ + if (dst.empty()) { \ + SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'. Empty vector!", \ + value.c_str(), src_key); \ + exit(-1); \ + } \ } while (0) // Read a string -#define SHERPA_ONNX_READ_META_DATA_STR(dst, src_key) \ - do { \ - auto value = \ - meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \ - if (!value) { \ - SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ - exit(-1); \ - } \ - \ - dst = value.get(); \ - if (dst.empty()) { \ - SHERPA_ONNX_LOGE("Invalid value for '%s'\n", src_key); \ - exit(-1); \ - } \ +#define SHERPA_ONNX_READ_META_DATA_STR(dst, src_key) \ + do { \ + auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \ + if (value.empty()) { \ + SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ + exit(-1); \ + } \ + \ + dst = std::move(value); \ + if (dst.empty()) { \ + SHERPA_ONNX_LOGE("Invalid value for '%s'\n", src_key); \ + exit(-1); \ + } \ } while (0) -#define SHERPA_ONNX_READ_META_DATA_STR_ALLOW_EMPTY(dst, src_key) \ - do { \ - auto value = \ - meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \ - if (!value) { \ - SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ - exit(-1); \ - } \ - \ - dst = value.get(); \ +#define SHERPA_ONNX_READ_META_DATA_STR_ALLOW_EMPTY(dst, src_key) \ + do { \ + auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \ + if (value.empty()) { \ + SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ + exit(-1); \ + } \ + \ + dst = std::move(value); \ } while (0) -#define SHERPA_ONNX_READ_META_DATA_STR_WITH_DEFAULT(dst, src_key, \ - default_value) \ - do { \ - auto value = \ - meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \ - if (!value) { \ - dst = default_value; \ - } else { \ - dst = value.get(); \ - if (dst.empty()) { \ - SHERPA_ONNX_LOGE("Invalid value for '%s'\n", src_key); \ - exit(-1); \ - } \ - } \ +#define SHERPA_ONNX_READ_META_DATA_STR_WITH_DEFAULT(dst, src_key, \ + default_value) \ + do { \ + auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \ + if (value.empty()) { \ + dst = default_value; \ + } else { \ + dst = std::move(value); \ + if (dst.empty()) { \ + SHERPA_ONNX_LOGE("Invalid value for '%s'\n", src_key); \ + exit(-1); \ + } \ + } \ } while (0) #define SHERPA_ONNX_EXIT(code) exit(code) diff --git a/sherpa-onnx/csrc/offline-ced-model.cc b/sherpa-onnx/csrc/offline-ced-model.cc index 538fe5bdb..d6dd35290 100644 --- a/sherpa-onnx/csrc/offline-ced-model.cc +++ b/sherpa-onnx/csrc/offline-ced-model.cc @@ -46,7 +46,7 @@ class OfflineCEDModel::Impl { int32_t NumEventClasses() const { return num_event_classes_; } - OrtAllocator *Allocator() const { return allocator_; } + OrtAllocator *Allocator() { return allocator_; } private: void Init(void *model_data, size_t model_data_length) { diff --git a/sherpa-onnx/csrc/offline-ct-transformer-model.cc b/sherpa-onnx/csrc/offline-ct-transformer-model.cc index 2ce593b3e..d616484b4 100644 --- a/sherpa-onnx/csrc/offline-ct-transformer-model.cc +++ b/sherpa-onnx/csrc/offline-ct-transformer-model.cc @@ -44,7 +44,7 @@ class OfflineCtTransformerModel::Impl { return std::move(ans[0]); } - OrtAllocator *Allocator() const { return allocator_; } + OrtAllocator *Allocator() { return allocator_; } const OfflineCtTransformerModelMetaData &GetModelMetadata() const { return meta_data_; diff --git a/sherpa-onnx/csrc/offline-ctc-model.cc b/sherpa-onnx/csrc/offline-ctc-model.cc index 9d1e05d9b..2cbd936ea 100644 --- a/sherpa-onnx/csrc/offline-ctc-model.cc +++ b/sherpa-onnx/csrc/offline-ctc-model.cc @@ -53,8 +53,8 @@ static ModelType GetModelType(char *model_data, size_t model_data_length, Ort::AllocatorWithDefaultOptions allocator; auto model_type = - meta_data.LookupCustomMetadataMapAllocated("model_type", allocator); - if (!model_type) { + LookupCustomModelMetaData(meta_data, "model_type", allocator); + if (model_type.empty()) { SHERPA_ONNX_LOGE( "No model_type in the metadata!\n" "If you are using models from NeMo, please refer to\n" @@ -74,22 +74,22 @@ static ModelType GetModelType(char *model_data, size_t model_data_length, return ModelType::kUnknown; } - if (model_type.get() == std::string("EncDecCTCModelBPE")) { + if (model_type == "EncDecCTCModelBPE") { return ModelType::kEncDecCTCModelBPE; - } else if (model_type.get() == std::string("EncDecCTCModel")) { + } else if (model_type == "EncDecCTCModel") { return ModelType::kEncDecCTCModel; - } else if (model_type.get() == std::string("EncDecHybridRNNTCTCBPEModel")) { + } else if (model_type == "EncDecHybridRNNTCTCBPEModel") { return ModelType::kEncDecHybridRNNTCTCBPEModel; - } else if (model_type.get() == std::string("tdnn")) { + } else if (model_type == "tdnn") { return ModelType::kTdnn; - } else if (model_type.get() == std::string("zipformer2_ctc")) { + } else if (model_type == "zipformer2_ctc") { return ModelType::kZipformerCtc; - } else if (model_type.get() == std::string("wenet_ctc")) { + } else if (model_type == "wenet_ctc") { return ModelType::kWenetCtc; - } else if (model_type.get() == std::string("telespeech_ctc")) { + } else if (model_type == "telespeech_ctc") { return ModelType::kTeleSpeechCtc; } else { - SHERPA_ONNX_LOGE("Unsupported model_type: %s", model_type.get()); + SHERPA_ONNX_LOGE("Unsupported model_type: %s", model_type.c_str()); return ModelType::kUnknown; } } diff --git a/sherpa-onnx/csrc/offline-moonshine-model.cc b/sherpa-onnx/csrc/offline-moonshine-model.cc index ab71d000f..bf9624d4d 100644 --- a/sherpa-onnx/csrc/offline-moonshine-model.cc +++ b/sherpa-onnx/csrc/offline-moonshine-model.cc @@ -155,7 +155,7 @@ class OfflineMoonshineModel::Impl { return {std::move(cached_decoder_out[0]), std::move(next_states)}; } - OrtAllocator *Allocator() const { return allocator_; } + OrtAllocator *Allocator() { return allocator_; } private: void InitPreprocessor(void *model_data, size_t model_data_length) { diff --git a/sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.cc b/sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.cc index 708cb4b4f..14dc7dbe4 100644 --- a/sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.cc +++ b/sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.cc @@ -68,7 +68,7 @@ class OfflineNemoEncDecCtcModel::Impl { int32_t SubsamplingFactor() const { return subsampling_factor_; } - OrtAllocator *Allocator() const { return allocator_; } + OrtAllocator *Allocator() { return allocator_; } std::string FeatureNormalizationMethod() const { return normalize_type_; } diff --git a/sherpa-onnx/csrc/offline-paraformer-model.cc b/sherpa-onnx/csrc/offline-paraformer-model.cc index ce1851062..9c61cb350 100644 --- a/sherpa-onnx/csrc/offline-paraformer-model.cc +++ b/sherpa-onnx/csrc/offline-paraformer-model.cc @@ -56,7 +56,7 @@ class OfflineParaformerModel::Impl { const std::vector &InverseStdDev() const { return inv_stddev_; } - OrtAllocator *Allocator() const { return allocator_; } + OrtAllocator *Allocator() { return allocator_; } private: void Init(void *model_data, size_t model_data_length) { diff --git a/sherpa-onnx/csrc/offline-recognizer-impl.cc b/sherpa-onnx/csrc/offline-recognizer-impl.cc index 07887df60..f89bb9bec 100644 --- a/sherpa-onnx/csrc/offline-recognizer-impl.cc +++ b/sherpa-onnx/csrc/offline-recognizer-impl.cc @@ -121,9 +121,9 @@ std::unique_ptr OfflineRecognizerImpl::Create( Ort::AllocatorWithDefaultOptions allocator; // used in the macro below - auto model_type_ptr = - meta_data.LookupCustomMetadataMapAllocated("model_type", allocator); - if (!model_type_ptr) { + auto model_type = + LookupCustomModelMetaData(meta_data, "model_type", allocator); + if (!model_type.empty()) { SHERPA_ONNX_LOGE( "No model_type in the metadata!\n\n" "Please refer to the following URLs to add metadata" @@ -164,7 +164,6 @@ std::unique_ptr OfflineRecognizerImpl::Create( "\n"); exit(-1); } - std::string model_type(model_type_ptr.get()); if (model_type == "conformer" || model_type == "zipformer" || model_type == "zipformer2") { @@ -301,9 +300,9 @@ std::unique_ptr OfflineRecognizerImpl::Create( Ort::AllocatorWithDefaultOptions allocator; // used in the macro below - auto model_type_ptr = - meta_data.LookupCustomMetadataMapAllocated("model_type", allocator); - if (!model_type_ptr) { + auto model_type = + LookupCustomModelMetaData(meta_data, "model_type", allocator); + if (model_type.empty()) { SHERPA_ONNX_LOGE( "No model_type in the metadata!\n\n" "Please refer to the following URLs to add metadata" @@ -344,7 +343,6 @@ std::unique_ptr OfflineRecognizerImpl::Create( "\n"); exit(-1); } - std::string model_type(model_type_ptr.get()); if (model_type == "conformer" || model_type == "zipformer" || model_type == "zipformer2") { diff --git a/sherpa-onnx/csrc/offline-sense-voice-model.cc b/sherpa-onnx/csrc/offline-sense-voice-model.cc index 24903a41a..a914ccf4a 100644 --- a/sherpa-onnx/csrc/offline-sense-voice-model.cc +++ b/sherpa-onnx/csrc/offline-sense-voice-model.cc @@ -56,7 +56,7 @@ class OfflineSenseVoiceModel::Impl { return meta_data_; } - OrtAllocator *Allocator() const { return allocator_; } + OrtAllocator *Allocator() { return allocator_; } private: void Init(void *model_data, size_t model_data_length) { diff --git a/sherpa-onnx/csrc/offline-tdnn-ctc-model.cc b/sherpa-onnx/csrc/offline-tdnn-ctc-model.cc index ea91d1c55..d7db0040c 100644 --- a/sherpa-onnx/csrc/offline-tdnn-ctc-model.cc +++ b/sherpa-onnx/csrc/offline-tdnn-ctc-model.cc @@ -63,7 +63,7 @@ class OfflineTdnnCtcModel::Impl { int32_t VocabSize() const { return vocab_size_; } - OrtAllocator *Allocator() const { return allocator_; } + OrtAllocator *Allocator() { return allocator_; } private: void Init(void *model_data, size_t model_data_length) { diff --git a/sherpa-onnx/csrc/offline-telespeech-ctc-model.cc b/sherpa-onnx/csrc/offline-telespeech-ctc-model.cc index 68c0afbe8..aeb918cd3 100644 --- a/sherpa-onnx/csrc/offline-telespeech-ctc-model.cc +++ b/sherpa-onnx/csrc/offline-telespeech-ctc-model.cc @@ -69,7 +69,7 @@ class OfflineTeleSpeechCtcModel::Impl { int32_t SubsamplingFactor() const { return subsampling_factor_; } - OrtAllocator *Allocator() const { return allocator_; } + OrtAllocator *Allocator() { return allocator_; } private: void Init(void *model_data, size_t model_data_length) { diff --git a/sherpa-onnx/csrc/offline-transducer-model.cc b/sherpa-onnx/csrc/offline-transducer-model.cc index 6a297347d..910ae3475 100644 --- a/sherpa-onnx/csrc/offline-transducer-model.cc +++ b/sherpa-onnx/csrc/offline-transducer-model.cc @@ -95,11 +95,11 @@ class OfflineTransducerModel::Impl { int32_t VocabSize() const { return vocab_size_; } int32_t ContextSize() const { return context_size_; } int32_t SubsamplingFactor() const { return 4; } - OrtAllocator *Allocator() const { return allocator_; } + OrtAllocator *Allocator() { return allocator_; } Ort::Value BuildDecoderInput( const std::vector &results, - int32_t end_index) const { + int32_t end_index) { assert(end_index <= results.size()); int32_t batch_size = end_index; @@ -122,7 +122,7 @@ class OfflineTransducerModel::Impl { } Ort::Value BuildDecoderInput(const std::vector &results, - int32_t end_index) const { + int32_t end_index) { assert(end_index <= results.size()); int32_t batch_size = end_index; diff --git a/sherpa-onnx/csrc/offline-transducer-nemo-model.cc b/sherpa-onnx/csrc/offline-transducer-nemo-model.cc index 5332a835e..7dd5d31b8 100644 --- a/sherpa-onnx/csrc/offline-transducer-nemo-model.cc +++ b/sherpa-onnx/csrc/offline-transducer-nemo-model.cc @@ -123,7 +123,7 @@ class OfflineTransducerNeMoModel::Impl { return std::move(logit[0]); } - std::vector GetDecoderInitStates(int32_t batch_size) const { + std::vector GetDecoderInitStates(int32_t batch_size) { std::array s0_shape{pred_rnn_layers_, batch_size, pred_hidden_}; Ort::Value s0 = Ort::Value::CreateTensor(allocator_, s0_shape.data(), s0_shape.size()); @@ -149,7 +149,7 @@ class OfflineTransducerNeMoModel::Impl { int32_t SubsamplingFactor() const { return subsampling_factor_; } int32_t VocabSize() const { return vocab_size_; } - OrtAllocator *Allocator() const { return allocator_; } + OrtAllocator *Allocator() { return allocator_; } std::string FeatureNormalizationMethod() const { return normalize_type_; } diff --git a/sherpa-onnx/csrc/offline-wenet-ctc-model.cc b/sherpa-onnx/csrc/offline-wenet-ctc-model.cc index 93fdffab8..d696aa1c7 100644 --- a/sherpa-onnx/csrc/offline-wenet-ctc-model.cc +++ b/sherpa-onnx/csrc/offline-wenet-ctc-model.cc @@ -47,7 +47,7 @@ class OfflineWenetCtcModel::Impl { int32_t SubsamplingFactor() const { return subsampling_factor_; } - OrtAllocator *Allocator() const { return allocator_; } + OrtAllocator *Allocator() { return allocator_; } private: void Init(void *model_data, size_t model_data_length) { diff --git a/sherpa-onnx/csrc/offline-whisper-model.cc b/sherpa-onnx/csrc/offline-whisper-model.cc index 485eaf93c..0747a329b 100644 --- a/sherpa-onnx/csrc/offline-whisper-model.cc +++ b/sherpa-onnx/csrc/offline-whisper-model.cc @@ -188,7 +188,7 @@ class OfflineWhisperModel::Impl { return {std::move(n_layer_self_k_cache), std::move(n_layer_self_v_cache)}; } - OrtAllocator *Allocator() const { return allocator_; } + OrtAllocator *Allocator() { return allocator_; } const std::vector &GetInitialTokens() const { return sot_sequence_; } diff --git a/sherpa-onnx/csrc/offline-zipformer-audio-tagging-model.cc b/sherpa-onnx/csrc/offline-zipformer-audio-tagging-model.cc index 8a2e80dc2..7ddf6d9b3 100644 --- a/sherpa-onnx/csrc/offline-zipformer-audio-tagging-model.cc +++ b/sherpa-onnx/csrc/offline-zipformer-audio-tagging-model.cc @@ -47,7 +47,7 @@ class OfflineZipformerAudioTaggingModel::Impl { int32_t NumEventClasses() const { return num_event_classes_; } - OrtAllocator *Allocator() const { return allocator_; } + OrtAllocator *Allocator() { return allocator_; } private: void Init(void *model_data, size_t model_data_length) { diff --git a/sherpa-onnx/csrc/offline-zipformer-ctc-model.cc b/sherpa-onnx/csrc/offline-zipformer-ctc-model.cc index 8db9439e4..a783ce506 100644 --- a/sherpa-onnx/csrc/offline-zipformer-ctc-model.cc +++ b/sherpa-onnx/csrc/offline-zipformer-ctc-model.cc @@ -48,7 +48,7 @@ class OfflineZipformerCtcModel::Impl { int32_t VocabSize() const { return vocab_size_; } int32_t SubsamplingFactor() const { return 4; } - OrtAllocator *Allocator() const { return allocator_; } + OrtAllocator *Allocator() { return allocator_; } private: void Init(void *model_data, size_t model_data_length) { diff --git a/sherpa-onnx/csrc/online-cnn-bilstm-model.cc b/sherpa-onnx/csrc/online-cnn-bilstm-model.cc index ce8da377e..f4fb3c8f9 100644 --- a/sherpa-onnx/csrc/online-cnn-bilstm-model.cc +++ b/sherpa-onnx/csrc/online-cnn-bilstm-model.cc @@ -47,7 +47,7 @@ class OnlineCNNBiLSTMModel::Impl { return {std::move(ans[0]), std::move(ans[1])}; } - OrtAllocator *Allocator() const { return allocator_; } + OrtAllocator *Allocator() { return allocator_; } const OnlineCNNBiLSTMModelMetaData &GetModelMetadata() const { return meta_data_; diff --git a/sherpa-onnx/csrc/online-conformer-transducer-model.cc b/sherpa-onnx/csrc/online-conformer-transducer-model.cc index 7c252f5a4..2bceffc7d 100644 --- a/sherpa-onnx/csrc/online-conformer-transducer-model.cc +++ b/sherpa-onnx/csrc/online-conformer-transducer-model.cc @@ -163,8 +163,11 @@ std::vector OnlineConformerTransducerModel::StackStates( conv_vec[i] = &states[i][1]; } - Ort::Value attn = Cat(allocator_, attn_vec, 2); - Ort::Value conv = Cat(allocator_, conv_vec, 2); + auto allocator = + const_cast(this)->allocator_; + + Ort::Value attn = Cat(allocator, attn_vec, 2); + Ort::Value conv = Cat(allocator, conv_vec, 2); std::vector ans; ans.reserve(2); @@ -183,8 +186,11 @@ OnlineConformerTransducerModel::UnStackStates( std::vector> ans(batch_size); - std::vector attn_vec = Unbind(allocator_, &states[0], 2); - std::vector conv_vec = Unbind(allocator_, &states[1], 2); + auto allocator = + const_cast(this)->allocator_; + + std::vector attn_vec = Unbind(allocator, &states[0], 2); + std::vector conv_vec = Unbind(allocator, &states[1], 2); assert(attn_vec.size() == batch_size); assert(conv_vec.size() == batch_size); diff --git a/sherpa-onnx/csrc/online-lstm-transducer-model.cc b/sherpa-onnx/csrc/online-lstm-transducer-model.cc index 094cc933c..b9ef9ca5b 100644 --- a/sherpa-onnx/csrc/online-lstm-transducer-model.cc +++ b/sherpa-onnx/csrc/online-lstm-transducer-model.cc @@ -158,9 +158,10 @@ std::vector OnlineLstmTransducerModel::StackStates( h_buf[i] = &states[i][0]; c_buf[i] = &states[i][1]; } + auto allocator = const_cast(this)->allocator_; - Ort::Value h = Cat(allocator_, h_buf, 1); - Ort::Value c = Cat(allocator_, c_buf, 1); + Ort::Value h = Cat(allocator, h_buf, 1); + Ort::Value c = Cat(allocator, c_buf, 1); std::vector ans; ans.reserve(2); @@ -177,8 +178,10 @@ std::vector> OnlineLstmTransducerModel::UnStackStates( std::vector> ans(batch_size); - std::vector h_vec = Unbind(allocator_, &states[0], 1); - std::vector c_vec = Unbind(allocator_, &states[1], 1); + auto allocator = const_cast(this)->allocator_; + + std::vector h_vec = Unbind(allocator, &states[0], 1); + std::vector c_vec = Unbind(allocator, &states[1], 1); assert(h_vec.size() == batch_size); assert(c_vec.size() == batch_size); diff --git a/sherpa-onnx/csrc/online-nemo-ctc-model.cc b/sherpa-onnx/csrc/online-nemo-ctc-model.cc index d93ff73b1..172ee69f4 100644 --- a/sherpa-onnx/csrc/online-nemo-ctc-model.cc +++ b/sherpa-onnx/csrc/online-nemo-ctc-model.cc @@ -102,7 +102,7 @@ class OnlineNeMoCtcModel::Impl { int32_t ChunkShift() const { return chunk_shift_; } - OrtAllocator *Allocator() const { return allocator_; } + OrtAllocator *Allocator() { return allocator_; } // Return a vector containing 3 tensors // - cache_last_channel @@ -119,7 +119,7 @@ class OnlineNeMoCtcModel::Impl { } std::vector StackStates( - std::vector> states) const { + std::vector> states) { int32_t batch_size = static_cast(states.size()); if (batch_size == 1) { return std::move(states[0]); @@ -157,6 +157,8 @@ class OnlineNeMoCtcModel::Impl { std::vector states) const { assert(states.size() == 3); + auto allocator = const_cast(this)->allocator_; + std::vector> ans; auto shape = states[0].GetTensorTypeAndShapeInfo().GetShape(); @@ -171,9 +173,9 @@ class OnlineNeMoCtcModel::Impl { for (int32_t i = 0; i != 3; ++i) { std::vector v; if (i == 2) { - v = Unbind(allocator_, &states[i], 0); + v = Unbind(allocator, &states[i], 0); } else { - v = Unbind(allocator_, &states[i], 0); + v = Unbind(allocator, &states[i], 0); } assert(v.size() == batch_size); diff --git a/sherpa-onnx/csrc/online-paraformer-model.cc b/sherpa-onnx/csrc/online-paraformer-model.cc index 9397ff75b..d7d2e436d 100644 --- a/sherpa-onnx/csrc/online-paraformer-model.cc +++ b/sherpa-onnx/csrc/online-paraformer-model.cc @@ -105,7 +105,7 @@ class OnlineParaformerModel::Impl { const std::vector &InverseStdDev() const { return inv_stddev_; } - OrtAllocator *Allocator() const { return allocator_; } + OrtAllocator *Allocator() { return allocator_; } private: void InitEncoder(void *model_data, size_t model_data_length) { diff --git a/sherpa-onnx/csrc/online-rnn-lm.cc b/sherpa-onnx/csrc/online-rnn-lm.cc index 1b13d3a2d..2a44ddbe0 100644 --- a/sherpa-onnx/csrc/online-rnn-lm.cc +++ b/sherpa-onnx/csrc/online-rnn-lm.cc @@ -5,10 +5,10 @@ #include "sherpa-onnx/csrc/online-rnn-lm.h" +#include #include #include #include -#include #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/macros.h" @@ -53,49 +53,49 @@ class OnlineRnnLM::Impl { // classic rescore function void ComputeLMScore(float scale, int32_t context_size, - std::vector *hyps) { - Ort::AllocatorWithDefaultOptions allocator; - - for (auto &hyp : *hyps) { - for (auto &h_m : hyp) { - auto &h = h_m.second; - auto &ys = h.ys; - const int32_t token_num_in_chunk = - ys.size() - context_size - h.cur_scored_pos - 1; - - if (token_num_in_chunk < 1) { - continue; - } - - if (h.nn_lm_states.empty()) { - h.nn_lm_states = Convert(GetInitStates()); - } - - if (token_num_in_chunk >= h.lm_rescore_min_chunk) { - std::array x_shape{1, token_num_in_chunk}; - - Ort::Value x = Ort::Value::CreateTensor( - allocator, x_shape.data(), x_shape.size()); - int64_t *p_x = x.GetTensorMutableData(); - std::copy(ys.begin() + context_size + h.cur_scored_pos, - ys.end() - 1, p_x); - - // streaming forward by NN LM - auto out = ScoreToken(std::move(x), - Convert(std::move(h.nn_lm_states))); - - // update NN LM score in hyp - const float *p_nll = out.first.GetTensorData(); - h.lm_log_prob = -scale * (*p_nll); - - // update NN LM states in hyp - h.nn_lm_states = Convert(std::move(out.second)); - - h.cur_scored_pos += token_num_in_chunk; - } + std::vector *hyps) { + Ort::AllocatorWithDefaultOptions allocator; + + for (auto &hyp : *hyps) { + for (auto &h_m : hyp) { + auto &h = h_m.second; + auto &ys = h.ys; + const int32_t token_num_in_chunk = + ys.size() - context_size - h.cur_scored_pos - 1; + + if (token_num_in_chunk < 1) { + continue; + } + + if (h.nn_lm_states.empty()) { + h.nn_lm_states = Convert(GetInitStates()); + } + + if (token_num_in_chunk >= h.lm_rescore_min_chunk) { + std::array x_shape{1, token_num_in_chunk}; + + Ort::Value x = Ort::Value::CreateTensor( + allocator, x_shape.data(), x_shape.size()); + int64_t *p_x = x.GetTensorMutableData(); + std::copy(ys.begin() + context_size + h.cur_scored_pos, ys.end() - 1, + p_x); + + // streaming forward by NN LM + auto out = + ScoreToken(std::move(x), Convert(std::move(h.nn_lm_states))); + + // update NN LM score in hyp + const float *p_nll = out.first.GetTensorData(); + h.lm_log_prob = -scale * (*p_nll); + + // update NN LM states in hyp + h.nn_lm_states = Convert(std::move(out.second)); + + h.cur_scored_pos += token_num_in_chunk; } } } + } std::pair> ScoreToken( Ort::Value x, std::vector states) { @@ -125,7 +125,7 @@ class OnlineRnnLM::Impl { } // get init states for classic rescore - std::vector GetInitStates() const { + std::vector GetInitStates() { std::vector ans; ans.reserve(init_states_.size()); @@ -226,7 +226,7 @@ std::pair> OnlineRnnLM::ScoreToken( // classic rescore scores void OnlineRnnLM::ComputeLMScore(float scale, int32_t context_size, - std::vector *hyps) { + std::vector *hyps) { return impl_->ComputeLMScore(scale, context_size, hyps); } @@ -235,5 +235,4 @@ void OnlineRnnLM::ComputeLMScoreSF(float scale, Hypothesis *hyp) { return impl_->ComputeLMScoreSF(scale, hyp); } - } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/online-transducer-model.cc b/sherpa-onnx/csrc/online-transducer-model.cc index 16577dd49..51a9aef3c 100644 --- a/sherpa-onnx/csrc/online-transducer-model.cc +++ b/sherpa-onnx/csrc/online-transducer-model.cc @@ -54,8 +54,8 @@ static ModelType GetModelType(char *model_data, size_t model_data_length, Ort::AllocatorWithDefaultOptions allocator; auto model_type = - meta_data.LookupCustomMetadataMapAllocated("model_type", allocator); - if (!model_type) { + LookupCustomModelMetaData(meta_data, "model_type", allocator); + if (model_type.empty()) { SHERPA_ONNX_LOGE( "No model_type in the metadata!\n" "Please make sure you are using the latest export-onnx.py from icefall " @@ -63,16 +63,16 @@ static ModelType GetModelType(char *model_data, size_t model_data_length, return ModelType::kUnknown; } - if (model_type.get() == std::string("conformer")) { + if (model_type == "conformer") { return ModelType::kConformer; - } else if (model_type.get() == std::string("lstm")) { + } else if (model_type == "lstm") { return ModelType::kLstm; - } else if (model_type.get() == std::string("zipformer")) { + } else if (model_type == "zipformer") { return ModelType::kZipformer; - } else if (model_type.get() == std::string("zipformer2")) { + } else if (model_type == "zipformer2") { return ModelType::kZipformer2; } else { - SHERPA_ONNX_LOGE("Unsupported model_type: %s", model_type.get()); + SHERPA_ONNX_LOGE("Unsupported model_type: %s", model_type.c_str()); return ModelType::kUnknown; } } diff --git a/sherpa-onnx/csrc/online-transducer-nemo-model.cc b/sherpa-onnx/csrc/online-transducer-nemo-model.cc index 4e12da44c..264593a1c 100644 --- a/sherpa-onnx/csrc/online-transducer-nemo-model.cc +++ b/sherpa-onnx/csrc/online-transducer-nemo-model.cc @@ -197,7 +197,7 @@ class OnlineTransducerNeMoModel::Impl { int32_t VocabSize() const { return vocab_size_; } - OrtAllocator *Allocator() const { return allocator_; } + OrtAllocator *Allocator() { return allocator_; } std::string FeatureNormalizationMethod() const { return normalize_type_; } @@ -224,6 +224,8 @@ class OnlineTransducerNeMoModel::Impl { std::vector ans; + auto allocator = const_cast(this)->allocator_; + // stack cache_last_channel std::vector buf(batch_size); @@ -239,9 +241,9 @@ class OnlineTransducerNeMoModel::Impl { Ort::Value c{nullptr}; if (i == 2) { - c = Cat(allocator_, buf, 0); + c = Cat(allocator, buf, 0); } else { - c = Cat(allocator_, buf, 0); + c = Cat(allocator, buf, 0); } ans.push_back(std::move(c)); @@ -251,7 +253,7 @@ class OnlineTransducerNeMoModel::Impl { } std::vector> UnStackStates( - std::vector states) const { + std::vector states) { assert(states.size() == 3); std::vector> ans; diff --git a/sherpa-onnx/csrc/online-wenet-ctc-model.cc b/sherpa-onnx/csrc/online-wenet-ctc-model.cc index 1b1605183..cce322aa4 100644 --- a/sherpa-onnx/csrc/online-wenet-ctc-model.cc +++ b/sherpa-onnx/csrc/online-wenet-ctc-model.cc @@ -101,7 +101,7 @@ class OnlineWenetCtcModel::Impl { return config_.wenet_ctc.chunk_size * subsampling_factor_; } - OrtAllocator *Allocator() const { return allocator_; } + OrtAllocator *Allocator() { return allocator_; } // Return a vector containing 3 tensors // - attn_cache diff --git a/sherpa-onnx/csrc/online-zipformer-transducer-model.cc b/sherpa-onnx/csrc/online-zipformer-transducer-model.cc index 324b2b088..36e2d9dbd 100644 --- a/sherpa-onnx/csrc/online-zipformer-transducer-model.cc +++ b/sherpa-onnx/csrc/online-zipformer-transducer-model.cc @@ -179,12 +179,15 @@ std::vector OnlineZipformerTransducerModel::StackStates( std::vector ans; ans.reserve(states[0].size()); + auto allocator = + const_cast(this)->allocator_; + // cached_len for (int32_t i = 0; i != num_encoders; ++i) { for (int32_t n = 0; n != batch_size; ++n) { buf[n] = &states[n][i]; } - auto v = Cat(allocator_, buf, 1); // (num_layers, 1) + auto v = Cat(allocator, buf, 1); // (num_layers, 1) ans.push_back(std::move(v)); } @@ -193,7 +196,7 @@ std::vector OnlineZipformerTransducerModel::StackStates( for (int32_t n = 0; n != batch_size; ++n) { buf[n] = &states[n][num_encoders + i]; } - auto v = Cat(allocator_, buf, 1); // (num_layers, 1, encoder_dims) + auto v = Cat(allocator, buf, 1); // (num_layers, 1, encoder_dims) ans.push_back(std::move(v)); } @@ -203,7 +206,7 @@ std::vector OnlineZipformerTransducerModel::StackStates( buf[n] = &states[n][num_encoders * 2 + i]; } // (num_layers, left_context_len, 1, attention_dims) - auto v = Cat(allocator_, buf, 2); + auto v = Cat(allocator, buf, 2); ans.push_back(std::move(v)); } @@ -213,7 +216,7 @@ std::vector OnlineZipformerTransducerModel::StackStates( buf[n] = &states[n][num_encoders * 3 + i]; } // (num_layers, left_context_len, 1, attention_dims/2) - auto v = Cat(allocator_, buf, 2); + auto v = Cat(allocator, buf, 2); ans.push_back(std::move(v)); } @@ -223,7 +226,7 @@ std::vector OnlineZipformerTransducerModel::StackStates( buf[n] = &states[n][num_encoders * 4 + i]; } // (num_layers, left_context_len, 1, attention_dims/2) - auto v = Cat(allocator_, buf, 2); + auto v = Cat(allocator, buf, 2); ans.push_back(std::move(v)); } @@ -233,7 +236,7 @@ std::vector OnlineZipformerTransducerModel::StackStates( buf[n] = &states[n][num_encoders * 5 + i]; } // (num_layers, 1, encoder_dims, cnn_module_kernels-1) - auto v = Cat(allocator_, buf, 1); + auto v = Cat(allocator, buf, 1); ans.push_back(std::move(v)); } @@ -243,7 +246,7 @@ std::vector OnlineZipformerTransducerModel::StackStates( buf[n] = &states[n][num_encoders * 6 + i]; } // (num_layers, 1, encoder_dims, cnn_module_kernels-1) - auto v = Cat(allocator_, buf, 1); + auto v = Cat(allocator, buf, 1); ans.push_back(std::move(v)); } @@ -258,12 +261,15 @@ OnlineZipformerTransducerModel::UnStackStates( int32_t batch_size = states[0].GetTensorTypeAndShapeInfo().GetShape()[1]; int32_t num_encoders = num_encoder_layers_.size(); + auto allocator = + const_cast(this)->allocator_; + std::vector> ans; ans.resize(batch_size); // cached_len for (int32_t i = 0; i != num_encoders; ++i) { - auto v = Unbind(allocator_, &states[i], 1); + auto v = Unbind(allocator, &states[i], 1); assert(v.size() == batch_size); for (int32_t n = 0; n != batch_size; ++n) { @@ -273,7 +279,7 @@ OnlineZipformerTransducerModel::UnStackStates( // cached_avg for (int32_t i = num_encoders; i != 2 * num_encoders; ++i) { - auto v = Unbind(allocator_, &states[i], 1); + auto v = Unbind(allocator, &states[i], 1); assert(v.size() == batch_size); for (int32_t n = 0; n != batch_size; ++n) { @@ -283,7 +289,7 @@ OnlineZipformerTransducerModel::UnStackStates( // cached_key for (int32_t i = 2 * num_encoders; i != 3 * num_encoders; ++i) { - auto v = Unbind(allocator_, &states[i], 2); + auto v = Unbind(allocator, &states[i], 2); assert(v.size() == batch_size); for (int32_t n = 0; n != batch_size; ++n) { @@ -293,7 +299,7 @@ OnlineZipformerTransducerModel::UnStackStates( // cached_val for (int32_t i = 3 * num_encoders; i != 4 * num_encoders; ++i) { - auto v = Unbind(allocator_, &states[i], 2); + auto v = Unbind(allocator, &states[i], 2); assert(v.size() == batch_size); for (int32_t n = 0; n != batch_size; ++n) { @@ -303,7 +309,7 @@ OnlineZipformerTransducerModel::UnStackStates( // cached_val2 for (int32_t i = 4 * num_encoders; i != 5 * num_encoders; ++i) { - auto v = Unbind(allocator_, &states[i], 2); + auto v = Unbind(allocator, &states[i], 2); assert(v.size() == batch_size); for (int32_t n = 0; n != batch_size; ++n) { @@ -313,7 +319,7 @@ OnlineZipformerTransducerModel::UnStackStates( // cached_conv1 for (int32_t i = 5 * num_encoders; i != 6 * num_encoders; ++i) { - auto v = Unbind(allocator_, &states[i], 1); + auto v = Unbind(allocator, &states[i], 1); assert(v.size() == batch_size); for (int32_t n = 0; n != batch_size; ++n) { @@ -323,7 +329,7 @@ OnlineZipformerTransducerModel::UnStackStates( // cached_conv2 for (int32_t i = 6 * num_encoders; i != 7 * num_encoders; ++i) { - auto v = Unbind(allocator_, &states[i], 1); + auto v = Unbind(allocator, &states[i], 1); assert(v.size() == batch_size); for (int32_t n = 0; n != batch_size; ++n) { diff --git a/sherpa-onnx/csrc/online-zipformer2-ctc-model.cc b/sherpa-onnx/csrc/online-zipformer2-ctc-model.cc index 04699a56b..8f0708ad1 100644 --- a/sherpa-onnx/csrc/online-zipformer2-ctc-model.cc +++ b/sherpa-onnx/csrc/online-zipformer2-ctc-model.cc @@ -70,7 +70,7 @@ class OnlineZipformer2CtcModel::Impl { int32_t ChunkShift() const { return decode_chunk_len_; } - OrtAllocator *Allocator() const { return allocator_; } + OrtAllocator *Allocator() { return allocator_; } // Return a vector containing 3 tensors // - attn_cache @@ -86,7 +86,7 @@ class OnlineZipformer2CtcModel::Impl { } std::vector StackStates( - std::vector> states) const { + std::vector> states) { int32_t batch_size = static_cast(states.size()); std::vector buf(batch_size); @@ -159,7 +159,7 @@ class OnlineZipformer2CtcModel::Impl { } std::vector> UnStackStates( - std::vector states) const { + std::vector states) { int32_t m = std::accumulate(num_encoder_layers_.begin(), num_encoder_layers_.end(), 0); assert(states.size() == m * 6 + 2); diff --git a/sherpa-onnx/csrc/online-zipformer2-transducer-model.cc b/sherpa-onnx/csrc/online-zipformer2-transducer-model.cc index 0782f06fc..03c68474c 100644 --- a/sherpa-onnx/csrc/online-zipformer2-transducer-model.cc +++ b/sherpa-onnx/csrc/online-zipformer2-transducer-model.cc @@ -185,6 +185,9 @@ std::vector OnlineZipformer2TransducerModel::StackStates( std::vector buf(batch_size); + auto allocator = + const_cast(this)->allocator_; + std::vector ans; int32_t num_states = static_cast(states[0].size()); ans.reserve(num_states); @@ -194,42 +197,42 @@ std::vector OnlineZipformer2TransducerModel::StackStates( for (int32_t n = 0; n != batch_size; ++n) { buf[n] = &states[n][6 * i]; } - auto v = Cat(allocator_, buf, 1); + auto v = Cat(allocator, buf, 1); ans.push_back(std::move(v)); } { for (int32_t n = 0; n != batch_size; ++n) { buf[n] = &states[n][6 * i + 1]; } - auto v = Cat(allocator_, buf, 1); + auto v = Cat(allocator, buf, 1); ans.push_back(std::move(v)); } { for (int32_t n = 0; n != batch_size; ++n) { buf[n] = &states[n][6 * i + 2]; } - auto v = Cat(allocator_, buf, 1); + auto v = Cat(allocator, buf, 1); ans.push_back(std::move(v)); } { for (int32_t n = 0; n != batch_size; ++n) { buf[n] = &states[n][6 * i + 3]; } - auto v = Cat(allocator_, buf, 1); + auto v = Cat(allocator, buf, 1); ans.push_back(std::move(v)); } { for (int32_t n = 0; n != batch_size; ++n) { buf[n] = &states[n][6 * i + 4]; } - auto v = Cat(allocator_, buf, 0); + auto v = Cat(allocator, buf, 0); ans.push_back(std::move(v)); } { for (int32_t n = 0; n != batch_size; ++n) { buf[n] = &states[n][6 * i + 5]; } - auto v = Cat(allocator_, buf, 0); + auto v = Cat(allocator, buf, 0); ans.push_back(std::move(v)); } } @@ -238,7 +241,7 @@ std::vector OnlineZipformer2TransducerModel::StackStates( for (int32_t n = 0; n != batch_size; ++n) { buf[n] = &states[n][num_states - 2]; } - auto v = Cat(allocator_, buf, 0); + auto v = Cat(allocator, buf, 0); ans.push_back(std::move(v)); } @@ -246,7 +249,7 @@ std::vector OnlineZipformer2TransducerModel::StackStates( for (int32_t n = 0; n != batch_size; ++n) { buf[n] = &states[n][num_states - 1]; } - auto v = Cat(allocator_, buf, 0); + auto v = Cat(allocator, buf, 0); ans.push_back(std::move(v)); } return ans; @@ -261,12 +264,15 @@ OnlineZipformer2TransducerModel::UnStackStates( int32_t batch_size = states[0].GetTensorTypeAndShapeInfo().GetShape()[1]; + auto allocator = + const_cast(this)->allocator_; + std::vector> ans; ans.resize(batch_size); for (int32_t i = 0; i != m; ++i) { { - auto v = Unbind(allocator_, &states[i * 6], 1); + auto v = Unbind(allocator, &states[i * 6], 1); assert(static_cast(v.size()) == batch_size); for (int32_t n = 0; n != batch_size; ++n) { @@ -274,7 +280,7 @@ OnlineZipformer2TransducerModel::UnStackStates( } } { - auto v = Unbind(allocator_, &states[i * 6 + 1], 1); + auto v = Unbind(allocator, &states[i * 6 + 1], 1); assert(static_cast(v.size()) == batch_size); for (int32_t n = 0; n != batch_size; ++n) { @@ -282,7 +288,7 @@ OnlineZipformer2TransducerModel::UnStackStates( } } { - auto v = Unbind(allocator_, &states[i * 6 + 2], 1); + auto v = Unbind(allocator, &states[i * 6 + 2], 1); assert(static_cast(v.size()) == batch_size); for (int32_t n = 0; n != batch_size; ++n) { @@ -290,7 +296,7 @@ OnlineZipformer2TransducerModel::UnStackStates( } } { - auto v = Unbind(allocator_, &states[i * 6 + 3], 1); + auto v = Unbind(allocator, &states[i * 6 + 3], 1); assert(static_cast(v.size()) == batch_size); for (int32_t n = 0; n != batch_size; ++n) { @@ -298,7 +304,7 @@ OnlineZipformer2TransducerModel::UnStackStates( } } { - auto v = Unbind(allocator_, &states[i * 6 + 4], 0); + auto v = Unbind(allocator, &states[i * 6 + 4], 0); assert(static_cast(v.size()) == batch_size); for (int32_t n = 0; n != batch_size; ++n) { @@ -306,7 +312,7 @@ OnlineZipformer2TransducerModel::UnStackStates( } } { - auto v = Unbind(allocator_, &states[i * 6 + 5], 0); + auto v = Unbind(allocator, &states[i * 6 + 5], 0); assert(static_cast(v.size()) == batch_size); for (int32_t n = 0; n != batch_size; ++n) { @@ -316,7 +322,7 @@ OnlineZipformer2TransducerModel::UnStackStates( } { - auto v = Unbind(allocator_, &states[m * 6], 0); + auto v = Unbind(allocator, &states[m * 6], 0); assert(static_cast(v.size()) == batch_size); for (int32_t n = 0; n != batch_size; ++n) { @@ -324,7 +330,7 @@ OnlineZipformer2TransducerModel::UnStackStates( } } { - auto v = Unbind(allocator_, &states[m * 6 + 1], 0); + auto v = Unbind(allocator, &states[m * 6 + 1], 0); assert(static_cast(v.size()) == batch_size); for (int32_t n = 0; n != batch_size; ++n) { diff --git a/sherpa-onnx/csrc/onnx-utils.cc b/sherpa-onnx/csrc/onnx-utils.cc index 0f637020a..0dc69fc8f 100644 --- a/sherpa-onnx/csrc/onnx-utils.cc +++ b/sherpa-onnx/csrc/onnx-utils.cc @@ -21,6 +21,36 @@ namespace sherpa_onnx { +static std::string GetInputName(Ort::Session *sess, size_t index, + OrtAllocator *allocator) { +// Note(fangjun): We only tested 1.17.1 and 1.11.0 +// For other versions, we may need to change it +#if ORT_API_VERSION >= 17 + auto v = sess->GetInputNameAllocated(index, allocator); + return v.get(); +#else + auto v = sess->GetInputName(index, allocator); + std::string ans = v; + allocator->Free(allocator, v); + return ans; +#endif +} + +static std::string GetOutputName(Ort::Session *sess, size_t index, + OrtAllocator *allocator) { +// Note(fangjun): We only tested 1.17.1 and 1.11.0 +// For other versions, we may need to change it +#if ORT_API_VERSION >= 17 + auto v = sess->GetOutputNameAllocated(index, allocator); + return v.get(); +#else + auto v = sess->GetOutputName(index, allocator); + std::string ans = v; + allocator->Free(allocator, v); + return ans; +#endif +} + void GetInputNames(Ort::Session *sess, std::vector *input_names, std::vector *input_names_ptr) { Ort::AllocatorWithDefaultOptions allocator; @@ -28,8 +58,7 @@ void GetInputNames(Ort::Session *sess, std::vector *input_names, input_names->resize(node_count); input_names_ptr->resize(node_count); for (size_t i = 0; i != node_count; ++i) { - auto tmp = sess->GetInputNameAllocated(i, allocator); - (*input_names)[i] = tmp.get(); + (*input_names)[i] = GetInputName(sess, i, allocator); (*input_names_ptr)[i] = (*input_names)[i].c_str(); } } @@ -41,8 +70,7 @@ void GetOutputNames(Ort::Session *sess, std::vector *output_names, output_names->resize(node_count); output_names_ptr->resize(node_count); for (size_t i = 0; i != node_count; ++i) { - auto tmp = sess->GetOutputNameAllocated(i, allocator); - (*output_names)[i] = tmp.get(); + (*output_names)[i] = GetOutputName(sess, i, allocator); (*output_names_ptr)[i] = (*output_names)[i].c_str(); } } @@ -78,12 +106,24 @@ Ort::Value GetEncoderOutFrame(OrtAllocator *allocator, Ort::Value *encoder_out, void PrintModelMetadata(std::ostream &os, const Ort::ModelMetadata &meta_data) { Ort::AllocatorWithDefaultOptions allocator; +#if ORT_API_VERSION >= 17 std::vector v = meta_data.GetCustomMetadataMapKeysAllocated(allocator); for (const auto &key : v) { auto p = meta_data.LookupCustomMetadataMapAllocated(key.get(), allocator); os << key.get() << "=" << p.get() << "\n"; } +#else + int64_t num_keys = 0; + char **keys = meta_data.GetCustomMetadataMapKeys(allocator, num_keys); + for (int32_t i = 0; i < num_keys; ++i) { + auto v = LookupCustomModelMetaData(meta_data, keys[i], allocator); + os << keys[i] << "=" << v << "\n"; + allocator.Free(keys[i]); + } + + allocator.Free(keys); +#endif } Ort::Value Clone(OrtAllocator *allocator, const Ort::Value *v) { @@ -361,4 +401,20 @@ std::vector Convert(std::vector values) { return ans; } +std::string LookupCustomModelMetaData(const Ort::ModelMetadata &meta_data, + const char *key, + OrtAllocator *allocator) { +// Note(fangjun): We only tested 1.17.1 and 1.11.0 +// For other versions, we may need to change it +#if ORT_API_VERSION >= 17 + auto v = meta_data.LookupCustomMetadataMapAllocated(key, allocator); + return v.get(); +#else + auto v = meta_data.LookupCustomMetadataMap(key, allocator); + std::string ans = v; + allocator->Free(allocator, v); + return ans; +#endif +} + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/onnx-utils.h b/sherpa-onnx/csrc/onnx-utils.h index 98eb25137..8a19a2baf 100644 --- a/sherpa-onnx/csrc/onnx-utils.h +++ b/sherpa-onnx/csrc/onnx-utils.h @@ -59,6 +59,9 @@ void GetOutputNames(Ort::Session *sess, std::vector *output_names, Ort::Value GetEncoderOutFrame(OrtAllocator *allocator, Ort::Value *encoder_out, int32_t t); +std::string LookupCustomModelMetaData(const Ort::ModelMetadata &meta_data, + const char *key, OrtAllocator *allocator); + void PrintModelMetadata(std::ostream &os, const Ort::ModelMetadata &meta_data); // NOLINT diff --git a/sherpa-onnx/csrc/session.cc b/sherpa-onnx/csrc/session.cc index 9c5eb2b1a..160797c6e 100644 --- a/sherpa-onnx/csrc/session.cc +++ b/sherpa-onnx/csrc/session.cc @@ -60,6 +60,7 @@ Ort::SessionOptions GetSessionOptionsImpl( case Provider::kCPU: break; // nothing to do for the CPU provider case Provider::kXnnpack: { +#if ORT_API_VERSION >= 17 if (std::find(available_providers.begin(), available_providers.end(), "XnnpackExecutionProvider") != available_providers.end()) { sess_opts.AppendExecutionProvider("XNNPACK"); @@ -67,6 +68,11 @@ Ort::SessionOptions GetSessionOptionsImpl( SHERPA_ONNX_LOGE("Available providers: %s. Fallback to cpu!", os.str().c_str()); } +#else + SHERPA_ONNX_LOGE( + "Does not support xnnpack for onnxruntime: %d. Fallback to cpu!", + static_cast(ORT_API_VERSION)); +#endif break; } case Provider::kTRT: { diff --git a/sherpa-onnx/csrc/speaker-embedding-extractor-impl.cc b/sherpa-onnx/csrc/speaker-embedding-extractor-impl.cc index 4dafce91d..b9591d624 100644 --- a/sherpa-onnx/csrc/speaker-embedding-extractor-impl.cc +++ b/sherpa-onnx/csrc/speaker-embedding-extractor-impl.cc @@ -40,8 +40,8 @@ static ModelType GetModelType(char *model_data, size_t model_data_length, Ort::AllocatorWithDefaultOptions allocator; auto model_type = - meta_data.LookupCustomMetadataMapAllocated("framework", allocator); - if (!model_type) { + LookupCustomModelMetaData(meta_data, "framework", allocator); + if (model_type.empty()) { SHERPA_ONNX_LOGE( "No model_type in the metadata!\n" "Please make sure you have added metadata to the model.\n\n" @@ -52,14 +52,14 @@ static ModelType GetModelType(char *model_data, size_t model_data_length, return ModelType::kUnknown; } - if (model_type.get() == std::string("wespeaker")) { + if (model_type == "wespeaker") { return ModelType::kWeSpeaker; - } else if (model_type.get() == std::string("3d-speaker")) { + } else if (model_type == "3d-speaker") { return ModelType::k3dSpeaker; - } else if (model_type.get() == std::string("nemo")) { + } else if (model_type == "nemo") { return ModelType::kNeMo; } else { - SHERPA_ONNX_LOGE("Unsupported model_type: %s", model_type.get()); + SHERPA_ONNX_LOGE("Unsupported model_type: %s", model_type.c_str()); return ModelType::kUnknown; } } diff --git a/sherpa-onnx/csrc/speaker-embedding-extractor-nemo-model.cc b/sherpa-onnx/csrc/speaker-embedding-extractor-nemo-model.cc index 2e481b20f..1b60e2469 100644 --- a/sherpa-onnx/csrc/speaker-embedding-extractor-nemo-model.cc +++ b/sherpa-onnx/csrc/speaker-embedding-extractor-nemo-model.cc @@ -53,7 +53,7 @@ class SpeakerEmbeddingExtractorNeMoModel::Impl { return std::move(outputs[0]); } - OrtAllocator *Allocator() const { return allocator_; } + OrtAllocator *Allocator() { return allocator_; } const SpeakerEmbeddingExtractorNeMoModelMetaData &GetMetaData() const { return meta_data_; diff --git a/sherpa-onnx/csrc/spoken-language-identification-impl.cc b/sherpa-onnx/csrc/spoken-language-identification-impl.cc index 109b48789..5b29df484 100644 --- a/sherpa-onnx/csrc/spoken-language-identification-impl.cc +++ b/sherpa-onnx/csrc/spoken-language-identification-impl.cc @@ -42,8 +42,8 @@ static ModelType GetModelType(char *model_data, size_t model_data_length, Ort::AllocatorWithDefaultOptions allocator; auto model_type = - meta_data.LookupCustomMetadataMapAllocated("model_type", allocator); - if (!model_type) { + LookupCustomModelMetaData(meta_data, "model_type", allocator); + if (model_type.empty()) { SHERPA_ONNX_LOGE( "No model_type in the metadata!\n" "Please make sure you have added metadata to the model.\n\n" @@ -54,11 +54,10 @@ static ModelType GetModelType(char *model_data, size_t model_data_length, return ModelType::kUnknown; } - auto model_type_str = std::string(model_type.get()); - if (model_type_str.find("whisper") == 0) { + if (model_type.find("whisper") == 0) { return ModelType::kWhisper; } else { - SHERPA_ONNX_LOGE("Unsupported model_type: %s", model_type.get()); + SHERPA_ONNX_LOGE("Unsupported model_type: %s", model_type.c_str()); return ModelType::kUnknown; } } diff --git a/sherpa-onnx/csrc/symbol-table.cc b/sherpa-onnx/csrc/symbol-table.cc index 173b060b4..77b976431 100644 --- a/sherpa-onnx/csrc/symbol-table.cc +++ b/sherpa-onnx/csrc/symbol-table.cc @@ -29,20 +29,19 @@ namespace { const char *ws = " \t\n\r\f\v"; // trim from end of string (right) -inline std::string &TrimRight(std::string &s, const char *t = ws) { - s.erase(s.find_last_not_of(t) + 1); - return s; +inline void TrimRight(std::string *s, const char *t = ws) { + s->erase(s->find_last_not_of(t) + 1); } // trim from beginning of string (left) -inline std::string &TrimLeft(std::string &s, const char *t = ws) { - s.erase(0, s.find_first_not_of(t)); - return s; +inline void TrimLeft(std::string *s, const char *t = ws) { + s->erase(0, s->find_first_not_of(t)); } // trim from both ends of string (right then left) -inline std::string &Trim(std::string &s, const char *t = ws) { - return TrimLeft(TrimRight(s, t), t); +inline void Trim(std::string *s, const char *t = ws) { + TrimRight(s, t); + TrimLeft(s, t); } } // namespace @@ -56,7 +55,7 @@ std::unordered_map ReadTokens( std::string sym; int32_t id = -1; while (std::getline(is, line)) { - Trim(line); + Trim(&line); std::istringstream iss(line); iss >> sym; if (iss.eof()) { From c5205f08bf5650e5441ea5b7692bff2664c06b32 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 1 Nov 2024 11:40:13 +0800 Subject: [PATCH 061/183] Add an example for computing RTF about streaming ASR. (#1501) --- cxx-api-examples/CMakeLists.txt | 3 + .../streaming-zipformer-rtf-cxx-api.cc | 111 ++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 cxx-api-examples/streaming-zipformer-rtf-cxx-api.cc diff --git a/cxx-api-examples/CMakeLists.txt b/cxx-api-examples/CMakeLists.txt index e7e722660..cc4082e87 100644 --- a/cxx-api-examples/CMakeLists.txt +++ b/cxx-api-examples/CMakeLists.txt @@ -3,6 +3,9 @@ include_directories(${CMAKE_SOURCE_DIR}) add_executable(streaming-zipformer-cxx-api ./streaming-zipformer-cxx-api.cc) target_link_libraries(streaming-zipformer-cxx-api sherpa-onnx-cxx-api) +add_executable(streaming-zipformer-rtf-cxx-api ./streaming-zipformer-rtf-cxx-api.cc) +target_link_libraries(streaming-zipformer-rtf-cxx-api sherpa-onnx-cxx-api) + add_executable(whisper-cxx-api ./whisper-cxx-api.cc) target_link_libraries(whisper-cxx-api sherpa-onnx-cxx-api) diff --git a/cxx-api-examples/streaming-zipformer-rtf-cxx-api.cc b/cxx-api-examples/streaming-zipformer-rtf-cxx-api.cc new file mode 100644 index 000000000..9ec69aaba --- /dev/null +++ b/cxx-api-examples/streaming-zipformer-rtf-cxx-api.cc @@ -0,0 +1,111 @@ +// cxx-api-examples/streaming-zipformer-rtf-cxx-api.cc +// Copyright (c) 2024 Xiaomi Corporation + +// +// This file demonstrates how to use streaming Zipformer +// with sherpa-onnx's C++ API. +// +// clang-format off +// +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 +// tar xvf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 +// rm sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 +// +// clang-format on + +#include // NOLINT +#include +#include + +#include "sherpa-onnx/c-api/cxx-api.h" + +int32_t main(int argc, char *argv[]) { + int32_t num_runs = 1; + if (argc == 2) { + num_runs = atoi(argv[1]); + if (num_runs < 0) { + num_runs = 1; + } + } + + using namespace sherpa_onnx::cxx; // NOLINT + OnlineRecognizerConfig config; + + // please see + // https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#csukuangfj-sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-bilingual-chinese-english + config.model_config.transducer.encoder = + "./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/" + "encoder-epoch-99-avg-1.int8.onnx"; + + // Note: We recommend not using int8.onnx for the decoder. + config.model_config.transducer.decoder = + "./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/" + "decoder-epoch-99-avg-1.onnx"; + + config.model_config.transducer.joiner = + "./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/" + "joiner-epoch-99-avg-1.int8.onnx"; + + config.model_config.tokens = + "./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt"; + + config.model_config.num_threads = 1; + + std::cout << "Loading model\n"; + OnlineRecognizer recongizer = OnlineRecognizer::Create(config); + if (!recongizer.Get()) { + std::cerr << "Please check your config\n"; + return -1; + } + std::cout << "Loading model done\n"; + + std::string wave_filename = + "./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/test_wavs/" + "0.wav"; + Wave wave = ReadWave(wave_filename); + if (wave.samples.empty()) { + std::cerr << "Failed to read: '" << wave_filename << "'\n"; + return -1; + } + + std::cout << "Start recognition\n"; + float total_elapsed_seconds = 0; + OnlineRecognizerResult result; + for (int32_t i = 0; i < num_runs; ++i) { + const auto begin = std::chrono::steady_clock::now(); + + OnlineStream stream = recongizer.CreateStream(); + stream.AcceptWaveform(wave.sample_rate, wave.samples.data(), + wave.samples.size()); + stream.InputFinished(); + + while (recongizer.IsReady(&stream)) { + recongizer.Decode(&stream); + } + + result = recongizer.GetResult(&stream); + + auto end = std::chrono::steady_clock::now(); + float elapsed_seconds = + std::chrono::duration_cast(end - begin) + .count() / + 1000.; + printf("Run %d/%d, elapsed seconds: %.3f\n", i, num_runs, elapsed_seconds); + total_elapsed_seconds += elapsed_seconds; + } + float average_elapsed_secodns = total_elapsed_seconds / num_runs; + float duration = wave.samples.size() / static_cast(wave.sample_rate); + float rtf = total_elapsed_seconds / num_runs / duration; + + std::cout << "text: " << result.text << "\n"; + printf("Number of threads: %d\n", config.model_config.num_threads); + printf("Duration: %.3fs\n", duration); + printf("Total Elapsed seconds: %.3fs\n", total_elapsed_seconds); + printf("Num runs: %d\n", num_runs); + printf("Elapsed seconds per run: %.3f/%d=%.3f\n", total_elapsed_seconds, + num_runs, average_elapsed_secodns); + printf("(Real time factor) RTF = %.3f / %.3f = %.3f\n", + average_elapsed_secodns, duration, rtf); + + return 0; +} From f0cced1f37280fdcc3d7d69937ec019ba9fac9a5 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sun, 3 Nov 2024 19:15:11 +0800 Subject: [PATCH 062/183] Publish pre-built wheels with CUDA support for Linux aarch64. (#1507) --- .../workflows/build-wheels-aarch64-cuda.yaml | 120 ++++++++++++++++++ .../streaming-zipformer-rtf-cxx-api.cc | 23 +++- 2 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build-wheels-aarch64-cuda.yaml diff --git a/.github/workflows/build-wheels-aarch64-cuda.yaml b/.github/workflows/build-wheels-aarch64-cuda.yaml new file mode 100644 index 000000000..9c226c431 --- /dev/null +++ b/.github/workflows/build-wheels-aarch64-cuda.yaml @@ -0,0 +1,120 @@ +name: build-wheels-aarch64-cuda + +on: + push: + branches: + - wheel + workflow_dispatch: + +env: + SHERPA_ONNX_IS_IN_GITHUB_ACTIONS: 1 + +concurrency: + group: build-wheels-aarch64-cuda-${{ github.ref }} + cancel-in-progress: true + +jobs: + build_wheels_aarch64_cuda: + name: ${{ matrix.manylinux }} ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: ["cp37", "cp38", "cp39", "cp310", "cp311", "cp312", "cp313"] + manylinux: [manylinux2014] #, manylinux_2_28] + + steps: + - uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: all + + # see https://cibuildwheel.readthedocs.io/en/stable/changelog/ + # for a list of versions + - name: Build wheels + uses: pypa/cibuildwheel@v2.21.3 + env: + CIBW_BEFORE_ALL: | + git clone --depth 1 --branch v1.2.12 https://github.com/alsa-project/alsa-lib + cd alsa-lib + ./gitcompile + cd .. + echo "PWD" + ls -lh /project/alsa-lib/src/.libs + + CIBW_ENVIRONMENT: CPLUS_INCLUDE_PATH=/project/alsa-lib/include:$CPLUS_INCLUDE_PATH SHERPA_ONNX_ALSA_LIB_DIR=/project/alsa-lib/src/.libs LD_LIBRARY_PATH=/project/build/bdist.linux-x86_64/wheel/sherpa_onnx/lib:$SHERPA_ONNX_ALSA_LIB_DIR SHERPA_ONNX_MAKE_ARGS="VERBOSE=1" SHERPA_ONNX_ENABLE_ALSA=1 SHERPA_ONNX_ENABLE_GPU=ON + CIBW_BUILD: "${{ matrix.python-version}}-* " + CIBW_SKIP: "cp27-* cp35-* cp36-* *-win32 pp* *-musllinux* *-manylinux_i686" + CIBW_BUILD_VERBOSITY: 3 + CIBW_ARCHS_LINUX: aarch64 + CIBW_MANYLINUX_AARCH64_IMAGE: quay.io/pypa/${{ matrix.manylinux }}_aarch64 + # From onnxruntime >= 1.17.0, it drops support for CentOS 7.0 and it supports only manylinux_2_28. + # manylinux_2_24 is no longer supported + + - name: Display wheels + shell: bash + run: | + ls -lh ./wheelhouse/ + + - name: Install patchelf + if: matrix.os == 'ubuntu-latest' + shell: bash + run: | + sudo apt-get update -q + sudo apt-get install -q -y patchelf + patchelf --help + + - name: Patch wheels + shell: bash + if: matrix.os == 'ubuntu-latest' + run: | + mkdir ./wheels + sudo ./scripts/wheel/patch_wheel.py --in-dir ./wheelhouse --out-dir ./wheels + + ls -lh ./wheels/ + rm -rf ./wheelhouse + mv ./wheels ./wheelhouse + + - name: Publish to huggingface + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v3 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + + rm -rf huggingface + export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false + + SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" + + d=cuda/$SHERPA_ONNX_VERSION + + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface + cd huggingface + git fetch + git pull + git merge -m "merge remote" --ff origin main + + mkdir -p $d + + cp -v ../wheelhouse/*.whl $d/ + + git status + git add . + git commit -m "add more wheels" + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-wheels main + + - uses: actions/upload-artifact@v4 + with: + name: wheel-${{ matrix.python-version }}-${{ matrix.manylinux }} + path: ./wheelhouse/*.whl diff --git a/cxx-api-examples/streaming-zipformer-rtf-cxx-api.cc b/cxx-api-examples/streaming-zipformer-rtf-cxx-api.cc index 9ec69aaba..2e74d30be 100644 --- a/cxx-api-examples/streaming-zipformer-rtf-cxx-api.cc +++ b/cxx-api-examples/streaming-zipformer-rtf-cxx-api.cc @@ -7,10 +7,28 @@ // // clang-format off // +// cd /path/sherpa-onnx/ +// mkdir build +// cd build +// cmake .. +// make +// // wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 // tar xvf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 // rm sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 // +// # 1. Test on CPU, run once +// +// ./bin/streaming-zipformer-rtf-cxx-api +// +// # 2. Test on CPU, run 10 times +// +// ./bin/streaming-zipformer-rtf-cxx-api 10 +// +// # 3. Test on GPU, run 10 times +// +// ./bin/streaming-zipformer-rtf-cxx-api 10 cuda +// // clang-format on #include // NOLINT @@ -21,13 +39,15 @@ int32_t main(int argc, char *argv[]) { int32_t num_runs = 1; - if (argc == 2) { + if (argc >= 2) { num_runs = atoi(argv[1]); if (num_runs < 0) { num_runs = 1; } } + bool use_gpu = (argc == 3); + using namespace sherpa_onnx::cxx; // NOLINT OnlineRecognizerConfig config; @@ -50,6 +70,7 @@ int32_t main(int argc, char *argv[]) { "./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt"; config.model_config.num_threads = 1; + config.model_config.provider = use_gpu ? "cuda" : "cpu"; std::cout << "Loading model\n"; OnlineRecognizer recongizer = OnlineRecognizer::Create(config); From 6ee8c99c5d3563727ba04b4c20c4fe2c8cbd2954 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sun, 3 Nov 2024 19:47:04 +0800 Subject: [PATCH 063/183] Fix building (#1508) --- sherpa-onnx/csrc/offline-recognizer-impl.cc | 2 +- sherpa-onnx/csrc/onnx-utils.cc | 8 ++++---- sherpa-onnx/csrc/session.cc | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sherpa-onnx/csrc/offline-recognizer-impl.cc b/sherpa-onnx/csrc/offline-recognizer-impl.cc index f89bb9bec..99c41d307 100644 --- a/sherpa-onnx/csrc/offline-recognizer-impl.cc +++ b/sherpa-onnx/csrc/offline-recognizer-impl.cc @@ -123,7 +123,7 @@ std::unique_ptr OfflineRecognizerImpl::Create( auto model_type = LookupCustomModelMetaData(meta_data, "model_type", allocator); - if (!model_type.empty()) { + if (model_type.empty()) { SHERPA_ONNX_LOGE( "No model_type in the metadata!\n\n" "Please refer to the following URLs to add metadata" diff --git a/sherpa-onnx/csrc/onnx-utils.cc b/sherpa-onnx/csrc/onnx-utils.cc index 0dc69fc8f..31b6554c6 100644 --- a/sherpa-onnx/csrc/onnx-utils.cc +++ b/sherpa-onnx/csrc/onnx-utils.cc @@ -25,7 +25,7 @@ static std::string GetInputName(Ort::Session *sess, size_t index, OrtAllocator *allocator) { // Note(fangjun): We only tested 1.17.1 and 1.11.0 // For other versions, we may need to change it -#if ORT_API_VERSION >= 17 +#if ORT_API_VERSION >= 12 auto v = sess->GetInputNameAllocated(index, allocator); return v.get(); #else @@ -40,7 +40,7 @@ static std::string GetOutputName(Ort::Session *sess, size_t index, OrtAllocator *allocator) { // Note(fangjun): We only tested 1.17.1 and 1.11.0 // For other versions, we may need to change it -#if ORT_API_VERSION >= 17 +#if ORT_API_VERSION >= 12 auto v = sess->GetOutputNameAllocated(index, allocator); return v.get(); #else @@ -106,7 +106,7 @@ Ort::Value GetEncoderOutFrame(OrtAllocator *allocator, Ort::Value *encoder_out, void PrintModelMetadata(std::ostream &os, const Ort::ModelMetadata &meta_data) { Ort::AllocatorWithDefaultOptions allocator; -#if ORT_API_VERSION >= 17 +#if ORT_API_VERSION >= 12 std::vector v = meta_data.GetCustomMetadataMapKeysAllocated(allocator); for (const auto &key : v) { @@ -406,7 +406,7 @@ std::string LookupCustomModelMetaData(const Ort::ModelMetadata &meta_data, OrtAllocator *allocator) { // Note(fangjun): We only tested 1.17.1 and 1.11.0 // For other versions, we may need to change it -#if ORT_API_VERSION >= 17 +#if ORT_API_VERSION >= 12 auto v = meta_data.LookupCustomMetadataMapAllocated(key, allocator); return v.get(); #else diff --git a/sherpa-onnx/csrc/session.cc b/sherpa-onnx/csrc/session.cc index 160797c6e..cd9eb516f 100644 --- a/sherpa-onnx/csrc/session.cc +++ b/sherpa-onnx/csrc/session.cc @@ -60,7 +60,7 @@ Ort::SessionOptions GetSessionOptionsImpl( case Provider::kCPU: break; // nothing to do for the CPU provider case Provider::kXnnpack: { -#if ORT_API_VERSION >= 17 +#if ORT_API_VERSION >= 12 if (std::find(available_providers.begin(), available_providers.end(), "XnnpackExecutionProvider") != available_providers.end()) { sess_opts.AppendExecutionProvider("XNNPACK"); From 4eeb336f5980d48173ffabd585860a09f13b379d Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 4 Nov 2024 07:54:19 +0800 Subject: [PATCH 064/183] Export the English TTS model from MeloTTS (#1509) --- .../workflows/export-melo-tts-to-onnx.yaml | 66 +++++- scripts/melo-tts/README.md | 3 +- scripts/melo-tts/export-onnx-en.py | 221 ++++++++++++++++++ scripts/melo-tts/export-onnx.py | 3 + scripts/melo-tts/run.sh | 20 ++ sherpa-onnx/csrc/macros.h | 4 - sherpa-onnx/csrc/melo-tts-lexicon.cc | 67 ++++-- sherpa-onnx/csrc/melo-tts-lexicon.h | 3 + sherpa-onnx/csrc/offline-tts-vits-impl.h | 4 + sherpa-onnx/csrc/offline-tts-vits-model.cc | 6 +- sherpa-onnx/csrc/onnx-utils.cc | 4 +- 11 files changed, 372 insertions(+), 29 deletions(-) create mode 100755 scripts/melo-tts/export-onnx-en.py diff --git a/.github/workflows/export-melo-tts-to-onnx.yaml b/.github/workflows/export-melo-tts-to-onnx.yaml index 42cc28d6a..d0715b95a 100644 --- a/.github/workflows/export-melo-tts-to-onnx.yaml +++ b/.github/workflows/export-melo-tts-to-onnx.yaml @@ -40,7 +40,7 @@ jobs: name: test.wav path: scripts/melo-tts/test.wav - - name: Publish to huggingface + - name: Publish to huggingface (Chinese + English) env: HF_TOKEN: ${{ secrets.HF_TOKEN }} uses: nick-fields/retry@v3 @@ -61,14 +61,14 @@ jobs: git fetch git pull echo "pwd: $PWD" - ls -lh ../scripts/melo-tts + ls -lh ../scripts/melo-tts/zh_en rm -rf ./ - cp -v ../scripts/melo-tts/*.onnx . - cp -v ../scripts/melo-tts/lexicon.txt . - cp -v ../scripts/melo-tts/tokens.txt . - cp -v ../scripts/melo-tts/README.md . + cp -v ../scripts/melo-tts/zh_en/*.onnx . + cp -v ../scripts/melo-tts/zh_en/lexicon.txt . + cp -v ../scripts/melo-tts/zh_en/tokens.txt . + cp -v ../scripts/melo-tts/zh_en/README.md . curl -SL -O https://raw.githubusercontent.com/myshell-ai/MeloTTS/main/LICENSE @@ -102,6 +102,60 @@ jobs: tar cjvf $dst.tar.bz2 $dst rm -rf $dst + - name: Publish to huggingface (English) + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v3 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + + rm -rf huggingface + export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false + + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/vits-melo-tts-en huggingface + cd huggingface + git fetch + git pull + echo "pwd: $PWD" + ls -lh ../scripts/melo-tts/en + + rm -rf ./ + + cp -v ../scripts/melo-tts/en/*.onnx . + cp -v ../scripts/melo-tts/en/lexicon.txt . + cp -v ../scripts/melo-tts/en/tokens.txt . + cp -v ../scripts/melo-tts/en/README.md . + + curl -SL -O https://raw.githubusercontent.com/myshell-ai/MeloTTS/main/LICENSE + + git lfs track "*.onnx" + git add . + + ls -lh + + git status + + git diff + + git commit -m "add models" + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/vits-melo-tts-en main || true + + cd .. + + rm -rf huggingface/.git* + dst=vits-melo-tts-en + + mv huggingface $dst + + tar cjvf $dst.tar.bz2 $dst + rm -rf $dst + - name: Release uses: svenstaro/upload-release-action@v2 with: diff --git a/scripts/melo-tts/README.md b/scripts/melo-tts/README.md index 802af0608..146aa4632 100644 --- a/scripts/melo-tts/README.md +++ b/scripts/melo-tts/README.md @@ -3,4 +3,5 @@ Models in this directory are converted from https://github.com/myshell-ai/MeloTTS -Note there is only a single female speaker in the model. +Note there is only a single female speaker in the model for Chinese+English TTS. +TTS model, whereas there are 5 female speakers in the model For English TTS. diff --git a/scripts/melo-tts/export-onnx-en.py b/scripts/melo-tts/export-onnx-en.py new file mode 100755 index 000000000..e81bcff6d --- /dev/null +++ b/scripts/melo-tts/export-onnx-en.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +# This model exports the English-only TTS model. +# It has 5 speakers. +# {'EN-US': 0, 'EN-BR': 1, 'EN_INDIA': 2, 'EN-AU': 3, 'EN-Default': 4} + +from typing import Any, Dict + +import onnx +import torch +from melo.api import TTS +from melo.text import language_id_map, language_tone_start_map +from melo.text.chinese import pinyin_to_symbol_map +from melo.text.english import eng_dict, refine_syllables +from pypinyin import Style, lazy_pinyin, phrases_dict, pinyin_dict + + +def generate_tokens(symbol_list): + with open("tokens.txt", "w", encoding="utf-8") as f: + for i, s in enumerate(symbol_list): + f.write(f"{s} {i}\n") + + +def add_new_english_words(lexicon): + """ + Args: + lexicon: + Please modify it in-place. + """ + + # Please have a look at + # https://github.com/myshell-ai/MeloTTS/blob/main/melo/text/cmudict.rep + + # We give several examples below about how to add new words + + # Example 1. Add a new word kaldi + + # It does not contain the word kaldi in cmudict.rep + # so if we add the following line to cmudict.rep + # + # KALDI K AH0 - L D IH0 + # + # then we need to change the lexicon like below + lexicon["kaldi"] = [["K", "AH0"], ["L", "D", "IH0"]] + # + # K AH0 and L D IH0 are separated by a dash "-", so + # ["K", "AH0"] is a in list and ["L", "D", "IH0"] is in a separate list + + # Note: Either kaldi or KALDI is fine. You can use either lowercase or + # uppercase or both + + # Example 2. Add a new word SF + # + # If we add the following line to cmudict.rep + # + # SF EH1 S - EH1 F + # + # to cmudict.rep, then we need to change the lexicon like below: + lexicon["SF"] = [["EH1", "S"], ["EH1", "F"]] + + # Please add your new words here + + # No need to return lexicon since it is changed in-place + + +def generate_lexicon(): + add_new_english_words(eng_dict) + with open("lexicon.txt", "w", encoding="utf-8") as f: + for word in eng_dict: + phones, tones = refine_syllables(eng_dict[word]) + tones = [t + language_tone_start_map["EN"] for t in tones] + tones = [str(t) for t in tones] + + phones = " ".join(phones) + tones = " ".join(tones) + + f.write(f"{word.lower()} {phones} {tones}\n") + + +def add_meta_data(filename: str, meta_data: Dict[str, Any]): + """Add meta data to an ONNX model. It is changed in-place. + + Args: + filename: + Filename of the ONNX model to be changed. + meta_data: + Key-value pairs. + """ + model = onnx.load(filename) + while len(model.metadata_props): + model.metadata_props.pop() + + for key, value in meta_data.items(): + meta = model.metadata_props.add() + meta.key = key + meta.value = str(value) + + onnx.save(model, filename) + + +class ModelWrapper(torch.nn.Module): + def __init__(self, model: "SynthesizerTrn"): + super().__init__() + self.model = model + self.lang_id = language_id_map[model.language] + + def forward( + self, + x, + x_lengths, + tones, + sid, + noise_scale, + length_scale, + noise_scale_w, + max_len=None, + ): + """ + Args: + x: A 1-D array of dtype np.int64. Its shape is (token_numbers,) + tones: A 1-D array of dtype np.int64. Its shape is (token_numbers,) + lang_id: A 1-D array of dtype np.int64. Its shape is (token_numbers,) + sid: an integer + """ + bert = torch.zeros(x.shape[0], 1024, x.shape[1], dtype=torch.float32) + ja_bert = torch.zeros(x.shape[0], 768, x.shape[1], dtype=torch.float32) + lang_id = torch.zeros_like(x) + lang_id[:, 1::2] = self.lang_id + return self.model.model.infer( + x=x, + x_lengths=x_lengths, + sid=sid, + tone=tones, + language=lang_id, + bert=bert, + ja_bert=ja_bert, + noise_scale=noise_scale, + noise_scale_w=noise_scale_w, + length_scale=length_scale, + )[0] + + +def main(): + generate_lexicon() + + language = "EN" + model = TTS(language=language, device="cpu") + + generate_tokens(model.hps["symbols"]) + + torch_model = ModelWrapper(model) + + opset_version = 13 + x = torch.randint(low=0, high=10, size=(60,), dtype=torch.int64) + print(x.shape) + x_lengths = torch.tensor([x.size(0)], dtype=torch.int64) + sid = torch.tensor([1], dtype=torch.int64) + tones = torch.zeros_like(x) + + noise_scale = torch.tensor([1.0], dtype=torch.float32) + length_scale = torch.tensor([1.0], dtype=torch.float32) + noise_scale_w = torch.tensor([1.0], dtype=torch.float32) + + x = x.unsqueeze(0) + tones = tones.unsqueeze(0) + + filename = "model.onnx" + + torch.onnx.export( + torch_model, + ( + x, + x_lengths, + tones, + sid, + noise_scale, + length_scale, + noise_scale_w, + ), + filename, + opset_version=opset_version, + input_names=[ + "x", + "x_lengths", + "tones", + "sid", + "noise_scale", + "length_scale", + "noise_scale_w", + ], + output_names=["y"], + dynamic_axes={ + "x": {0: "N", 1: "L"}, + "x_lengths": {0: "N"}, + "tones": {0: "N", 1: "L"}, + "y": {0: "N", 1: "S", 2: "T"}, + }, + ) + + meta_data = { + "model_type": "melo-vits", + "comment": "melo", + "version": 2, + "language": "English", + "add_blank": int(model.hps.data.add_blank), + "n_speakers": len(model.hps.data.spk2id), # 5 + "jieba": 0, + "sample_rate": model.hps.data.sampling_rate, + "bert_dim": 1024, + "ja_bert_dim": 768, + "speaker_id": 0, + "lang_id": language_id_map[model.language], + "tone_start": language_tone_start_map[model.language], + "url": "https://github.com/myshell-ai/MeloTTS", + "license": "MIT license", + "description": "MeloTTS is a high-quality multi-lingual text-to-speech library by MyShell.ai", + } + add_meta_data(filename, meta_data) + + +if __name__ == "__main__": + main() diff --git a/scripts/melo-tts/export-onnx.py b/scripts/melo-tts/export-onnx.py index 6db8335a7..69d9066ee 100755 --- a/scripts/melo-tts/export-onnx.py +++ b/scripts/melo-tts/export-onnx.py @@ -1,4 +1,7 @@ #!/usr/bin/env python3 +# This script export ZH_EN TTS model, which supports both Chinese and English. +# This model has only 1 speaker. + from typing import Any, Dict import onnx diff --git a/scripts/melo-tts/run.sh b/scripts/melo-tts/run.sh index b8fee07ed..e3ba3c062 100755 --- a/scripts/melo-tts/run.sh +++ b/scripts/melo-tts/run.sh @@ -38,4 +38,24 @@ tail tokens.txt ./test.py +mkdir zh_en +mv -v *.onnx zh_en/ +mv -v lexicon.txt zh_en +mv -v tokens.txt zh_en +cp -v README.md zh_en + +ls -lh +echo "---" +ls -lh zh_en + +./export-onnx-en.py + +mkdir en +mv -v *.onnx en/ +mv -v lexicon.txt en +mv -v tokens.txt en +cp -v README.md en + +ls -lh en + ls -lh diff --git a/sherpa-onnx/csrc/macros.h b/sherpa-onnx/csrc/macros.h index 506e63e13..7cd4d4627 100644 --- a/sherpa-onnx/csrc/macros.h +++ b/sherpa-onnx/csrc/macros.h @@ -152,10 +152,6 @@ #define SHERPA_ONNX_READ_META_DATA_STR_ALLOW_EMPTY(dst, src_key) \ do { \ auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \ - if (value.empty()) { \ - SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ - exit(-1); \ - } \ \ dst = std::move(value); \ } while (0) diff --git a/sherpa-onnx/csrc/melo-tts-lexicon.cc b/sherpa-onnx/csrc/melo-tts-lexicon.cc index a605e6c38..29857824f 100644 --- a/sherpa-onnx/csrc/melo-tts-lexicon.cc +++ b/sherpa-onnx/csrc/melo-tts-lexicon.cc @@ -48,6 +48,20 @@ class MeloTtsLexicon::Impl { } } + Impl(const std::string &lexicon, const std::string &tokens, + const OfflineTtsVitsModelMetaData &meta_data, bool debug) + : meta_data_(meta_data), debug_(debug) { + { + std::ifstream is(tokens); + InitTokens(is); + } + + { + std::ifstream is(lexicon); + InitLexicon(is); + } + } + std::vector ConvertTextToTokenIds(const std::string &_text) const { std::string text = ToLowerCase(_text); // see @@ -65,21 +79,39 @@ class MeloTtsLexicon::Impl { s = std::regex_replace(s, punct_re4, "!"); std::vector words; - bool is_hmm = true; - jieba_->Cut(text, words, is_hmm); - - if (debug_) { - SHERPA_ONNX_LOGE("input text: %s", text.c_str()); - SHERPA_ONNX_LOGE("after replacing punctuations: %s", s.c_str()); - - std::ostringstream os; - std::string sep = ""; - for (const auto &w : words) { - os << sep << w; - sep = "_"; - } + if (jieba_) { + bool is_hmm = true; + jieba_->Cut(text, words, is_hmm); + + if (debug_) { + SHERPA_ONNX_LOGE("input text: %s", text.c_str()); + SHERPA_ONNX_LOGE("after replacing punctuations: %s", s.c_str()); + + std::ostringstream os; + std::string sep = ""; + for (const auto &w : words) { + os << sep << w; + sep = "_"; + } - SHERPA_ONNX_LOGE("after jieba processing: %s", os.str().c_str()); + SHERPA_ONNX_LOGE("after jieba processing: %s", os.str().c_str()); + } + } else { + words = SplitUtf8(text); + + if (debug_) { + fprintf(stderr, "Input text in string (lowercase): %s\n", text.c_str()); + fprintf(stderr, "Input text in bytes (lowercase):"); + for (uint8_t c : text) { + fprintf(stderr, " %02x", c); + } + fprintf(stderr, "\n"); + fprintf(stderr, "After splitting to words:"); + for (const auto &w : words) { + fprintf(stderr, " %s", w.c_str()); + } + fprintf(stderr, "\n"); + } } std::vector ans; @@ -241,6 +273,7 @@ class MeloTtsLexicon::Impl { {std::move(word), TokenIDs{std::move(ids64), std::move(tone_list)}}); } + // For Chinese+English MeloTTS word2ids_["呣"] = word2ids_["母"]; word2ids_["嗯"] = word2ids_["恩"]; } @@ -268,6 +301,12 @@ MeloTtsLexicon::MeloTtsLexicon(const std::string &lexicon, : impl_(std::make_unique(lexicon, tokens, dict_dir, meta_data, debug)) {} +MeloTtsLexicon::MeloTtsLexicon(const std::string &lexicon, + const std::string &tokens, + const OfflineTtsVitsModelMetaData &meta_data, + bool debug) + : impl_(std::make_unique(lexicon, tokens, meta_data, debug)) {} + std::vector MeloTtsLexicon::ConvertTextToTokenIds( const std::string &text, const std::string & /*unused_voice = ""*/) const { return impl_->ConvertTextToTokenIds(text); diff --git a/sherpa-onnx/csrc/melo-tts-lexicon.h b/sherpa-onnx/csrc/melo-tts-lexicon.h index 261f3412e..da0644be2 100644 --- a/sherpa-onnx/csrc/melo-tts-lexicon.h +++ b/sherpa-onnx/csrc/melo-tts-lexicon.h @@ -22,6 +22,9 @@ class MeloTtsLexicon : public OfflineTtsFrontend { const std::string &dict_dir, const OfflineTtsVitsModelMetaData &meta_data, bool debug); + MeloTtsLexicon(const std::string &lexicon, const std::string &tokens, + const OfflineTtsVitsModelMetaData &meta_data, bool debug); + std::vector ConvertTextToTokenIds( const std::string &text, const std::string &unused_voice = "") const override; diff --git a/sherpa-onnx/csrc/offline-tts-vits-impl.h b/sherpa-onnx/csrc/offline-tts-vits-impl.h index fb43a88ab..72b21a3b5 100644 --- a/sherpa-onnx/csrc/offline-tts-vits-impl.h +++ b/sherpa-onnx/csrc/offline-tts-vits-impl.h @@ -349,6 +349,10 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { config_.model.vits.lexicon, config_.model.vits.tokens, config_.model.vits.dict_dir, model_->GetMetaData(), config_.model.debug); + } else if (meta_data.is_melo_tts && meta_data.language == "English") { + frontend_ = std::make_unique( + config_.model.vits.lexicon, config_.model.vits.tokens, + model_->GetMetaData(), config_.model.debug); } else if (meta_data.jieba && !config_.model.vits.dict_dir.empty()) { frontend_ = std::make_unique( config_.model.vits.lexicon, config_.model.vits.tokens, diff --git a/sherpa-onnx/csrc/offline-tts-vits-model.cc b/sherpa-onnx/csrc/offline-tts-vits-model.cc index c97cecf80..90e0993d6 100644 --- a/sherpa-onnx/csrc/offline-tts-vits-model.cc +++ b/sherpa-onnx/csrc/offline-tts-vits-model.cc @@ -46,8 +46,10 @@ class OfflineTtsVitsModel::Impl { } Ort::Value Run(Ort::Value x, Ort::Value tones, int64_t sid, float speed) { - // For MeloTTS, we hardcode sid to the one contained in the meta data - sid = meta_data_.speaker_id; + if (meta_data_.num_speakers == 1) { + // For MeloTTS, we hardcode sid to the one contained in the meta data + sid = meta_data_.speaker_id; + } auto memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault); diff --git a/sherpa-onnx/csrc/onnx-utils.cc b/sherpa-onnx/csrc/onnx-utils.cc index 31b6554c6..5e346a69e 100644 --- a/sherpa-onnx/csrc/onnx-utils.cc +++ b/sherpa-onnx/csrc/onnx-utils.cc @@ -408,10 +408,10 @@ std::string LookupCustomModelMetaData(const Ort::ModelMetadata &meta_data, // For other versions, we may need to change it #if ORT_API_VERSION >= 12 auto v = meta_data.LookupCustomMetadataMapAllocated(key, allocator); - return v.get(); + return v ? v.get() : ""; #else auto v = meta_data.LookupCustomMetadataMap(key, allocator); - std::string ans = v; + std::string ans = v ? v : ""; allocator->Free(allocator, v); return ans; #endif From 86b1856c207761d999228c2f14f013908b5047c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=B5=E5=B0=8F=E5=87=A1?= <2672931+whyb@users.noreply.github.com> Date: Tue, 5 Nov 2024 20:34:12 +0800 Subject: [PATCH 065/183] Reduce vad-sense-voice example code. (#1510) --- c-api-examples/vad-sense-voice-c-api.c | 46 ++++++-------------------- 1 file changed, 11 insertions(+), 35 deletions(-) diff --git a/c-api-examples/vad-sense-voice-c-api.c b/c-api-examples/vad-sense-voice-c-api.c index ee9504d1a..eeddfce88 100644 --- a/c-api-examples/vad-sense-voice-c-api.c +++ b/c-api-examples/vad-sense-voice-c-api.c @@ -99,11 +99,16 @@ int32_t main() { int32_t window_size = vadConfig.silero_vad.window_size; int32_t i = 0; - - while (i + window_size < wave->num_samples) { - SherpaOnnxVoiceActivityDetectorAcceptWaveform(vad, wave->samples + i, - window_size); - i += window_size; + int is_eof = 0; + + while (!is_eof) { + if (i + window_size < wave->num_samples) { + SherpaOnnxVoiceActivityDetectorAcceptWaveform(vad, wave->samples + i, + window_size); + } else { + SherpaOnnxVoiceActivityDetectorFlush(vad); + is_eof = 1; + } while (!SherpaOnnxVoiceActivityDetectorEmpty(vad)) { const SherpaOnnxSpeechSegment *segment = @@ -132,36 +137,7 @@ int32_t main() { SherpaOnnxDestroySpeechSegment(segment); SherpaOnnxVoiceActivityDetectorPop(vad); } - } - - SherpaOnnxVoiceActivityDetectorFlush(vad); - - while (!SherpaOnnxVoiceActivityDetectorEmpty(vad)) { - const SherpaOnnxSpeechSegment *segment = - SherpaOnnxVoiceActivityDetectorFront(vad); - - const SherpaOnnxOfflineStream *stream = - SherpaOnnxCreateOfflineStream(recognizer); - - SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate, segment->samples, - segment->n); - - SherpaOnnxDecodeOfflineStream(recognizer, stream); - - const SherpaOnnxOfflineRecognizerResult *result = - SherpaOnnxGetOfflineStreamResult(stream); - - float start = segment->start / 16000.0f; - float duration = segment->n / 16000.0f; - float stop = start + duration; - - fprintf(stderr, "%.3f -- %.3f: %s\n", start, stop, result->text); - - SherpaOnnxDestroyOfflineRecognizerResult(result); - SherpaOnnxDestroyOfflineStream(stream); - - SherpaOnnxDestroySpeechSegment(segment); - SherpaOnnxVoiceActivityDetectorPop(vad); + i += window_size; } SherpaOnnxDestroyOfflineRecognizer(recognizer); From f94cca71cf45bf980eca0830ff8ee91f2029a846 Mon Sep 17 00:00:00 2001 From: VEP Date: Fri, 8 Nov 2024 19:04:34 +0800 Subject: [PATCH 066/183] Fix: Reset sample-buffer after processing (#1521) Co-authored-by: VEP <517138883@qq.com> --- .../two-pass-speech-recognition-from-microphone.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python-api-examples/two-pass-speech-recognition-from-microphone.py b/python-api-examples/two-pass-speech-recognition-from-microphone.py index aa2245c5c..7d5017f58 100755 --- a/python-api-examples/two-pass-speech-recognition-from-microphone.py +++ b/python-api-examples/two-pass-speech-recognition-from-microphone.py @@ -427,7 +427,6 @@ def main(): ) print("\r{}:{}".format(segment_id, result), flush=True) segment_id += 1 - else: sample_buffers = [] first_recognizer.reset(stream) From f97daed4080132a2bedbff77cf0919d737d18eaf Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 8 Nov 2024 21:07:36 +0800 Subject: [PATCH 067/183] Fixes #1512 (#1522) --- .github/workflows/build-xcframework.yaml | 7 +++++++ build-ios-shared.sh | 10 +++++++--- sherpa-onnx/csrc/CMakeLists.txt | 2 +- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-xcframework.yaml b/.github/workflows/build-xcframework.yaml index 90953010e..8fcfafd43 100644 --- a/.github/workflows/build-xcframework.yaml +++ b/.github/workflows/build-xcframework.yaml @@ -43,6 +43,13 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Build iOS shared + if: matrix.with_tts == 'ON' + shell: bash + run: | + export CMAKE_VERBOSE_MAKEFILE=ON + ./build-ios-shared.sh + - name: Build iOS if: matrix.with_tts == 'ON' shell: bash diff --git a/build-ios-shared.sh b/build-ios-shared.sh index caf81decc..90b2e9a5d 100755 --- a/build-ios-shared.sh +++ b/build-ios-shared.sh @@ -19,6 +19,10 @@ if [ "$SHERPA_ONNX_GITHUB_MIRROW" == true ]; then SHERPA_ONNX_GITHUB=hub.nuaa.cf fi +if [ ! -z CMAKE_VERBOSE_MAKEFILE ]; then + CMAKE_VERBOSE_MAKEFILE=ON +fi + if [ ! -f $onnxruntime_dir/onnxruntime.xcframework/ios-arm64/onnxruntime.a ]; then mkdir -p $onnxruntime_dir pushd $onnxruntime_dir @@ -50,7 +54,7 @@ if [[ ! -f build/simulator_x86_64/install/lib/libsherpa-onnx-c-api.dylib ]]; the -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ -DBUILD_ESPEAK_NG_EXE=OFF \ -DBUILD_ESPEAK_NG_TESTS=OFF \ - -S .. \ + -S .. -D CMAKE_VERBOSE_MAKEFILE=$CMAKE_VERBOSE_MAKEFILE \ -DCMAKE_TOOLCHAIN_FILE=./toolchains/ios.toolchain.cmake \ -DPLATFORM=SIMULATOR64 \ -DENABLE_BITCODE=0 \ @@ -81,7 +85,7 @@ if [[ ! -f build/simulator_arm64/install/lib/libsherpa-onnx-c-api.dylib ]]; then -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ -DBUILD_ESPEAK_NG_EXE=OFF \ -DBUILD_ESPEAK_NG_TESTS=OFF \ - -S .. \ + -S .. -D CMAKE_VERBOSE_MAKEFILE=$CMAKE_VERBOSE_MAKEFILE \ -DCMAKE_TOOLCHAIN_FILE=./toolchains/ios.toolchain.cmake \ -DPLATFORM=SIMULATORARM64 \ -DENABLE_BITCODE=0 \ @@ -114,7 +118,7 @@ if [[ ! -f build/os64/install/lib/libsherpa-onnx-c-api.dylib ]]; then -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ -DBUILD_ESPEAK_NG_EXE=OFF \ -DBUILD_ESPEAK_NG_TESTS=OFF \ - -S .. \ + -S .. -D CMAKE_VERBOSE_MAKEFILE=$CMAKE_VERBOSE_MAKEFILE \ -DCMAKE_TOOLCHAIN_FILE=./toolchains/ios.toolchain.cmake \ -DPLATFORM=OS64 \ -DENABLE_BITCODE=0 \ diff --git a/sherpa-onnx/csrc/CMakeLists.txt b/sherpa-onnx/csrc/CMakeLists.txt index f34c78609..d241ee1ca 100644 --- a/sherpa-onnx/csrc/CMakeLists.txt +++ b/sherpa-onnx/csrc/CMakeLists.txt @@ -214,7 +214,7 @@ if(SHERPA_ONNX_ENABLE_GPU) ) endif() -if(BUILD_SHARED_LIBS) +if(BUILD_SHARED_LIBS AND NOT DEFINED onnxruntime_lib_files) target_link_libraries(sherpa-onnx-core onnxruntime) else() target_link_libraries(sherpa-onnx-core ${onnxruntime_lib_files}) From 4fab3f2e2fae7e07ab18328d1a504cfc20bd134c Mon Sep 17 00:00:00 2001 From: VEP Date: Fri, 8 Nov 2024 21:28:04 +0800 Subject: [PATCH 068/183] Revert: [#1521] No need to reset sample-buffer (#1524) Co-authored-by: VEP <517138883@qq.com> --- .../two-pass-speech-recognition-from-microphone.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python-api-examples/two-pass-speech-recognition-from-microphone.py b/python-api-examples/two-pass-speech-recognition-from-microphone.py index 7d5017f58..aa2245c5c 100755 --- a/python-api-examples/two-pass-speech-recognition-from-microphone.py +++ b/python-api-examples/two-pass-speech-recognition-from-microphone.py @@ -427,6 +427,7 @@ def main(): ) print("\r{}:{}".format(segment_id, result), flush=True) segment_id += 1 + else: sample_buffers = [] first_recognizer.reset(stream) From a16c9aff8bbc1ccbc79593adcf5d6a39de8ae5f3 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 13 Nov 2024 00:04:16 +0800 Subject: [PATCH 069/183] Add Lazarus example for Moonshine models. (#1532) --- build-ios-shared.sh | 2 + cmake/piper-phonemize.cmake | 16 +++--- .../generate_subtitles/my_init.pas | 54 +++++++++++++++++++ scripts/lazarus/generate-subtitles.py | 14 +++++ 4 files changed, 78 insertions(+), 8 deletions(-) diff --git a/build-ios-shared.sh b/build-ios-shared.sh index 90b2e9a5d..2af1b36f0 100755 --- a/build-ios-shared.sh +++ b/build-ios-shared.sh @@ -21,6 +21,8 @@ fi if [ ! -z CMAKE_VERBOSE_MAKEFILE ]; then CMAKE_VERBOSE_MAKEFILE=ON +else + CMAKE_VERBOSE_MAKEFILE=OFF fi if [ ! -f $onnxruntime_dir/onnxruntime.xcframework/ios-arm64/onnxruntime.a ]; then diff --git a/cmake/piper-phonemize.cmake b/cmake/piper-phonemize.cmake index 9c9c71f5a..0e11fd176 100644 --- a/cmake/piper-phonemize.cmake +++ b/cmake/piper-phonemize.cmake @@ -1,18 +1,18 @@ function(download_piper_phonemize) include(FetchContent) - set(piper_phonemize_URL "https://github.com/csukuangfj/piper-phonemize/archive/38ee199dcc49c7b6de89f7ebfb32ed682763fa1b.zip") - set(piper_phonemize_URL2 "https://hf-mirror.com/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/piper-phonemize-38ee199dcc49c7b6de89f7ebfb32ed682763fa1b.zip") - set(piper_phonemize_HASH "SHA256=ab4d06ca76047e1585c63c482f39ffead5315785345055360703cc9382c5e74b") + set(piper_phonemize_URL "https://github.com/csukuangfj/piper-phonemize/archive/78a788e0b719013401572d70fef372e77bff8e43.zip") + set(piper_phonemize_URL2 "https://hf-mirror.com/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/piper-phonemize-78a788e0b719013401572d70fef372e77bff8e43.zip") + set(piper_phonemize_HASH "SHA256=89641a46489a4898754643ce57bda9c9b54b4ca46485fdc02bf0dc84b866645d") # If you don't have access to the Internet, # please pre-download kaldi-decoder set(possible_file_locations - $ENV{HOME}/Downloads/piper-phonemize-38ee199dcc49c7b6de89f7ebfb32ed682763fa1b.zip - ${CMAKE_SOURCE_DIR}/piper-phonemize-38ee199dcc49c7b6de89f7ebfb32ed682763fa1b.zip - ${CMAKE_BINARY_DIR}/piper-phonemize-38ee199dcc49c7b6de89f7ebfb32ed682763fa1b.zip - /tmp/piper-phonemize-38ee199dcc49c7b6de89f7ebfb32ed682763fa1b.zip - /star-fj/fangjun/download/github/piper-phonemize-38ee199dcc49c7b6de89f7ebfb32ed682763fa1b.zip + $ENV{HOME}/Downloads/piper-phonemize-78a788e0b719013401572d70fef372e77bff8e43.zip + ${CMAKE_SOURCE_DIR}/piper-phonemize-78a788e0b719013401572d70fef372e77bff8e43.zip + ${CMAKE_BINARY_DIR}/piper-phonemize-78a788e0b719013401572d70fef372e77bff8e43.zip + /tmp/piper-phonemize-78a788e0b719013401572d70fef372e77bff8e43.zip + /star-fj/fangjun/download/github/piper-phonemize-78a788e0b719013401572d70fef372e77bff8e43.zip ) foreach(f IN LISTS possible_file_locations) diff --git a/lazarus-examples/generate_subtitles/my_init.pas b/lazarus-examples/generate_subtitles/my_init.pas index d57448b6d..d01cb6081 100644 --- a/lazarus-examples/generate_subtitles/my_init.pas +++ b/lazarus-examples/generate_subtitles/my_init.pas @@ -159,6 +159,30 @@ function CreateOfflineRecognizerWhisper( Result := TSherpaOnnxOfflineRecognizer.Create(Config); end; +function CreateOfflineRecognizerMoonshine( + Tokens: AnsiString; + Preprocessor: AnsiString; + Encoder: AnsiString; + UncachedDecoder: AnsiString; + CachedDecoder: AnsiString): TSherpaOnnxOfflineRecognizer; +var + Config: TSherpaOnnxOfflineRecognizerConfig; +begin + Initialize(Config); + + Config.ModelConfig.Moonshine.Preprocessor := Preprocessor; + Config.ModelConfig.Moonshine.Encoder := Encoder; + Config.ModelConfig.Moonshine.UncachedDecoder := UncachedDecoder; + Config.ModelConfig.Moonshine.CachedDecoder := CachedDecoder; + + Config.ModelConfig.Tokens := Tokens; + Config.ModelConfig.Provider := 'cpu'; + Config.ModelConfig.NumThreads := 2; + Config.ModelConfig.Debug := False; + + Result := TSherpaOnnxOfflineRecognizer.Create(Config); +end; + constructor TMyInitThread.Create(CreateSuspended : boolean; ModelDirectory: AnsiString); begin inherited Create(CreateSuspended); @@ -193,6 +217,11 @@ procedure TMyInitThread.Execute; NeMoTransducerEncoder: AnsiString; NeMoTransducerDecoder: AnsiString; NeMoTransducerJoiner: AnsiString; + + MoonshinePreprocessor: AnsiString; + MoonshineEncoder: AnsiString; + MoonshineUncachedDecoder: AnsiString; + MoonshineCachedDecoder: AnsiString; begin VadFilename := ModelDir + 'silero_vad.onnx'; Tokens := ModelDir + 'tokens.txt'; @@ -292,6 +321,24 @@ procedure TMyInitThread.Execute; NeMoTransducerDecoder := ModelDir + 'nemo-transducer-decoder.onnx'; NeMoTransducerJoiner := ModelDir + 'nemo-transducer-joiner.onnx'; + { + Please Visit + https://k2-fsa.github.io/sherpa/onnx/moonshine/models.html + to download a Moonshine model. + + Note that you have to rename model files after downloading. The following + is an example. + + mv preprocess.onnx moonshine-preprocessor.onnx + mv encode.int8.onnx moonshine-encoder.onnx + mv uncached_decode.int8.onnx moonshine-uncached-decoder.onnx + mv cached_decode.int8.onnx moonshine-cached-decoder.onnx + } + MoonshinePreprocessor := ModelDir + 'moonshine-preprocessor.onnx'; + MoonshineEncoder := ModelDir + 'moonshine-encoder.onnx'; + MoonshineUncachedDecoder := ModelDir + 'moonshine-uncached-decoder.onnx'; + MoonshineCachedDecoder := ModelDir + 'moonshine-cached-decoder.onnx'; + if not FileExists(VadFilename) then begin Status := VadFilename + ' does not exist! Please download it from' + @@ -344,6 +391,13 @@ procedure TMyInitThread.Execute; NeMoTransducerEncoder, NeMoTransducerDecoder, NeMoTransducerJoiner, 'nemo_transducer'); Msg := 'NeMo transducer'; end + else if FileExists(MoonshinePreprocessor) and FileExists(MoonshineEncoder) and FileExists(MoonshineUncachedDecoder) and FileExists(MoonshineCachedDecoder) then + begin + Form1.OfflineRecognizer := CreateOfflineRecognizerMoonshine(Tokens, + MoonshinePreprocessor, MoonshineEncoder, MoonshineUncachedDecoder, + MoonshineCachedDecoder); + Msg := 'Moonshine'; + end else begin Status := 'Please download at least one non-streaming speech recognition model first.'; diff --git a/scripts/lazarus/generate-subtitles.py b/scripts/lazarus/generate-subtitles.py index 6459886c1..b7d7f62a4 100755 --- a/scripts/lazarus/generate-subtitles.py +++ b/scripts/lazarus/generate-subtitles.py @@ -50,6 +50,20 @@ def get_models(): popd """, ), + Model( + model_name="sherpa-onnx-moonshine-tiny-en-int8", + lang="en", + short_name="moonshine_tiny", + cmd=""" + pushd $model_name + mv -v preprocess.onnx moonshine-preprocessor.onnx + mv -v encode.int8.onnx moonshine-encoder.onnx + mv -v uncached_decode.int8.onnx moonshine-uncached-decoder.onnx + mv -v cached_decode.int8.onnx moonshine-cached-decoder.onnx + + popd + """, + ), Model( model_name="sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17", lang="zh_en_ko_ja_yue", From 3f777b3fe36f7ba05d706b14523f7bbf4e592db7 Mon Sep 17 00:00:00 2001 From: Aero Date: Wed, 13 Nov 2024 00:04:57 +0800 Subject: [PATCH 070/183] Add isolate_tts demo (#1529) --- flutter-examples/tts/lib/isolate_tts.dart | 246 ++++++++++++++++++++++ flutter-examples/tts/lib/main.dart | 8 +- flutter-examples/tts/lib/model.dart | 11 +- flutter-examples/tts/lib/tts.dart | 234 ++++++++++---------- flutter-examples/tts/pubspec.yaml | 6 + 5 files changed, 376 insertions(+), 129 deletions(-) create mode 100644 flutter-examples/tts/lib/isolate_tts.dart diff --git a/flutter-examples/tts/lib/isolate_tts.dart b/flutter-examples/tts/lib/isolate_tts.dart new file mode 100644 index 000000000..950503c3c --- /dev/null +++ b/flutter-examples/tts/lib/isolate_tts.dart @@ -0,0 +1,246 @@ +import 'dart:io'; +import 'dart:isolate'; + +import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; +import 'package:media_kit/media_kit.dart'; +import 'package:path/path.dart' as p; +import 'package:path_provider/path_provider.dart'; +import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; + +import 'utils.dart'; + +class _IsolateTask { + final SendPort sendPort; + + RootIsolateToken? rootIsolateToken; + + _IsolateTask(this.sendPort, this.rootIsolateToken); +} + +class _PortModel { + final String method; + + final SendPort? sendPort; + dynamic data; + + _PortModel({ + required this.method, + this.sendPort, + this.data, + }); +} + +class _TtsManager { + /// 主进程通信端口 + final ReceivePort receivePort; + + final Isolate isolate; + + final SendPort isolatePort; + + _TtsManager({ + required this.receivePort, + required this.isolate, + required this.isolatePort, + }); +} + +class IsolateTts { + static late final _TtsManager _ttsManager; + + /// 获取线程里的通信端口 + static SendPort get _sendPort => _ttsManager.isolatePort; + + static late sherpa_onnx.OfflineTts _tts; + + static late Player _player; + + static Future init() async { + ReceivePort port = ReceivePort(); + RootIsolateToken? rootIsolateToken = RootIsolateToken.instance; + + Isolate isolate = await Isolate.spawn( + _isolateEntry, + _IsolateTask(port.sendPort, rootIsolateToken), + errorsAreFatal: false, + ); + port.listen((msg) async { + if (msg is SendPort) { + print(11); + _ttsManager = _TtsManager(receivePort: port, isolate: isolate, isolatePort: msg); + return; + } + }); + } + + static Future _isolateEntry(_IsolateTask task) async { + if (task.rootIsolateToken != null) { + BackgroundIsolateBinaryMessenger.ensureInitialized(task.rootIsolateToken!); + } + MediaKit.ensureInitialized(); + _player = Player(); + sherpa_onnx.initBindings(); + final receivePort = ReceivePort(); + task.sendPort.send(receivePort.sendPort); + + String modelDir = ''; + String modelName = ''; + String ruleFsts = ''; + String ruleFars = ''; + String lexicon = ''; + String dataDir = ''; + String dictDir = ''; + + // Example 7 + // https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models + // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-melo-tts-zh_en.tar.bz2 + modelDir = 'vits-melo-tts-zh_en'; + modelName = 'model.onnx'; + lexicon = 'lexicon.txt'; + dictDir = 'vits-melo-tts-zh_en/dict'; + + if (modelName == '') { + throw Exception('You are supposed to select a model by changing the code before you run the app'); + } + + final Directory directory = await getApplicationDocumentsDirectory(); + modelName = p.join(directory.path, modelDir, modelName); + + if (ruleFsts != '') { + final all = ruleFsts.split(','); + var tmp = []; + for (final f in all) { + tmp.add(p.join(directory.path, f)); + } + ruleFsts = tmp.join(','); + } + + if (ruleFars != '') { + final all = ruleFars.split(','); + var tmp = []; + for (final f in all) { + tmp.add(p.join(directory.path, f)); + } + ruleFars = tmp.join(','); + } + + if (lexicon != '') { + lexicon = p.join(directory.path, modelDir, lexicon); + } + + if (dataDir != '') { + dataDir = p.join(directory.path, dataDir); + } + + if (dictDir != '') { + dictDir = p.join(directory.path, dictDir); + } + + final tokens = p.join(directory.path, modelDir, 'tokens.txt'); + + final vits = sherpa_onnx.OfflineTtsVitsModelConfig( + model: modelName, + lexicon: lexicon, + tokens: tokens, + dataDir: dataDir, + dictDir: dictDir, + ); + + final modelConfig = sherpa_onnx.OfflineTtsModelConfig( + vits: vits, + numThreads: 2, + debug: true, + provider: 'cpu', + ); + + final config = sherpa_onnx.OfflineTtsConfig( + model: modelConfig, + ruleFsts: ruleFsts, + ruleFars: ruleFars, + maxNumSenetences: 1, + ); + // print(config); + receivePort.listen((msg) async { + print(msg); + if (msg is _PortModel) { + switch (msg.method) { + case 'generate': + { + _PortModel _v = msg; + final stopwatch = Stopwatch(); + stopwatch.start(); + final audio = _tts.generate(text: _v.data['text'], sid: _v.data['sid'], speed: _v.data['speed']); + final suffix = '-sid-${_v.data['sid']}-speed-${_v.data['sid'].toStringAsPrecision(2)}'; + final filename = await generateWaveFilename(suffix); + + final ok = sherpa_onnx.writeWave( + filename: filename, + samples: audio.samples, + sampleRate: audio.sampleRate, + ); + + if (ok) { + stopwatch.stop(); + double elapsed = stopwatch.elapsed.inMilliseconds.toDouble(); + + double waveDuration = audio.samples.length.toDouble() / audio.sampleRate.toDouble(); + + print('Saved to\n$filename\n' + 'Elapsed: ${(elapsed / 1000).toStringAsPrecision(4)} s\n' + 'Wave duration: ${waveDuration.toStringAsPrecision(4)} s\n' + 'RTF: ${(elapsed / 1000).toStringAsPrecision(4)}/${waveDuration.toStringAsPrecision(4)} ' + '= ${(elapsed / 1000 / waveDuration).toStringAsPrecision(3)} '); + + await _player.open(Media('file:///$filename')); + await _player.play(); + } + } + break; + } + } + }); + _tts = sherpa_onnx.OfflineTts(config); + } + + static Future generate({required String text, int sid = 0, double speed = 1.0}) async { + ReceivePort receivePort = ReceivePort(); + _sendPort.send(_PortModel( + method: 'generate', + data: {'text': text, 'sid': sid, 'speed': speed}, + sendPort: receivePort.sendPort, + )); + await receivePort.first; + receivePort.close(); + } +} + +/// 这里是页面 +class IsolateTtsView extends StatefulWidget { + const IsolateTtsView({super.key}); + + @override + State createState() => _IsolateTtsViewState(); +} + +class _IsolateTtsViewState extends State { + @override + void initState() { + super.initState(); + IsolateTts.init(); + } + + @override + Widget build(BuildContext context) { + return Scaffold( + body: Center( + child: ElevatedButton( + onPressed: () { + IsolateTts.generate(text: '这是已退出的 isolate TTS'); + }, + child: Text('Isolate TTS'), + ), + ), + ); + } +} diff --git a/flutter-examples/tts/lib/main.dart b/flutter-examples/tts/lib/main.dart index 91bc120e8..78042254a 100644 --- a/flutter-examples/tts/lib/main.dart +++ b/flutter-examples/tts/lib/main.dart @@ -1,8 +1,9 @@ // Copyright (c) 2024 Xiaomi Corporation import 'package:flutter/material.dart'; -import './tts.dart'; import './info.dart'; +import './tts.dart'; +import 'isolate_tts.dart'; void main() { runApp(const MyApp()); @@ -38,6 +39,7 @@ class _MyHomePageState extends State { final List _tabs = [ TtsScreen(), InfoScreen(), + IsolateTtsView(), ]; @override Widget build(BuildContext context) { @@ -62,6 +64,10 @@ class _MyHomePageState extends State { icon: Icon(Icons.info), label: 'Info', ), + BottomNavigationBarItem( + icon: Icon(Icons.multiline_chart), + label: 'isolate', + ), ], ), ); diff --git a/flutter-examples/tts/lib/model.dart b/flutter-examples/tts/lib/model.dart index 16ada98c3..7a53934cc 100644 --- a/flutter-examples/tts/lib/model.dart +++ b/flutter-examples/tts/lib/model.dart @@ -79,17 +79,16 @@ Future createOfflineTts() async { // Example 7 // https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-melo-tts-zh_en.tar.bz2 - // modelDir = 'vits-melo-tts-zh_en'; - // modelName = 'model.onnx'; - // lexicon = 'lexicon.txt'; - // dictDir = 'vits-melo-tts-zh_en/dict'; + modelDir = 'vits-melo-tts-zh_en'; + modelName = 'model.onnx'; + lexicon = 'lexicon.txt'; + dictDir = 'vits-melo-tts-zh_en/dict'; // ============================================================ // Please don't change the remaining part of this function // ============================================================ if (modelName == '') { - throw Exception( - 'You are supposed to select a model by changing the code before you run the app'); + throw Exception('You are supposed to select a model by changing the code before you run the app'); } final Directory directory = await getApplicationDocumentsDirectory(); diff --git a/flutter-examples/tts/lib/tts.dart b/flutter-examples/tts/lib/tts.dart index 342bf070b..cdf799612 100644 --- a/flutter-examples/tts/lib/tts.dart +++ b/flutter-examples/tts/lib/tts.dart @@ -77,9 +77,7 @@ class _TtsScreenState extends State { onTapOutside: (PointerDownEvent event) { FocusManager.instance.primaryFocus?.unfocus(); }, - inputFormatters: [ - FilteringTextInputFormatter.digitsOnly - ]), + inputFormatters: [FilteringTextInputFormatter.digitsOnly]), Slider( // decoration: InputDecoration( // labelText: "speech speed", @@ -108,125 +106,117 @@ class _TtsScreenState extends State { }, ), const SizedBox(height: 5), - Row( - mainAxisAlignment: MainAxisAlignment.center, - children: [ - OutlinedButton( - child: Text("Generate"), - onPressed: () async { - await _init(); - await _player?.stop(); - - setState(() { - _maxSpeakerID = _tts?.numSpeakers ?? 0; - if (_maxSpeakerID > 0) { - _maxSpeakerID -= 1; - } - }); - - if (_tts == null) { - _controller_hint.value = TextEditingValue( - text: 'Failed to initialize tts', - ); - return; - } - - _controller_hint.value = TextEditingValue( - text: '', - ); - - final text = _controller_text_input.text.trim(); - if (text == '') { - _controller_hint.value = TextEditingValue( - text: 'Please first input your text to generate', - ); - return; - } - - final sid = - int.tryParse(_controller_sid.text.trim()) ?? 0; - - final stopwatch = Stopwatch(); - stopwatch.start(); - final audio = - _tts!.generate(text: text, sid: sid, speed: _speed); - final suffix = - '-sid-$sid-speed-${_speed.toStringAsPrecision(2)}'; - final filename = await generateWaveFilename(suffix); - - final ok = sherpa_onnx.writeWave( - filename: filename, - samples: audio.samples, - sampleRate: audio.sampleRate, - ); - - if (ok) { - stopwatch.stop(); - double elapsed = - stopwatch.elapsed.inMilliseconds.toDouble(); - - double waveDuration = - audio.samples.length.toDouble() / - audio.sampleRate.toDouble(); - - _controller_hint.value = TextEditingValue( - text: 'Saved to\n$filename\n' - 'Elapsed: ${(elapsed / 1000).toStringAsPrecision(4)} s\n' - 'Wave duration: ${waveDuration.toStringAsPrecision(4)} s\n' - 'RTF: ${(elapsed / 1000).toStringAsPrecision(4)}/${waveDuration.toStringAsPrecision(4)} ' - '= ${(elapsed / 1000 / waveDuration).toStringAsPrecision(3)} ', - ); - _lastFilename = filename; - - await _player?.play(DeviceFileSource(_lastFilename)); - } else { - _controller_hint.value = TextEditingValue( - text: 'Failed to save generated audio', - ); - } - }, - ), - const SizedBox(width: 5), - OutlinedButton( - child: Text("Clear"), - onPressed: () { - _controller_text_input.value = TextEditingValue( - text: '', - ); - - _controller_hint.value = TextEditingValue( - text: '', - ); - }, - ), - const SizedBox(width: 5), - OutlinedButton( - child: Text("Play"), - onPressed: () async { - if (_lastFilename == '') { - _controller_hint.value = TextEditingValue( - text: 'No generated wave file found', - ); - return; - } - await _player?.stop(); - await _player?.play(DeviceFileSource(_lastFilename)); - _controller_hint.value = TextEditingValue( - text: 'Playing\n$_lastFilename', - ); - }, - ), - const SizedBox(width: 5), - OutlinedButton( - child: Text("Stop"), - onPressed: () async { - await _player?.stop(); - _controller_hint.value = TextEditingValue( - text: '', - ); - }, - ), - ]), + Row(mainAxisAlignment: MainAxisAlignment.center, children: [ + OutlinedButton( + child: Text("Generate"), + onPressed: () async { + await _init(); + await _player?.stop(); + + setState(() { + _maxSpeakerID = _tts?.numSpeakers ?? 0; + if (_maxSpeakerID > 0) { + _maxSpeakerID -= 1; + } + }); + + if (_tts == null) { + _controller_hint.value = TextEditingValue( + text: 'Failed to initialize tts', + ); + return; + } + + _controller_hint.value = TextEditingValue( + text: '', + ); + + final text = _controller_text_input.text.trim(); + if (text == '') { + _controller_hint.value = TextEditingValue( + text: 'Please first input your text to generate', + ); + return; + } + + final sid = int.tryParse(_controller_sid.text.trim()) ?? 0; + + final stopwatch = Stopwatch(); + stopwatch.start(); + final audio = _tts!.generate(text: text, sid: sid, speed: _speed); + final suffix = '-sid-$sid-speed-${_speed.toStringAsPrecision(2)}'; + final filename = await generateWaveFilename(suffix); + + final ok = sherpa_onnx.writeWave( + filename: filename, + samples: audio.samples, + sampleRate: audio.sampleRate, + ); + + if (ok) { + stopwatch.stop(); + double elapsed = stopwatch.elapsed.inMilliseconds.toDouble(); + + double waveDuration = audio.samples.length.toDouble() / audio.sampleRate.toDouble(); + + _controller_hint.value = TextEditingValue( + text: 'Saved to\n$filename\n' + 'Elapsed: ${(elapsed / 1000).toStringAsPrecision(4)} s\n' + 'Wave duration: ${waveDuration.toStringAsPrecision(4)} s\n' + 'RTF: ${(elapsed / 1000).toStringAsPrecision(4)}/${waveDuration.toStringAsPrecision(4)} ' + '= ${(elapsed / 1000 / waveDuration).toStringAsPrecision(3)} ', + ); + _lastFilename = filename; + + await _player?.play(DeviceFileSource(_lastFilename)); + } else { + _controller_hint.value = TextEditingValue( + text: 'Failed to save generated audio', + ); + } + }, + ), + const SizedBox(width: 5), + OutlinedButton( + child: Text("Clear"), + onPressed: () { + _controller_text_input.value = TextEditingValue( + text: '', + ); + + _controller_hint.value = TextEditingValue( + text: '', + ); + }, + ), + const SizedBox(width: 5), + OutlinedButton( + child: Text("Play"), + onPressed: () async { + if (_lastFilename == '') { + _controller_hint.value = TextEditingValue( + text: 'No generated wave file found', + ); + return; + } + await _player?.stop(); + await _player?.play(DeviceFileSource(_lastFilename)); + _controller_hint.value = TextEditingValue( + text: 'Playing\n$_lastFilename', + ); + }, + ), + const SizedBox(width: 5), + OutlinedButton( + child: Text("Stop"), + onPressed: () async { + await _player?.stop(); + _controller_hint.value = TextEditingValue( + text: '', + ); + }, + ), + ]), const SizedBox(height: 5), TextField( decoration: InputDecoration( diff --git a/flutter-examples/tts/pubspec.yaml b/flutter-examples/tts/pubspec.yaml index d3b1e26f6..b65d08381 100644 --- a/flutter-examples/tts/pubspec.yaml +++ b/flutter-examples/tts/pubspec.yaml @@ -24,6 +24,12 @@ dependencies: url_launcher: 6.2.6 url_launcher_linux: 3.1.0 audioplayers: ^5.0.0 + media_kit: + media_kit_libs_video: flutter: uses-material-design: true + + assets: + - assets/vits-melo-tts-zh_en/ + - assets/vits-melo-tts-zh_en/dict/ \ No newline at end of file From 8436ba834c9131a1e10f031174e2644fc85f90ad Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 13 Nov 2024 21:06:50 +0800 Subject: [PATCH 071/183] Add WebAssembly example for VAD + Moonshine models. (#1535) --- README.md | 6 +++++- scripts/wasm/generate-vad-asr.py | 18 ++++++++++++++++++ wasm/vad-asr/app-vad-asr.js | 7 +++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ebc90bca8..20882d636 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,7 @@ We also have spaces built using WebAssembly. They are listed below: |Real-time speech recognition (English) |[Click me][wasm-hf-streaming-asr-en-zipformer] |[地址][wasm-ms-streaming-asr-en-zipformer]| |VAD + speech recognition (Chinese + English + Korean + Japanese + Cantonese) with [SenseVoice][SenseVoice]|[Click me][wasm-hf-vad-asr-zh-en-ko-ja-yue-sense-voice]| [地址][wasm-ms-vad-asr-zh-en-ko-ja-yue-sense-voice]| |VAD + speech recognition (English) with [Whisper][Whisper] tiny.en|[Click me][wasm-hf-vad-asr-en-whisper-tiny-en]| [地址][wasm-ms-vad-asr-en-whisper-tiny-en]| +|VAD + speech recognition (English) with [Moonshine tiny][Moonshine tiny]|[Click me][wasm-hf-vad-asr-en-moonshine-tiny-en]| [地址][wasm-ms-vad-asr-en-moonshine-tiny-en]| |VAD + speech recognition (English) with Zipformer trained with [GigaSpeech][GigaSpeech] |[Click me][wasm-hf-vad-asr-en-zipformer-gigaspeech]| [地址][wasm-ms-vad-asr-en-zipformer-gigaspeech]| |VAD + speech recognition (Chinese) with Zipformer trained with [WenetSpeech][WenetSpeech] |[Click me][wasm-hf-vad-asr-zh-zipformer-wenetspeech]| [地址][wasm-ms-vad-asr-zh-zipformer-wenetspeech]| |VAD + speech recognition (Japanese) with Zipformer trained with [ReazonSpeech][ReazonSpeech]|[Click me][wasm-hf-vad-asr-ja-zipformer-reazonspeech]| [地址][wasm-ms-vad-asr-ja-zipformer-reazonspeech]| @@ -240,7 +241,7 @@ for more models. The following table lists only **SOME** of them. |Name | Supported Languages| Description| |-----|-----|----| |[Whisper tiny.en](https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2)|English| See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/whisper/tiny.en.html)| -|[Moonshine tiny](https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2)|English|See [also](https://github.com/usefulsensors/moonshine)| +|[Moonshine tiny][Moonshine tiny]|English|See [also](https://github.com/usefulsensors/moonshine)| |[sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17][sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17]|Chinese, Cantonese, English, Korean, Japanese| 支持多种中文方言. See [also](https://k2-fsa.github.io/sherpa/onnx/sense-voice/index.html)| |[sherpa-onnx-paraformer-zh-2024-03-09][sherpa-onnx-paraformer-zh-2024-03-09]|Chinese, English| 也支持多种中文方言. See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-paraformer/paraformer-models.html#csukuangfj-sherpa-onnx-paraformer-zh-2024-03-09-chinese-english)| |[sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01][sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01]|Japanese|See [also](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01-japanese)| @@ -320,6 +321,8 @@ Video demo in Chinese: [爆了！炫神教你开打字挂！真正影响胜率 [wasm-ms-vad-asr-zh-en-ko-ja-yue-sense-voice]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-zh-en-jp-ko-cantonese-sense-voice [wasm-hf-vad-asr-en-whisper-tiny-en]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-en-whisper-tiny [wasm-ms-vad-asr-en-whisper-tiny-en]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-en-whisper-tiny +[wasm-hf-vad-asr-en-moonshine-tiny-en]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-en-moonshine-tiny +[wasm-ms-vad-asr-en-moonshine-tiny-en]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-vad-asr-sherpa-onnx-en-moonshine-tiny [wasm-hf-vad-asr-en-zipformer-gigaspeech]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-en-zipformer-gigaspeech [wasm-ms-vad-asr-en-zipformer-gigaspeech]: https://www.modelscope.cn/studios/k2-fsa/web-assembly-vad-asr-sherpa-onnx-en-zipformer-gigaspeech [wasm-hf-vad-asr-zh-zipformer-wenetspeech]: https://huggingface.co/spaces/k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-zipformer-wenetspeech @@ -405,3 +408,4 @@ Video demo in Chinese: [爆了！炫神教你开打字挂！真正影响胜率 [sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04.tar.bz2 [sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 [sherpa-onnx-streaming-zipformer-fr-2023-04-14]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-fr-2023-04-14.tar.bz2 +[Moonshine tiny]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 diff --git a/scripts/wasm/generate-vad-asr.py b/scripts/wasm/generate-vad-asr.py index 4c0099af8..6176e0840 100755 --- a/scripts/wasm/generate-vad-asr.py +++ b/scripts/wasm/generate-vad-asr.py @@ -51,6 +51,24 @@ def get_models(): git diff """, ), + Model( + model_name="sherpa-onnx-moonshine-tiny-en-int8", + hf="k2-fsa/web-assembly-vad-asr-sherpa-onnx-en-moonshine-tiny", + ms="csukuangfj/web-assembly-vad-asr-sherpa-onnx-en-moonshine-tiny", + short_name="vad-asr-en-moonshine_tiny", + cmd=""" + pushd $model_name + mv -v preprocess.onnx ../moonshine-preprocessor.onnx + mv -v encode.int8.onnx ../moonshine-encoder.onnx + mv -v uncached_decode.int8.onnx ../moonshine-uncached-decoder.onnx + mv -v cached_decode.int8.onnx ../moonshine-cached-decoder.onnx + mv -v tokens.txt ../ + popd + rm -rf $model_name + sed -i.bak 's/Zipformer/Moonshine tiny supporting English 英文/g' ../index.html + git diff + """, + ), Model( model_name="sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17", hf="k2-fsa/web-assembly-vad-asr-sherpa-onnx-zh-en-ja-ko-cantonese-sense-voice", diff --git a/wasm/vad-asr/app-vad-asr.js b/wasm/vad-asr/app-vad-asr.js index 5cb172e64..68b7b7da1 100644 --- a/wasm/vad-asr/app-vad-asr.js +++ b/wasm/vad-asr/app-vad-asr.js @@ -111,6 +111,13 @@ function initOfflineRecognizer() { }; } else if (fileExists('telespeech.onnx')) { config.modelConfig.telespeechCtc = './telespeech.onnx'; + } else if (fileExists('moonshine-preprocessor.onnx')) { + config.modelConfig.moonshine = { + preprocessor: './moonshine-preprocessor.onnx', + encoder: './moonshine-encoder.onnx', + uncachedDecoder: './moonshine-uncached-decoder.onnx', + cachedDecoder: './moonshine-cached-decoder.onnx' + }; } else { console.log('Please specify a model.'); alert('Please specify a model.'); From c34ab3559187a065f0e607c3d120ed203f1a4bcc Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 14 Nov 2024 20:57:35 +0800 Subject: [PATCH 072/183] Add Android APK for streaming Paraformer ASR (#1538) --- .github/workflows/apk-asr.yaml | 4 ++-- scripts/apk/generate-asr-apk-script.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/apk-asr.yaml b/.github/workflows/apk-asr.yaml index c51ce1e20..e49b179c8 100644 --- a/.github/workflows/apk-asr.yaml +++ b/.github/workflows/apk-asr.yaml @@ -23,8 +23,8 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - total: ["3"] - index: ["0", "1", "2"] + total: ["6"] + index: ["0", "1", "2", "3", "4", "5"] steps: - uses: actions/checkout@v4 diff --git a/scripts/apk/generate-asr-apk-script.py b/scripts/apk/generate-asr-apk-script.py index 608532176..bda643c01 100755 --- a/scripts/apk/generate-asr-apk-script.py +++ b/scripts/apk/generate-asr-apk-script.py @@ -243,6 +243,23 @@ def get_models(): ls -lh + popd + """, + ), + Model( + model_name="sherpa-onnx-streaming-paraformer-bilingual-zh-en", + idx=5, + lang="zh_en", + short_name="paraformer", + cmd=""" + pushd $model_name + rm -fv decoder.onnx + rm -fv encoder.onnx + + rm -rfv test_wavs + + ls -lh + popd """, ), From b28b0c81b19d7b392ebca37dc07d9af893545667 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 15 Nov 2024 16:06:17 +0800 Subject: [PATCH 073/183] Support static build for windows arm64. (#1539) --- .github/workflows/windows-arm64.yaml | 2 +- cmake/onnxruntime-win-arm64-static.cmake | 72 ++++++++++++++++++++++++ cmake/onnxruntime.cmake | 7 ++- 3 files changed, 77 insertions(+), 4 deletions(-) create mode 100644 cmake/onnxruntime-win-arm64-static.cmake diff --git a/.github/workflows/windows-arm64.yaml b/.github/workflows/windows-arm64.yaml index a6d2a96da..105260ce2 100644 --- a/.github/workflows/windows-arm64.yaml +++ b/.github/workflows/windows-arm64.yaml @@ -34,7 +34,7 @@ jobs: fail-fast: false matrix: os: [windows-latest] - shared_lib: [ON] + shared_lib: [ON, OFF] with_tts: [ON, OFF] steps: diff --git a/cmake/onnxruntime-win-arm64-static.cmake b/cmake/onnxruntime-win-arm64-static.cmake new file mode 100644 index 000000000..0ebbfc29a --- /dev/null +++ b/cmake/onnxruntime-win-arm64-static.cmake @@ -0,0 +1,72 @@ +# Copyright (c) 2022-2023 Xiaomi Corporation +message(STATUS "CMAKE_SYSTEM_NAME: ${CMAKE_SYSTEM_NAME}") +message(STATUS "CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}") +message(STATUS "CMAKE_VS_PLATFORM_NAME: ${CMAKE_VS_PLATFORM_NAME}") + +if(NOT CMAKE_SYSTEM_NAME STREQUAL Windows) + message(FATAL_ERROR "This file is for Windows only. Given: ${CMAKE_SYSTEM_NAME}") +endif() + +if(NOT (CMAKE_VS_PLATFORM_NAME STREQUAL ARM64 OR CMAKE_VS_PLATFORM_NAME STREQUAL arm64)) + message(FATAL_ERROR "This file is for Windows arm64 only. Given: ${CMAKE_VS_PLATFORM_NAME}") +endif() + +if(BUILD_SHARED_LIBS) + message(FATAL_ERROR "This file is for building static libraries. BUILD_SHARED_LIBS: ${BUILD_SHARED_LIBS}") +endif() + +if(NOT CMAKE_BUILD_TYPE STREQUAL Release) + message(FATAL_ERROR "This file is for building a release version on Windows arm64") +endif() + +set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.17.1/onnxruntime-win-arm64-static_lib-1.17.1.tar.bz2") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-win-arm64-static_lib-1.17.1.tar.bz2") +set(onnxruntime_HASH "SHA256=534ab5bb8b5495ce45fed866cf3ec9034f89f2057a0152e49120b1088003a17e") + +# If you don't have access to the Internet, +# please download onnxruntime to one of the following locations. +# You can add more if you want. +set(possible_file_locations + $ENV{HOME}/Downloads/onnxruntime-win-arm64-static_lib-1.17.1.tar.bz2 + ${CMAKE_SOURCE_DIR}/onnxruntime-win-arm64-static_lib-1.17.1.tar.bz2 + ${CMAKE_BINARY_DIR}/onnxruntime-win-arm64-static_lib-1.17.1.tar.bz2 + /tmp/onnxruntime-win-arm64-static_lib-1.17.1.tar.bz2 +) + +foreach(f IN LISTS possible_file_locations) + if(EXISTS ${f}) + set(onnxruntime_URL "${f}") + file(TO_CMAKE_PATH "${onnxruntime_URL}" onnxruntime_URL) + message(STATUS "Found local downloaded onnxruntime: ${onnxruntime_URL}") + set(onnxruntime_URL2) + break() + endif() +endforeach() + +FetchContent_Declare(onnxruntime + URL + ${onnxruntime_URL} + ${onnxruntime_URL2} + URL_HASH ${onnxruntime_HASH} +) + +FetchContent_GetProperties(onnxruntime) +if(NOT onnxruntime_POPULATED) + message(STATUS "Downloading onnxruntime from ${onnxruntime_URL}") + FetchContent_Populate(onnxruntime) +endif() +message(STATUS "onnxruntime is downloaded to ${onnxruntime_SOURCE_DIR}") + +# for static libraries, we use onnxruntime_lib_files directly below +include_directories(${onnxruntime_SOURCE_DIR}/include) + +file(GLOB onnxruntime_lib_files "${onnxruntime_SOURCE_DIR}/lib/*.lib") + +set(onnxruntime_lib_files ${onnxruntime_lib_files} PARENT_SCOPE) + +message(STATUS "onnxruntime lib files: ${onnxruntime_lib_files}") +if(SHERPA_ONNX_ENABLE_PYTHON) + install(FILES ${onnxruntime_lib_files} DESTINATION ..) +else() + install(FILES ${onnxruntime_lib_files} DESTINATION lib) +endif() diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake index 8453b96bd..14b47bd21 100644 --- a/cmake/onnxruntime.cmake +++ b/cmake/onnxruntime.cmake @@ -91,10 +91,11 @@ function(download_onnxruntime) endif() elseif(CMAKE_VS_PLATFORM_NAME STREQUAL ARM64 OR CMAKE_VS_PLATFORM_NAME STREQUAL arm64) # for 64-bit windows (arm64) - if(NOT BUILD_SHARED_LIBS) - message(FATAL_ERROR "Please pass -DBUILD_SHARED_LIBS=ON to cmake") + if(BUILD_SHARED_LIBS) + include(onnxruntime-win-arm64) + else() + include(onnxruntime-win-arm64-static) endif() - include(onnxruntime-win-arm64) else() # for 64-bit windows (x64) if(SHERPA_ONNX_ENABLE_DIRECTML) From e993c0853824ff3bdc8332ff2a1416402fd97755 Mon Sep 17 00:00:00 2001 From: Anders Xiao Date: Sat, 16 Nov 2024 11:57:10 +0800 Subject: [PATCH 074/183] fix windows build (#1546) --- cmake/onnxruntime.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake index 14b47bd21..6ed15c29c 100644 --- a/cmake/onnxruntime.cmake +++ b/cmake/onnxruntime.cmake @@ -152,6 +152,8 @@ if(SHERPA_ONNX_USE_PRE_INSTALLED_ONNXRUNTIME_IF_AVAILABLE) if(DEFINED ENV{SHERPA_ONNXRUNTIME_LIB_DIR}) if(APPLE) set(location_onnxruntime_lib $ENV{SHERPA_ONNXRUNTIME_LIB_DIR}/libonnxruntime.dylib) + elseif(WIN32) + set(location_onnxruntime_lib $ENV{SHERPA_ONNXRUNTIME_LIB_DIR}/onnxruntime.lib) else() set(location_onnxruntime_lib $ENV{SHERPA_ONNXRUNTIME_LIB_DIR}/libonnxruntime.so) endif() @@ -198,6 +200,7 @@ if(location_onnxruntime_header_dir AND location_onnxruntime_lib) add_library(onnxruntime SHARED IMPORTED) set_target_properties(onnxruntime PROPERTIES IMPORTED_LOCATION ${location_onnxruntime_lib} + IMPORTED_IMPLIB ${location_onnxruntime_lib} INTERFACE_INCLUDE_DIRECTORIES "${location_onnxruntime_header_dir}" ) if(SHERPA_ONNX_ENABLE_GPU AND location_onnxruntime_cuda_lib) From 9a480126481ee2d0bd12d9a4a07d2afae89f8723 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sat, 16 Nov 2024 16:42:02 +0800 Subject: [PATCH 075/183] Use xcframework for Flutter iOS plugin. (#1547) --- .github/workflows/release-dart-package.yaml | 7 +- CHANGELOG.md | 16 ++ CMakeLists.txt | 5 +- build-ios-shared.sh | 146 +++++++++++++++++- .../add-punctuations/pubspec.yaml | 2 +- dart-api-examples/audio-tagging/pubspec.yaml | 2 +- .../keyword-spotter/pubspec.yaml | 2 +- .../non-streaming-asr/pubspec.yaml | 2 +- .../speaker-diarization/pubspec.yaml | 2 +- .../speaker-identification/pubspec.yaml | 2 +- dart-api-examples/streaming-asr/pubspec.yaml | 2 +- dart-api-examples/tts/pubspec.yaml | 2 +- .../vad-with-non-streaming-asr/pubspec.yaml | 2 +- dart-api-examples/vad/pubspec.yaml | 2 +- flutter-examples/README.md | 10 +- flutter-examples/streaming_asr/pubspec.yaml | 4 +- flutter-examples/tts/lib/model.dart | 11 +- flutter-examples/tts/pubspec.yaml | 4 +- flutter/sherpa_onnx/lib/sherpa_onnx.dart | 10 +- flutter/sherpa_onnx/pubspec.yaml | 12 +- .../ios/sherpa_onnx_ios.podspec | 5 +- .../macos/sherpa_onnx_macos.podspec | 2 +- new-release.sh | 11 +- nodejs-addon-examples/package.json | 2 +- 24 files changed, 216 insertions(+), 49 deletions(-) diff --git a/.github/workflows/release-dart-package.yaml b/.github/workflows/release-dart-package.yaml index f590403fe..cc830e2c2 100644 --- a/.github/workflows/release-dart-package.yaml +++ b/.github/workflows/release-dart-package.yaml @@ -481,11 +481,8 @@ jobs: - name: Copy pre-built libs shell: bash run: | - echo "----ios-arm64----" - cp -v build-ios-shared/ios-arm64/libsherpa-onnx-c-api.dylib flutter/sherpa_onnx_ios/ios/ - cp -v build-ios-shared/ios-onnxruntime/onnxruntime.xcframework/ios-arm64/onnxruntime.a flutter/sherpa_onnx_ios/ios/libonnxruntime.a - - ls -lh flutter/sherpa_onnx_ios/ios/libonnxruntime.a + echo "----ios arm64 and arm64_x64_simulator----" + cp -av build-ios-shared/sherpa_onnx.xcframework flutter/sherpa_onnx_ios/ios/ mv -v flutter/sherpa_onnx_ios /tmp/to_be_published diff --git a/CHANGELOG.md b/CHANGELOG.md index 8787fda3a..669f2e405 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,19 @@ +## 1.10.31 + +* Publish pre-built wheels for Python 3.13 (#1485) +* Publish pre-built macos xcframework (#1490) +* Fix reading tokens.txt on Windows. (#1497) +* Add two-pass ASR Android APKs for Moonshine models. (#1499) +* Support building GPU-capable sherpa-onnx on Linux aarch64. (#1500) +* Publish pre-built wheels with CUDA support for Linux aarch64. (#1507) +* Export the English TTS model from MeloTTS (#1509) +* Add Lazarus example for Moonshine models. (#1532) +* Add isolate_tts demo (#1529) +* Add WebAssembly example for VAD + Moonshine models. (#1535) +* Add Android APK for streaming Paraformer ASR (#1538) +* Support static build for windows arm64. (#1539) +* Use xcframework for Flutter iOS plugin to support iOS simulators. + ## 1.10.30 * Fix building node-addon for Windows x86. (#1469) diff --git a/CMakeLists.txt b/CMakeLists.txt index ac357b746..4adc3dca5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,10 +8,9 @@ set(CMAKE_POLICY_DEFAULT_CMP0069 NEW) project(sherpa-onnx) # Remember to update -# ./nodejs-addon-examples -# ./dart-api-examples/ # ./CHANGELOG.md -set(SHERPA_ONNX_VERSION "1.10.30") +# ./new-release.sh +set(SHERPA_ONNX_VERSION "1.10.31") # Disable warning about # diff --git a/build-ios-shared.sh b/build-ios-shared.sh index 2af1b36f0..c58bb300d 100755 --- a/build-ios-shared.sh +++ b/build-ios-shared.sh @@ -76,6 +76,8 @@ if [[ ! -f build/simulator_x86_64/install/lib/libsherpa-onnx-c-api.dylib ]]; the -B build/simulator_x86_64 cmake --build build/simulator_x86_64 -j 4 --target install +else + echo "Skip building for simulator (x86_64)" fi echo "Building for simulator (arm64)" @@ -107,6 +109,8 @@ if [[ ! -f build/simulator_arm64/install/lib/libsherpa-onnx-c-api.dylib ]]; then -B build/simulator_arm64 cmake --build build/simulator_arm64 -j 4 --target install +else + echo "Skip building for simulator (arm64)" fi echo "Building for arm64" @@ -140,11 +144,149 @@ if [[ ! -f build/os64/install/lib/libsherpa-onnx-c-api.dylib ]]; then -B build/os64 cmake --build build/os64 -j 4 --target install +else + echo "Skip building for arm64" fi echo "Collect dynamic libraries " mkdir -p ios-arm64 ios-arm64-simulator ios-x86_64-simulator cp -v ./build/os64/install/lib/libsherpa-onnx-c-api.dylib ios-arm64/ -cp -v ./build/simulator_arm64/install/lib/libsherpa-onnx-c-api.dylib ios-arm64-simulator -cp -v .//build/simulator_x86_64/install/lib/libsherpa-onnx-c-api.dylib ios-x86_64-simulator +cp -v ./build/simulator_arm64/install/lib/libsherpa-onnx-c-api.dylib ios-arm64-simulator/ +cp -v .//build/simulator_x86_64/install/lib/libsherpa-onnx-c-api.dylib ios-x86_64-simulator/ + +# see https://github.com/k2-fsa/sherpa-onnx/issues/1172#issuecomment-2439662662 +rm -rf ios-arm64_x86_64-simulator +mkdir ios-arm64_x86_64-simulator + +lipo \ + -create \ + ios-arm64-simulator/libsherpa-onnx-c-api.dylib \ + ios-x86_64-simulator/libsherpa-onnx-c-api.dylib \ + -output \ + ios-arm64_x86_64-simulator/libsherpa-onnx-c-api.dylib + +pushd ios-arm64 +rm -rf sherpa_onnx.framework +mkdir sherpa_onnx.framework + +lipo \ + -create \ + libsherpa-onnx-c-api.dylib \ + -output \ + sherpa_onnx + +mv sherpa_onnx sherpa_onnx.framework/ +cd sherpa_onnx.framework + +install_name_tool \ + -change @rpath/libsherpa-onnx-c-api.dylib @rpath/sherpa_onnx.framework/sherpa_onnx \ + sherpa_onnx + +install_name_tool \ + -id "@rpath/sherpa_onnx.framework/sherpa_onnx" \ + sherpa_onnx + +chmod +x sherpa_onnx +popd + +pushd ios-arm64_x86_64-simulator +rm -rf sherpa_onnx.framework +mkdir sherpa_onnx.framework + +lipo \ + -create \ + libsherpa-onnx-c-api.dylib \ + -output \ + sherpa_onnx + +mv sherpa_onnx sherpa_onnx.framework/ +cd sherpa_onnx.framework +install_name_tool \ + -change @rpath/libsherpa-onnx-c-api.dylib @rpath/sherpa_onnx.framework/sherpa_onnx \ + sherpa_onnx + +install_name_tool \ + -id "@rpath/sherpa_onnx.framework/sherpa_onnx" \ + sherpa_onnx + +chmod +x sherpa_onnx +popd + +for d in ios-arm64_x86_64-simulator ios-arm64; do + dst=$d/sherpa_onnx.framework + + # The Info.plist is modified from + # https://github.com/Spicely/flutter_openim_sdk_ffi/blob/main/ios/openim_sdk_ffi.framework/Info.plist + cat >$dst/Info.plist < + + + + CFBundleName + sherpa_onnx + DTSDKName + iphoneos17.0 + DTXcode + 1501 + DTSDKBuild + 21A326 + CFBundleDevelopmentRegion + en + CFBundleVersion + 1 + BuildMachineOSBuild + 23B81 + DTPlatformName + iphoneos + CFBundlePackageType + FMWK + CFBundleShortVersionString + 1.10.31 + CFBundleSupportedPlatforms + + iPhoneOS + + CFBundleInfoDictionaryVersion + 6.0 + CFBundleExecutable + sherpa_onnx + DTCompiler + com.apple.compilers.llvm.clang.1_0 + UIRequiredDeviceCapabilities + + arm64 + + MinimumOSVersion + 12.0 + CFBundleIdentifier + com.k2fsa.sherpa.onnx + UIDeviceFamily + + 1 + 2 + + CFBundleSignature + ???? + DTPlatformVersion + 17.0 + DTXcodeBuild + 15A507 + DTPlatformBuild + 21A326 + + +EOF +done + +rm -rf sherpa_onnx.xcframework +xcodebuild -create-xcframework \ + -framework ios-arm64/sherpa_onnx.framework \ + -framework ios-arm64_x86_64-simulator/sherpa_onnx.framework \ + -output sherpa_onnx.xcframework + +cd sherpa_onnx.xcframework +echo "PWD: $PWD" +ls -lh +echo "---" +ls -lh */* diff --git a/dart-api-examples/add-punctuations/pubspec.yaml b/dart-api-examples/add-punctuations/pubspec.yaml index 9cf4cf28e..55e8c5d98 100644 --- a/dart-api-examples/add-punctuations/pubspec.yaml +++ b/dart-api-examples/add-punctuations/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.30 + sherpa_onnx: ^1.10.31 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/audio-tagging/pubspec.yaml b/dart-api-examples/audio-tagging/pubspec.yaml index 06aff0384..a79e925b1 100644 --- a/dart-api-examples/audio-tagging/pubspec.yaml +++ b/dart-api-examples/audio-tagging/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.30 + sherpa_onnx: ^1.10.31 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/keyword-spotter/pubspec.yaml b/dart-api-examples/keyword-spotter/pubspec.yaml index 28d010ac8..d79f68486 100644 --- a/dart-api-examples/keyword-spotter/pubspec.yaml +++ b/dart-api-examples/keyword-spotter/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.30 + sherpa_onnx: ^1.10.31 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/non-streaming-asr/pubspec.yaml b/dart-api-examples/non-streaming-asr/pubspec.yaml index 71a1f05ed..3c61a0cb2 100644 --- a/dart-api-examples/non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.30 + sherpa_onnx: ^1.10.31 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/speaker-diarization/pubspec.yaml b/dart-api-examples/speaker-diarization/pubspec.yaml index 28e0c39d6..6fb714d10 100644 --- a/dart-api-examples/speaker-diarization/pubspec.yaml +++ b/dart-api-examples/speaker-diarization/pubspec.yaml @@ -8,7 +8,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.30 + sherpa_onnx: ^1.10.31 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/speaker-identification/pubspec.yaml b/dart-api-examples/speaker-identification/pubspec.yaml index 21cbea554..36969fb1d 100644 --- a/dart-api-examples/speaker-identification/pubspec.yaml +++ b/dart-api-examples/speaker-identification/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.30 + sherpa_onnx: ^1.10.31 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/streaming-asr/pubspec.yaml b/dart-api-examples/streaming-asr/pubspec.yaml index 9f837d92e..ec6270297 100644 --- a/dart-api-examples/streaming-asr/pubspec.yaml +++ b/dart-api-examples/streaming-asr/pubspec.yaml @@ -11,7 +11,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.30 + sherpa_onnx: ^1.10.31 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/tts/pubspec.yaml b/dart-api-examples/tts/pubspec.yaml index 607db08ee..ed55db1eb 100644 --- a/dart-api-examples/tts/pubspec.yaml +++ b/dart-api-examples/tts/pubspec.yaml @@ -8,7 +8,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.30 + sherpa_onnx: ^1.10.31 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml index 9328dfca2..6d331d8f5 100644 --- a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.30 + sherpa_onnx: ^1.10.31 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad/pubspec.yaml b/dart-api-examples/vad/pubspec.yaml index 45ee33487..b16bcb64b 100644 --- a/dart-api-examples/vad/pubspec.yaml +++ b/dart-api-examples/vad/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.30 + sherpa_onnx: ^1.10.31 path: ^1.9.0 args: ^2.5.0 diff --git a/flutter-examples/README.md b/flutter-examples/README.md index b7ed66715..3bb6b52c5 100644 --- a/flutter-examples/README.md +++ b/flutter-examples/README.md @@ -136,14 +136,16 @@ flutter create --platforms ios ./ Connect your iPhone to the computer, and run `flutter devices`, which will print: ```bash -Found 3 connected devices: - iPhone (mobile) • 00008030-001064212E85802E • ios • iOS 16.3 20D47 - macOS (desktop) • macos • darwin-x64 • macOS 13.1 22C65 darwin-x64 - Chrome (web) • chrome • web-javascript • Google Chrome 126.0.6478.127 +Found 4 connected devices: + iPhone 14 (mobile) • 634110C4-168D-408F-A938-D7FC62222579 • ios • com.apple.CoreSimulator.SimRuntime.iOS-16-2 (simulator) + iPhone (mobile) • 00008030-001064212E85802E • ios • iOS 16.3 20D47 + macOS (desktop) • macos • darwin-x64 • macOS 13.1 22C65 darwin-x64 + Chrome (web) • chrome • web-javascript • Google Chrome 126.0.6478.127 No wireless devices were found. Run "flutter emulators" to list and start any available device emulators. +(E.g., flutter emulators --launch ios) If you expected another device to be detected, please run "flutter doctor" to diagnose potential issues. You may also try increasing the time to wait for connected devices with the "--device-timeout" flag. Visit https://flutter.dev/setup/ for troubleshooting tips. diff --git a/flutter-examples/streaming_asr/pubspec.yaml b/flutter-examples/streaming_asr/pubspec.yaml index 68b000976..1eb29cbd4 100644 --- a/flutter-examples/streaming_asr/pubspec.yaml +++ b/flutter-examples/streaming_asr/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' -version: 1.10.30 +version: 1.10.31 topics: - speech-recognition @@ -31,7 +31,7 @@ dependencies: record: ^5.1.0 url_launcher: ^6.2.6 - sherpa_onnx: ^1.10.30 + sherpa_onnx: ^1.10.31 # sherpa_onnx: # path: ../../flutter/sherpa_onnx diff --git a/flutter-examples/tts/lib/model.dart b/flutter-examples/tts/lib/model.dart index 7a53934cc..16ada98c3 100644 --- a/flutter-examples/tts/lib/model.dart +++ b/flutter-examples/tts/lib/model.dart @@ -79,16 +79,17 @@ Future createOfflineTts() async { // Example 7 // https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-melo-tts-zh_en.tar.bz2 - modelDir = 'vits-melo-tts-zh_en'; - modelName = 'model.onnx'; - lexicon = 'lexicon.txt'; - dictDir = 'vits-melo-tts-zh_en/dict'; + // modelDir = 'vits-melo-tts-zh_en'; + // modelName = 'model.onnx'; + // lexicon = 'lexicon.txt'; + // dictDir = 'vits-melo-tts-zh_en/dict'; // ============================================================ // Please don't change the remaining part of this function // ============================================================ if (modelName == '') { - throw Exception('You are supposed to select a model by changing the code before you run the app'); + throw Exception( + 'You are supposed to select a model by changing the code before you run the app'); } final Directory directory = await getApplicationDocumentsDirectory(); diff --git a/flutter-examples/tts/pubspec.yaml b/flutter-examples/tts/pubspec.yaml index b65d08381..669a1ed4d 100644 --- a/flutter-examples/tts/pubspec.yaml +++ b/flutter-examples/tts/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' # Remove this line if you wish to publish to pub.dev -version: 1.10.30 +version: 1.10.31 environment: sdk: ">=2.17.0 <4.0.0" @@ -18,7 +18,7 @@ dependencies: cupertino_icons: ^1.0.6 path_provider: ^2.1.3 path: ^1.9.0 - sherpa_onnx: ^1.10.30 + sherpa_onnx: ^1.10.31 # sherpa_onnx: # path: ../../flutter/sherpa_onnx url_launcher: 6.2.6 diff --git a/flutter/sherpa_onnx/lib/sherpa_onnx.dart b/flutter/sherpa_onnx/lib/sherpa_onnx.dart index 9fcd2872f..b9fb7dd53 100644 --- a/flutter/sherpa_onnx/lib/sherpa_onnx.dart +++ b/flutter/sherpa_onnx/lib/sherpa_onnx.dart @@ -25,7 +25,7 @@ String? _path; // https://github.com/flutter/codelabs/blob/main/ffigen_codelab/step_05/lib/ffigen_app.dart // https://api.flutter.dev/flutter/dart-io/Platform-class.html final DynamicLibrary _dylib = () { - if (Platform.isMacOS || Platform.isIOS) { + if (Platform.isMacOS) { if (_path == null) { return DynamicLibrary.open('libsherpa-onnx-c-api.dylib'); } else { @@ -33,6 +33,14 @@ final DynamicLibrary _dylib = () { } } + if (Platform.isIOS) { + if (_path == null) { + return DynamicLibrary.open('sherpa_onnx.framework/sherpa_onnx'); + } else { + return DynamicLibrary.open('$_path/sherpa_onnx.framework/sherpa_onnx'); + } + } + if (Platform.isAndroid || Platform.isLinux) { if (_path == null) { return DynamicLibrary.open('libsherpa-onnx-c-api.so'); diff --git a/flutter/sherpa_onnx/pubspec.yaml b/flutter/sherpa_onnx/pubspec.yaml index 9e367fa64..675ed844a 100644 --- a/flutter/sherpa_onnx/pubspec.yaml +++ b/flutter/sherpa_onnx/pubspec.yaml @@ -17,7 +17,7 @@ topics: - voice-activity-detection # remember to change the version in ../sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec -version: 1.10.30 +version: 1.10.31 homepage: https://github.com/k2-fsa/sherpa-onnx @@ -30,23 +30,23 @@ dependencies: flutter: sdk: flutter - sherpa_onnx_android: ^1.10.30 + sherpa_onnx_android: ^1.10.31 # sherpa_onnx_android: # path: ../sherpa_onnx_android - sherpa_onnx_macos: ^1.10.30 + sherpa_onnx_macos: ^1.10.31 # sherpa_onnx_macos: # path: ../sherpa_onnx_macos - sherpa_onnx_linux: ^1.10.30 + sherpa_onnx_linux: ^1.10.31 # sherpa_onnx_linux: # path: ../sherpa_onnx_linux - sherpa_onnx_windows: ^1.10.30 + sherpa_onnx_windows: ^1.10.31 # sherpa_onnx_windows: # path: ../sherpa_onnx_windows - sherpa_onnx_ios: ^1.10.30 + sherpa_onnx_ios: ^1.10.31 # sherpa_onnx_ios: # path: ../sherpa_onnx_ios diff --git a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec index e0e0a9769..11930c4e7 100644 --- a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec +++ b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec @@ -7,7 +7,7 @@ # https://groups.google.com/g/dart-ffi/c/nUATMBy7r0c Pod::Spec.new do |s| s.name = 'sherpa_onnx_ios' - s.version = '1.10.30' + s.version = '1.10.31' s.summary = 'A new Flutter FFI plugin project.' s.description = <<-DESC A new Flutter FFI plugin project. @@ -23,7 +23,8 @@ A new Flutter FFI plugin project. s.source = { :path => '.' } s.dependency 'Flutter' s.platform = :ios, '12.0' - s.ios.vendored_libraries = '*.dylib', '*.a' + s.preserve_paths = 'sherpa_onnx.xcframework/**/*' + s.vendored_frameworks = 'sherpa_onnx.xcframework' # Flutter.framework does not contain a i386 slice. s.pod_target_xcconfig = { diff --git a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec index f74dddad9..95b0970b3 100644 --- a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec +++ b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec @@ -4,7 +4,7 @@ # Pod::Spec.new do |s| s.name = 'sherpa_onnx_macos' - s.version = '1.10.30' + s.version = '1.10.31' s.summary = 'sherpa-onnx Flutter FFI plugin project.' s.description = <<-DESC sherpa-onnx Flutter FFI plugin project. diff --git a/new-release.sh b/new-release.sh index 48095178e..7e6ed56b5 100755 --- a/new-release.sh +++ b/new-release.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.29/1\.10\.30/g' {} \; -find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.29/1\.10\.30/g' {} \; -find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.29/1\.10\.30/g' {} \; -find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.29/1\.10\.30/g' {} \; -find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.29/1\.10\.30/g' {} \; +sed -i.bak 's/1\.10\.30/1\.10\.31/g' ./build-ios-shared.sh +find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.30/1\.10\.31/g' {} \; +find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.30/1\.10\.31/g' {} \; +find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.30/1\.10\.31/g' {} \; +find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.30/1\.10\.31/g' {} \; +find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.30/1\.10\.31/g' {} \; diff --git a/nodejs-addon-examples/package.json b/nodejs-addon-examples/package.json index 3afbc2f3b..ee4404123 100644 --- a/nodejs-addon-examples/package.json +++ b/nodejs-addon-examples/package.json @@ -1,5 +1,5 @@ { "dependencies": { - "sherpa-onnx-node": "^1.10.30" + "sherpa-onnx-node": "^1.10.31" } } From e424cc9e0df1a4f03bd0da6845af6f3b891d8518 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 20 Nov 2024 10:06:49 +0800 Subject: [PATCH 076/183] Support cross-compiling for HarmonyOS (#1553) --- .github/workflows/harmony-os.yaml | 159 ++++++++++++++++++++++++++++++ CMakeLists.txt | 7 +- build-ohos-arm64-v8a.sh | 132 +++++++++++++++++++++++++ build-ohos-armeabi-v7a.sh | 136 +++++++++++++++++++++++++ build-ohos-x86-64.sh | 132 +++++++++++++++++++++++++ 5 files changed, 565 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/harmony-os.yaml create mode 100755 build-ohos-arm64-v8a.sh create mode 100755 build-ohos-armeabi-v7a.sh create mode 100755 build-ohos-x86-64.sh diff --git a/.github/workflows/harmony-os.yaml b/.github/workflows/harmony-os.yaml new file mode 100644 index 000000000..e1a2ae1a2 --- /dev/null +++ b/.github/workflows/harmony-os.yaml @@ -0,0 +1,159 @@ +name: harmony-os + +on: + push: + branches: + - master + - ohos + tags: + - 'v[0-9]+.[0-9]+.[0-9]+*' + + workflow_dispatch: + +concurrency: + group: harmony-os-${{ github.ref }} + cancel-in-progress: true + +jobs: + harmony_os: + name: Harmony OS ${{ matrix.arch }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + arch: [arm64-v8a, armeabi-v7a, x86_64] + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ohos-${{ matrix.arch }} + + - name: cache-toolchain + id: cache-toolchain-ohos + uses: actions/cache@v4 + with: + path: command-line-tools + key: commandline-tools-linux-x64-5.0.5.200.zip + + - name: Download toolchain + if: steps.cache-toolchain-ohos.outputs.cache-hit != 'true' + shell: bash + run: | + curl -SL -O https://huggingface.co/csukuangfj/harmonyos-commandline-tools/resolve/main/commandline-tools-linux-x64-5.0.5.200.zip + unzip commandline-tools-linux-x64-5.0.5.200.zip + rm commandline-tools-linux-x64-5.0.5.200.zip + + - name: Set environment variable + shell: bash + run: | + echo "$GITHUB_WORKSPACE/command-line-tools/sdk/default/openharmony/native/build-tools/cmake/bin" >> "$GITHUB_PATH" + which cmake + + cmake --version + + ls -lh $GITHUB_WORKSPACE/command-line-tools/sdk/default/openharmony/native/build/cmake/ohos.toolchain.cmake + + echo "====" + cat $GITHUB_WORKSPACE/command-line-tools/sdk/default/openharmony/native/build/cmake/ohos.toolchain.cmake + echo "====" + + # echo "$GITHUB_WORKSPACE/command-line-tools/sdk/default/openharmony/native/llvm/bin" >> "$GITHUB_PATH" + + ls -lh $GITHUB_WORKSPACE/command-line-tools/sdk/default/openharmony/native/llvm/bin/ + echo "--" + ls -lh $GITHUB_WORKSPACE/command-line-tools/sdk/default/openharmony/native/llvm/bin/*unknown* + + cat $GITHUB_PATH + + # /home/runner/work/onnxruntime-libs/onnxruntime-libs/command-line-tools/sdk/default/openharmony/native/llvm/bin/aarch64-unknown-linux-ohos-clang -v || true + export PATH=$PWD/command-line-tools/sdk/default/openharmony/native/llvm/bin:$PATH + echo "path: $PATH" + + which aarch64-unknown-linux-ohos-clang++ || true + which aarch64-unknown-linux-ohos-clang || true + + aarch64-unknown-linux-ohos-clang++ --version || true + aarch64-unknown-linux-ohos-clang --version || true + + which armv7-unknown-linux-ohos-clang++ + which armv7-unknown-linux-ohos-clang + + armv7-unknown-linux-ohos-clang++ --version + armv7-unknown-linux-ohos-clang --version + + which x86_64-unknown-linux-ohos-clang++ + which x86_64-unknown-linux-ohos-clang + + x86_64-unknown-linux-ohos-clang++ --version + x86_64-unknown-linux-ohos-clang --version + + - name: Build ${{ matrix.arch }} + shell: bash + run: | + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + cmake --version + + arch=${{ matrix.arch }} + + echo "arch: $arch" + + export OHOS_SDK_NATIVE_DIR="$GITHUB_WORKSPACE/command-line-tools/sdk/default/openharmony/native" + + if [[ $arch == arm64-v8a ]]; then + ./build-ohos-arm64-v8a.sh + elif [[ $arch == armeabi-v7a ]]; then + ./build-ohos-armeabi-v7a.sh + elif [[ $arch == x86_64 ]]; then + ./build-ohos-x86-64.sh + else + echo "Unknown arch $arch" + fi + + - name: Collect result for ${{ matrix.arch }} + shell: bash + run: | + SHERPA_ONNX_VERSION=v$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + echo "SHERPA_ONNX_VERSION=$SHERPA_ONNX_VERSION" >> "$GITHUB_ENV" + + arch=${{ matrix.arch }} + d=sherpa-onnx-$SHERPA_ONNX_VERSION-ohos-$arch + if [[ $arch == x86_64 ]]; then + cd ./build-ohos-x86-64 + else + cd ./build-ohos-$arch + fi + + mv install $d + tar cjfv $d.tar.bz2 $d + + ls -lh $d/lib + + + file $d/lib/* + + readelf -d $d/lib/libsherpa-onnx-c-api.so + + mv $d.tar.bz2 ../ + + - uses: actions/upload-artifact@v4 + with: + name: sherpa-onnx-ohos-${{ matrix.arch }} + path: ./*.tar.bz2 + + - name: Release jar + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + overwrite: true + file: ./*.tar.bz2 + # repo_name: k2-fsa/sherpa-onnx + # repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} + # tag: v1.10.23 diff --git a/CMakeLists.txt b/CMakeLists.txt index 4adc3dca5..d6f94c668 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -125,6 +125,11 @@ if(MSVC) ) endif() +if(CMAKE_SYSTEM_NAME STREQUAL OHOS) + set(CMAKE_CXX_FLAGS "-Wno-unused-command-line-argument ${CMAKE_CXX_FLAGS}") + set(CMAKE_C_FLAGS "-Wno-unused-command-line-argument ${CMAKE_C_FLAGS}") +endif() + message(STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") message(STATUS "CMAKE_INSTALL_PREFIX: ${CMAKE_INSTALL_PREFIX}") message(STATUS "BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}") @@ -263,7 +268,7 @@ message(STATUS "C++ Standard version: ${CMAKE_CXX_STANDARD}") include(CheckIncludeFileCXX) -if(UNIX AND NOT APPLE AND NOT SHERPA_ONNX_ENABLE_WASM AND NOT CMAKE_SYSTEM_NAME STREQUAL Android) +if(UNIX AND NOT APPLE AND NOT SHERPA_ONNX_ENABLE_WASM AND NOT CMAKE_SYSTEM_NAME STREQUAL Android AND NOT CMAKE_SYSTEM_NAME STREQUAL OHOS) check_include_file_cxx(alsa/asoundlib.h SHERPA_ONNX_HAS_ALSA) if(SHERPA_ONNX_HAS_ALSA) message(STATUS "With Alsa") diff --git a/build-ohos-arm64-v8a.sh b/build-ohos-arm64-v8a.sh new file mode 100755 index 000000000..cedbd495f --- /dev/null +++ b/build-ohos-arm64-v8a.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +set -ex + +dir=$PWD/build-ohos-arm64-v8a + +mkdir -p $dir +cd $dir + +# Please first download the commandline tools from +# https://developer.huawei.com/consumer/cn/download/ +# +# Example filename on Linux: commandline-tools-linux-x64-5.0.5.200.zip +# You can also download it from https://hf-mirror.com/csukuangfj/harmonyos-commandline-tools/tree/main + +# mkdir /star-fj/fangjun/software/huawei +# cd /star-fj/fangjun/software/huawei +# wget https://hf-mirror.com/csukuangfj/harmonyos-commandline-tools/resolve/main/commandline-tools-linux-x64-5.0.5.200.zip +# unzip commandline-tools-linux-x64-5.0.5.200.zip +# rm commandline-tools-linux-x64-5.0.5.200.zip +if [ -z $OHOS_SDK_NATIVE_DIR ]; then + OHOS_SDK_NATIVE_DIR=/star-fj/fangjun/software/huawei/command-line-tools/sdk/default/openharmony/native/ + export PATH=$OHOS_SDK_NATIVE_DIR/build-tools/cmake/bin:$PATH + # You can find the following content inside OHOS_SDK_NATIVE_DIR + # ls -lh /star-fj/fangjun/software/huawei/command-line-tools/sdk/default/openharmony/native/ + # total 524K + # -rw-r--r-- 1 kuangfangjun root 501K Jan 1 2001 NOTICE.txt + # drwxr-xr-x 3 kuangfangjun root 0 Nov 6 22:36 build + # drwxr-xr-x 3 kuangfangjun root 0 Nov 6 22:36 build-tools + # -rw-r--r-- 1 kuangfangjun root 371 Jan 1 2001 compatible_config.json + # drwxr-xr-x 4 kuangfangjun root 0 Nov 6 22:36 docs + # drwxr-xr-x 10 kuangfangjun root 0 Nov 6 22:36 llvm + # -rw-r--r-- 1 kuangfangjun root 16K Jan 1 2001 nativeapi_syscap_config.json + # -rw-r--r-- 1 kuangfangjun root 5.9K Jan 1 2001 ndk_system_capability.json + # -rw-r--r-- 1 kuangfangjun root 167 Jan 1 2001 oh-uni-package.json + # drwxr-xr-x 3 kuangfangjun root 0 Nov 6 22:36 sysroot +fi + +# If you don't want to install commandline tools, you can install the SDK +# using DevEco Studio. The following uses API version 10 as an example and +# it has installed the SDK to +# /Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native +# +# Remember to select ``native`` when you install the SDK +if [ ! -d $OHOS_SDK_NATIVE_DIR ]; then + OHOS_SDK_NATIVE_DIR=/Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native + # export PATH=$OHOS_SDK_NATIVE_DIR/build-tools/cmake/bin:$PATH + # ls -lh /Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native/ + # total 1560 + # -rw-r--r-- 1 fangjun staff 764K Jan 1 2001 NOTICE.txt + # drwxr-xr-x 3 fangjun staff 96B Nov 19 22:42 build + # drwxr-xr-x 3 fangjun staff 96B Nov 19 22:42 build-tools + # drwxr-xr-x 10 fangjun staff 320B Nov 19 22:42 llvm + # -rw-r--r-- 1 fangjun staff 4.0K Jan 1 2001 nativeapi_syscap_config.json + # -rw-r--r-- 1 fangjun staff 1.9K Jan 1 2001 ndk_system_capability.json + # -rw-r--r-- 1 fangjun staff 169B Jan 1 2001 oh-uni-package.json + # drwxr-xr-x 3 fangjun staff 96B Nov 19 22:42 sysroot +fi + +if [ ! -d $OHOS_SDK_NATIVE_DIR ]; then + echo "Please first download Command Line Tools for HarmonyOS" + exit 1 +fi + +export PATH=$OHOS_SDK_NATIVE_DIR/llvm/bin:$PATH + +OHOS_TOOLCHAIN_FILE=$OHOS_SDK_NATIVE_DIR/build/cmake/ohos.toolchain.cmake + +if [ ! -f $OHOS_TOOLCHAIN_FILE ]; then + echo "$OHOS_TOOLCHAIN_FILE does not exist" + echo "Please first download Command Line Tools for HarmonyOS" + exit 1 +fi + + +sleep 1 +onnxruntime_version=1.16.3 +onnxruntime_dir=onnxruntime-ohos-arm64-v8a-$onnxruntime_version + +if [ ! -f $onnxruntime_dir/lib/libonnxruntime.so ]; then + # wget -c https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${onnxruntime_version}/$onnxruntime_dir.zip + wget -c https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/$onnxruntime_dir.zip + unzip $onnxruntime_dir.zip + rm $onnxruntime_dir.zip +fi + +export SHERPA_ONNXRUNTIME_LIB_DIR=$dir/$onnxruntime_dir/lib +export SHERPA_ONNXRUNTIME_INCLUDE_DIR=$dir/$onnxruntime_dir/include + +echo "SHERPA_ONNXRUNTIME_LIB_DIR: $SHERPA_ONNXRUNTIME_LIB_DIR" +echo "SHERPA_ONNXRUNTIME_INCLUDE_DIR $SHERPA_ONNXRUNTIME_INCLUDE_DIR" + +if [ -z $SHERPA_ONNX_ENABLE_TTS ]; then + SHERPA_ONNX_ENABLE_TTS=ON +fi + +if [ -z $SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION ]; then + SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=ON +fi + +if [ -z $SHERPA_ONNX_ENABLE_BINARY ]; then + SHERPA_ONNX_ENABLE_BINARY=OFF +fi + +cmake \ + -DOHOS_ARCH=arm64-v8a \ + -DCMAKE_TOOLCHAIN_FILE=$OHOS_TOOLCHAIN_FILE \ + -DSHERPA_ONNX_ENABLE_TTS=$SHERPA_ONNX_ENABLE_TTS \ + -DSHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=$SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION \ + -DSHERPA_ONNX_ENABLE_BINARY=$SHERPA_ONNX_ENABLE_BINARY \ + -DBUILD_PIPER_PHONMIZE_EXE=OFF \ + -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ + -DBUILD_ESPEAK_NG_EXE=OFF \ + -DBUILD_ESPEAK_NG_TESTS=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + -DSHERPA_ONNX_ENABLE_JNI=OFF \ + -DSHERPA_ONNX_ENABLE_C_API=ON \ + -DCMAKE_INSTALL_PREFIX=./install \ + .. + +# make VERBOSE=1 -j4 +make -j2 +make install/strip +cp -fv $onnxruntime_dir/lib/libonnxruntime.so install/lib +cp -fv $OHOS_SDK_NATIVE_DIR/llvm/lib/aarch64-linux-ohos/libc++_shared.so install/lib + +rm -rf install/share +rm -rf install/lib/pkgconfig diff --git a/build-ohos-armeabi-v7a.sh b/build-ohos-armeabi-v7a.sh new file mode 100755 index 000000000..ee1d560ba --- /dev/null +++ b/build-ohos-armeabi-v7a.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +set -ex + +dir=$PWD/build-ohos-armeabi-v7a + +mkdir -p $dir +cd $dir + +# Please first download the commandline tools from +# https://developer.huawei.com/consumer/cn/download/ +# +# Example filename on Linux: commandline-tools-linux-x64-5.0.5.200.zip +# You can also download it from https://hf-mirror.com/csukuangfj/harmonyos-commandline-tools/tree/main + +# mkdir /star-fj/fangjun/software/huawei +# cd /star-fj/fangjun/software/huawei +# wget https://hf-mirror.com/csukuangfj/harmonyos-commandline-tools/resolve/main/commandline-tools-linux-x64-5.0.5.200.zip +# unzip commandline-tools-linux-x64-5.0.5.200.zip +# rm commandline-tools-linux-x64-5.0.5.200.zip +if [ -z $OHOS_SDK_NATIVE_DIR ]; then + OHOS_SDK_NATIVE_DIR=/star-fj/fangjun/software/huawei/command-line-tools/sdk/default/openharmony/native/ + export PATH=$OHOS_SDK_NATIVE_DIR/build-tools/cmake/bin:$PATH + # You can find the following content inside OHOS_SDK_NATIVE_DIR + # ls -lh /star-fj/fangjun/software/huawei/command-line-tools/sdk/default/openharmony/native/ + # total 524K + # -rw-r--r-- 1 kuangfangjun root 501K Jan 1 2001 NOTICE.txt + # drwxr-xr-x 3 kuangfangjun root 0 Nov 6 22:36 build + # drwxr-xr-x 3 kuangfangjun root 0 Nov 6 22:36 build-tools + # -rw-r--r-- 1 kuangfangjun root 371 Jan 1 2001 compatible_config.json + # drwxr-xr-x 4 kuangfangjun root 0 Nov 6 22:36 docs + # drwxr-xr-x 10 kuangfangjun root 0 Nov 6 22:36 llvm + # -rw-r--r-- 1 kuangfangjun root 16K Jan 1 2001 nativeapi_syscap_config.json + # -rw-r--r-- 1 kuangfangjun root 5.9K Jan 1 2001 ndk_system_capability.json + # -rw-r--r-- 1 kuangfangjun root 167 Jan 1 2001 oh-uni-package.json + # drwxr-xr-x 3 kuangfangjun root 0 Nov 6 22:36 sysroot +fi + +# If you don't want to install commandline tools, you can install the SDK +# using DevEco Studio. The following uses API version 10 as an example and +# it has installed the SDK to +# /Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native +# +# Remember to select ``native`` when you install the SDK +if [ ! -d $OHOS_SDK_NATIVE_DIR ]; then + OHOS_SDK_NATIVE_DIR=/Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native + # export PATH=$OHOS_SDK_NATIVE_DIR/build-tools/cmake/bin:$PATH + # ls -lh /Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native/ + # total 1560 + # -rw-r--r-- 1 fangjun staff 764K Jan 1 2001 NOTICE.txt + # drwxr-xr-x 3 fangjun staff 96B Nov 19 22:42 build + # drwxr-xr-x 3 fangjun staff 96B Nov 19 22:42 build-tools + # drwxr-xr-x 10 fangjun staff 320B Nov 19 22:42 llvm + # -rw-r--r-- 1 fangjun staff 4.0K Jan 1 2001 nativeapi_syscap_config.json + # -rw-r--r-- 1 fangjun staff 1.9K Jan 1 2001 ndk_system_capability.json + # -rw-r--r-- 1 fangjun staff 169B Jan 1 2001 oh-uni-package.json + # drwxr-xr-x 3 fangjun staff 96B Nov 19 22:42 sysroot +fi + +if [ ! -d $OHOS_SDK_NATIVE_DIR ]; then + echo "Please first download Command Line Tools for HarmonyOS" + exit 1 +fi + +export PATH=$OHOS_SDK_NATIVE_DIR/llvm/bin:$PATH + +OHOS_TOOLCHAIN_FILE=$OHOS_SDK_NATIVE_DIR/build/cmake/ohos.toolchain.cmake + +if [ ! -f $OHOS_TOOLCHAIN_FILE ]; then + echo "$OHOS_TOOLCHAIN_FILE does not exist" + echo "Please first download Command Line Tools for HarmonyOS" + exit 1 +fi + + +sleep 1 +onnxruntime_version=1.16.3 +onnxruntime_dir=onnxruntime-ohos-armeabi-v7a-$onnxruntime_version + +if [ ! -f $onnxruntime_dir/lib/libonnxruntime.so ]; then + # wget -c https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${onnxruntime_version}/$onnxruntime_dir.zip + wget -c https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/$onnxruntime_dir.zip + unzip $onnxruntime_dir.zip + rm $onnxruntime_dir.zip +fi + +export SHERPA_ONNXRUNTIME_LIB_DIR=$dir/$onnxruntime_dir/lib +export SHERPA_ONNXRUNTIME_INCLUDE_DIR=$dir/$onnxruntime_dir/include + +echo "SHERPA_ONNXRUNTIME_LIB_DIR: $SHERPA_ONNXRUNTIME_LIB_DIR" +echo "SHERPA_ONNXRUNTIME_INCLUDE_DIR $SHERPA_ONNXRUNTIME_INCLUDE_DIR" + +if [ -z $SHERPA_ONNX_ENABLE_TTS ]; then + SHERPA_ONNX_ENABLE_TTS=ON +fi + +if [ -z $SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION ]; then + SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=ON +fi + +if [ -z $SHERPA_ONNX_ENABLE_BINARY ]; then + SHERPA_ONNX_ENABLE_BINARY=OFF +fi + +# See https://github.com/llvm/llvm-project/issues/57732 +# we need to use -mfloat-abi=hard +cmake \ + -DOHOS_ARCH=armeabi-v7a \ + -DCMAKE_CXX_FLAGS="-mfloat-abi=hard" \ + -DCMAKE_C_FLAGS="-mfloat-abi=hard" \ + -DCMAKE_TOOLCHAIN_FILE=$OHOS_TOOLCHAIN_FILE \ + -DSHERPA_ONNX_ENABLE_TTS=$SHERPA_ONNX_ENABLE_TTS \ + -DSHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=$SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION \ + -DSHERPA_ONNX_ENABLE_BINARY=$SHERPA_ONNX_ENABLE_BINARY \ + -DBUILD_PIPER_PHONMIZE_EXE=OFF \ + -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ + -DBUILD_ESPEAK_NG_EXE=OFF \ + -DBUILD_ESPEAK_NG_TESTS=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + -DSHERPA_ONNX_ENABLE_JNI=OFF \ + -DSHERPA_ONNX_ENABLE_C_API=ON \ + -DCMAKE_INSTALL_PREFIX=./install \ + .. + +# make VERBOSE=1 -j4 +make -j2 +make install/strip +cp -fv $onnxruntime_dir/lib/libonnxruntime.so install/lib +cp -fv $OHOS_SDK_NATIVE_DIR/llvm/lib/arm-linux-ohos/libc++_shared.so install/lib + +rm -rf install/share +rm -rf install/lib/pkgconfig diff --git a/build-ohos-x86-64.sh b/build-ohos-x86-64.sh new file mode 100755 index 000000000..6c8787dc0 --- /dev/null +++ b/build-ohos-x86-64.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +set -ex + +dir=$PWD/build-ohos-x86-64 + +mkdir -p $dir +cd $dir + +# Please first download the commandline tools from +# https://developer.huawei.com/consumer/cn/download/ +# +# Example filename on Linux: commandline-tools-linux-x64-5.0.5.200.zip +# You can also download it from https://hf-mirror.com/csukuangfj/harmonyos-commandline-tools/tree/main + +# mkdir /star-fj/fangjun/software/huawei +# cd /star-fj/fangjun/software/huawei +# wget https://hf-mirror.com/csukuangfj/harmonyos-commandline-tools/resolve/main/commandline-tools-linux-x64-5.0.5.200.zip +# unzip commandline-tools-linux-x64-5.0.5.200.zip +# rm commandline-tools-linux-x64-5.0.5.200.zip +if [ -z $OHOS_SDK_NATIVE_DIR ]; then + OHOS_SDK_NATIVE_DIR=/star-fj/fangjun/software/huawei/command-line-tools/sdk/default/openharmony/native/ + export PATH=$OHOS_SDK_NATIVE_DIR/build-tools/cmake/bin:$PATH + # You can find the following content inside OHOS_SDK_NATIVE_DIR + # ls -lh /star-fj/fangjun/software/huawei/command-line-tools/sdk/default/openharmony/native/ + # total 524K + # -rw-r--r-- 1 kuangfangjun root 501K Jan 1 2001 NOTICE.txt + # drwxr-xr-x 3 kuangfangjun root 0 Nov 6 22:36 build + # drwxr-xr-x 3 kuangfangjun root 0 Nov 6 22:36 build-tools + # -rw-r--r-- 1 kuangfangjun root 371 Jan 1 2001 compatible_config.json + # drwxr-xr-x 4 kuangfangjun root 0 Nov 6 22:36 docs + # drwxr-xr-x 10 kuangfangjun root 0 Nov 6 22:36 llvm + # -rw-r--r-- 1 kuangfangjun root 16K Jan 1 2001 nativeapi_syscap_config.json + # -rw-r--r-- 1 kuangfangjun root 5.9K Jan 1 2001 ndk_system_capability.json + # -rw-r--r-- 1 kuangfangjun root 167 Jan 1 2001 oh-uni-package.json + # drwxr-xr-x 3 kuangfangjun root 0 Nov 6 22:36 sysroot +fi + +# If you don't want to install commandline tools, you can install the SDK +# using DevEco Studio. The following uses API version 10 as an example and +# it has installed the SDK to +# /Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native +# +# Remember to select ``native`` when you install the SDK +if [ ! -d $OHOS_SDK_NATIVE_DIR ]; then + OHOS_SDK_NATIVE_DIR=/Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native + # export PATH=$OHOS_SDK_NATIVE_DIR/build-tools/cmake/bin:$PATH + # ls -lh /Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native/ + # total 1560 + # -rw-r--r-- 1 fangjun staff 764K Jan 1 2001 NOTICE.txt + # drwxr-xr-x 3 fangjun staff 96B Nov 19 22:42 build + # drwxr-xr-x 3 fangjun staff 96B Nov 19 22:42 build-tools + # drwxr-xr-x 10 fangjun staff 320B Nov 19 22:42 llvm + # -rw-r--r-- 1 fangjun staff 4.0K Jan 1 2001 nativeapi_syscap_config.json + # -rw-r--r-- 1 fangjun staff 1.9K Jan 1 2001 ndk_system_capability.json + # -rw-r--r-- 1 fangjun staff 169B Jan 1 2001 oh-uni-package.json + # drwxr-xr-x 3 fangjun staff 96B Nov 19 22:42 sysroot +fi + +if [ ! -d $OHOS_SDK_NATIVE_DIR ]; then + echo "Please first download Command Line Tools for HarmonyOS" + exit 1 +fi + +export PATH=$OHOS_SDK_NATIVE_DIR/llvm/bin:$PATH + +OHOS_TOOLCHAIN_FILE=$OHOS_SDK_NATIVE_DIR/build/cmake/ohos.toolchain.cmake + +if [ ! -f $OHOS_TOOLCHAIN_FILE ]; then + echo "$OHOS_TOOLCHAIN_FILE does not exist" + echo "Please first download Command Line Tools for HarmonyOS" + exit 1 +fi + + +sleep 1 +onnxruntime_version=1.16.3 +onnxruntime_dir=onnxruntime-ohos-x86_64-$onnxruntime_version + +if [ ! -f $onnxruntime_dir/lib/libonnxruntime.so ]; then + # wget -c https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${onnxruntime_version}/$onnxruntime_dir.zip + wget -c https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/$onnxruntime_dir.zip + unzip $onnxruntime_dir.zip + rm $onnxruntime_dir.zip +fi + +export SHERPA_ONNXRUNTIME_LIB_DIR=$dir/$onnxruntime_dir/lib +export SHERPA_ONNXRUNTIME_INCLUDE_DIR=$dir/$onnxruntime_dir/include + +echo "SHERPA_ONNXRUNTIME_LIB_DIR: $SHERPA_ONNXRUNTIME_LIB_DIR" +echo "SHERPA_ONNXRUNTIME_INCLUDE_DIR $SHERPA_ONNXRUNTIME_INCLUDE_DIR" + +if [ -z $SHERPA_ONNX_ENABLE_TTS ]; then + SHERPA_ONNX_ENABLE_TTS=ON +fi + +if [ -z $SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION ]; then + SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=ON +fi + +if [ -z $SHERPA_ONNX_ENABLE_BINARY ]; then + SHERPA_ONNX_ENABLE_BINARY=OFF +fi + +cmake \ + -DOHOS_ARCH=x86_64 \ + -DCMAKE_TOOLCHAIN_FILE=$OHOS_TOOLCHAIN_FILE \ + -DSHERPA_ONNX_ENABLE_TTS=$SHERPA_ONNX_ENABLE_TTS \ + -DSHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=$SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION \ + -DSHERPA_ONNX_ENABLE_BINARY=$SHERPA_ONNX_ENABLE_BINARY \ + -DBUILD_PIPER_PHONMIZE_EXE=OFF \ + -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ + -DBUILD_ESPEAK_NG_EXE=OFF \ + -DBUILD_ESPEAK_NG_TESTS=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + -DSHERPA_ONNX_ENABLE_JNI=OFF \ + -DSHERPA_ONNX_ENABLE_C_API=ON \ + -DCMAKE_INSTALL_PREFIX=./install \ + .. + +# make VERBOSE=1 -j4 +make -j2 +make install/strip +cp -fv $onnxruntime_dir/lib/libonnxruntime.so install/lib +cp -fv $OHOS_SDK_NATIVE_DIR/llvm/lib/x86_64-linux-ohos/libc++_shared.so install/lib + +rm -rf install/share +rm -rf install/lib/pkgconfig From 31d6206fde528a7bca7a7610e944f405b880ae89 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sun, 24 Nov 2024 16:29:24 +0800 Subject: [PATCH 077/183] HarmonyOS support for VAD. (#1561) --- build-ohos-arm64-v8a.sh | 46 +++++++++++---------- build-ohos-armeabi-v7a.sh | 32 +++++--------- build-ohos-x86-64.sh | 46 +++++++++++---------- sherpa-onnx/c-api/c-api.cc | 38 ++++++++++++++++- sherpa-onnx/c-api/c-api.h | 15 +++++++ sherpa-onnx/csrc/CMakeLists.txt | 6 +++ sherpa-onnx/csrc/macros.h | 12 ++++++ sherpa-onnx/csrc/onnx-utils.cc | 36 ++++++++++++++++ sherpa-onnx/csrc/onnx-utils.h | 9 ++++ sherpa-onnx/csrc/silero-vad-model.cc | 11 ++++- sherpa-onnx/csrc/silero-vad-model.h | 8 ++++ sherpa-onnx/csrc/vad-model.cc | 8 ++++ sherpa-onnx/csrc/vad-model.h | 9 ++++ sherpa-onnx/csrc/voice-activity-detector.cc | 12 +++++- sherpa-onnx/csrc/voice-activity-detector.h | 10 +++++ 15 files changed, 229 insertions(+), 69 deletions(-) diff --git a/build-ohos-arm64-v8a.sh b/build-ohos-arm64-v8a.sh index cedbd495f..182575bc1 100755 --- a/build-ohos-arm64-v8a.sh +++ b/build-ohos-arm64-v8a.sh @@ -19,7 +19,6 @@ cd $dir # rm commandline-tools-linux-x64-5.0.5.200.zip if [ -z $OHOS_SDK_NATIVE_DIR ]; then OHOS_SDK_NATIVE_DIR=/star-fj/fangjun/software/huawei/command-line-tools/sdk/default/openharmony/native/ - export PATH=$OHOS_SDK_NATIVE_DIR/build-tools/cmake/bin:$PATH # You can find the following content inside OHOS_SDK_NATIVE_DIR # ls -lh /star-fj/fangjun/software/huawei/command-line-tools/sdk/default/openharmony/native/ # total 524K @@ -35,32 +34,39 @@ if [ -z $OHOS_SDK_NATIVE_DIR ]; then # drwxr-xr-x 3 kuangfangjun root 0 Nov 6 22:36 sysroot fi -# If you don't want to install commandline tools, you can install the SDK -# using DevEco Studio. The following uses API version 10 as an example and -# it has installed the SDK to -# /Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native -# -# Remember to select ``native`` when you install the SDK if [ ! -d $OHOS_SDK_NATIVE_DIR ]; then - OHOS_SDK_NATIVE_DIR=/Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native - # export PATH=$OHOS_SDK_NATIVE_DIR/build-tools/cmake/bin:$PATH - # ls -lh /Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native/ - # total 1560 - # -rw-r--r-- 1 fangjun staff 764K Jan 1 2001 NOTICE.txt - # drwxr-xr-x 3 fangjun staff 96B Nov 19 22:42 build - # drwxr-xr-x 3 fangjun staff 96B Nov 19 22:42 build-tools - # drwxr-xr-x 10 fangjun staff 320B Nov 19 22:42 llvm - # -rw-r--r-- 1 fangjun staff 4.0K Jan 1 2001 nativeapi_syscap_config.json - # -rw-r--r-- 1 fangjun staff 1.9K Jan 1 2001 ndk_system_capability.json - # -rw-r--r-- 1 fangjun staff 169B Jan 1 2001 oh-uni-package.json - # drwxr-xr-x 3 fangjun staff 96B Nov 19 22:42 sysroot + OHOS_SDK_NATIVE_DIR=/Users/fangjun/software/command-line-tools/sdk/default/openharmony/native + # (py38) fangjuns-MacBook-Pro:software fangjun$ ls -lh command-line-tools/sdk/default/openharmony/native/ + # total 752 + # -rw-r--r-- 1 fangjun staff 341K Jan 1 2001 NOTICE.txt + # drwxr-xr-x 3 fangjun staff 96B Nov 6 21:17 build + # drwxr-xr-x 3 fangjun staff 96B Nov 6 21:18 build-tools + # -rw-r--r-- 1 fangjun staff 371B Jan 1 2001 compatible_config.json + # drwxr-xr-x 10 fangjun staff 320B Nov 6 21:18 llvm + # -rw-r--r-- 1 fangjun staff 16K Jan 1 2001 nativeapi_syscap_config.json + # -rw-r--r-- 1 fangjun staff 5.9K Jan 1 2001 ndk_system_capability.json + # -rw-r--r-- 1 fangjun staff 167B Jan 1 2001 oh-uni-package.json + # drwxr-xr-x 3 fangjun staff 96B Nov 6 21:17 sysroot fi if [ ! -d $OHOS_SDK_NATIVE_DIR ]; then echo "Please first download Command Line Tools for HarmonyOS" + echo "See https://developer.huawei.com/consumer/cn/download/" + echo "or" + echo "https://hf-mirror.com/csukuangfj/harmonyos-commandline-tools/tree/main" + exit 1 +fi + +if [ ! -f $OHOS_SDK_NATIVE_DIR/llvm/bin/aarch64-unknown-linux-ohos-clang ]; then + echo "$OHOS_SDK_NATIVE_DIR/llvm/bin/aarch64-unknown-linux-ohos-clang does not exist" + echo "Please first download Command Line Tools for HarmonyOS" + echo "See https://developer.huawei.com/consumer/cn/download/" + echo "or" + echo "https://hf-mirror.com/csukuangfj/harmonyos-commandline-tools/tree/main" exit 1 fi +export PATH=$OHOS_SDK_NATIVE_DIR/build-tools/cmake/bin:$PATH export PATH=$OHOS_SDK_NATIVE_DIR/llvm/bin:$PATH OHOS_TOOLCHAIN_FILE=$OHOS_SDK_NATIVE_DIR/build/cmake/ohos.toolchain.cmake @@ -71,7 +77,6 @@ if [ ! -f $OHOS_TOOLCHAIN_FILE ]; then exit 1 fi - sleep 1 onnxruntime_version=1.16.3 onnxruntime_dir=onnxruntime-ohos-arm64-v8a-$onnxruntime_version @@ -126,7 +131,6 @@ cmake \ make -j2 make install/strip cp -fv $onnxruntime_dir/lib/libonnxruntime.so install/lib -cp -fv $OHOS_SDK_NATIVE_DIR/llvm/lib/aarch64-linux-ohos/libc++_shared.so install/lib rm -rf install/share rm -rf install/lib/pkgconfig diff --git a/build-ohos-armeabi-v7a.sh b/build-ohos-armeabi-v7a.sh index ee1d560ba..e0a2ac883 100755 --- a/build-ohos-armeabi-v7a.sh +++ b/build-ohos-armeabi-v7a.sh @@ -35,32 +35,24 @@ if [ -z $OHOS_SDK_NATIVE_DIR ]; then # drwxr-xr-x 3 kuangfangjun root 0 Nov 6 22:36 sysroot fi -# If you don't want to install commandline tools, you can install the SDK -# using DevEco Studio. The following uses API version 10 as an example and -# it has installed the SDK to -# /Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native -# -# Remember to select ``native`` when you install the SDK if [ ! -d $OHOS_SDK_NATIVE_DIR ]; then - OHOS_SDK_NATIVE_DIR=/Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native - # export PATH=$OHOS_SDK_NATIVE_DIR/build-tools/cmake/bin:$PATH - # ls -lh /Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native/ - # total 1560 - # -rw-r--r-- 1 fangjun staff 764K Jan 1 2001 NOTICE.txt - # drwxr-xr-x 3 fangjun staff 96B Nov 19 22:42 build - # drwxr-xr-x 3 fangjun staff 96B Nov 19 22:42 build-tools - # drwxr-xr-x 10 fangjun staff 320B Nov 19 22:42 llvm - # -rw-r--r-- 1 fangjun staff 4.0K Jan 1 2001 nativeapi_syscap_config.json - # -rw-r--r-- 1 fangjun staff 1.9K Jan 1 2001 ndk_system_capability.json - # -rw-r--r-- 1 fangjun staff 169B Jan 1 2001 oh-uni-package.json - # drwxr-xr-x 3 fangjun staff 96B Nov 19 22:42 sysroot + echo "Please first download Command Line Tools for HarmonyOS" + echo "See https://developer.huawei.com/consumer/cn/download/" + echo "or" + echo "https://hf-mirror.com/csukuangfj/harmonyos-commandline-tools/tree/main" + exit 1 fi -if [ ! -d $OHOS_SDK_NATIVE_DIR ]; then +if [ ! -f $OHOS_SDK_NATIVE_DIR/llvm/bin/armv7-unknown-linux-ohos-clang ]; then + echo "$OHOS_SDK_NATIVE_DIR/llvm/bin/armv7-unknown-linux-ohos-clang does not exist" echo "Please first download Command Line Tools for HarmonyOS" + echo "See https://developer.huawei.com/consumer/cn/download/" + echo "or" + echo "https://hf-mirror.com/csukuangfj/harmonyos-commandline-tools/tree/main" exit 1 fi +export PATH=$OHOS_SDK_NATIVE_DIR/build-tools/cmake/bin:$PATH export PATH=$OHOS_SDK_NATIVE_DIR/llvm/bin:$PATH OHOS_TOOLCHAIN_FILE=$OHOS_SDK_NATIVE_DIR/build/cmake/ohos.toolchain.cmake @@ -71,7 +63,6 @@ if [ ! -f $OHOS_TOOLCHAIN_FILE ]; then exit 1 fi - sleep 1 onnxruntime_version=1.16.3 onnxruntime_dir=onnxruntime-ohos-armeabi-v7a-$onnxruntime_version @@ -130,7 +121,6 @@ cmake \ make -j2 make install/strip cp -fv $onnxruntime_dir/lib/libonnxruntime.so install/lib -cp -fv $OHOS_SDK_NATIVE_DIR/llvm/lib/arm-linux-ohos/libc++_shared.so install/lib rm -rf install/share rm -rf install/lib/pkgconfig diff --git a/build-ohos-x86-64.sh b/build-ohos-x86-64.sh index 6c8787dc0..58ff5da5a 100755 --- a/build-ohos-x86-64.sh +++ b/build-ohos-x86-64.sh @@ -19,7 +19,6 @@ cd $dir # rm commandline-tools-linux-x64-5.0.5.200.zip if [ -z $OHOS_SDK_NATIVE_DIR ]; then OHOS_SDK_NATIVE_DIR=/star-fj/fangjun/software/huawei/command-line-tools/sdk/default/openharmony/native/ - export PATH=$OHOS_SDK_NATIVE_DIR/build-tools/cmake/bin:$PATH # You can find the following content inside OHOS_SDK_NATIVE_DIR # ls -lh /star-fj/fangjun/software/huawei/command-line-tools/sdk/default/openharmony/native/ # total 524K @@ -35,32 +34,39 @@ if [ -z $OHOS_SDK_NATIVE_DIR ]; then # drwxr-xr-x 3 kuangfangjun root 0 Nov 6 22:36 sysroot fi -# If you don't want to install commandline tools, you can install the SDK -# using DevEco Studio. The following uses API version 10 as an example and -# it has installed the SDK to -# /Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native -# -# Remember to select ``native`` when you install the SDK if [ ! -d $OHOS_SDK_NATIVE_DIR ]; then - OHOS_SDK_NATIVE_DIR=/Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native - # export PATH=$OHOS_SDK_NATIVE_DIR/build-tools/cmake/bin:$PATH - # ls -lh /Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native/ - # total 1560 - # -rw-r--r-- 1 fangjun staff 764K Jan 1 2001 NOTICE.txt - # drwxr-xr-x 3 fangjun staff 96B Nov 19 22:42 build - # drwxr-xr-x 3 fangjun staff 96B Nov 19 22:42 build-tools - # drwxr-xr-x 10 fangjun staff 320B Nov 19 22:42 llvm - # -rw-r--r-- 1 fangjun staff 4.0K Jan 1 2001 nativeapi_syscap_config.json - # -rw-r--r-- 1 fangjun staff 1.9K Jan 1 2001 ndk_system_capability.json - # -rw-r--r-- 1 fangjun staff 169B Jan 1 2001 oh-uni-package.json - # drwxr-xr-x 3 fangjun staff 96B Nov 19 22:42 sysroot + OHOS_SDK_NATIVE_DIR=/Users/fangjun/software/command-line-tools/sdk/default/openharmony/native + # (py38) fangjuns-MacBook-Pro:software fangjun$ ls -lh command-line-tools/sdk/default/openharmony/native/ + # total 752 + # -rw-r--r-- 1 fangjun staff 341K Jan 1 2001 NOTICE.txt + # drwxr-xr-x 3 fangjun staff 96B Nov 6 21:17 build + # drwxr-xr-x 3 fangjun staff 96B Nov 6 21:18 build-tools + # -rw-r--r-- 1 fangjun staff 371B Jan 1 2001 compatible_config.json + # drwxr-xr-x 10 fangjun staff 320B Nov 6 21:18 llvm + # -rw-r--r-- 1 fangjun staff 16K Jan 1 2001 nativeapi_syscap_config.json + # -rw-r--r-- 1 fangjun staff 5.9K Jan 1 2001 ndk_system_capability.json + # -rw-r--r-- 1 fangjun staff 167B Jan 1 2001 oh-uni-package.json + # drwxr-xr-x 3 fangjun staff 96B Nov 6 21:17 sysroot fi if [ ! -d $OHOS_SDK_NATIVE_DIR ]; then echo "Please first download Command Line Tools for HarmonyOS" + echo "See https://developer.huawei.com/consumer/cn/download/" + echo "or" + echo "https://hf-mirror.com/csukuangfj/harmonyos-commandline-tools/tree/main" + exit 1 +fi + +if [ ! -f $OHOS_SDK_NATIVE_DIR/llvm/bin/x86_64-unknown-linux-ohos-clang ]; then + echo "$OHOS_SDK_NATIVE_DIR/llvm/bin/x86_64-unknown-linux-ohos-clang does not exist" + echo "Please first download Command Line Tools for HarmonyOS" + echo "See https://developer.huawei.com/consumer/cn/download/" + echo "or" + echo "https://hf-mirror.com/csukuangfj/harmonyos-commandline-tools/tree/main" exit 1 fi +export PATH=$OHOS_SDK_NATIVE_DIR/build-tools/cmake/bin:$PATH export PATH=$OHOS_SDK_NATIVE_DIR/llvm/bin:$PATH OHOS_TOOLCHAIN_FILE=$OHOS_SDK_NATIVE_DIR/build/cmake/ohos.toolchain.cmake @@ -71,7 +77,6 @@ if [ ! -f $OHOS_TOOLCHAIN_FILE ]; then exit 1 fi - sleep 1 onnxruntime_version=1.16.3 onnxruntime_dir=onnxruntime-ohos-x86_64-$onnxruntime_version @@ -126,7 +131,6 @@ cmake \ make -j2 make install/strip cp -fv $onnxruntime_dir/lib/libonnxruntime.so install/lib -cp -fv $OHOS_SDK_NATIVE_DIR/llvm/lib/x86_64-linux-ohos/libc++_shared.so install/lib rm -rf install/share rm -rf install/lib/pkgconfig diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index f01b4917f..d2091bc56 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -11,6 +11,10 @@ #include #include +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/audio-tagging.h" #include "sherpa-onnx/csrc/circular-buffer.h" #include "sherpa-onnx/csrc/display.h" @@ -917,8 +921,8 @@ struct SherpaOnnxVoiceActivityDetector { std::unique_ptr impl; }; -SherpaOnnxVoiceActivityDetector *SherpaOnnxCreateVoiceActivityDetector( - const SherpaOnnxVadModelConfig *config, float buffer_size_in_seconds) { +sherpa_onnx::VadModelConfig GetVadModelConfig( + const SherpaOnnxVadModelConfig *config) { sherpa_onnx::VadModelConfig vad_config; vad_config.silero_vad.model = SHERPA_ONNX_OR(config->silero_vad.model, ""); @@ -947,9 +951,20 @@ SherpaOnnxVoiceActivityDetector *SherpaOnnxCreateVoiceActivityDetector( vad_config.debug = SHERPA_ONNX_OR(config->debug, false); if (vad_config.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", vad_config.ToString().c_str()); +#else SHERPA_ONNX_LOGE("%s", vad_config.ToString().c_str()); +#endif } + return vad_config; +} + +SherpaOnnxVoiceActivityDetector *SherpaOnnxCreateVoiceActivityDetector( + const SherpaOnnxVadModelConfig *config, float buffer_size_in_seconds) { + auto vad_config = GetVadModelConfig(config); + if (!vad_config.Validate()) { SHERPA_ONNX_LOGE("Errors in config"); return nullptr; @@ -962,6 +977,25 @@ SherpaOnnxVoiceActivityDetector *SherpaOnnxCreateVoiceActivityDetector( return p; } +#ifdef __OHOS__ +SherpaOnnxVoiceActivityDetector *SherpaOnnxCreateVoiceActivityDetectorOHOS( + const SherpaOnnxVadModelConfig *config, float buffer_size_in_seconds, + NativeResourceManager *mgr) { + if (mgr == nullptr) { + return SherpaOnnxCreateVoiceActivityDetector(config, + buffer_size_in_seconds); + } + + auto vad_config = GetVadModelConfig(config); + + SherpaOnnxVoiceActivityDetector *p = new SherpaOnnxVoiceActivityDetector; + p->impl = std::make_unique( + mgr, vad_config, buffer_size_in_seconds); + + return p; +} +#endif + void SherpaOnnxDestroyVoiceActivityDetector( SherpaOnnxVoiceActivityDetector *p) { delete p; diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index 8b4b65786..6ea5c27fb 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -841,6 +841,21 @@ SHERPA_ONNX_API SherpaOnnxVoiceActivityDetector * SherpaOnnxCreateVoiceActivityDetector(const SherpaOnnxVadModelConfig *config, float buffer_size_in_seconds); +#ifdef __OHOS__ + +// Return an instance of VoiceActivityDetector. +// The user has to use SherpaOnnxDestroyVoiceActivityDetector() to free +// the returned pointer to avoid memory leak. +// +// It is for HarmonyOS +typedef struct NativeResourceManager NativeResourceManager; + +SHERPA_ONNX_API SherpaOnnxVoiceActivityDetector * +SherpaOnnxCreateVoiceActivityDetectorOHOS( + const SherpaOnnxVadModelConfig *config, float buffer_size_in_seconds, + NativeResourceManager *mgr); +#endif + SHERPA_ONNX_API void SherpaOnnxDestroyVoiceActivityDetector( SherpaOnnxVoiceActivityDetector *p); diff --git a/sherpa-onnx/csrc/CMakeLists.txt b/sherpa-onnx/csrc/CMakeLists.txt index d241ee1ca..b3c1617e3 100644 --- a/sherpa-onnx/csrc/CMakeLists.txt +++ b/sherpa-onnx/csrc/CMakeLists.txt @@ -207,6 +207,12 @@ target_link_libraries(sherpa-onnx-core kaldi-decoder-core ssentencepiece_core ) +if(DEFINED OHOS AND x${OHOS} STREQUAL xOHOS) + target_link_libraries(sherpa-onnx-core + hilog_ndk.z + rawfile.z + ) +endif() if(SHERPA_ONNX_ENABLE_GPU) target_link_libraries(sherpa-onnx-core diff --git a/sherpa-onnx/csrc/macros.h b/sherpa-onnx/csrc/macros.h index 7cd4d4627..ac11d6a79 100644 --- a/sherpa-onnx/csrc/macros.h +++ b/sherpa-onnx/csrc/macros.h @@ -8,6 +8,16 @@ #include #include +#if __OHOS__ +#include "hilog/log.h" + +#undef LOG_DOMAIN +#undef LOG_TAG + +// https://gitee.com/openharmony/docs/blob/145a084f0b742e4325915e32f8184817927d1251/en/contribute/OpenHarmony-Log-guide.md#hilog-api-usage-specifications +#define LOG_DOMAIN 0x6666 +#define LOG_TAG "sherpa_onnx" +#endif #if __ANDROID_API__ >= 8 #include "android/log.h" @@ -19,6 +29,8 @@ fprintf(stderr, "\n"); \ __android_log_print(ANDROID_LOG_WARN, "sherpa-onnx", ##__VA_ARGS__); \ } while (0) +#elif defined(__OHOS__) +#define SHERPA_ONNX_LOGE(...) OH_LOG_INFO(LOG_APP, ##__VA_ARGS__) #elif SHERPA_ONNX_ENABLE_WASM #define SHERPA_ONNX_LOGE(...) \ do { \ diff --git a/sherpa-onnx/csrc/onnx-utils.cc b/sherpa-onnx/csrc/onnx-utils.cc index 5e346a69e..6d7e26846 100644 --- a/sherpa-onnx/csrc/onnx-utils.cc +++ b/sherpa-onnx/csrc/onnx-utils.cc @@ -7,9 +7,13 @@ #include #include #include +#include #include #include #include +#include + +#include "sherpa-onnx/csrc/macros.h" #if __ANDROID_API__ >= 9 #include "android/asset_manager.h" @@ -326,6 +330,38 @@ std::vector ReadFile(AAssetManager *mgr, const std::string &filename) { } #endif +#if __OHOS__ +std::vector ReadFile(NativeResourceManager *mgr, + const std::string &filename) { + std::unique_ptr fp( + OH_ResourceManager_OpenRawFile(mgr, filename.c_str()), + OH_ResourceManager_CloseRawFile); + + if (!fp) { + std::ostringstream os; + os << "Read file '" << filename << "' failed."; + SHERPA_ONNX_LOGE("%s", os.str().c_str()); + return {}; + } + + auto len = static_cast(OH_ResourceManager_GetRawFileSize(fp.get())); + + std::vector buffer(len); + + int32_t n = OH_ResourceManager_ReadRawFile(fp.get(), buffer.data(), len); + + if (n != len) { + std::ostringstream os; + os << "Read file '" << filename << "' failed. Number of bytes read: " << n + << ". Expected bytes to read: " << len; + SHERPA_ONNX_LOGE("%s", os.str().c_str()); + return {}; + } + + return buffer; +} +#endif + Ort::Value Repeat(OrtAllocator *allocator, Ort::Value *cur_encoder_out, const std::vector &hyps_num_split) { std::vector cur_encoder_out_shape = diff --git a/sherpa-onnx/csrc/onnx-utils.h b/sherpa-onnx/csrc/onnx-utils.h index 8a19a2baf..c978fbb77 100644 --- a/sherpa-onnx/csrc/onnx-utils.h +++ b/sherpa-onnx/csrc/onnx-utils.h @@ -22,6 +22,10 @@ #include "android/asset_manager_jni.h" #endif +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "onnxruntime_cxx_api.h" // NOLINT namespace sherpa_onnx { @@ -103,6 +107,11 @@ std::vector ReadFile(const std::string &filename); std::vector ReadFile(AAssetManager *mgr, const std::string &filename); #endif +#if __OHOS__ +std::vector ReadFile(NativeResourceManager *mgr, + const std::string &filename); +#endif + // TODO(fangjun): Document it Ort::Value Repeat(OrtAllocator *allocator, Ort::Value *cur_encoder_out, const std::vector &hyps_num_split); diff --git a/sherpa-onnx/csrc/silero-vad-model.cc b/sherpa-onnx/csrc/silero-vad-model.cc index 66841d56d..10f8027ba 100644 --- a/sherpa-onnx/csrc/silero-vad-model.cc +++ b/sherpa-onnx/csrc/silero-vad-model.cc @@ -37,8 +37,9 @@ class SileroVadModel::Impl { min_speech_samples_ = sample_rate_ * config_.silero_vad.min_speech_duration; } -#if __ANDROID_API__ >= 9 - Impl(AAssetManager *mgr, const VadModelConfig &config) +#if __ANDROID_API__ >= 9 || defined(__OHOS__) + template + Impl(Manager *mgr, const VadModelConfig &config) : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), @@ -437,6 +438,12 @@ SileroVadModel::SileroVadModel(AAssetManager *mgr, const VadModelConfig &config) : impl_(std::make_unique(mgr, config)) {} #endif +#if __OHOS__ +SileroVadModel::SileroVadModel(NativeResourceManager *mgr, + const VadModelConfig &config) + : impl_(std::make_unique(mgr, config)) {} +#endif + SileroVadModel::~SileroVadModel() = default; void SileroVadModel::Reset() { return impl_->Reset(); } diff --git a/sherpa-onnx/csrc/silero-vad-model.h b/sherpa-onnx/csrc/silero-vad-model.h index 169cb7244..7c4c2f90f 100644 --- a/sherpa-onnx/csrc/silero-vad-model.h +++ b/sherpa-onnx/csrc/silero-vad-model.h @@ -11,6 +11,10 @@ #include "android/asset_manager_jni.h" #endif +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/vad-model.h" namespace sherpa_onnx { @@ -23,6 +27,10 @@ class SileroVadModel : public VadModel { SileroVadModel(AAssetManager *mgr, const VadModelConfig &config); #endif +#if __OHOS__ + SileroVadModel(NativeResourceManager *mgr, const VadModelConfig &config); +#endif + ~SileroVadModel() override; // reset the internal model states diff --git a/sherpa-onnx/csrc/vad-model.cc b/sherpa-onnx/csrc/vad-model.cc index be9a5e7fe..658745046 100644 --- a/sherpa-onnx/csrc/vad-model.cc +++ b/sherpa-onnx/csrc/vad-model.cc @@ -21,4 +21,12 @@ std::unique_ptr VadModel::Create(AAssetManager *mgr, } #endif +#if __OHOS__ +std::unique_ptr VadModel::Create(NativeResourceManager *mgr, + const VadModelConfig &config) { + // TODO(fangjun): Support other VAD models. + return std::make_unique(mgr, config); +} +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/vad-model.h b/sherpa-onnx/csrc/vad-model.h index 81028f2f1..3a425b193 100644 --- a/sherpa-onnx/csrc/vad-model.h +++ b/sherpa-onnx/csrc/vad-model.h @@ -11,6 +11,10 @@ #include "android/asset_manager_jni.h" #endif +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/vad-model-config.h" namespace sherpa_onnx { @@ -26,6 +30,11 @@ class VadModel { const VadModelConfig &config); #endif +#if __OHOS__ + static std::unique_ptr Create(NativeResourceManager *mgr, + const VadModelConfig &config); +#endif + // reset the internal model states virtual void Reset() = 0; diff --git a/sherpa-onnx/csrc/voice-activity-detector.cc b/sherpa-onnx/csrc/voice-activity-detector.cc index c20d3476d..8c6038b7a 100644 --- a/sherpa-onnx/csrc/voice-activity-detector.cc +++ b/sherpa-onnx/csrc/voice-activity-detector.cc @@ -22,8 +22,9 @@ class VoiceActivityDetector::Impl { Init(); } -#if __ANDROID_API__ >= 9 - Impl(AAssetManager *mgr, const VadModelConfig &config, +#if __ANDROID_API__ >= 9 || defined(__OHOS__) + template + Impl(Manager *mgr, const VadModelConfig &config, float buffer_size_in_seconds = 60) : model_(VadModel::Create(mgr, config)), config_(config), @@ -184,6 +185,13 @@ VoiceActivityDetector::VoiceActivityDetector( : impl_(std::make_unique(mgr, config, buffer_size_in_seconds)) {} #endif +#if __OHOS__ +VoiceActivityDetector::VoiceActivityDetector( + NativeResourceManager *mgr, const VadModelConfig &config, + float buffer_size_in_seconds /*= 60*/) + : impl_(std::make_unique(mgr, config, buffer_size_in_seconds)) {} +#endif + VoiceActivityDetector::~VoiceActivityDetector() = default; void VoiceActivityDetector::AcceptWaveform(const float *samples, int32_t n) { diff --git a/sherpa-onnx/csrc/voice-activity-detector.h b/sherpa-onnx/csrc/voice-activity-detector.h index 9eb53c554..9cc1a8897 100644 --- a/sherpa-onnx/csrc/voice-activity-detector.h +++ b/sherpa-onnx/csrc/voice-activity-detector.h @@ -12,6 +12,10 @@ #include "android/asset_manager_jni.h" #endif +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/vad-model-config.h" namespace sherpa_onnx { @@ -31,6 +35,12 @@ class VoiceActivityDetector { float buffer_size_in_seconds = 60); #endif +#if __OHOS__ + VoiceActivityDetector(NativeResourceManager *mgr, + const VadModelConfig &config, + float buffer_size_in_seconds = 60); +#endif + ~VoiceActivityDetector(); void AcceptWaveform(const float *samples, int32_t n); From a4b79f077c889e0edf5f5591b6e018e2b1760ca6 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 26 Nov 2024 13:45:15 +0800 Subject: [PATCH 078/183] Fix flutter ios (#1563) --- CHANGELOG.md | 6 ++++++ CMakeLists.txt | 2 +- build-ios-shared.sh | 11 +++++++++-- dart-api-examples/add-punctuations/pubspec.yaml | 2 +- dart-api-examples/audio-tagging/pubspec.yaml | 2 +- dart-api-examples/keyword-spotter/pubspec.yaml | 2 +- dart-api-examples/non-streaming-asr/pubspec.yaml | 2 +- dart-api-examples/speaker-diarization/pubspec.yaml | 2 +- .../speaker-identification/pubspec.yaml | 2 +- dart-api-examples/streaming-asr/pubspec.yaml | 2 +- dart-api-examples/tts/pubspec.yaml | 2 +- .../vad-with-non-streaming-asr/pubspec.yaml | 2 +- dart-api-examples/vad/pubspec.yaml | 2 +- flutter-examples/streaming_asr/pubspec.yaml | 4 ++-- flutter-examples/tts/pubspec.yaml | 4 ++-- flutter/sherpa_onnx/pubspec.yaml | 12 ++++++------ flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec | 4 ++-- .../macos/sherpa_onnx_macos.podspec | 2 +- new-release.sh | 12 ++++++------ nodejs-addon-examples/package.json | 2 +- 20 files changed, 46 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 669f2e405..6493a95e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 1.10.32 + +* Support cross-compiling for HarmonyOS (#1553) +* HarmonyOS support for VAD. (#1561) +* Fix publishing flutter iOS app to appstore. + ## 1.10.31 * Publish pre-built wheels for Python 3.13 (#1485) diff --git a/CMakeLists.txt b/CMakeLists.txt index d6f94c668..f5b626b8c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ project(sherpa-onnx) # Remember to update # ./CHANGELOG.md # ./new-release.sh -set(SHERPA_ONNX_VERSION "1.10.31") +set(SHERPA_ONNX_VERSION "1.10.32") # Disable warning about # diff --git a/build-ios-shared.sh b/build-ios-shared.sh index c58bb300d..9ac67848e 100755 --- a/build-ios-shared.sh +++ b/build-ios-shared.sh @@ -242,7 +242,7 @@ for d in ios-arm64_x86_64-simulator ios-arm64; do CFBundlePackageType FMWK CFBundleShortVersionString - 1.10.31 + 1.10.32 CFBundleSupportedPlatforms iPhoneOS @@ -258,7 +258,7 @@ for d in ios-arm64_x86_64-simulator ios-arm64; do arm64 MinimumOSVersion - 12.0 + 13.0 CFBundleIdentifier com.k2fsa.sherpa.onnx UIDeviceFamily @@ -274,6 +274,13 @@ for d in ios-arm64_x86_64-simulator ios-arm64; do 15A507 DTPlatformBuild 21A326 + SupportedArchitectures + + arm64 + x86_64 + + SupportedPlatform + ios EOF diff --git a/dart-api-examples/add-punctuations/pubspec.yaml b/dart-api-examples/add-punctuations/pubspec.yaml index 55e8c5d98..fef4dd729 100644 --- a/dart-api-examples/add-punctuations/pubspec.yaml +++ b/dart-api-examples/add-punctuations/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.31 + sherpa_onnx: ^1.10.32 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/audio-tagging/pubspec.yaml b/dart-api-examples/audio-tagging/pubspec.yaml index a79e925b1..fa359d89d 100644 --- a/dart-api-examples/audio-tagging/pubspec.yaml +++ b/dart-api-examples/audio-tagging/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.31 + sherpa_onnx: ^1.10.32 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/keyword-spotter/pubspec.yaml b/dart-api-examples/keyword-spotter/pubspec.yaml index d79f68486..ffa074a23 100644 --- a/dart-api-examples/keyword-spotter/pubspec.yaml +++ b/dart-api-examples/keyword-spotter/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.31 + sherpa_onnx: ^1.10.32 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/non-streaming-asr/pubspec.yaml b/dart-api-examples/non-streaming-asr/pubspec.yaml index 3c61a0cb2..52ee859e6 100644 --- a/dart-api-examples/non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.31 + sherpa_onnx: ^1.10.32 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/speaker-diarization/pubspec.yaml b/dart-api-examples/speaker-diarization/pubspec.yaml index 6fb714d10..6706e28d1 100644 --- a/dart-api-examples/speaker-diarization/pubspec.yaml +++ b/dart-api-examples/speaker-diarization/pubspec.yaml @@ -8,7 +8,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.31 + sherpa_onnx: ^1.10.32 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/speaker-identification/pubspec.yaml b/dart-api-examples/speaker-identification/pubspec.yaml index 36969fb1d..4b47846b8 100644 --- a/dart-api-examples/speaker-identification/pubspec.yaml +++ b/dart-api-examples/speaker-identification/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.31 + sherpa_onnx: ^1.10.32 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/streaming-asr/pubspec.yaml b/dart-api-examples/streaming-asr/pubspec.yaml index ec6270297..cfc9037ab 100644 --- a/dart-api-examples/streaming-asr/pubspec.yaml +++ b/dart-api-examples/streaming-asr/pubspec.yaml @@ -11,7 +11,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.31 + sherpa_onnx: ^1.10.32 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/tts/pubspec.yaml b/dart-api-examples/tts/pubspec.yaml index ed55db1eb..9cf7049bc 100644 --- a/dart-api-examples/tts/pubspec.yaml +++ b/dart-api-examples/tts/pubspec.yaml @@ -8,7 +8,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.31 + sherpa_onnx: ^1.10.32 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml index 6d331d8f5..ca0d08cad 100644 --- a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.31 + sherpa_onnx: ^1.10.32 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad/pubspec.yaml b/dart-api-examples/vad/pubspec.yaml index b16bcb64b..cd1acfbd2 100644 --- a/dart-api-examples/vad/pubspec.yaml +++ b/dart-api-examples/vad/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.31 + sherpa_onnx: ^1.10.32 path: ^1.9.0 args: ^2.5.0 diff --git a/flutter-examples/streaming_asr/pubspec.yaml b/flutter-examples/streaming_asr/pubspec.yaml index 1eb29cbd4..680d9d346 100644 --- a/flutter-examples/streaming_asr/pubspec.yaml +++ b/flutter-examples/streaming_asr/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' -version: 1.10.31 +version: 1.10.32 topics: - speech-recognition @@ -31,7 +31,7 @@ dependencies: record: ^5.1.0 url_launcher: ^6.2.6 - sherpa_onnx: ^1.10.31 + sherpa_onnx: ^1.10.32 # sherpa_onnx: # path: ../../flutter/sherpa_onnx diff --git a/flutter-examples/tts/pubspec.yaml b/flutter-examples/tts/pubspec.yaml index 669a1ed4d..f8eca471a 100644 --- a/flutter-examples/tts/pubspec.yaml +++ b/flutter-examples/tts/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' # Remove this line if you wish to publish to pub.dev -version: 1.10.31 +version: 1.10.32 environment: sdk: ">=2.17.0 <4.0.0" @@ -18,7 +18,7 @@ dependencies: cupertino_icons: ^1.0.6 path_provider: ^2.1.3 path: ^1.9.0 - sherpa_onnx: ^1.10.31 + sherpa_onnx: ^1.10.32 # sherpa_onnx: # path: ../../flutter/sherpa_onnx url_launcher: 6.2.6 diff --git a/flutter/sherpa_onnx/pubspec.yaml b/flutter/sherpa_onnx/pubspec.yaml index 675ed844a..f5da2dbb6 100644 --- a/flutter/sherpa_onnx/pubspec.yaml +++ b/flutter/sherpa_onnx/pubspec.yaml @@ -17,7 +17,7 @@ topics: - voice-activity-detection # remember to change the version in ../sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec -version: 1.10.31 +version: 1.10.32 homepage: https://github.com/k2-fsa/sherpa-onnx @@ -30,23 +30,23 @@ dependencies: flutter: sdk: flutter - sherpa_onnx_android: ^1.10.31 + sherpa_onnx_android: ^1.10.32 # sherpa_onnx_android: # path: ../sherpa_onnx_android - sherpa_onnx_macos: ^1.10.31 + sherpa_onnx_macos: ^1.10.32 # sherpa_onnx_macos: # path: ../sherpa_onnx_macos - sherpa_onnx_linux: ^1.10.31 + sherpa_onnx_linux: ^1.10.32 # sherpa_onnx_linux: # path: ../sherpa_onnx_linux - sherpa_onnx_windows: ^1.10.31 + sherpa_onnx_windows: ^1.10.32 # sherpa_onnx_windows: # path: ../sherpa_onnx_windows - sherpa_onnx_ios: ^1.10.31 + sherpa_onnx_ios: ^1.10.32 # sherpa_onnx_ios: # path: ../sherpa_onnx_ios diff --git a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec index 11930c4e7..e840a2225 100644 --- a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec +++ b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec @@ -7,7 +7,7 @@ # https://groups.google.com/g/dart-ffi/c/nUATMBy7r0c Pod::Spec.new do |s| s.name = 'sherpa_onnx_ios' - s.version = '1.10.31' + s.version = '1.10.32' s.summary = 'A new Flutter FFI plugin project.' s.description = <<-DESC A new Flutter FFI plugin project. @@ -22,7 +22,7 @@ A new Flutter FFI plugin project. # `../src/*` so that the C sources can be shared among all target platforms. s.source = { :path => '.' } s.dependency 'Flutter' - s.platform = :ios, '12.0' + s.platform = :ios, '13.0' s.preserve_paths = 'sherpa_onnx.xcframework/**/*' s.vendored_frameworks = 'sherpa_onnx.xcframework' diff --git a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec index 95b0970b3..100966033 100644 --- a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec +++ b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec @@ -4,7 +4,7 @@ # Pod::Spec.new do |s| s.name = 'sherpa_onnx_macos' - s.version = '1.10.31' + s.version = '1.10.32' s.summary = 'sherpa-onnx Flutter FFI plugin project.' s.description = <<-DESC sherpa-onnx Flutter FFI plugin project. diff --git a/new-release.sh b/new-release.sh index 7e6ed56b5..eaf868971 100755 --- a/new-release.sh +++ b/new-release.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash -sed -i.bak 's/1\.10\.30/1\.10\.31/g' ./build-ios-shared.sh -find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.30/1\.10\.31/g' {} \; -find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.30/1\.10\.31/g' {} \; -find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.30/1\.10\.31/g' {} \; -find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.30/1\.10\.31/g' {} \; -find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.30/1\.10\.31/g' {} \; +sed -i.bak 's/1\.10\.31/1\.10\.32/g' ./build-ios-shared.sh +find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.31/1\.10\.32/g' {} \; +find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.31/1\.10\.32/g' {} \; +find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.31/1\.10\.32/g' {} \; +find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.31/1\.10\.32/g' {} \; +find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.31/1\.10\.32/g' {} \; diff --git a/nodejs-addon-examples/package.json b/nodejs-addon-examples/package.json index ee4404123..47bed5b88 100644 --- a/nodejs-addon-examples/package.json +++ b/nodejs-addon-examples/package.json @@ -1,5 +1,5 @@ { "dependencies": { - "sherpa-onnx-node": "^1.10.31" + "sherpa-onnx-node": "^1.10.32" } } From 298b6b6fda9b1616b6405a4fc0d7725c1a45fd65 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 26 Nov 2024 16:38:35 +0800 Subject: [PATCH 079/183] Add non-streaming ASR support for HarmonyOS. (#1564) --- sherpa-onnx/c-api/c-api.cc | 103 +++++++++++------- sherpa-onnx/c-api/c-api.h | 38 ++++--- sherpa-onnx/c-api/cxx-api.h | 5 - sherpa-onnx/csrc/offline-ctc-model.cc | 29 ++++- sherpa-onnx/csrc/offline-ctc-model.h | 10 +- sherpa-onnx/csrc/offline-lm.cc | 24 +++- sherpa-onnx/csrc/offline-lm.h | 10 +- sherpa-onnx/csrc/offline-moonshine-model.cc | 29 ++++- sherpa-onnx/csrc/offline-moonshine-model.h | 10 +- .../csrc/offline-nemo-enc-dec-ctc-model.cc | 33 +++++- .../csrc/offline-nemo-enc-dec-ctc-model.h | 11 +- sherpa-onnx/csrc/offline-paraformer-model.cc | 33 +++++- sherpa-onnx/csrc/offline-paraformer-model.h | 10 +- .../csrc/offline-recognizer-ctc-impl.h | 11 +- sherpa-onnx/csrc/offline-recognizer-impl.cc | 29 ++++- sherpa-onnx/csrc/offline-recognizer-impl.h | 14 +-- .../csrc/offline-recognizer-moonshine-impl.h | 11 +- .../csrc/offline-recognizer-paraformer-impl.h | 10 +- .../offline-recognizer-sense-voice-impl.h | 10 +- .../csrc/offline-recognizer-transducer-impl.h | 20 +--- .../offline-recognizer-transducer-nemo-impl.h | 10 +- .../csrc/offline-recognizer-whisper-impl.h | 15 +-- sherpa-onnx/csrc/offline-recognizer.cc | 27 ++++- sherpa-onnx/csrc/offline-recognizer.h | 18 +-- sherpa-onnx/csrc/offline-rnn-lm.cc | 29 ++++- sherpa-onnx/csrc/offline-rnn-lm.h | 10 +- sherpa-onnx/csrc/offline-sense-voice-model.cc | 33 +++++- sherpa-onnx/csrc/offline-sense-voice-model.h | 10 +- sherpa-onnx/csrc/offline-tdnn-ctc-model.cc | 33 +++++- sherpa-onnx/csrc/offline-tdnn-ctc-model.h | 10 +- .../csrc/offline-telespeech-ctc-model.cc | 33 +++++- .../csrc/offline-telespeech-ctc-model.h | 11 +- sherpa-onnx/csrc/offline-transducer-model.cc | 33 +++++- sherpa-onnx/csrc/offline-transducer-model.h | 10 +- .../csrc/offline-transducer-nemo-model.cc | 33 +++++- .../csrc/offline-transducer-nemo-model.h | 11 +- sherpa-onnx/csrc/offline-wenet-ctc-model.cc | 33 +++++- sherpa-onnx/csrc/offline-wenet-ctc-model.h | 10 +- sherpa-onnx/csrc/offline-whisper-model.cc | 47 ++++++-- sherpa-onnx/csrc/offline-whisper-model.h | 14 +-- .../csrc/offline-zipformer-ctc-model.cc | 33 +++++- .../csrc/offline-zipformer-ctc-model.h | 11 +- sherpa-onnx/csrc/silero-vad-model.cc | 32 ++++-- sherpa-onnx/csrc/silero-vad-model.h | 18 +-- sherpa-onnx/csrc/symbol-table.cc | 19 +++- sherpa-onnx/csrc/symbol-table.h | 10 +- sherpa-onnx/csrc/vad-model.cc | 25 +++-- sherpa-onnx/csrc/vad-model.h | 19 +--- sherpa-onnx/csrc/voice-activity-detector.cc | 35 ++++-- sherpa-onnx/csrc/voice-activity-detector.h | 20 +--- 50 files changed, 648 insertions(+), 454 deletions(-) diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index d2091bc56..c9dafe84b 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -352,27 +352,7 @@ struct SherpaOnnxOfflineStream { : impl(std::move(p)) {} }; -static sherpa_onnx::OfflineRecognizerConfig convertConfig( - const SherpaOnnxOfflineRecognizerConfig *config); - -const SherpaOnnxOfflineRecognizer *SherpaOnnxCreateOfflineRecognizer( - const SherpaOnnxOfflineRecognizerConfig *config) { - sherpa_onnx::OfflineRecognizerConfig recognizer_config = - convertConfig(config); - - if (!recognizer_config.Validate()) { - SHERPA_ONNX_LOGE("Errors in config"); - return nullptr; - } - - SherpaOnnxOfflineRecognizer *recognizer = new SherpaOnnxOfflineRecognizer; - - recognizer->impl = - std::make_unique(recognizer_config); - - return recognizer; -} -sherpa_onnx::OfflineRecognizerConfig convertConfig( +static sherpa_onnx::OfflineRecognizerConfig GetOfflineRecognizerConfig( const SherpaOnnxOfflineRecognizerConfig *config) { sherpa_onnx::OfflineRecognizerConfig recognizer_config; @@ -491,17 +471,39 @@ sherpa_onnx::OfflineRecognizerConfig convertConfig( recognizer_config.rule_fars = SHERPA_ONNX_OR(config->rule_fars, ""); if (config->model_config.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", recognizer_config.ToString().c_str()); +#else SHERPA_ONNX_LOGE("%s", recognizer_config.ToString().c_str()); +#endif } return recognizer_config; } +const SherpaOnnxOfflineRecognizer *SherpaOnnxCreateOfflineRecognizer( + const SherpaOnnxOfflineRecognizerConfig *config) { + sherpa_onnx::OfflineRecognizerConfig recognizer_config = + GetOfflineRecognizerConfig(config); + + if (!recognizer_config.Validate()) { + SHERPA_ONNX_LOGE("Errors in config"); + return nullptr; + } + + SherpaOnnxOfflineRecognizer *recognizer = new SherpaOnnxOfflineRecognizer; + + recognizer->impl = + std::make_unique(recognizer_config); + + return recognizer; +} + void SherpaOnnxOfflineRecognizerSetConfig( const SherpaOnnxOfflineRecognizer *recognizer, const SherpaOnnxOfflineRecognizerConfig *config) { sherpa_onnx::OfflineRecognizerConfig recognizer_config = - convertConfig(config); + GetOfflineRecognizerConfig(config); recognizer->impl->SetConfig(recognizer_config); } @@ -977,25 +979,6 @@ SherpaOnnxVoiceActivityDetector *SherpaOnnxCreateVoiceActivityDetector( return p; } -#ifdef __OHOS__ -SherpaOnnxVoiceActivityDetector *SherpaOnnxCreateVoiceActivityDetectorOHOS( - const SherpaOnnxVadModelConfig *config, float buffer_size_in_seconds, - NativeResourceManager *mgr) { - if (mgr == nullptr) { - return SherpaOnnxCreateVoiceActivityDetector(config, - buffer_size_in_seconds); - } - - auto vad_config = GetVadModelConfig(config); - - SherpaOnnxVoiceActivityDetector *p = new SherpaOnnxVoiceActivityDetector; - p->impl = std::make_unique( - mgr, vad_config, buffer_size_in_seconds); - - return p; -} -#endif - void SherpaOnnxDestroyVoiceActivityDetector( SherpaOnnxVoiceActivityDetector *p) { delete p; @@ -1891,4 +1874,42 @@ SherpaOnnxOfflineSpeakerDiarizationProcessWithCallbackNoArg( return ans; } +#ifdef __OHOS__ + +const SherpaOnnxOfflineRecognizer *SherpaOnnxCreateOfflineRecognizerOHOS( + const SherpaOnnxOfflineRecognizerConfig *config, + NativeResourceManager *mgr) { + if (mgr == nullptr) { + return SherpaOnnxCreateOfflineRecognizer(config); + } + + sherpa_onnx::OfflineRecognizerConfig recognizer_config = + GetOfflineRecognizerConfig(config); + + SherpaOnnxOfflineRecognizer *recognizer = new SherpaOnnxOfflineRecognizer; + + recognizer->impl = + std::make_unique(mgr, recognizer_config); + + return recognizer; +} + +SherpaOnnxVoiceActivityDetector *SherpaOnnxCreateVoiceActivityDetectorOHOS( + const SherpaOnnxVadModelConfig *config, float buffer_size_in_seconds, + NativeResourceManager *mgr) { + if (mgr == nullptr) { + return SherpaOnnxCreateVoiceActivityDetector(config, + buffer_size_in_seconds); + } + + auto vad_config = GetVadModelConfig(config); + + SherpaOnnxVoiceActivityDetector *p = new SherpaOnnxVoiceActivityDetector; + p->impl = std::make_unique( + mgr, vad_config, buffer_size_in_seconds); + + return p; +} +#endif + #endif diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index 6ea5c27fb..31e86b8e5 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -841,21 +841,6 @@ SHERPA_ONNX_API SherpaOnnxVoiceActivityDetector * SherpaOnnxCreateVoiceActivityDetector(const SherpaOnnxVadModelConfig *config, float buffer_size_in_seconds); -#ifdef __OHOS__ - -// Return an instance of VoiceActivityDetector. -// The user has to use SherpaOnnxDestroyVoiceActivityDetector() to free -// the returned pointer to avoid memory leak. -// -// It is for HarmonyOS -typedef struct NativeResourceManager NativeResourceManager; - -SHERPA_ONNX_API SherpaOnnxVoiceActivityDetector * -SherpaOnnxCreateVoiceActivityDetectorOHOS( - const SherpaOnnxVadModelConfig *config, float buffer_size_in_seconds, - NativeResourceManager *mgr); -#endif - SHERPA_ONNX_API void SherpaOnnxDestroyVoiceActivityDetector( SherpaOnnxVoiceActivityDetector *p); @@ -1537,6 +1522,29 @@ SherpaOnnxOfflineSpeakerDiarizationProcessWithCallbackNoArg( SHERPA_ONNX_API void SherpaOnnxOfflineSpeakerDiarizationDestroyResult( const SherpaOnnxOfflineSpeakerDiarizationResult *r); +#ifdef __OHOS__ + +// It is for HarmonyOS +typedef struct NativeResourceManager NativeResourceManager; + +/// @param config Config for the recognizer. +/// @return Return a pointer to the recognizer. The user has to invoke +// SherpaOnnxDestroyOfflineRecognizer() to free it to avoid memory +// leak. +SHERPA_ONNX_API const SherpaOnnxOfflineRecognizer * +SherpaOnnxCreateOfflineRecognizerOHOS( + const SherpaOnnxOfflineRecognizerConfig *config, + NativeResourceManager *mgr); + +// Return an instance of VoiceActivityDetector. +// The user has to use SherpaOnnxDestroyVoiceActivityDetector() to free +// the returned pointer to avoid memory leak. +SHERPA_ONNX_API SherpaOnnxVoiceActivityDetector * +SherpaOnnxCreateVoiceActivityDetectorOHOS( + const SherpaOnnxVadModelConfig *config, float buffer_size_in_seconds, + NativeResourceManager *mgr); +#endif + #if defined(__GNUC__) #pragma GCC diagnostic pop #endif diff --git a/sherpa-onnx/c-api/cxx-api.h b/sherpa-onnx/c-api/cxx-api.h index ade52b9aa..2a476efa1 100644 --- a/sherpa-onnx/c-api/cxx-api.h +++ b/sherpa-onnx/c-api/cxx-api.h @@ -214,11 +214,6 @@ struct SHERPA_ONNX_API OfflineTdnnModelConfig { std::string model; }; -struct SHERPA_ONNX_API SherpaOnnxOfflineLMConfig { - std::string model; - float scale = 1.0; -}; - struct SHERPA_ONNX_API OfflineSenseVoiceModelConfig { std::string model; std::string language; diff --git a/sherpa-onnx/csrc/offline-ctc-model.cc b/sherpa-onnx/csrc/offline-ctc-model.cc index 2cbd936ea..daff5654a 100644 --- a/sherpa-onnx/csrc/offline-ctc-model.cc +++ b/sherpa-onnx/csrc/offline-ctc-model.cc @@ -9,6 +9,15 @@ #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.h" #include "sherpa-onnx/csrc/offline-tdnn-ctc-model.h" @@ -48,7 +57,11 @@ static ModelType GetModelType(char *model_data, size_t model_data_length, if (debug) { std::ostringstream os; PrintModelMetadata(os, meta_data); - SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s\n", os.str().c_str()); +#else + SHERPA_ONNX_LOGE("%s\n", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; @@ -144,10 +157,9 @@ std::unique_ptr OfflineCtcModel::Create( return nullptr; } -#if __ANDROID_API__ >= 9 - +template std::unique_ptr OfflineCtcModel::Create( - AAssetManager *mgr, const OfflineModelConfig &config) { + Manager *mgr, const OfflineModelConfig &config) { // TODO(fangjun): Refactor it. We don't need to use model_type here ModelType model_type = ModelType::kUnknown; @@ -196,6 +208,15 @@ std::unique_ptr OfflineCtcModel::Create( return nullptr; } + +#if __ANDROID_API__ >= 9 +template std::unique_ptr OfflineCtcModel::Create( + AAssetManager *mgr, const OfflineModelConfig &config); +#endif + +#if __OHOS__ +template std::unique_ptr OfflineCtcModel::Create( + NativeResourceManager *mgr, const OfflineModelConfig &config); #endif } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-ctc-model.h b/sherpa-onnx/csrc/offline-ctc-model.h index ead532b48..5ad4fcdcf 100644 --- a/sherpa-onnx/csrc/offline-ctc-model.h +++ b/sherpa-onnx/csrc/offline-ctc-model.h @@ -8,11 +8,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/offline-model-config.h" @@ -25,10 +20,9 @@ class OfflineCtcModel { static std::unique_ptr Create( const OfflineModelConfig &config); -#if __ANDROID_API__ >= 9 + template static std::unique_ptr Create( - AAssetManager *mgr, const OfflineModelConfig &config); -#endif + Manager *mgr, const OfflineModelConfig &config); /** Run the forward method of the model. * diff --git a/sherpa-onnx/csrc/offline-lm.cc b/sherpa-onnx/csrc/offline-lm.cc index e199ea5e4..0c42b7ff9 100644 --- a/sherpa-onnx/csrc/offline-lm.cc +++ b/sherpa-onnx/csrc/offline-lm.cc @@ -8,6 +8,15 @@ #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/offline-rnn-lm.h" namespace sherpa_onnx { @@ -16,12 +25,11 @@ std::unique_ptr OfflineLM::Create(const OfflineLMConfig &config) { return std::make_unique(config); } -#if __ANDROID_API__ >= 9 -std::unique_ptr OfflineLM::Create(AAssetManager *mgr, +template +std::unique_ptr OfflineLM::Create(Manager *mgr, const OfflineLMConfig &config) { return std::make_unique(mgr, config); } -#endif void OfflineLM::ComputeLMScore(float scale, int32_t context_size, std::vector *hyps) { @@ -75,4 +83,14 @@ void OfflineLM::ComputeLMScore(float scale, int32_t context_size, } } +#if __ANDROID_API__ >= 9 +template std::unique_ptr OfflineLM::Create( + AAssetManager *mgr, const OfflineLMConfig &config); +#endif + +#if __OHOS__ +template std::unique_ptr OfflineLM::Create( + NativeResourceManager *mgr, const OfflineLMConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-lm.h b/sherpa-onnx/csrc/offline-lm.h index 07082c149..a9af82020 100644 --- a/sherpa-onnx/csrc/offline-lm.h +++ b/sherpa-onnx/csrc/offline-lm.h @@ -8,11 +8,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/hypothesis.h" #include "sherpa-onnx/csrc/offline-lm-config.h" @@ -25,10 +20,9 @@ class OfflineLM { static std::unique_ptr Create(const OfflineLMConfig &config); -#if __ANDROID_API__ >= 9 - static std::unique_ptr Create(AAssetManager *mgr, + template + static std::unique_ptr Create(Manager *mgr, const OfflineLMConfig &config); -#endif /** Rescore a batch of sentences. * diff --git a/sherpa-onnx/csrc/offline-moonshine-model.cc b/sherpa-onnx/csrc/offline-moonshine-model.cc index bf9624d4d..dbd18a92d 100644 --- a/sherpa-onnx/csrc/offline-moonshine-model.cc +++ b/sherpa-onnx/csrc/offline-moonshine-model.cc @@ -8,6 +8,15 @@ #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/onnx-utils.h" #include "sherpa-onnx/csrc/session.h" @@ -43,8 +52,8 @@ class OfflineMoonshineModel::Impl { } } -#if __ANDROID_API__ >= 9 - Impl(AAssetManager *mgr, const OfflineModelConfig &config) + template + Impl(Manager *mgr, const OfflineModelConfig &config) : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), @@ -69,7 +78,6 @@ class OfflineMoonshineModel::Impl { InitCachedDecoder(buf.data(), buf.size()); } } -#endif Ort::Value ForwardPreprocessor(Ort::Value audio) { auto features = preprocessor_sess_->Run( @@ -242,11 +250,10 @@ class OfflineMoonshineModel::Impl { OfflineMoonshineModel::OfflineMoonshineModel(const OfflineModelConfig &config) : impl_(std::make_unique(config)) {} -#if __ANDROID_API__ >= 9 -OfflineMoonshineModel::OfflineMoonshineModel(AAssetManager *mgr, +template +OfflineMoonshineModel::OfflineMoonshineModel(Manager *mgr, const OfflineModelConfig &config) : impl_(std::make_unique(mgr, config)) {} -#endif OfflineMoonshineModel::~OfflineMoonshineModel() = default; @@ -279,4 +286,14 @@ OrtAllocator *OfflineMoonshineModel::Allocator() const { return impl_->Allocator(); } +#if __ANDROID_API__ >= 9 +template OfflineMoonshineModel::OfflineMoonshineModel( + AAssetManager *mgr, const OfflineModelConfig &config); +#endif + +#if __OHOS__ +template OfflineMoonshineModel::OfflineMoonshineModel( + NativeResourceManager *mgr, const OfflineModelConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-moonshine-model.h b/sherpa-onnx/csrc/offline-moonshine-model.h index 7065b1445..a8f9b408d 100644 --- a/sherpa-onnx/csrc/offline-moonshine-model.h +++ b/sherpa-onnx/csrc/offline-moonshine-model.h @@ -9,11 +9,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/offline-model-config.h" @@ -25,9 +20,8 @@ class OfflineMoonshineModel { public: explicit OfflineMoonshineModel(const OfflineModelConfig &config); -#if __ANDROID_API__ >= 9 - OfflineMoonshineModel(AAssetManager *mgr, const OfflineModelConfig &config); -#endif + template + OfflineMoonshineModel(Manager *mgr, const OfflineModelConfig &config); ~OfflineMoonshineModel(); diff --git a/sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.cc b/sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.cc index 14dc7dbe4..18db415b4 100644 --- a/sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.cc +++ b/sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.cc @@ -4,6 +4,15 @@ #include "sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.h" +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/onnx-utils.h" #include "sherpa-onnx/csrc/session.h" @@ -23,8 +32,8 @@ class OfflineNemoEncDecCtcModel::Impl { Init(buf.data(), buf.size()); } -#if __ANDROID_API__ >= 9 - Impl(AAssetManager *mgr, const OfflineModelConfig &config) + template + Impl(Manager *mgr, const OfflineModelConfig &config) : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), @@ -32,7 +41,6 @@ class OfflineNemoEncDecCtcModel::Impl { auto buf = ReadFile(mgr, config_.nemo_ctc.model); Init(buf.data(), buf.size()); } -#endif std::vector Forward(Ort::Value features, Ort::Value features_length) { @@ -88,7 +96,11 @@ class OfflineNemoEncDecCtcModel::Impl { if (config_.debug) { std::ostringstream os; PrintModelMetadata(os, meta_data); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s\n", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s\n", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; // used in the macro below @@ -126,11 +138,10 @@ OfflineNemoEncDecCtcModel::OfflineNemoEncDecCtcModel( const OfflineModelConfig &config) : impl_(std::make_unique(config)) {} -#if __ANDROID_API__ >= 9 +template OfflineNemoEncDecCtcModel::OfflineNemoEncDecCtcModel( - AAssetManager *mgr, const OfflineModelConfig &config) + Manager *mgr, const OfflineModelConfig &config) : impl_(std::make_unique(mgr, config)) {} -#endif OfflineNemoEncDecCtcModel::~OfflineNemoEncDecCtcModel() = default; @@ -156,4 +167,14 @@ std::string OfflineNemoEncDecCtcModel::FeatureNormalizationMethod() const { bool OfflineNemoEncDecCtcModel::IsGigaAM() const { return impl_->IsGigaAM(); } +#if __ANDROID_API__ >= 9 +template OfflineNemoEncDecCtcModel::OfflineNemoEncDecCtcModel( + AAssetManager *mgr, const OfflineModelConfig &config); +#endif + +#if __OHOS__ +template OfflineNemoEncDecCtcModel::OfflineNemoEncDecCtcModel( + NativeResourceManager *mgr, const OfflineModelConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.h b/sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.h index c961e694e..08ff068c1 100644 --- a/sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.h +++ b/sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.h @@ -8,11 +8,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/offline-ctc-model.h" #include "sherpa-onnx/csrc/offline-model-config.h" @@ -29,10 +24,8 @@ class OfflineNemoEncDecCtcModel : public OfflineCtcModel { public: explicit OfflineNemoEncDecCtcModel(const OfflineModelConfig &config); -#if __ANDROID_API__ >= 9 - OfflineNemoEncDecCtcModel(AAssetManager *mgr, - const OfflineModelConfig &config); -#endif + template + OfflineNemoEncDecCtcModel(Manager *mgr, const OfflineModelConfig &config); ~OfflineNemoEncDecCtcModel() override; diff --git a/sherpa-onnx/csrc/offline-paraformer-model.cc b/sherpa-onnx/csrc/offline-paraformer-model.cc index 9c61cb350..c8c65c8c9 100644 --- a/sherpa-onnx/csrc/offline-paraformer-model.cc +++ b/sherpa-onnx/csrc/offline-paraformer-model.cc @@ -8,6 +8,15 @@ #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/onnx-utils.h" #include "sherpa-onnx/csrc/session.h" @@ -26,8 +35,8 @@ class OfflineParaformerModel::Impl { Init(buf.data(), buf.size()); } -#if __ANDROID_API__ >= 9 - Impl(AAssetManager *mgr, const OfflineModelConfig &config) + template + Impl(Manager *mgr, const OfflineModelConfig &config) : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), @@ -35,7 +44,6 @@ class OfflineParaformerModel::Impl { auto buf = ReadFile(mgr, config_.paraformer.model); Init(buf.data(), buf.size()); } -#endif std::vector Forward(Ort::Value features, Ort::Value features_length) { @@ -72,7 +80,11 @@ class OfflineParaformerModel::Impl { if (config_.debug) { std::ostringstream os; PrintModelMetadata(os, meta_data); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s\n", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s\n", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; // used in the macro below @@ -109,11 +121,10 @@ class OfflineParaformerModel::Impl { OfflineParaformerModel::OfflineParaformerModel(const OfflineModelConfig &config) : impl_(std::make_unique(config)) {} -#if __ANDROID_API__ >= 9 -OfflineParaformerModel::OfflineParaformerModel(AAssetManager *mgr, +template +OfflineParaformerModel::OfflineParaformerModel(Manager *mgr, const OfflineModelConfig &config) : impl_(std::make_unique(mgr, config)) {} -#endif OfflineParaformerModel::~OfflineParaformerModel() = default; @@ -141,4 +152,14 @@ OrtAllocator *OfflineParaformerModel::Allocator() const { return impl_->Allocator(); } +#if __ANDROID_API__ >= 9 +template OfflineParaformerModel::OfflineParaformerModel( + AAssetManager *mgr, const OfflineModelConfig &config); +#endif + +#if __OHOS__ +template OfflineParaformerModel::OfflineParaformerModel( + NativeResourceManager *mgr, const OfflineModelConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-paraformer-model.h b/sherpa-onnx/csrc/offline-paraformer-model.h index d5c2329f6..219f80796 100644 --- a/sherpa-onnx/csrc/offline-paraformer-model.h +++ b/sherpa-onnx/csrc/offline-paraformer-model.h @@ -7,11 +7,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/offline-model-config.h" @@ -21,9 +16,8 @@ class OfflineParaformerModel { public: explicit OfflineParaformerModel(const OfflineModelConfig &config); -#if __ANDROID_API__ >= 9 - OfflineParaformerModel(AAssetManager *mgr, const OfflineModelConfig &config); -#endif + template + OfflineParaformerModel(Manager *mgr, const OfflineModelConfig &config); ~OfflineParaformerModel(); diff --git a/sherpa-onnx/csrc/offline-recognizer-ctc-impl.h b/sherpa-onnx/csrc/offline-recognizer-ctc-impl.h index 1199ff109..2721ecdf3 100644 --- a/sherpa-onnx/csrc/offline-recognizer-ctc-impl.h +++ b/sherpa-onnx/csrc/offline-recognizer-ctc-impl.h @@ -12,11 +12,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/offline-ctc-decoder.h" #include "sherpa-onnx/csrc/offline-ctc-fst-decoder.h" #include "sherpa-onnx/csrc/offline-ctc-greedy-search-decoder.h" @@ -80,16 +75,14 @@ class OfflineRecognizerCtcImpl : public OfflineRecognizerImpl { Init(); } -#if __ANDROID_API__ >= 9 - OfflineRecognizerCtcImpl(AAssetManager *mgr, - const OfflineRecognizerConfig &config) + template + OfflineRecognizerCtcImpl(Manager *mgr, const OfflineRecognizerConfig &config) : OfflineRecognizerImpl(mgr, config), config_(config), symbol_table_(mgr, config_.model_config.tokens), model_(OfflineCtcModel::Create(mgr, config_.model_config)) { Init(); } -#endif void Init() { if (!config_.model_config.telespeech_ctc.empty()) { diff --git a/sherpa-onnx/csrc/offline-recognizer-impl.cc b/sherpa-onnx/csrc/offline-recognizer-impl.cc index 99c41d307..2a7a8dab9 100644 --- a/sherpa-onnx/csrc/offline-recognizer-impl.cc +++ b/sherpa-onnx/csrc/offline-recognizer-impl.cc @@ -13,6 +13,10 @@ #include "android/asset_manager.h" #include "android/asset_manager_jni.h" +#elif __OHOS__ +#include + +#include "rawfile/raw_file_manager.h" #endif #include "fst/extensions/far/far.h" @@ -211,9 +215,9 @@ std::unique_ptr OfflineRecognizerImpl::Create( exit(-1); } -#if __ANDROID_API__ >= 9 +template std::unique_ptr OfflineRecognizerImpl::Create( - AAssetManager *mgr, const OfflineRecognizerConfig &config) { + Manager *mgr, const OfflineRecognizerConfig &config) { if (!config.model_config.sense_voice.model.empty()) { return std::make_unique(mgr, config); } @@ -389,7 +393,6 @@ std::unique_ptr OfflineRecognizerImpl::Create( exit(-1); } -#endif OfflineRecognizerImpl::OfflineRecognizerImpl( const OfflineRecognizerConfig &config) @@ -436,9 +439,9 @@ OfflineRecognizerImpl::OfflineRecognizerImpl( } } -#if __ANDROID_API__ >= 9 +template OfflineRecognizerImpl::OfflineRecognizerImpl( - AAssetManager *mgr, const OfflineRecognizerConfig &config) + Manager *mgr, const OfflineRecognizerConfig &config) : config_(config) { if (!config.rule_fsts.empty()) { std::vector files; @@ -482,7 +485,6 @@ OfflineRecognizerImpl::OfflineRecognizerImpl( } // for (const auto &f : files) } // if (!config.rule_fars.empty()) } -#endif std::string OfflineRecognizerImpl::ApplyInverseTextNormalization( std::string text) const { @@ -499,4 +501,19 @@ void OfflineRecognizerImpl::SetConfig(const OfflineRecognizerConfig &config) { config_ = config; } +#if __ANDROID_API__ >= 9 +template OfflineRecognizerImpl::OfflineRecognizerImpl( + AAssetManager *mgr, const OfflineRecognizerConfig &config); + +template std::unique_ptr OfflineRecognizerImpl::Create( + AAssetManager *mgr, const OfflineRecognizerConfig &config); +#endif + +#if __OHOS__ +template OfflineRecognizerImpl::OfflineRecognizerImpl( + NativeResourceManager *mgr, const OfflineRecognizerConfig &config); +template std::unique_ptr OfflineRecognizerImpl::Create( + NativeResourceManager *mgr, const OfflineRecognizerConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-recognizer-impl.h b/sherpa-onnx/csrc/offline-recognizer-impl.h index 32010bf70..8a6e6fcce 100644 --- a/sherpa-onnx/csrc/offline-recognizer-impl.h +++ b/sherpa-onnx/csrc/offline-recognizer-impl.h @@ -9,11 +9,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "kaldifst/csrc/text-normalizer.h" #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/offline-recognizer.h" @@ -28,13 +23,12 @@ class OfflineRecognizerImpl { static std::unique_ptr Create( const OfflineRecognizerConfig &config); -#if __ANDROID_API__ >= 9 - OfflineRecognizerImpl(AAssetManager *mgr, - const OfflineRecognizerConfig &config); + template + OfflineRecognizerImpl(Manager *mgr, const OfflineRecognizerConfig &config); + template static std::unique_ptr Create( - AAssetManager *mgr, const OfflineRecognizerConfig &config); -#endif + Manager *mgr, const OfflineRecognizerConfig &config); virtual ~OfflineRecognizerImpl() = default; diff --git a/sherpa-onnx/csrc/offline-recognizer-moonshine-impl.h b/sherpa-onnx/csrc/offline-recognizer-moonshine-impl.h index 7d52a41b2..deec9852d 100644 --- a/sherpa-onnx/csrc/offline-recognizer-moonshine-impl.h +++ b/sherpa-onnx/csrc/offline-recognizer-moonshine-impl.h @@ -12,11 +12,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/offline-model-config.h" #include "sherpa-onnx/csrc/offline-moonshine-decoder.h" #include "sherpa-onnx/csrc/offline-moonshine-greedy-search-decoder.h" @@ -59,8 +54,8 @@ class OfflineRecognizerMoonshineImpl : public OfflineRecognizerImpl { Init(); } -#if __ANDROID_API__ >= 9 - OfflineRecognizerMoonshineImpl(AAssetManager *mgr, + template + OfflineRecognizerMoonshineImpl(Manager *mgr, const OfflineRecognizerConfig &config) : OfflineRecognizerImpl(mgr, config), config_(config), @@ -70,8 +65,6 @@ class OfflineRecognizerMoonshineImpl : public OfflineRecognizerImpl { Init(); } -#endif - void Init() { if (config_.decoding_method == "greedy_search") { decoder_ = diff --git a/sherpa-onnx/csrc/offline-recognizer-paraformer-impl.h b/sherpa-onnx/csrc/offline-recognizer-paraformer-impl.h index 525c92cc2..5b80c99e3 100644 --- a/sherpa-onnx/csrc/offline-recognizer-paraformer-impl.h +++ b/sherpa-onnx/csrc/offline-recognizer-paraformer-impl.h @@ -11,11 +11,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/offline-model-config.h" #include "sherpa-onnx/csrc/offline-paraformer-decoder.h" #include "sherpa-onnx/csrc/offline-paraformer-greedy-search-decoder.h" @@ -105,8 +100,8 @@ class OfflineRecognizerParaformerImpl : public OfflineRecognizerImpl { InitFeatConfig(); } -#if __ANDROID_API__ >= 9 - OfflineRecognizerParaformerImpl(AAssetManager *mgr, + template + OfflineRecognizerParaformerImpl(Manager *mgr, const OfflineRecognizerConfig &config) : OfflineRecognizerImpl(mgr, config), config_(config), @@ -124,7 +119,6 @@ class OfflineRecognizerParaformerImpl : public OfflineRecognizerImpl { InitFeatConfig(); } -#endif std::unique_ptr CreateStream() const override { return std::make_unique(config_.feat_config); diff --git a/sherpa-onnx/csrc/offline-recognizer-sense-voice-impl.h b/sherpa-onnx/csrc/offline-recognizer-sense-voice-impl.h index 6cebf23c7..7ee5e41c0 100644 --- a/sherpa-onnx/csrc/offline-recognizer-sense-voice-impl.h +++ b/sherpa-onnx/csrc/offline-recognizer-sense-voice-impl.h @@ -11,11 +11,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/offline-ctc-greedy-search-decoder.h" #include "sherpa-onnx/csrc/offline-model-config.h" #include "sherpa-onnx/csrc/offline-recognizer-impl.h" @@ -83,8 +78,8 @@ class OfflineRecognizerSenseVoiceImpl : public OfflineRecognizerImpl { InitFeatConfig(); } -#if __ANDROID_API__ >= 9 - OfflineRecognizerSenseVoiceImpl(AAssetManager *mgr, + template + OfflineRecognizerSenseVoiceImpl(Manager *mgr, const OfflineRecognizerConfig &config) : OfflineRecognizerImpl(mgr, config), config_(config), @@ -103,7 +98,6 @@ class OfflineRecognizerSenseVoiceImpl : public OfflineRecognizerImpl { InitFeatConfig(); } -#endif std::unique_ptr CreateStream() const override { return std::make_unique(config_.feat_config); diff --git a/sherpa-onnx/csrc/offline-recognizer-transducer-impl.h b/sherpa-onnx/csrc/offline-recognizer-transducer-impl.h index 05759ac5b..64f3798fa 100644 --- a/sherpa-onnx/csrc/offline-recognizer-transducer-impl.h +++ b/sherpa-onnx/csrc/offline-recognizer-transducer-impl.h @@ -14,11 +14,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/context-graph.h" #include "sherpa-onnx/csrc/log.h" #include "sherpa-onnx/csrc/macros.h" @@ -109,9 +104,9 @@ class OfflineRecognizerTransducerImpl : public OfflineRecognizerImpl { } } -#if __ANDROID_API__ >= 9 + template explicit OfflineRecognizerTransducerImpl( - AAssetManager *mgr, const OfflineRecognizerConfig &config) + Manager *mgr, const OfflineRecognizerConfig &config) : OfflineRecognizerImpl(mgr, config), config_(config), symbol_table_(mgr, config_.model_config.tokens), @@ -148,7 +143,6 @@ class OfflineRecognizerTransducerImpl : public OfflineRecognizerImpl { exit(-1); } } -#endif std::unique_ptr CreateStream( const std::string &hotwords) const override { @@ -246,10 +240,7 @@ class OfflineRecognizerTransducerImpl : public OfflineRecognizerImpl { } } - OfflineRecognizerConfig GetConfig() const override { - return config_; - } - + OfflineRecognizerConfig GetConfig() const override { return config_; } void InitHotwords() { // each line in hotwords_file contains space-separated words @@ -271,8 +262,8 @@ class OfflineRecognizerTransducerImpl : public OfflineRecognizerImpl { hotwords_, config_.hotwords_score, boost_scores_); } -#if __ANDROID_API__ >= 9 - void InitHotwords(AAssetManager *mgr) { + template + void InitHotwords(Manager *mgr) { // each line in hotwords_file contains space-separated words auto buf = ReadFile(mgr, config_.hotwords_file); @@ -294,7 +285,6 @@ class OfflineRecognizerTransducerImpl : public OfflineRecognizerImpl { hotwords_graph_ = std::make_shared( hotwords_, config_.hotwords_score, boost_scores_); } -#endif private: OfflineRecognizerConfig config_; diff --git a/sherpa-onnx/csrc/offline-recognizer-transducer-nemo-impl.h b/sherpa-onnx/csrc/offline-recognizer-transducer-nemo-impl.h index 6727b0983..167d1021e 100644 --- a/sherpa-onnx/csrc/offline-recognizer-transducer-nemo-impl.h +++ b/sherpa-onnx/csrc/offline-recognizer-transducer-nemo-impl.h @@ -14,11 +14,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/offline-recognizer-impl.h" #include "sherpa-onnx/csrc/offline-recognizer.h" @@ -57,9 +52,9 @@ class OfflineRecognizerTransducerNeMoImpl : public OfflineRecognizerImpl { PostInit(); } -#if __ANDROID_API__ >= 9 + template explicit OfflineRecognizerTransducerNeMoImpl( - AAssetManager *mgr, const OfflineRecognizerConfig &config) + Manager *mgr, const OfflineRecognizerConfig &config) : OfflineRecognizerImpl(mgr, config), config_(config), symbol_table_(mgr, config_.model_config.tokens), @@ -76,7 +71,6 @@ class OfflineRecognizerTransducerNeMoImpl : public OfflineRecognizerImpl { PostInit(); } -#endif std::unique_ptr CreateStream() const override { return std::make_unique(config_.feat_config); diff --git a/sherpa-onnx/csrc/offline-recognizer-whisper-impl.h b/sherpa-onnx/csrc/offline-recognizer-whisper-impl.h index 023700e77..fc6cc74ed 100644 --- a/sherpa-onnx/csrc/offline-recognizer-whisper-impl.h +++ b/sherpa-onnx/csrc/offline-recognizer-whisper-impl.h @@ -12,11 +12,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/offline-model-config.h" #include "sherpa-onnx/csrc/offline-recognizer-impl.h" #include "sherpa-onnx/csrc/offline-recognizer.h" @@ -60,8 +55,8 @@ class OfflineRecognizerWhisperImpl : public OfflineRecognizerImpl { Init(); } -#if __ANDROID_API__ >= 9 - OfflineRecognizerWhisperImpl(AAssetManager *mgr, + template + OfflineRecognizerWhisperImpl(Manager *mgr, const OfflineRecognizerConfig &config) : OfflineRecognizerImpl(mgr, config), config_(config), @@ -71,8 +66,6 @@ class OfflineRecognizerWhisperImpl : public OfflineRecognizerImpl { Init(); } -#endif - void Init() { // tokens.txt from whisper is base64 encoded, so we need to decode it symbol_table_.ApplyBase64Decode(); @@ -105,9 +98,7 @@ class OfflineRecognizerWhisperImpl : public OfflineRecognizerImpl { config_.model_config.whisper = config.model_config.whisper; } - OfflineRecognizerConfig GetConfig() const override { - return config_; - } + OfflineRecognizerConfig GetConfig() const override { return config_; } private: void DecodeStream(OfflineStream *s) const { diff --git a/sherpa-onnx/csrc/offline-recognizer.cc b/sherpa-onnx/csrc/offline-recognizer.cc index f73e35ad6..80bc20900 100644 --- a/sherpa-onnx/csrc/offline-recognizer.cc +++ b/sherpa-onnx/csrc/offline-recognizer.cc @@ -6,11 +6,21 @@ #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/file-utils.h" #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/offline-lm-config.h" #include "sherpa-onnx/csrc/offline-recognizer-impl.h" #include "sherpa-onnx/csrc/text-utils.h" + namespace sherpa_onnx { void OfflineRecognizerConfig::Register(ParseOptions *po) { @@ -132,11 +142,10 @@ std::string OfflineRecognizerConfig::ToString() const { return os.str(); } -#if __ANDROID_API__ >= 9 -OfflineRecognizer::OfflineRecognizer(AAssetManager *mgr, +template +OfflineRecognizer::OfflineRecognizer(Manager *mgr, const OfflineRecognizerConfig &config) : impl_(OfflineRecognizerImpl::Create(mgr, config)) {} -#endif OfflineRecognizer::OfflineRecognizer(const OfflineRecognizerConfig &config) : impl_(OfflineRecognizerImpl::Create(config)) {} @@ -157,11 +166,21 @@ void OfflineRecognizer::DecodeStreams(OfflineStream **ss, int32_t n) const { } void OfflineRecognizer::SetConfig(const OfflineRecognizerConfig &config) { - impl_->SetConfig(config); + impl_->SetConfig(config); } OfflineRecognizerConfig OfflineRecognizer::GetConfig() const { return impl_->GetConfig(); } +#if __ANDROID_API__ >= 9 +template OfflineRecognizer::OfflineRecognizer( + AAssetManager *mgr, const OfflineRecognizerConfig &config); +#endif + +#if __OHOS__ +template OfflineRecognizer::OfflineRecognizer( + NativeResourceManager *mgr, const OfflineRecognizerConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-recognizer.h b/sherpa-onnx/csrc/offline-recognizer.h index 8f0b47a08..3c78ea9b6 100644 --- a/sherpa-onnx/csrc/offline-recognizer.h +++ b/sherpa-onnx/csrc/offline-recognizer.h @@ -9,11 +9,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/features.h" #include "sherpa-onnx/csrc/offline-ctc-fst-decoder-config.h" #include "sherpa-onnx/csrc/offline-lm-config.h" @@ -82,9 +77,8 @@ class OfflineRecognizer { public: ~OfflineRecognizer(); -#if __ANDROID_API__ >= 9 - OfflineRecognizer(AAssetManager *mgr, const OfflineRecognizerConfig &config); -#endif + template + OfflineRecognizer(Manager *mgr, const OfflineRecognizerConfig &config); explicit OfflineRecognizer(const OfflineRecognizerConfig &config); @@ -120,10 +114,10 @@ class OfflineRecognizer { void DecodeStreams(OfflineStream **ss, int32_t n) const; /** Onnxruntime Session objects are not affected by this method. - * The exact behavior can be defined by a specific recognizer impl. - * For instance, for the whisper recognizer, you can retrieve the language and task from - * the config and ignore any remaining fields in `config`. - */ + * The exact behavior can be defined by a specific recognizer impl. + * For instance, for the whisper recognizer, you can retrieve the language and + * task from the config and ignore any remaining fields in `config`. + */ void SetConfig(const OfflineRecognizerConfig &config); OfflineRecognizerConfig GetConfig() const; diff --git a/sherpa-onnx/csrc/offline-rnn-lm.cc b/sherpa-onnx/csrc/offline-rnn-lm.cc index 70cd0ad47..665b775ba 100644 --- a/sherpa-onnx/csrc/offline-rnn-lm.cc +++ b/sherpa-onnx/csrc/offline-rnn-lm.cc @@ -8,6 +8,15 @@ #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/onnx-utils.h" @@ -27,8 +36,8 @@ class OfflineRnnLM::Impl { Init(buf.data(), buf.size()); } -#if __ANDROID_API__ >= 9 - Impl(AAssetManager *mgr, const OfflineLMConfig &config) + template + Impl(Manager *mgr, const OfflineLMConfig &config) : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_{GetSessionOptions(config)}, @@ -36,7 +45,6 @@ class OfflineRnnLM::Impl { auto buf = ReadFile(mgr, config_.model); Init(buf.data(), buf.size()); } -#endif Ort::Value Rescore(Ort::Value x, Ort::Value x_lens) { std::array inputs = {std::move(x), std::move(x_lens)}; @@ -76,10 +84,9 @@ class OfflineRnnLM::Impl { OfflineRnnLM::OfflineRnnLM(const OfflineLMConfig &config) : impl_(std::make_unique(config)) {} -#if __ANDROID_API__ >= 9 -OfflineRnnLM::OfflineRnnLM(AAssetManager *mgr, const OfflineLMConfig &config) +template +OfflineRnnLM::OfflineRnnLM(Manager *mgr, const OfflineLMConfig &config) : impl_(std::make_unique(mgr, config)) {} -#endif OfflineRnnLM::~OfflineRnnLM() = default; @@ -87,4 +94,14 @@ Ort::Value OfflineRnnLM::Rescore(Ort::Value x, Ort::Value x_lens) { return impl_->Rescore(std::move(x), std::move(x_lens)); } +#if __ANDROID_API__ >= 9 +template OfflineRnnLM::OfflineRnnLM(AAssetManager *mgr, + const OfflineLMConfig &config); +#endif + +#if __OHOS__ +template OfflineRnnLM::OfflineRnnLM(NativeResourceManager *mgr, + const OfflineLMConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-rnn-lm.h b/sherpa-onnx/csrc/offline-rnn-lm.h index f4dd904f7..2d16a19e9 100644 --- a/sherpa-onnx/csrc/offline-rnn-lm.h +++ b/sherpa-onnx/csrc/offline-rnn-lm.h @@ -7,11 +7,6 @@ #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/offline-lm-config.h" #include "sherpa-onnx/csrc/offline-lm.h" @@ -24,9 +19,8 @@ class OfflineRnnLM : public OfflineLM { explicit OfflineRnnLM(const OfflineLMConfig &config); -#if __ANDROID_API__ >= 9 - OfflineRnnLM(AAssetManager *mgr, const OfflineLMConfig &config); -#endif + template + OfflineRnnLM(Manager *mgr, const OfflineLMConfig &config); /** Rescore a batch of sentences. * diff --git a/sherpa-onnx/csrc/offline-sense-voice-model.cc b/sherpa-onnx/csrc/offline-sense-voice-model.cc index a914ccf4a..04e7cd224 100644 --- a/sherpa-onnx/csrc/offline-sense-voice-model.cc +++ b/sherpa-onnx/csrc/offline-sense-voice-model.cc @@ -8,6 +8,15 @@ #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/onnx-utils.h" #include "sherpa-onnx/csrc/session.h" @@ -26,8 +35,8 @@ class OfflineSenseVoiceModel::Impl { Init(buf.data(), buf.size()); } -#if __ANDROID_API__ >= 9 - Impl(AAssetManager *mgr, const OfflineModelConfig &config) + template + Impl(Manager *mgr, const OfflineModelConfig &config) : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), @@ -35,7 +44,6 @@ class OfflineSenseVoiceModel::Impl { auto buf = ReadFile(mgr, config_.sense_voice.model); Init(buf.data(), buf.size()); } -#endif Ort::Value Forward(Ort::Value features, Ort::Value features_length, Ort::Value language, Ort::Value text_norm) { @@ -72,7 +80,11 @@ class OfflineSenseVoiceModel::Impl { if (config_.debug) { std::ostringstream os; PrintModelMetadata(os, meta_data); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s\n", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s\n", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; // used in the macro below @@ -129,11 +141,10 @@ class OfflineSenseVoiceModel::Impl { OfflineSenseVoiceModel::OfflineSenseVoiceModel(const OfflineModelConfig &config) : impl_(std::make_unique(config)) {} -#if __ANDROID_API__ >= 9 -OfflineSenseVoiceModel::OfflineSenseVoiceModel(AAssetManager *mgr, +template +OfflineSenseVoiceModel::OfflineSenseVoiceModel(Manager *mgr, const OfflineModelConfig &config) : impl_(std::make_unique(mgr, config)) {} -#endif OfflineSenseVoiceModel::~OfflineSenseVoiceModel() = default; @@ -154,4 +165,14 @@ OrtAllocator *OfflineSenseVoiceModel::Allocator() const { return impl_->Allocator(); } +#if __ANDROID_API__ >= 9 +template OfflineSenseVoiceModel::OfflineSenseVoiceModel( + AAssetManager *mgr, const OfflineModelConfig &config); +#endif + +#if __OHOS__ +template OfflineSenseVoiceModel::OfflineSenseVoiceModel( + NativeResourceManager *mgr, const OfflineModelConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-sense-voice-model.h b/sherpa-onnx/csrc/offline-sense-voice-model.h index 29d31b286..e82680c56 100644 --- a/sherpa-onnx/csrc/offline-sense-voice-model.h +++ b/sherpa-onnx/csrc/offline-sense-voice-model.h @@ -7,11 +7,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/offline-model-config.h" #include "sherpa-onnx/csrc/offline-sense-voice-model-meta-data.h" @@ -22,9 +17,8 @@ class OfflineSenseVoiceModel { public: explicit OfflineSenseVoiceModel(const OfflineModelConfig &config); -#if __ANDROID_API__ >= 9 - OfflineSenseVoiceModel(AAssetManager *mgr, const OfflineModelConfig &config); -#endif + template + OfflineSenseVoiceModel(Manager *mgr, const OfflineModelConfig &config); ~OfflineSenseVoiceModel(); diff --git a/sherpa-onnx/csrc/offline-tdnn-ctc-model.cc b/sherpa-onnx/csrc/offline-tdnn-ctc-model.cc index d7db0040c..de441c481 100644 --- a/sherpa-onnx/csrc/offline-tdnn-ctc-model.cc +++ b/sherpa-onnx/csrc/offline-tdnn-ctc-model.cc @@ -6,6 +6,15 @@ #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/onnx-utils.h" #include "sherpa-onnx/csrc/session.h" @@ -25,8 +34,8 @@ class OfflineTdnnCtcModel::Impl { Init(buf.data(), buf.size()); } -#if __ANDROID_API__ >= 9 - Impl(AAssetManager *mgr, const OfflineModelConfig &config) + template + Impl(Manager *mgr, const OfflineModelConfig &config) : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), @@ -34,7 +43,6 @@ class OfflineTdnnCtcModel::Impl { auto buf = ReadFile(mgr, config_.tdnn.model); Init(buf.data(), buf.size()); } -#endif std::vector Forward(Ort::Value features) { auto nnet_out = @@ -79,7 +87,11 @@ class OfflineTdnnCtcModel::Impl { if (config_.debug) { std::ostringstream os; PrintModelMetadata(os, meta_data); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s\n", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s\n", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; // used in the macro below @@ -106,11 +118,10 @@ class OfflineTdnnCtcModel::Impl { OfflineTdnnCtcModel::OfflineTdnnCtcModel(const OfflineModelConfig &config) : impl_(std::make_unique(config)) {} -#if __ANDROID_API__ >= 9 -OfflineTdnnCtcModel::OfflineTdnnCtcModel(AAssetManager *mgr, +template +OfflineTdnnCtcModel::OfflineTdnnCtcModel(Manager *mgr, const OfflineModelConfig &config) : impl_(std::make_unique(mgr, config)) {} -#endif OfflineTdnnCtcModel::~OfflineTdnnCtcModel() = default; @@ -125,4 +136,14 @@ OrtAllocator *OfflineTdnnCtcModel::Allocator() const { return impl_->Allocator(); } +#if __ANDROID_API__ >= 9 +template OfflineTdnnCtcModel::OfflineTdnnCtcModel( + AAssetManager *mgr, const OfflineModelConfig &config); +#endif + +#if __OHOS__ +template OfflineTdnnCtcModel::OfflineTdnnCtcModel( + NativeResourceManager *mgr, const OfflineModelConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-tdnn-ctc-model.h b/sherpa-onnx/csrc/offline-tdnn-ctc-model.h index b6b5c7e59..0c2e43afe 100644 --- a/sherpa-onnx/csrc/offline-tdnn-ctc-model.h +++ b/sherpa-onnx/csrc/offline-tdnn-ctc-model.h @@ -7,11 +7,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/offline-ctc-model.h" #include "sherpa-onnx/csrc/offline-model-config.h" @@ -27,9 +22,8 @@ class OfflineTdnnCtcModel : public OfflineCtcModel { public: explicit OfflineTdnnCtcModel(const OfflineModelConfig &config); -#if __ANDROID_API__ >= 9 - OfflineTdnnCtcModel(AAssetManager *mgr, const OfflineModelConfig &config); -#endif + template + OfflineTdnnCtcModel(Manager *mgr, const OfflineModelConfig &config); ~OfflineTdnnCtcModel() override; diff --git a/sherpa-onnx/csrc/offline-telespeech-ctc-model.cc b/sherpa-onnx/csrc/offline-telespeech-ctc-model.cc index aeb918cd3..d87e47a0d 100644 --- a/sherpa-onnx/csrc/offline-telespeech-ctc-model.cc +++ b/sherpa-onnx/csrc/offline-telespeech-ctc-model.cc @@ -4,6 +4,15 @@ #include "sherpa-onnx/csrc/offline-telespeech-ctc-model.h" +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/onnx-utils.h" #include "sherpa-onnx/csrc/session.h" @@ -23,8 +32,8 @@ class OfflineTeleSpeechCtcModel::Impl { Init(buf.data(), buf.size()); } -#if __ANDROID_API__ >= 9 - Impl(AAssetManager *mgr, const OfflineModelConfig &config) + template + Impl(Manager *mgr, const OfflineModelConfig &config) : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), @@ -32,7 +41,6 @@ class OfflineTeleSpeechCtcModel::Impl { auto buf = ReadFile(mgr, config_.telespeech_ctc); Init(buf.data(), buf.size()); } -#endif std::vector Forward(Ort::Value features, Ort::Value /*features_length*/) { @@ -85,7 +93,11 @@ class OfflineTeleSpeechCtcModel::Impl { if (config_.debug) { std::ostringstream os; PrintModelMetadata(os, meta_data); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s\n", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s\n", os.str().c_str()); +#endif } { @@ -117,11 +129,10 @@ OfflineTeleSpeechCtcModel::OfflineTeleSpeechCtcModel( const OfflineModelConfig &config) : impl_(std::make_unique(config)) {} -#if __ANDROID_API__ >= 9 +template OfflineTeleSpeechCtcModel::OfflineTeleSpeechCtcModel( - AAssetManager *mgr, const OfflineModelConfig &config) + Manager *mgr, const OfflineModelConfig &config) : impl_(std::make_unique(mgr, config)) {} -#endif OfflineTeleSpeechCtcModel::~OfflineTeleSpeechCtcModel() = default; @@ -141,4 +152,14 @@ OrtAllocator *OfflineTeleSpeechCtcModel::Allocator() const { return impl_->Allocator(); } +#if __ANDROID_API__ >= 9 +template OfflineTeleSpeechCtcModel::OfflineTeleSpeechCtcModel( + AAssetManager *mgr, const OfflineModelConfig &config); +#endif + +#if __OHOS__ +template OfflineTeleSpeechCtcModel::OfflineTeleSpeechCtcModel( + NativeResourceManager *mgr, const OfflineModelConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-telespeech-ctc-model.h b/sherpa-onnx/csrc/offline-telespeech-ctc-model.h index 42ef300ff..3fe701a02 100644 --- a/sherpa-onnx/csrc/offline-telespeech-ctc-model.h +++ b/sherpa-onnx/csrc/offline-telespeech-ctc-model.h @@ -8,11 +8,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/offline-ctc-model.h" #include "sherpa-onnx/csrc/offline-model-config.h" @@ -31,10 +26,8 @@ class OfflineTeleSpeechCtcModel : public OfflineCtcModel { public: explicit OfflineTeleSpeechCtcModel(const OfflineModelConfig &config); -#if __ANDROID_API__ >= 9 - OfflineTeleSpeechCtcModel(AAssetManager *mgr, - const OfflineModelConfig &config); -#endif + template + OfflineTeleSpeechCtcModel(Manager *mgr, const OfflineModelConfig &config); ~OfflineTeleSpeechCtcModel() override; diff --git a/sherpa-onnx/csrc/offline-transducer-model.cc b/sherpa-onnx/csrc/offline-transducer-model.cc index 910ae3475..da519cc2f 100644 --- a/sherpa-onnx/csrc/offline-transducer-model.cc +++ b/sherpa-onnx/csrc/offline-transducer-model.cc @@ -8,6 +8,15 @@ #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/offline-transducer-decoder.h" #include "sherpa-onnx/csrc/onnx-utils.h" @@ -38,8 +47,8 @@ class OfflineTransducerModel::Impl { } } -#if __ANDROID_API__ >= 9 - Impl(AAssetManager *mgr, const OfflineModelConfig &config) + template + Impl(Manager *mgr, const OfflineModelConfig &config) : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), @@ -59,7 +68,6 @@ class OfflineTransducerModel::Impl { InitJoiner(buf.data(), buf.size()); } } -#endif std::pair RunEncoder(Ort::Value features, Ort::Value features_length) { @@ -161,7 +169,11 @@ class OfflineTransducerModel::Impl { std::ostringstream os; os << "---encoder---\n"; PrintModelMetadata(os, meta_data); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s\n", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s\n", os.str().c_str()); +#endif } } @@ -244,11 +256,10 @@ class OfflineTransducerModel::Impl { OfflineTransducerModel::OfflineTransducerModel(const OfflineModelConfig &config) : impl_(std::make_unique(config)) {} -#if __ANDROID_API__ >= 9 -OfflineTransducerModel::OfflineTransducerModel(AAssetManager *mgr, +template +OfflineTransducerModel::OfflineTransducerModel(Manager *mgr, const OfflineModelConfig &config) : impl_(std::make_unique(mgr, config)) {} -#endif OfflineTransducerModel::~OfflineTransducerModel() = default; @@ -291,4 +302,14 @@ Ort::Value OfflineTransducerModel::BuildDecoderInput( return impl_->BuildDecoderInput(results, end_index); } +#if __ANDROID_API__ >= 9 +template OfflineTransducerModel::OfflineTransducerModel( + AAssetManager *mgr, const OfflineModelConfig &config); +#endif + +#if __OHOS__ +template OfflineTransducerModel::OfflineTransducerModel( + NativeResourceManager *mgr, const OfflineModelConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-transducer-model.h b/sherpa-onnx/csrc/offline-transducer-model.h index 31a238cb7..e8ca93a3c 100644 --- a/sherpa-onnx/csrc/offline-transducer-model.h +++ b/sherpa-onnx/csrc/offline-transducer-model.h @@ -8,11 +8,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/hypothesis.h" #include "sherpa-onnx/csrc/offline-model-config.h" @@ -25,9 +20,8 @@ class OfflineTransducerModel { public: explicit OfflineTransducerModel(const OfflineModelConfig &config); -#if __ANDROID_API__ >= 9 - OfflineTransducerModel(AAssetManager *mgr, const OfflineModelConfig &config); -#endif + template + OfflineTransducerModel(Manager *mgr, const OfflineModelConfig &config); ~OfflineTransducerModel(); diff --git a/sherpa-onnx/csrc/offline-transducer-nemo-model.cc b/sherpa-onnx/csrc/offline-transducer-nemo-model.cc index 7dd5d31b8..bd6f1ab57 100644 --- a/sherpa-onnx/csrc/offline-transducer-nemo-model.cc +++ b/sherpa-onnx/csrc/offline-transducer-nemo-model.cc @@ -9,6 +9,15 @@ #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/offline-transducer-decoder.h" #include "sherpa-onnx/csrc/onnx-utils.h" @@ -40,8 +49,8 @@ class OfflineTransducerNeMoModel::Impl { } } -#if __ANDROID_API__ >= 9 - Impl(AAssetManager *mgr, const OfflineModelConfig &config) + template + Impl(Manager *mgr, const OfflineModelConfig &config) : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), @@ -61,7 +70,6 @@ class OfflineTransducerNeMoModel::Impl { InitJoiner(buf.data(), buf.size()); } } -#endif std::vector RunEncoder(Ort::Value features, Ort::Value features_length) { @@ -172,7 +180,11 @@ class OfflineTransducerNeMoModel::Impl { std::ostringstream os; os << "---encoder---\n"; PrintModelMetadata(os, meta_data); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s\n", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s\n", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; // used in the macro below @@ -256,11 +268,10 @@ OfflineTransducerNeMoModel::OfflineTransducerNeMoModel( const OfflineModelConfig &config) : impl_(std::make_unique(config)) {} -#if __ANDROID_API__ >= 9 +template OfflineTransducerNeMoModel::OfflineTransducerNeMoModel( - AAssetManager *mgr, const OfflineModelConfig &config) + Manager *mgr, const OfflineModelConfig &config) : impl_(std::make_unique(mgr, config)) {} -#endif OfflineTransducerNeMoModel::~OfflineTransducerNeMoModel() = default; @@ -305,4 +316,14 @@ std::string OfflineTransducerNeMoModel::FeatureNormalizationMethod() const { bool OfflineTransducerNeMoModel::IsGigaAM() const { return impl_->IsGigaAM(); } +#if __ANDROID_API__ >= 9 +template OfflineTransducerNeMoModel::OfflineTransducerNeMoModel( + AAssetManager *mgr, const OfflineModelConfig &config); +#endif + +#if __OHOS__ +template OfflineTransducerNeMoModel::OfflineTransducerNeMoModel( + NativeResourceManager *mgr, const OfflineModelConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-transducer-nemo-model.h b/sherpa-onnx/csrc/offline-transducer-nemo-model.h index e4017a4c4..697f749e2 100644 --- a/sherpa-onnx/csrc/offline-transducer-nemo-model.h +++ b/sherpa-onnx/csrc/offline-transducer-nemo-model.h @@ -9,11 +9,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/offline-model-config.h" @@ -26,10 +21,8 @@ class OfflineTransducerNeMoModel { public: explicit OfflineTransducerNeMoModel(const OfflineModelConfig &config); -#if __ANDROID_API__ >= 9 - OfflineTransducerNeMoModel(AAssetManager *mgr, - const OfflineModelConfig &config); -#endif + template + OfflineTransducerNeMoModel(Manager *mgr, const OfflineModelConfig &config); ~OfflineTransducerNeMoModel(); diff --git a/sherpa-onnx/csrc/offline-wenet-ctc-model.cc b/sherpa-onnx/csrc/offline-wenet-ctc-model.cc index d696aa1c7..5a9397178 100644 --- a/sherpa-onnx/csrc/offline-wenet-ctc-model.cc +++ b/sherpa-onnx/csrc/offline-wenet-ctc-model.cc @@ -4,6 +4,15 @@ #include "sherpa-onnx/csrc/offline-wenet-ctc-model.h" +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/onnx-utils.h" #include "sherpa-onnx/csrc/session.h" @@ -23,8 +32,8 @@ class OfflineWenetCtcModel::Impl { Init(buf.data(), buf.size()); } -#if __ANDROID_API__ >= 9 - Impl(AAssetManager *mgr, const OfflineModelConfig &config) + template + Impl(Manager *mgr, const OfflineModelConfig &config) : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), @@ -32,7 +41,6 @@ class OfflineWenetCtcModel::Impl { auto buf = ReadFile(mgr, config_.wenet_ctc.model); Init(buf.data(), buf.size()); } -#endif std::vector Forward(Ort::Value features, Ort::Value features_length) { @@ -63,7 +71,11 @@ class OfflineWenetCtcModel::Impl { if (config_.debug) { std::ostringstream os; PrintModelMetadata(os, meta_data); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s\n", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s\n", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; // used in the macro below @@ -92,11 +104,10 @@ class OfflineWenetCtcModel::Impl { OfflineWenetCtcModel::OfflineWenetCtcModel(const OfflineModelConfig &config) : impl_(std::make_unique(config)) {} -#if __ANDROID_API__ >= 9 -OfflineWenetCtcModel::OfflineWenetCtcModel(AAssetManager *mgr, +template +OfflineWenetCtcModel::OfflineWenetCtcModel(Manager *mgr, const OfflineModelConfig &config) : impl_(std::make_unique(mgr, config)) {} -#endif OfflineWenetCtcModel::~OfflineWenetCtcModel() = default; @@ -115,4 +126,14 @@ OrtAllocator *OfflineWenetCtcModel::Allocator() const { return impl_->Allocator(); } +#if __ANDROID_API__ >= 9 +template OfflineWenetCtcModel::OfflineWenetCtcModel( + AAssetManager *mgr, const OfflineModelConfig &config); +#endif + +#if __OHOS__ +template OfflineWenetCtcModel::OfflineWenetCtcModel( + NativeResourceManager *mgr, const OfflineModelConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-wenet-ctc-model.h b/sherpa-onnx/csrc/offline-wenet-ctc-model.h index 4eb78b73a..ba5f21246 100644 --- a/sherpa-onnx/csrc/offline-wenet-ctc-model.h +++ b/sherpa-onnx/csrc/offline-wenet-ctc-model.h @@ -8,11 +8,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/offline-ctc-model.h" #include "sherpa-onnx/csrc/offline-model-config.h" @@ -31,9 +26,8 @@ class OfflineWenetCtcModel : public OfflineCtcModel { public: explicit OfflineWenetCtcModel(const OfflineModelConfig &config); -#if __ANDROID_API__ >= 9 - OfflineWenetCtcModel(AAssetManager *mgr, const OfflineModelConfig &config); -#endif + template + OfflineWenetCtcModel(Manager *mgr, const OfflineModelConfig &config); ~OfflineWenetCtcModel() override; diff --git a/sherpa-onnx/csrc/offline-whisper-model.cc b/sherpa-onnx/csrc/offline-whisper-model.cc index 0747a329b..360374cdb 100644 --- a/sherpa-onnx/csrc/offline-whisper-model.cc +++ b/sherpa-onnx/csrc/offline-whisper-model.cc @@ -11,6 +11,15 @@ #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/onnx-utils.h" #include "sherpa-onnx/csrc/session.h" @@ -52,8 +61,8 @@ class OfflineWhisperModel::Impl { } } -#if __ANDROID_API__ >= 9 - Impl(AAssetManager *mgr, const OfflineModelConfig &config) + template + Impl(Manager *mgr, const OfflineModelConfig &config) : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), @@ -69,7 +78,8 @@ class OfflineWhisperModel::Impl { } } - Impl(AAssetManager *mgr, const SpokenLanguageIdentificationConfig &config) + template + Impl(Manager *mgr, const SpokenLanguageIdentificationConfig &config) : lid_config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), @@ -84,7 +94,6 @@ class OfflineWhisperModel::Impl { InitDecoder(buf.data(), buf.size()); } } -#endif std::pair ForwardEncoder(Ort::Value features) { auto encoder_out = encoder_sess_->Run( @@ -237,7 +246,11 @@ class OfflineWhisperModel::Impl { std::ostringstream os; os << "---encoder---\n"; PrintModelMetadata(os, meta_data); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s\n", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s\n", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; // used in the macro below @@ -338,17 +351,16 @@ OfflineWhisperModel::OfflineWhisperModel( const SpokenLanguageIdentificationConfig &config) : impl_(std::make_unique(config)) {} -#if __ANDROID_API__ >= 9 -OfflineWhisperModel::OfflineWhisperModel(AAssetManager *mgr, +template +OfflineWhisperModel::OfflineWhisperModel(Manager *mgr, const OfflineModelConfig &config) : impl_(std::make_unique(mgr, config)) {} +template OfflineWhisperModel::OfflineWhisperModel( - AAssetManager *mgr, const SpokenLanguageIdentificationConfig &config) + Manager *mgr, const SpokenLanguageIdentificationConfig &config) : impl_(std::make_unique(mgr, config)) {} -#endif - OfflineWhisperModel::~OfflineWhisperModel() = default; std::pair OfflineWhisperModel::ForwardEncoder( @@ -453,4 +465,21 @@ void OfflineWhisperModel::NormalizeFeatures(float *features, int32_t num_frames, } } +#if __ANDROID_API__ >= 9 +template OfflineWhisperModel::OfflineWhisperModel( + AAssetManager *mgr, const OfflineModelConfig &config); + +template OfflineWhisperModel::OfflineWhisperModel( + AAssetManager *mgr, const SpokenLanguageIdentificationConfig &config); +#endif + +#if __OHOS__ +template OfflineWhisperModel::OfflineWhisperModel( + NativeResourceManager *mgr, const OfflineModelConfig &config); + +template OfflineWhisperModel::OfflineWhisperModel( + NativeResourceManager *mgr, + const SpokenLanguageIdentificationConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-whisper-model.h b/sherpa-onnx/csrc/offline-whisper-model.h index 866714bc5..60d76928c 100644 --- a/sherpa-onnx/csrc/offline-whisper-model.h +++ b/sherpa-onnx/csrc/offline-whisper-model.h @@ -11,11 +11,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/offline-model-config.h" #include "sherpa-onnx/csrc/spoken-language-identification.h" @@ -29,11 +24,12 @@ class OfflineWhisperModel { explicit OfflineWhisperModel( const SpokenLanguageIdentificationConfig &config); -#if __ANDROID_API__ >= 9 - OfflineWhisperModel(AAssetManager *mgr, const OfflineModelConfig &config); - OfflineWhisperModel(AAssetManager *mgr, + template + OfflineWhisperModel(Manager *mgr, const OfflineModelConfig &config); + + template + OfflineWhisperModel(Manager *mgr, const SpokenLanguageIdentificationConfig &config); -#endif ~OfflineWhisperModel(); diff --git a/sherpa-onnx/csrc/offline-zipformer-ctc-model.cc b/sherpa-onnx/csrc/offline-zipformer-ctc-model.cc index a783ce506..8cfa30c26 100644 --- a/sherpa-onnx/csrc/offline-zipformer-ctc-model.cc +++ b/sherpa-onnx/csrc/offline-zipformer-ctc-model.cc @@ -6,6 +6,15 @@ #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/onnx-utils.h" #include "sherpa-onnx/csrc/session.h" @@ -25,8 +34,8 @@ class OfflineZipformerCtcModel::Impl { Init(buf.data(), buf.size()); } -#if __ANDROID_API__ >= 9 - Impl(AAssetManager *mgr, const OfflineModelConfig &config) + template + Impl(Manager *mgr, const OfflineModelConfig &config) : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), @@ -34,7 +43,6 @@ class OfflineZipformerCtcModel::Impl { auto buf = ReadFile(mgr, config_.zipformer_ctc.model); Init(buf.data(), buf.size()); } -#endif std::vector Forward(Ort::Value features, Ort::Value features_length) { @@ -64,7 +72,11 @@ class OfflineZipformerCtcModel::Impl { if (config_.debug) { std::ostringstream os; PrintModelMetadata(os, meta_data); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s\n", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s\n", os.str().c_str()); +#endif } // get vocab size from the output[0].shape, which is (N, T, vocab_size) @@ -93,11 +105,10 @@ OfflineZipformerCtcModel::OfflineZipformerCtcModel( const OfflineModelConfig &config) : impl_(std::make_unique(config)) {} -#if __ANDROID_API__ >= 9 +template OfflineZipformerCtcModel::OfflineZipformerCtcModel( - AAssetManager *mgr, const OfflineModelConfig &config) + Manager *mgr, const OfflineModelConfig &config) : impl_(std::make_unique(mgr, config)) {} -#endif OfflineZipformerCtcModel::~OfflineZipformerCtcModel() = default; @@ -118,4 +129,14 @@ int32_t OfflineZipformerCtcModel::SubsamplingFactor() const { return impl_->SubsamplingFactor(); } +#if __ANDROID_API__ >= 9 +template OfflineZipformerCtcModel::OfflineZipformerCtcModel( + AAssetManager *mgr, const OfflineModelConfig &config); +#endif + +#if __OHOS__ +template OfflineZipformerCtcModel::OfflineZipformerCtcModel( + NativeResourceManager *mgr, const OfflineModelConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-zipformer-ctc-model.h b/sherpa-onnx/csrc/offline-zipformer-ctc-model.h index c4e835636..f76f96d54 100644 --- a/sherpa-onnx/csrc/offline-zipformer-ctc-model.h +++ b/sherpa-onnx/csrc/offline-zipformer-ctc-model.h @@ -7,11 +7,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/offline-ctc-model.h" #include "sherpa-onnx/csrc/offline-model-config.h" @@ -28,10 +23,8 @@ class OfflineZipformerCtcModel : public OfflineCtcModel { public: explicit OfflineZipformerCtcModel(const OfflineModelConfig &config); -#if __ANDROID_API__ >= 9 - OfflineZipformerCtcModel(AAssetManager *mgr, - const OfflineModelConfig &config); -#endif + template + OfflineZipformerCtcModel(Manager *mgr, const OfflineModelConfig &config); ~OfflineZipformerCtcModel() override; diff --git a/sherpa-onnx/csrc/silero-vad-model.cc b/sherpa-onnx/csrc/silero-vad-model.cc index 10f8027ba..80f0cbd65 100644 --- a/sherpa-onnx/csrc/silero-vad-model.cc +++ b/sherpa-onnx/csrc/silero-vad-model.cc @@ -8,6 +8,15 @@ #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/onnx-utils.h" #include "sherpa-onnx/csrc/session.h" @@ -37,7 +46,6 @@ class SileroVadModel::Impl { min_speech_samples_ = sample_rate_ * config_.silero_vad.min_speech_duration; } -#if __ANDROID_API__ >= 9 || defined(__OHOS__) template Impl(Manager *mgr, const VadModelConfig &config) : config_(config), @@ -59,7 +67,6 @@ class SileroVadModel::Impl { min_speech_samples_ = sample_rate_ * config_.silero_vad.min_speech_duration; } -#endif void Reset() { if (is_v5_) { @@ -433,16 +440,9 @@ class SileroVadModel::Impl { SileroVadModel::SileroVadModel(const VadModelConfig &config) : impl_(std::make_unique(config)) {} -#if __ANDROID_API__ >= 9 -SileroVadModel::SileroVadModel(AAssetManager *mgr, const VadModelConfig &config) +template +SileroVadModel::SileroVadModel(Manager *mgr, const VadModelConfig &config) : impl_(std::make_unique(mgr, config)) {} -#endif - -#if __OHOS__ -SileroVadModel::SileroVadModel(NativeResourceManager *mgr, - const VadModelConfig &config) - : impl_(std::make_unique(mgr, config)) {} -#endif SileroVadModel::~SileroVadModel() = default; @@ -472,4 +472,14 @@ void SileroVadModel::SetThreshold(float threshold) { impl_->SetThreshold(threshold); } +#if __ANDROID_API__ >= 9 +template SileroVadModel::SileroVadModel(AAssetManager *mgr, + const VadModelConfig &config); +#endif + +#if __OHOS__ +template SileroVadModel::SileroVadModel(NativeResourceManager *mgr, + const VadModelConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/silero-vad-model.h b/sherpa-onnx/csrc/silero-vad-model.h index 7c4c2f90f..839139995 100644 --- a/sherpa-onnx/csrc/silero-vad-model.h +++ b/sherpa-onnx/csrc/silero-vad-model.h @@ -6,15 +6,6 @@ #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - -#if __OHOS__ -#include "rawfile/raw_file_manager.h" -#endif - #include "sherpa-onnx/csrc/vad-model.h" namespace sherpa_onnx { @@ -23,13 +14,8 @@ class SileroVadModel : public VadModel { public: explicit SileroVadModel(const VadModelConfig &config); -#if __ANDROID_API__ >= 9 - SileroVadModel(AAssetManager *mgr, const VadModelConfig &config); -#endif - -#if __OHOS__ - SileroVadModel(NativeResourceManager *mgr, const VadModelConfig &config); -#endif + template + SileroVadModel(Manager *mgr, const VadModelConfig &config); ~SileroVadModel() override; diff --git a/sherpa-onnx/csrc/symbol-table.cc b/sherpa-onnx/csrc/symbol-table.cc index 77b976431..723fec68b 100644 --- a/sherpa-onnx/csrc/symbol-table.cc +++ b/sherpa-onnx/csrc/symbol-table.cc @@ -15,6 +15,10 @@ #include "android/asset_manager.h" #include "android/asset_manager_jni.h" +#elif __OHOS__ +#include + +#include "rawfile/raw_file_manager.h" #endif #include "sherpa-onnx/csrc/base64-decode.h" @@ -99,14 +103,13 @@ SymbolTable::SymbolTable(const std::string &filename, bool is_file) { } } -#if __ANDROID_API__ >= 9 -SymbolTable::SymbolTable(AAssetManager *mgr, const std::string &filename) { +template +SymbolTable::SymbolTable(Manager *mgr, const std::string &filename) { auto buf = ReadFile(mgr, filename); std::istrstream is(buf.data(), buf.size()); Init(is); } -#endif void SymbolTable::Init(std::istream &is) { sym2id_ = ReadTokens(is, &id2sym_); } @@ -169,4 +172,14 @@ void SymbolTable::ApplyBase64Decode() { } } +#if __ANDROID_API__ >= 9 +template SymbolTable::SymbolTable(AAssetManager *mgr, + const std::string &filename); +#endif + +#if __OHOS__ +template SymbolTable::SymbolTable(NativeResourceManager *mgr, + const std::string &filename); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/symbol-table.h b/sherpa-onnx/csrc/symbol-table.h index 75a96144e..20d8d206b 100644 --- a/sherpa-onnx/csrc/symbol-table.h +++ b/sherpa-onnx/csrc/symbol-table.h @@ -10,11 +10,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - namespace sherpa_onnx { // The same token can be mapped to different integer IDs, so @@ -39,9 +34,8 @@ class SymbolTable { /// Fields are separated by space(s). explicit SymbolTable(const std::string &filename, bool is_file = true); -#if __ANDROID_API__ >= 9 - SymbolTable(AAssetManager *mgr, const std::string &filename); -#endif + template + SymbolTable(Manager *mgr, const std::string &filename); /// Return a string representation of this symbol table std::string ToString() const; diff --git a/sherpa-onnx/csrc/vad-model.cc b/sherpa-onnx/csrc/vad-model.cc index 658745046..58203bb9c 100644 --- a/sherpa-onnx/csrc/vad-model.cc +++ b/sherpa-onnx/csrc/vad-model.cc @@ -4,6 +4,15 @@ #include "sherpa-onnx/csrc/vad-model.h" +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/silero-vad-model.h" namespace sherpa_onnx { @@ -13,20 +22,20 @@ std::unique_ptr VadModel::Create(const VadModelConfig &config) { return std::make_unique(config); } -#if __ANDROID_API__ >= 9 -std::unique_ptr VadModel::Create(AAssetManager *mgr, +template +std::unique_ptr VadModel::Create(Manager *mgr, const VadModelConfig &config) { // TODO(fangjun): Support other VAD models. return std::make_unique(mgr, config); } + +#if __ANDROID_API__ >= 9 +template std::unique_ptr VadModel::Create( + AAssetManager *mgr, const VadModelConfig &config); #endif #if __OHOS__ -std::unique_ptr VadModel::Create(NativeResourceManager *mgr, - const VadModelConfig &config) { - // TODO(fangjun): Support other VAD models. - return std::make_unique(mgr, config); -} +template std::unique_ptr VadModel::Create( + NativeResourceManager *mgr, const VadModelConfig &config); #endif - } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/vad-model.h b/sherpa-onnx/csrc/vad-model.h index 3a425b193..aac68bb17 100644 --- a/sherpa-onnx/csrc/vad-model.h +++ b/sherpa-onnx/csrc/vad-model.h @@ -6,15 +6,6 @@ #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - -#if __OHOS__ -#include "rawfile/raw_file_manager.h" -#endif - #include "sherpa-onnx/csrc/vad-model-config.h" namespace sherpa_onnx { @@ -25,15 +16,9 @@ class VadModel { static std::unique_ptr Create(const VadModelConfig &config); -#if __ANDROID_API__ >= 9 - static std::unique_ptr Create(AAssetManager *mgr, - const VadModelConfig &config); -#endif - -#if __OHOS__ - static std::unique_ptr Create(NativeResourceManager *mgr, + template + static std::unique_ptr Create(Manager *mgr, const VadModelConfig &config); -#endif // reset the internal model states virtual void Reset() = 0; diff --git a/sherpa-onnx/csrc/voice-activity-detector.cc b/sherpa-onnx/csrc/voice-activity-detector.cc index 8c6038b7a..8bc8f7c94 100644 --- a/sherpa-onnx/csrc/voice-activity-detector.cc +++ b/sherpa-onnx/csrc/voice-activity-detector.cc @@ -8,6 +8,15 @@ #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/circular-buffer.h" #include "sherpa-onnx/csrc/vad-model.h" @@ -22,7 +31,6 @@ class VoiceActivityDetector::Impl { Init(); } -#if __ANDROID_API__ >= 9 || defined(__OHOS__) template Impl(Manager *mgr, const VadModelConfig &config, float buffer_size_in_seconds = 60) @@ -31,7 +39,6 @@ class VoiceActivityDetector::Impl { buffer_(buffer_size_in_seconds * config.sample_rate) { Init(); } -#endif void AcceptWaveform(const float *samples, int32_t n) { if (buffer_.Size() > max_utterance_length_) { @@ -178,19 +185,11 @@ VoiceActivityDetector::VoiceActivityDetector( const VadModelConfig &config, float buffer_size_in_seconds /*= 60*/) : impl_(std::make_unique(config, buffer_size_in_seconds)) {} -#if __ANDROID_API__ >= 9 +template VoiceActivityDetector::VoiceActivityDetector( - AAssetManager *mgr, const VadModelConfig &config, + Manager *mgr, const VadModelConfig &config, float buffer_size_in_seconds /*= 60*/) : impl_(std::make_unique(mgr, config, buffer_size_in_seconds)) {} -#endif - -#if __OHOS__ -VoiceActivityDetector::VoiceActivityDetector( - NativeResourceManager *mgr, const VadModelConfig &config, - float buffer_size_in_seconds /*= 60*/) - : impl_(std::make_unique(mgr, config, buffer_size_in_seconds)) {} -#endif VoiceActivityDetector::~VoiceActivityDetector() = default; @@ -220,4 +219,16 @@ const VadModelConfig &VoiceActivityDetector::GetConfig() const { return impl_->GetConfig(); } +#if __ANDROID_API__ >= 9 +template VoiceActivityDetector::VoiceActivityDetector( + AAssetManager *mgr, const VadModelConfig &config, + float buffer_size_in_seconds = 60); +#endif + +#if __OHOS__ +template VoiceActivityDetector::VoiceActivityDetector( + NativeResourceManager *mgr, const VadModelConfig &config, + float buffer_size_in_seconds = 60); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/voice-activity-detector.h b/sherpa-onnx/csrc/voice-activity-detector.h index 9cc1a8897..015cedf04 100644 --- a/sherpa-onnx/csrc/voice-activity-detector.h +++ b/sherpa-onnx/csrc/voice-activity-detector.h @@ -7,15 +7,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - -#if __OHOS__ -#include "rawfile/raw_file_manager.h" -#endif - #include "sherpa-onnx/csrc/vad-model-config.h" namespace sherpa_onnx { @@ -30,16 +21,9 @@ class VoiceActivityDetector { explicit VoiceActivityDetector(const VadModelConfig &config, float buffer_size_in_seconds = 60); -#if __ANDROID_API__ >= 9 - VoiceActivityDetector(AAssetManager *mgr, const VadModelConfig &config, - float buffer_size_in_seconds = 60); -#endif - -#if __OHOS__ - VoiceActivityDetector(NativeResourceManager *mgr, - const VadModelConfig &config, + template + VoiceActivityDetector(Manager *mgr, const VadModelConfig &config, float buffer_size_in_seconds = 60); -#endif ~VoiceActivityDetector(); From 210122726912db22456f1d6cac96cf54971176ce Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 26 Nov 2024 18:36:56 +0800 Subject: [PATCH 080/183] Add streaming ASR support for HarmonyOS. (#1565) --- sherpa-onnx/c-api/c-api.cc | 28 +++++++++++++++- sherpa-onnx/c-api/c-api.h | 7 ++++ sherpa-onnx/csrc/offline-recognizer-impl.cc | 6 ++-- .../csrc/online-conformer-transducer-model.cc | 27 +++++++++++++-- .../csrc/online-conformer-transducer-model.h | 11 ++----- sherpa-onnx/csrc/online-ctc-model.cc | 23 +++++++++++-- sherpa-onnx/csrc/online-ctc-model.h | 10 ++---- .../csrc/online-lstm-transducer-model.cc | 23 +++++++++++-- .../csrc/online-lstm-transducer-model.h | 11 ++----- sherpa-onnx/csrc/online-nemo-ctc-model.cc | 30 +++++++++++++---- sherpa-onnx/csrc/online-nemo-ctc-model.h | 10 ++---- sherpa-onnx/csrc/online-paraformer-model.cc | 30 +++++++++++++---- sherpa-onnx/csrc/online-paraformer-model.h | 10 ++---- sherpa-onnx/csrc/online-recognizer-ctc-impl.h | 5 ++- sherpa-onnx/csrc/online-recognizer-impl.cc | 33 ++++++++++++++----- sherpa-onnx/csrc/online-recognizer-impl.h | 14 +++----- .../csrc/online-recognizer-paraformer-impl.h | 6 ++-- .../csrc/online-recognizer-transducer-impl.h | 15 +++------ .../online-recognizer-transducer-nemo-impl.h | 10 ++---- sherpa-onnx/csrc/online-recognizer.cc | 24 ++++++++++++-- sherpa-onnx/csrc/online-recognizer.h | 10 ++---- sherpa-onnx/csrc/online-transducer-model.cc | 21 ++++++++++-- sherpa-onnx/csrc/online-transducer-model.h | 10 ++---- .../csrc/online-transducer-nemo-model.cc | 30 +++++++++++++---- .../csrc/online-transducer-nemo-model.h | 11 ++----- sherpa-onnx/csrc/online-wenet-ctc-model.cc | 30 +++++++++++++---- sherpa-onnx/csrc/online-wenet-ctc-model.h | 10 ++---- .../csrc/online-zipformer-transducer-model.cc | 23 +++++++++++-- .../csrc/online-zipformer-transducer-model.h | 11 ++----- .../csrc/online-zipformer2-ctc-model.cc | 28 ++++++++++++---- .../csrc/online-zipformer2-ctc-model.h | 10 ++---- .../online-zipformer2-transducer-model.cc | 23 +++++++++++-- .../csrc/online-zipformer2-transducer-model.h | 10 ++---- sherpa-onnx/csrc/symbol-table.cc | 6 ++-- sherpa-onnx/python/csrc/vad-model.cc | 7 ++-- 35 files changed, 367 insertions(+), 206 deletions(-) diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index c9dafe84b..fbc3a010a 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -56,7 +56,7 @@ struct SherpaOnnxDisplay { #define SHERPA_ONNX_OR(x, y) (x ? x : y) -const SherpaOnnxOnlineRecognizer *SherpaOnnxCreateOnlineRecognizer( +static sherpa_onnx::OnlineRecognizerConfig GetOnlineRecognizerConfig( const SherpaOnnxOnlineRecognizerConfig *config) { sherpa_onnx::OnlineRecognizerConfig recognizer_config; @@ -151,9 +151,21 @@ const SherpaOnnxOnlineRecognizer *SherpaOnnxCreateOnlineRecognizer( recognizer_config.rule_fars = SHERPA_ONNX_OR(config->rule_fars, ""); if (config->model_config.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s\n", recognizer_config.ToString().c_str()); +#else SHERPA_ONNX_LOGE("%s\n", recognizer_config.ToString().c_str()); +#endif } + return recognizer_config; +} + +const SherpaOnnxOnlineRecognizer *SherpaOnnxCreateOnlineRecognizer( + const SherpaOnnxOnlineRecognizerConfig *config) { + sherpa_onnx::OnlineRecognizerConfig recognizer_config = + GetOnlineRecognizerConfig(config); + if (!recognizer_config.Validate()) { SHERPA_ONNX_LOGE("Errors in config!"); return nullptr; @@ -1876,6 +1888,20 @@ SherpaOnnxOfflineSpeakerDiarizationProcessWithCallbackNoArg( #ifdef __OHOS__ +const SherpaOnnxOnlineRecognizer *SherpaOnnxCreateOnlineRecognizerOHOS( + const SherpaOnnxOnlineRecognizerConfig *config, + NativeResourceManager *mgr) { + sherpa_onnx::OnlineRecognizerConfig recognizer_config = + GetOnlineRecognizerConfig(config); + + SherpaOnnxOnlineRecognizer *recognizer = new SherpaOnnxOnlineRecognizer; + + recognizer->impl = + std::make_unique(mgr, recognizer_config); + + return recognizer; +} + const SherpaOnnxOfflineRecognizer *SherpaOnnxCreateOfflineRecognizerOHOS( const SherpaOnnxOfflineRecognizerConfig *config, NativeResourceManager *mgr) { diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index 31e86b8e5..1feaab306 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -1527,6 +1527,13 @@ SHERPA_ONNX_API void SherpaOnnxOfflineSpeakerDiarizationDestroyResult( // It is for HarmonyOS typedef struct NativeResourceManager NativeResourceManager; +/// @param config Config for the recognizer. +/// @return Return a pointer to the recognizer. The user has to invoke +// SherpaOnnxDestroyOnlineRecognizer() to free it to avoid memory leak. +SHERPA_ONNX_API const SherpaOnnxOnlineRecognizer * +SherpaOnnxCreateOnlineRecognizerOHOS( + const SherpaOnnxOnlineRecognizerConfig *config, NativeResourceManager *mgr); + /// @param config Config for the recognizer. /// @return Return a pointer to the recognizer. The user has to invoke // SherpaOnnxDestroyOfflineRecognizer() to free it to avoid memory diff --git a/sherpa-onnx/csrc/offline-recognizer-impl.cc b/sherpa-onnx/csrc/offline-recognizer-impl.cc index 2a7a8dab9..b3789849c 100644 --- a/sherpa-onnx/csrc/offline-recognizer-impl.cc +++ b/sherpa-onnx/csrc/offline-recognizer-impl.cc @@ -5,17 +5,17 @@ #include "sherpa-onnx/csrc/offline-recognizer-impl.h" #include +#include #include #include #if __ANDROID_API__ >= 9 -#include #include "android/asset_manager.h" #include "android/asset_manager_jni.h" -#elif __OHOS__ -#include +#endif +#if __OHOS__ #include "rawfile/raw_file_manager.h" #endif diff --git a/sherpa-onnx/csrc/online-conformer-transducer-model.cc b/sherpa-onnx/csrc/online-conformer-transducer-model.cc index 2bceffc7d..519d1a935 100644 --- a/sherpa-onnx/csrc/online-conformer-transducer-model.cc +++ b/sherpa-onnx/csrc/online-conformer-transducer-model.cc @@ -17,6 +17,10 @@ #include "android/asset_manager_jni.h" #endif +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/cat.h" #include "sherpa-onnx/csrc/macros.h" @@ -50,9 +54,9 @@ OnlineConformerTransducerModel::OnlineConformerTransducerModel( } } -#if __ANDROID_API__ >= 9 +template OnlineConformerTransducerModel::OnlineConformerTransducerModel( - AAssetManager *mgr, const OnlineModelConfig &config) + Manager *mgr, const OnlineModelConfig &config) : env_(ORT_LOGGING_LEVEL_ERROR), config_(config), sess_opts_(GetSessionOptions(config)), @@ -72,7 +76,6 @@ OnlineConformerTransducerModel::OnlineConformerTransducerModel( InitJoiner(buf.data(), buf.size()); } } -#endif void OnlineConformerTransducerModel::InitEncoder(void *model_data, size_t model_data_length) { @@ -91,7 +94,11 @@ void OnlineConformerTransducerModel::InitEncoder(void *model_data, std::ostringstream os; os << "---encoder---\n"; PrintModelMetadata(os, meta_data); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; // used in the macro below @@ -121,7 +128,11 @@ void OnlineConformerTransducerModel::InitDecoder(void *model_data, std::ostringstream os; os << "---decoder---\n"; PrintModelMetadata(os, meta_data); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; // used in the macro below @@ -273,4 +284,14 @@ Ort::Value OnlineConformerTransducerModel::RunJoiner(Ort::Value encoder_out, return std::move(logit[0]); } +#if __ANDROID_API__ >= 9 +template OnlineConformerTransducerModel::OnlineConformerTransducerModel( + AAssetManager *mgr, const OnlineModelConfig &config); +#endif + +#if __OHOS__ +template OnlineConformerTransducerModel::OnlineConformerTransducerModel( + NativeResourceManager *mgr, const OnlineModelConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/online-conformer-transducer-model.h b/sherpa-onnx/csrc/online-conformer-transducer-model.h index bcf9e6eda..5c901b87a 100644 --- a/sherpa-onnx/csrc/online-conformer-transducer-model.h +++ b/sherpa-onnx/csrc/online-conformer-transducer-model.h @@ -10,11 +10,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/online-model-config.h" #include "sherpa-onnx/csrc/online-transducer-model.h" @@ -25,10 +20,8 @@ class OnlineConformerTransducerModel : public OnlineTransducerModel { public: explicit OnlineConformerTransducerModel(const OnlineModelConfig &config); -#if __ANDROID_API__ >= 9 - OnlineConformerTransducerModel(AAssetManager *mgr, - const OnlineModelConfig &config); -#endif + template + OnlineConformerTransducerModel(Manager *mgr, const OnlineModelConfig &config); std::vector StackStates( const std::vector> &states) const override; diff --git a/sherpa-onnx/csrc/online-ctc-model.cc b/sherpa-onnx/csrc/online-ctc-model.cc index a3a071a72..649c6b507 100644 --- a/sherpa-onnx/csrc/online-ctc-model.cc +++ b/sherpa-onnx/csrc/online-ctc-model.cc @@ -9,6 +9,15 @@ #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/online-nemo-ctc-model.h" #include "sherpa-onnx/csrc/online-wenet-ctc-model.h" @@ -31,10 +40,9 @@ std::unique_ptr OnlineCtcModel::Create( } } -#if __ANDROID_API__ >= 9 - +template std::unique_ptr OnlineCtcModel::Create( - AAssetManager *mgr, const OnlineModelConfig &config) { + Manager *mgr, const OnlineModelConfig &config) { if (!config.wenet_ctc.model.empty()) { return std::make_unique(mgr, config); } else if (!config.zipformer2_ctc.model.empty()) { @@ -46,6 +54,15 @@ std::unique_ptr OnlineCtcModel::Create( exit(-1); } } + +#if __ANDROID_API__ >= 9 +template std::unique_ptr OnlineCtcModel::Create( + AAssetManager *mgr, const OnlineModelConfig &config); +#endif + +#if __OHOS__ +template std::unique_ptr OnlineCtcModel::Create( + NativeResourceManager *mgr, const OnlineModelConfig &config); #endif } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/online-ctc-model.h b/sherpa-onnx/csrc/online-ctc-model.h index 17721752d..bd01bc543 100644 --- a/sherpa-onnx/csrc/online-ctc-model.h +++ b/sherpa-onnx/csrc/online-ctc-model.h @@ -8,11 +8,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/online-model-config.h" @@ -25,10 +20,9 @@ class OnlineCtcModel { static std::unique_ptr Create( const OnlineModelConfig &config); -#if __ANDROID_API__ >= 9 + template static std::unique_ptr Create( - AAssetManager *mgr, const OnlineModelConfig &config); -#endif + Manager *mgr, const OnlineModelConfig &config); // Return a list of tensors containing the initial states virtual std::vector GetInitStates() const = 0; diff --git a/sherpa-onnx/csrc/online-lstm-transducer-model.cc b/sherpa-onnx/csrc/online-lstm-transducer-model.cc index b9ef9ca5b..91b499fdf 100644 --- a/sherpa-onnx/csrc/online-lstm-transducer-model.cc +++ b/sherpa-onnx/csrc/online-lstm-transducer-model.cc @@ -16,6 +16,10 @@ #include "android/asset_manager_jni.h" #endif +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/cat.h" #include "sherpa-onnx/csrc/macros.h" @@ -48,9 +52,9 @@ OnlineLstmTransducerModel::OnlineLstmTransducerModel( } } -#if __ANDROID_API__ >= 9 +template OnlineLstmTransducerModel::OnlineLstmTransducerModel( - AAssetManager *mgr, const OnlineModelConfig &config) + Manager *mgr, const OnlineModelConfig &config) : env_(ORT_LOGGING_LEVEL_ERROR), config_(config), sess_opts_(GetSessionOptions(config)), @@ -70,7 +74,6 @@ OnlineLstmTransducerModel::OnlineLstmTransducerModel( InitJoiner(buf.data(), buf.size()); } } -#endif void OnlineLstmTransducerModel::InitEncoder(void *model_data, size_t model_data_length) { @@ -89,7 +92,11 @@ void OnlineLstmTransducerModel::InitEncoder(void *model_data, std::ostringstream os; os << "---encoder---\n"; PrintModelMetadata(os, meta_data); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; // used in the macro below @@ -261,4 +268,14 @@ Ort::Value OnlineLstmTransducerModel::RunJoiner(Ort::Value encoder_out, return std::move(logit[0]); } +#if __ANDROID_API__ >= 9 +template OnlineLstmTransducerModel::OnlineLstmTransducerModel( + AAssetManager *mgr, const OnlineModelConfig &config); +#endif + +#if __OHOS__ +template OnlineLstmTransducerModel::OnlineLstmTransducerModel( + NativeResourceManager *mgr, const OnlineModelConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/online-lstm-transducer-model.h b/sherpa-onnx/csrc/online-lstm-transducer-model.h index 24119f240..64ade36d0 100644 --- a/sherpa-onnx/csrc/online-lstm-transducer-model.h +++ b/sherpa-onnx/csrc/online-lstm-transducer-model.h @@ -9,11 +9,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/online-model-config.h" #include "sherpa-onnx/csrc/online-transducer-model.h" @@ -24,10 +19,8 @@ class OnlineLstmTransducerModel : public OnlineTransducerModel { public: explicit OnlineLstmTransducerModel(const OnlineModelConfig &config); -#if __ANDROID_API__ >= 9 - OnlineLstmTransducerModel(AAssetManager *mgr, - const OnlineModelConfig &config); -#endif + template + OnlineLstmTransducerModel(Manager *mgr, const OnlineModelConfig &config); std::vector StackStates( const std::vector> &states) const override; diff --git a/sherpa-onnx/csrc/online-nemo-ctc-model.cc b/sherpa-onnx/csrc/online-nemo-ctc-model.cc index 172ee69f4..716c7ee7e 100644 --- a/sherpa-onnx/csrc/online-nemo-ctc-model.cc +++ b/sherpa-onnx/csrc/online-nemo-ctc-model.cc @@ -13,6 +13,10 @@ #include "android/asset_manager_jni.h" #endif +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/cat.h" #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/onnx-utils.h" @@ -36,8 +40,8 @@ class OnlineNeMoCtcModel::Impl { } } -#if __ANDROID_API__ >= 9 - Impl(AAssetManager *mgr, const OnlineModelConfig &config) + template + Impl(Manager *mgr, const OnlineModelConfig &config) : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), @@ -47,7 +51,6 @@ class OnlineNeMoCtcModel::Impl { Init(buf.data(), buf.size()); } } -#endif std::vector Forward(Ort::Value x, std::vector states) { @@ -202,7 +205,11 @@ class OnlineNeMoCtcModel::Impl { if (config_.debug) { std::ostringstream os; PrintModelMetadata(os, meta_data); - SHERPA_ONNX_LOGE("%s\n", os.str().c_str()); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else + SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; // used in the macro below @@ -286,11 +293,10 @@ class OnlineNeMoCtcModel::Impl { OnlineNeMoCtcModel::OnlineNeMoCtcModel(const OnlineModelConfig &config) : impl_(std::make_unique(config)) {} -#if __ANDROID_API__ >= 9 -OnlineNeMoCtcModel::OnlineNeMoCtcModel(AAssetManager *mgr, +template +OnlineNeMoCtcModel::OnlineNeMoCtcModel(Manager *mgr, const OnlineModelConfig &config) : impl_(std::make_unique(mgr, config)) {} -#endif OnlineNeMoCtcModel::~OnlineNeMoCtcModel() = default; @@ -323,4 +329,14 @@ std::vector> OnlineNeMoCtcModel::UnStackStates( return impl_->UnStackStates(std::move(states)); } +#if __ANDROID_API__ >= 9 +template OnlineNeMoCtcModel::OnlineNeMoCtcModel( + AAssetManager *mgr, const OnlineModelConfig &config); +#endif + +#if __OHOS__ +template OnlineNeMoCtcModel::OnlineNeMoCtcModel( + NativeResourceManager *mgr, const OnlineModelConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/online-nemo-ctc-model.h b/sherpa-onnx/csrc/online-nemo-ctc-model.h index c8dd182e8..4e5f820b6 100644 --- a/sherpa-onnx/csrc/online-nemo-ctc-model.h +++ b/sherpa-onnx/csrc/online-nemo-ctc-model.h @@ -8,11 +8,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/online-ctc-model.h" #include "sherpa-onnx/csrc/online-model-config.h" @@ -23,9 +18,8 @@ class OnlineNeMoCtcModel : public OnlineCtcModel { public: explicit OnlineNeMoCtcModel(const OnlineModelConfig &config); -#if __ANDROID_API__ >= 9 - OnlineNeMoCtcModel(AAssetManager *mgr, const OnlineModelConfig &config); -#endif + template + OnlineNeMoCtcModel(Manager *mgr, const OnlineModelConfig &config); ~OnlineNeMoCtcModel() override; diff --git a/sherpa-onnx/csrc/online-paraformer-model.cc b/sherpa-onnx/csrc/online-paraformer-model.cc index d7d2e436d..b21bb9bcc 100644 --- a/sherpa-onnx/csrc/online-paraformer-model.cc +++ b/sherpa-onnx/csrc/online-paraformer-model.cc @@ -13,6 +13,10 @@ #include "android/asset_manager_jni.h" #endif +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/onnx-utils.h" #include "sherpa-onnx/csrc/session.h" @@ -38,8 +42,8 @@ class OnlineParaformerModel::Impl { } } -#if __ANDROID_API__ >= 9 - Impl(AAssetManager *mgr, const OnlineModelConfig &config) + template + Impl(Manager *mgr, const OnlineModelConfig &config) : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), @@ -54,7 +58,6 @@ class OnlineParaformerModel::Impl { InitDecoder(buf.data(), buf.size()); } } -#endif std::vector ForwardEncoder(Ort::Value features, Ort::Value features_length) { @@ -123,7 +126,11 @@ class OnlineParaformerModel::Impl { if (config_.debug) { std::ostringstream os; PrintModelMetadata(os, meta_data); - SHERPA_ONNX_LOGE("%s\n", os.str().c_str()); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else + SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; // used in the macro below @@ -191,11 +198,10 @@ class OnlineParaformerModel::Impl { OnlineParaformerModel::OnlineParaformerModel(const OnlineModelConfig &config) : impl_(std::make_unique(config)) {} -#if __ANDROID_API__ >= 9 -OnlineParaformerModel::OnlineParaformerModel(AAssetManager *mgr, +template +OnlineParaformerModel::OnlineParaformerModel(Manager *mgr, const OnlineModelConfig &config) : impl_(std::make_unique(mgr, config)) {} -#endif OnlineParaformerModel::~OnlineParaformerModel() = default; @@ -246,4 +252,14 @@ OrtAllocator *OnlineParaformerModel::Allocator() const { return impl_->Allocator(); } +#if __ANDROID_API__ >= 9 +template OnlineParaformerModel::OnlineParaformerModel( + AAssetManager *mgr, const OnlineModelConfig &config); +#endif + +#if __OHOS__ +template OnlineParaformerModel::OnlineParaformerModel( + NativeResourceManager *mgr, const OnlineModelConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/online-paraformer-model.h b/sherpa-onnx/csrc/online-paraformer-model.h index 3c018a72d..cbf2d7157 100644 --- a/sherpa-onnx/csrc/online-paraformer-model.h +++ b/sherpa-onnx/csrc/online-paraformer-model.h @@ -8,11 +8,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/online-model-config.h" @@ -22,9 +17,8 @@ class OnlineParaformerModel { public: explicit OnlineParaformerModel(const OnlineModelConfig &config); -#if __ANDROID_API__ >= 9 - OnlineParaformerModel(AAssetManager *mgr, const OnlineModelConfig &config); -#endif + template + OnlineParaformerModel(Manager *mgr, const OnlineModelConfig &config); ~OnlineParaformerModel(); diff --git a/sherpa-onnx/csrc/online-recognizer-ctc-impl.h b/sherpa-onnx/csrc/online-recognizer-ctc-impl.h index 76452138d..3560b1ab7 100644 --- a/sherpa-onnx/csrc/online-recognizer-ctc-impl.h +++ b/sherpa-onnx/csrc/online-recognizer-ctc-impl.h @@ -88,8 +88,8 @@ class OnlineRecognizerCtcImpl : public OnlineRecognizerImpl { InitDecoder(); } -#if __ANDROID_API__ >= 9 - explicit OnlineRecognizerCtcImpl(AAssetManager *mgr, + template + explicit OnlineRecognizerCtcImpl(Manager *mgr, const OnlineRecognizerConfig &config) : OnlineRecognizerImpl(mgr, config), config_(config), @@ -104,7 +104,6 @@ class OnlineRecognizerCtcImpl : public OnlineRecognizerImpl { InitDecoder(); } -#endif std::unique_ptr CreateStream() const override { auto stream = std::make_unique(config_.feat_config); diff --git a/sherpa-onnx/csrc/online-recognizer-impl.cc b/sherpa-onnx/csrc/online-recognizer-impl.cc index cf59cb539..27168b0f6 100644 --- a/sherpa-onnx/csrc/online-recognizer-impl.cc +++ b/sherpa-onnx/csrc/online-recognizer-impl.cc @@ -4,15 +4,18 @@ #include "sherpa-onnx/csrc/online-recognizer-impl.h" +#include #include #if __ANDROID_API__ >= 9 -#include - #include "android/asset_manager.h" #include "android/asset_manager_jni.h" #endif +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "fst/extensions/far/far.h" #include "kaldifst/csrc/kaldi-fst-io.h" #include "sherpa-onnx/csrc/macros.h" @@ -61,9 +64,9 @@ std::unique_ptr OnlineRecognizerImpl::Create( exit(-1); } -#if __ANDROID_API__ >= 9 +template std::unique_ptr OnlineRecognizerImpl::Create( - AAssetManager *mgr, const OnlineRecognizerConfig &config) { + Manager *mgr, const OnlineRecognizerConfig &config) { if (!config.model_config.transducer.encoder.empty()) { Ort::Env env(ORT_LOGGING_LEVEL_ERROR); @@ -97,7 +100,6 @@ std::unique_ptr OnlineRecognizerImpl::Create( SHERPA_ONNX_LOGE("Please specify a model"); exit(-1); } -#endif OnlineRecognizerImpl::OnlineRecognizerImpl(const OnlineRecognizerConfig &config) : config_(config) { @@ -143,8 +145,8 @@ OnlineRecognizerImpl::OnlineRecognizerImpl(const OnlineRecognizerConfig &config) } } -#if __ANDROID_API__ >= 9 -OnlineRecognizerImpl::OnlineRecognizerImpl(AAssetManager *mgr, +template +OnlineRecognizerImpl::OnlineRecognizerImpl(Manager *mgr, const OnlineRecognizerConfig &config) : config_(config) { if (!config.rule_fsts.empty()) { @@ -189,7 +191,6 @@ OnlineRecognizerImpl::OnlineRecognizerImpl(AAssetManager *mgr, } // for (const auto &f : files) } // if (!config.rule_fars.empty()) } -#endif std::string OnlineRecognizerImpl::ApplyInverseTextNormalization( std::string text) const { @@ -202,4 +203,20 @@ std::string OnlineRecognizerImpl::ApplyInverseTextNormalization( return text; } +#if __ANDROID_API__ >= 9 +template OnlineRecognizerImpl::OnlineRecognizerImpl( + AAssetManager *mgr, const OnlineRecognizerConfig &config); + +template std::unique_ptr OnlineRecognizerImpl::Create( + AAssetManager *mgr, const OnlineRecognizerConfig &config); +#endif + +#if __OHOS__ +template OnlineRecognizerImpl::OnlineRecognizerImpl( + NativeResourceManager *mgr, const OnlineRecognizerConfig &config); + +template std::unique_ptr OnlineRecognizerImpl::Create( + NativeResourceManager *mgr, const OnlineRecognizerConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/online-recognizer-impl.h b/sherpa-onnx/csrc/online-recognizer-impl.h index 8b569f3af..b7bda7862 100644 --- a/sherpa-onnx/csrc/online-recognizer-impl.h +++ b/sherpa-onnx/csrc/online-recognizer-impl.h @@ -9,11 +9,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "kaldifst/csrc/text-normalizer.h" #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/online-recognizer.h" @@ -28,13 +23,12 @@ class OnlineRecognizerImpl { static std::unique_ptr Create( const OnlineRecognizerConfig &config); -#if __ANDROID_API__ >= 9 - OnlineRecognizerImpl(AAssetManager *mgr, - const OnlineRecognizerConfig &config); + template + OnlineRecognizerImpl(Manager *mgr, const OnlineRecognizerConfig &config); + template static std::unique_ptr Create( - AAssetManager *mgr, const OnlineRecognizerConfig &config); -#endif + Manager *mgr, const OnlineRecognizerConfig &config); virtual ~OnlineRecognizerImpl() = default; diff --git a/sherpa-onnx/csrc/online-recognizer-paraformer-impl.h b/sherpa-onnx/csrc/online-recognizer-paraformer-impl.h index 8ef66ea74..1e02fe519 100644 --- a/sherpa-onnx/csrc/online-recognizer-paraformer-impl.h +++ b/sherpa-onnx/csrc/online-recognizer-paraformer-impl.h @@ -120,8 +120,8 @@ class OnlineRecognizerParaformerImpl : public OnlineRecognizerImpl { config_.feat_config.normalize_samples = false; } -#if __ANDROID_API__ >= 9 - explicit OnlineRecognizerParaformerImpl(AAssetManager *mgr, + template + explicit OnlineRecognizerParaformerImpl(Manager *mgr, const OnlineRecognizerConfig &config) : OnlineRecognizerImpl(mgr, config), config_(config), @@ -138,7 +138,7 @@ class OnlineRecognizerParaformerImpl : public OnlineRecognizerImpl { // [-32768, 32767], so we set normalize_samples to false config_.feat_config.normalize_samples = false; } -#endif + OnlineRecognizerParaformerImpl(const OnlineRecognizerParaformerImpl &) = delete; diff --git a/sherpa-onnx/csrc/online-recognizer-transducer-impl.h b/sherpa-onnx/csrc/online-recognizer-transducer-impl.h index 475a90185..2eac3cf84 100644 --- a/sherpa-onnx/csrc/online-recognizer-transducer-impl.h +++ b/sherpa-onnx/csrc/online-recognizer-transducer-impl.h @@ -14,11 +14,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/file-utils.h" #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/online-lm.h" @@ -130,8 +125,8 @@ class OnlineRecognizerTransducerImpl : public OnlineRecognizerImpl { } } -#if __ANDROID_API__ >= 9 - explicit OnlineRecognizerTransducerImpl(AAssetManager *mgr, + template + explicit OnlineRecognizerTransducerImpl(Manager *mgr, const OnlineRecognizerConfig &config) : OnlineRecognizerImpl(mgr, config), config_(config), @@ -178,7 +173,6 @@ class OnlineRecognizerTransducerImpl : public OnlineRecognizerImpl { exit(-1); } } -#endif std::unique_ptr CreateStream() const override { auto stream = @@ -429,8 +423,8 @@ class OnlineRecognizerTransducerImpl : public OnlineRecognizerImpl { hotwords_, config_.hotwords_score, boost_scores_); } -#if __ANDROID_API__ >= 9 - void InitHotwords(AAssetManager *mgr) { + template + void InitHotwords(Manager *mgr) { // each line in hotwords_file contains space-separated words auto buf = ReadFile(mgr, config_.hotwords_file); @@ -452,7 +446,6 @@ class OnlineRecognizerTransducerImpl : public OnlineRecognizerImpl { hotwords_graph_ = std::make_shared( hotwords_, config_.hotwords_score, boost_scores_); } -#endif void InitHotwordsFromBufStr() { // each line in hotwords_file contains space-separated words diff --git a/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h b/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h index 0a09fdb01..a3f2756c8 100644 --- a/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h +++ b/sherpa-onnx/csrc/online-recognizer-transducer-nemo-impl.h @@ -16,11 +16,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/online-recognizer-impl.h" #include "sherpa-onnx/csrc/online-recognizer.h" @@ -65,9 +60,9 @@ class OnlineRecognizerTransducerNeMoImpl : public OnlineRecognizerImpl { PostInit(); } -#if __ANDROID_API__ >= 9 + template explicit OnlineRecognizerTransducerNeMoImpl( - AAssetManager *mgr, const OnlineRecognizerConfig &config) + Manager *mgr, const OnlineRecognizerConfig &config) : OnlineRecognizerImpl(mgr, config), config_(config), symbol_table_(mgr, config.model_config.tokens), @@ -85,7 +80,6 @@ class OnlineRecognizerTransducerNeMoImpl : public OnlineRecognizerImpl { PostInit(); } -#endif std::unique_ptr CreateStream() const override { auto stream = std::make_unique(config_.feat_config); diff --git a/sherpa-onnx/csrc/online-recognizer.cc b/sherpa-onnx/csrc/online-recognizer.cc index ba7cba8ec..4ccc939da 100644 --- a/sherpa-onnx/csrc/online-recognizer.cc +++ b/sherpa-onnx/csrc/online-recognizer.cc @@ -13,6 +13,15 @@ #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/file-utils.h" #include "sherpa-onnx/csrc/online-recognizer-impl.h" #include "sherpa-onnx/csrc/text-utils.h" @@ -197,11 +206,10 @@ std::string OnlineRecognizerConfig::ToString() const { OnlineRecognizer::OnlineRecognizer(const OnlineRecognizerConfig &config) : impl_(OnlineRecognizerImpl::Create(config)) {} -#if __ANDROID_API__ >= 9 -OnlineRecognizer::OnlineRecognizer(AAssetManager *mgr, +template +OnlineRecognizer::OnlineRecognizer(Manager *mgr, const OnlineRecognizerConfig &config) : impl_(OnlineRecognizerImpl::Create(mgr, config)) {} -#endif OnlineRecognizer::~OnlineRecognizer() = default; @@ -238,4 +246,14 @@ bool OnlineRecognizer::IsEndpoint(OnlineStream *s) const { void OnlineRecognizer::Reset(OnlineStream *s) const { impl_->Reset(s); } +#if __ANDROID_API__ >= 9 +template OnlineRecognizer::OnlineRecognizer( + AAssetManager *mgr, const OnlineRecognizerConfig &config); +#endif + +#if __OHOS__ +template OnlineRecognizer::OnlineRecognizer( + NativeResourceManager *mgr, const OnlineRecognizerConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/online-recognizer.h b/sherpa-onnx/csrc/online-recognizer.h index 45e0f4237..8854fbd22 100644 --- a/sherpa-onnx/csrc/online-recognizer.h +++ b/sherpa-onnx/csrc/online-recognizer.h @@ -9,11 +9,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/endpoint.h" #include "sherpa-onnx/csrc/features.h" #include "sherpa-onnx/csrc/online-ctc-fst-decoder-config.h" @@ -149,9 +144,8 @@ class OnlineRecognizer { public: explicit OnlineRecognizer(const OnlineRecognizerConfig &config); -#if __ANDROID_API__ >= 9 - OnlineRecognizer(AAssetManager *mgr, const OnlineRecognizerConfig &config); -#endif + template + OnlineRecognizer(Manager *mgr, const OnlineRecognizerConfig &config); ~OnlineRecognizer(); diff --git a/sherpa-onnx/csrc/online-transducer-model.cc b/sherpa-onnx/csrc/online-transducer-model.cc index 51a9aef3c..a38b5893f 100644 --- a/sherpa-onnx/csrc/online-transducer-model.cc +++ b/sherpa-onnx/csrc/online-transducer-model.cc @@ -9,6 +9,10 @@ #include "android/asset_manager_jni.h" #endif +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include #include #include @@ -49,7 +53,11 @@ static ModelType GetModelType(char *model_data, size_t model_data_length, if (debug) { std::ostringstream os; PrintModelMetadata(os, meta_data); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; @@ -155,9 +163,9 @@ Ort::Value OnlineTransducerModel::BuildDecoderInput( return decoder_input; } -#if __ANDROID_API__ >= 9 +template std::unique_ptr OnlineTransducerModel::Create( - AAssetManager *mgr, const OnlineModelConfig &config) { + Manager *mgr, const OnlineModelConfig &config) { if (!config.model_type.empty()) { const auto &model_type = config.model_type; if (model_type == "conformer") { @@ -195,6 +203,15 @@ std::unique_ptr OnlineTransducerModel::Create( // unreachable code return nullptr; } + +#if __ANDROID_API__ >= 9 +template std::unique_ptr OnlineTransducerModel::Create( + Manager *mgr, const OnlineModelConfig &config); +#endif + +#if __OHOS__ +template std::unique_ptr OnlineTransducerModel::Create( + NativeResourceManager *mgr, const OnlineModelConfig &config); #endif } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/online-transducer-model.h b/sherpa-onnx/csrc/online-transducer-model.h index 3dacb4a50..f6404eccd 100644 --- a/sherpa-onnx/csrc/online-transducer-model.h +++ b/sherpa-onnx/csrc/online-transducer-model.h @@ -8,11 +8,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/hypothesis.h" #include "sherpa-onnx/csrc/online-model-config.h" @@ -30,10 +25,9 @@ class OnlineTransducerModel { static std::unique_ptr Create( const OnlineModelConfig &config); -#if __ANDROID_API__ >= 9 + template static std::unique_ptr Create( - AAssetManager *mgr, const OnlineModelConfig &config); -#endif + Manager *mgr, const OnlineModelConfig &config); /** Stack a list of individual states into a batch. * diff --git a/sherpa-onnx/csrc/online-transducer-nemo-model.cc b/sherpa-onnx/csrc/online-transducer-nemo-model.cc index 264593a1c..73c23fe31 100644 --- a/sherpa-onnx/csrc/online-transducer-nemo-model.cc +++ b/sherpa-onnx/csrc/online-transducer-nemo-model.cc @@ -20,6 +20,10 @@ #include "android/asset_manager_jni.h" #endif +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/cat.h" #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/online-transducer-decoder.h" @@ -54,8 +58,8 @@ class OnlineTransducerNeMoModel::Impl { } } -#if __ANDROID_API__ >= 9 - Impl(AAssetManager *mgr, const OnlineModelConfig &config) + template + Impl(Manager *mgr, const OnlineModelConfig &config) : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), @@ -75,7 +79,6 @@ class OnlineTransducerNeMoModel::Impl { InitJoiner(buf.data(), buf.size()); } } -#endif std::vector RunEncoder(Ort::Value features, std::vector states) { @@ -302,7 +305,11 @@ class OnlineTransducerNeMoModel::Impl { std::ostringstream os; os << "---encoder---\n"; PrintModelMetadata(os, meta_data); - SHERPA_ONNX_LOGE("%s\n", os.str().c_str()); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else + SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; // used in the macro below @@ -460,11 +467,10 @@ OnlineTransducerNeMoModel::OnlineTransducerNeMoModel( const OnlineModelConfig &config) : impl_(std::make_unique(config)) {} -#if __ANDROID_API__ >= 9 +template OnlineTransducerNeMoModel::OnlineTransducerNeMoModel( - AAssetManager *mgr, const OnlineModelConfig &config) + Manager *mgr, const OnlineModelConfig &config) : impl_(std::make_unique(mgr, config)) {} -#endif OnlineTransducerNeMoModel::~OnlineTransducerNeMoModel() = default; @@ -528,4 +534,14 @@ std::vector> OnlineTransducerNeMoModel::UnStackStates( return impl_->UnStackStates(std::move(states)); } +#if __ANDROID_API__ >= 9 +template OnlineTransducerNeMoModel::OnlineTransducerNeMoModel( + AAssetManager *mgr, const OnlineModelConfig &config); +#endif + +#if __OHOS__ +template OnlineTransducerNeMoModel::OnlineTransducerNeMoModel( + NativeResourceManager *mgr, const OnlineModelConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/online-transducer-nemo-model.h b/sherpa-onnx/csrc/online-transducer-nemo-model.h index e12814cc0..98390bb15 100644 --- a/sherpa-onnx/csrc/online-transducer-nemo-model.h +++ b/sherpa-onnx/csrc/online-transducer-nemo-model.h @@ -11,11 +11,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/online-model-config.h" @@ -28,10 +23,8 @@ class OnlineTransducerNeMoModel { public: explicit OnlineTransducerNeMoModel(const OnlineModelConfig &config); -#if __ANDROID_API__ >= 9 - OnlineTransducerNeMoModel(AAssetManager *mgr, - const OnlineModelConfig &config); -#endif + template + OnlineTransducerNeMoModel(Manager *mgr, const OnlineModelConfig &config); ~OnlineTransducerNeMoModel(); // A list of 3 tensors: diff --git a/sherpa-onnx/csrc/online-wenet-ctc-model.cc b/sherpa-onnx/csrc/online-wenet-ctc-model.cc index cce322aa4..bf468484c 100644 --- a/sherpa-onnx/csrc/online-wenet-ctc-model.cc +++ b/sherpa-onnx/csrc/online-wenet-ctc-model.cc @@ -13,6 +13,10 @@ #include "android/asset_manager_jni.h" #endif +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/onnx-utils.h" #include "sherpa-onnx/csrc/session.h" @@ -33,8 +37,8 @@ class OnlineWenetCtcModel::Impl { } } -#if __ANDROID_API__ >= 9 - Impl(AAssetManager *mgr, const OnlineModelConfig &config) + template + Impl(Manager *mgr, const OnlineModelConfig &config) : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), @@ -44,7 +48,6 @@ class OnlineWenetCtcModel::Impl { Init(buf.data(), buf.size()); } } -#endif std::vector Forward(Ort::Value x, std::vector states) { @@ -139,7 +142,11 @@ class OnlineWenetCtcModel::Impl { if (config_.debug) { std::ostringstream os; PrintModelMetadata(os, meta_data); - SHERPA_ONNX_LOGE("%s\n", os.str().c_str()); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else + SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; // used in the macro below @@ -212,11 +219,10 @@ class OnlineWenetCtcModel::Impl { OnlineWenetCtcModel::OnlineWenetCtcModel(const OnlineModelConfig &config) : impl_(std::make_unique(config)) {} -#if __ANDROID_API__ >= 9 -OnlineWenetCtcModel::OnlineWenetCtcModel(AAssetManager *mgr, +template +OnlineWenetCtcModel::OnlineWenetCtcModel(Manager *mgr, const OnlineModelConfig &config) : impl_(std::make_unique(mgr, config)) {} -#endif OnlineWenetCtcModel::~OnlineWenetCtcModel() = default; @@ -258,4 +264,14 @@ std::vector> OnlineWenetCtcModel::UnStackStates( return ans; } +#if __ANDROID_API__ >= 9 +template OnlineWenetCtcModel::OnlineWenetCtcModel( + AAssetManager *mgr, const OnlineModelConfig &config); +#endif + +#if __OHOS__ +template OnlineWenetCtcModel::OnlineWenetCtcModel( + NativeResourceManager *mgr, const OnlineModelConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/online-wenet-ctc-model.h b/sherpa-onnx/csrc/online-wenet-ctc-model.h index 1be1034cc..28458b68b 100644 --- a/sherpa-onnx/csrc/online-wenet-ctc-model.h +++ b/sherpa-onnx/csrc/online-wenet-ctc-model.h @@ -8,11 +8,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/online-ctc-model.h" #include "sherpa-onnx/csrc/online-model-config.h" @@ -23,9 +18,8 @@ class OnlineWenetCtcModel : public OnlineCtcModel { public: explicit OnlineWenetCtcModel(const OnlineModelConfig &config); -#if __ANDROID_API__ >= 9 - OnlineWenetCtcModel(AAssetManager *mgr, const OnlineModelConfig &config); -#endif + template + OnlineWenetCtcModel(Manager *mgr, const OnlineModelConfig &config); ~OnlineWenetCtcModel() override; diff --git a/sherpa-onnx/csrc/online-zipformer-transducer-model.cc b/sherpa-onnx/csrc/online-zipformer-transducer-model.cc index 36e2d9dbd..437600262 100644 --- a/sherpa-onnx/csrc/online-zipformer-transducer-model.cc +++ b/sherpa-onnx/csrc/online-zipformer-transducer-model.cc @@ -17,6 +17,10 @@ #include "android/asset_manager_jni.h" #endif +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/cat.h" #include "sherpa-onnx/csrc/macros.h" @@ -50,9 +54,9 @@ OnlineZipformerTransducerModel::OnlineZipformerTransducerModel( } } -#if __ANDROID_API__ >= 9 +template OnlineZipformerTransducerModel::OnlineZipformerTransducerModel( - AAssetManager *mgr, const OnlineModelConfig &config) + Manager *mgr, const OnlineModelConfig &config) : env_(ORT_LOGGING_LEVEL_ERROR), config_(config), sess_opts_(GetSessionOptions(config)), @@ -72,7 +76,6 @@ OnlineZipformerTransducerModel::OnlineZipformerTransducerModel( InitJoiner(buf.data(), buf.size()); } } -#endif void OnlineZipformerTransducerModel::InitEncoder(void *model_data, size_t model_data_length) { @@ -91,7 +94,11 @@ void OnlineZipformerTransducerModel::InitEncoder(void *model_data, std::ostringstream os; os << "---encoder---\n"; PrintModelMetadata(os, meta_data); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; // used in the macro below @@ -480,4 +487,14 @@ Ort::Value OnlineZipformerTransducerModel::RunJoiner(Ort::Value encoder_out, return std::move(logit[0]); } +#if __ANDROID_API__ >= 9 +template OnlineZipformerTransducerModel::OnlineZipformerTransducerModel( + AAssetManager *mgr, const OnlineModelConfig &config); +#endif + +#if __OHOS__ +template OnlineZipformerTransducerModel::OnlineZipformerTransducerModel( + NativeResourceManager *mgr, const OnlineModelConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/online-zipformer-transducer-model.h b/sherpa-onnx/csrc/online-zipformer-transducer-model.h index b2b7da040..9e4368a69 100644 --- a/sherpa-onnx/csrc/online-zipformer-transducer-model.h +++ b/sherpa-onnx/csrc/online-zipformer-transducer-model.h @@ -9,11 +9,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/online-model-config.h" #include "sherpa-onnx/csrc/online-transducer-model.h" @@ -24,10 +19,8 @@ class OnlineZipformerTransducerModel : public OnlineTransducerModel { public: explicit OnlineZipformerTransducerModel(const OnlineModelConfig &config); -#if __ANDROID_API__ >= 9 - OnlineZipformerTransducerModel(AAssetManager *mgr, - const OnlineModelConfig &config); -#endif + template + OnlineZipformerTransducerModel(Manager *mgr, const OnlineModelConfig &config); std::vector StackStates( const std::vector> &states) const override; diff --git a/sherpa-onnx/csrc/online-zipformer2-ctc-model.cc b/sherpa-onnx/csrc/online-zipformer2-ctc-model.cc index 8f0708ad1..298b90522 100644 --- a/sherpa-onnx/csrc/online-zipformer2-ctc-model.cc +++ b/sherpa-onnx/csrc/online-zipformer2-ctc-model.cc @@ -15,6 +15,10 @@ #include "android/asset_manager_jni.h" #endif +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/cat.h" #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/onnx-utils.h" @@ -37,8 +41,8 @@ class OnlineZipformer2CtcModel::Impl { } } -#if __ANDROID_API__ >= 9 - Impl(AAssetManager *mgr, const OnlineModelConfig &config) + template + Impl(Manager *mgr, const OnlineModelConfig &config) : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), @@ -48,7 +52,6 @@ class OnlineZipformer2CtcModel::Impl { Init(buf.data(), buf.size()); } } -#endif std::vector Forward(Ort::Value features, std::vector states) { @@ -255,7 +258,11 @@ class OnlineZipformer2CtcModel::Impl { std::ostringstream os; os << "---zipformer2_ctc---\n"; PrintModelMetadata(os, meta_data); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; // used in the macro below @@ -415,11 +422,10 @@ OnlineZipformer2CtcModel::OnlineZipformer2CtcModel( const OnlineModelConfig &config) : impl_(std::make_unique(config)) {} -#if __ANDROID_API__ >= 9 +template OnlineZipformer2CtcModel::OnlineZipformer2CtcModel( - AAssetManager *mgr, const OnlineModelConfig &config) + Manager *mgr, const OnlineModelConfig &config) : impl_(std::make_unique(mgr, config)) {} -#endif OnlineZipformer2CtcModel::~OnlineZipformer2CtcModel() = default; @@ -458,4 +464,14 @@ std::vector> OnlineZipformer2CtcModel::UnStackStates( return impl_->UnStackStates(std::move(states)); } +#if __ANDROID_API__ >= 9 +template OnlineZipformer2CtcModel::OnlineZipformer2CtcModel( + AAssetManager *mgr, const OnlineModelConfig &config); +#endif + +#if __OHOS__ +template OnlineZipformer2CtcModel::OnlineZipformer2CtcModel( + NativeResourceManager *mgr, const OnlineModelConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/online-zipformer2-ctc-model.h b/sherpa-onnx/csrc/online-zipformer2-ctc-model.h index 11b59e2bb..32ddf2122 100644 --- a/sherpa-onnx/csrc/online-zipformer2-ctc-model.h +++ b/sherpa-onnx/csrc/online-zipformer2-ctc-model.h @@ -8,11 +8,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/online-ctc-model.h" #include "sherpa-onnx/csrc/online-model-config.h" @@ -23,9 +18,8 @@ class OnlineZipformer2CtcModel : public OnlineCtcModel { public: explicit OnlineZipformer2CtcModel(const OnlineModelConfig &config); -#if __ANDROID_API__ >= 9 - OnlineZipformer2CtcModel(AAssetManager *mgr, const OnlineModelConfig &config); -#endif + template + OnlineZipformer2CtcModel(Manager *mgr, const OnlineModelConfig &config); ~OnlineZipformer2CtcModel() override; diff --git a/sherpa-onnx/csrc/online-zipformer2-transducer-model.cc b/sherpa-onnx/csrc/online-zipformer2-transducer-model.cc index 03c68474c..85a32ec3c 100644 --- a/sherpa-onnx/csrc/online-zipformer2-transducer-model.cc +++ b/sherpa-onnx/csrc/online-zipformer2-transducer-model.cc @@ -19,6 +19,10 @@ #include "android/asset_manager_jni.h" #endif +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/cat.h" #include "sherpa-onnx/csrc/macros.h" @@ -54,9 +58,9 @@ OnlineZipformer2TransducerModel::OnlineZipformer2TransducerModel( } } -#if __ANDROID_API__ >= 9 +template OnlineZipformer2TransducerModel::OnlineZipformer2TransducerModel( - AAssetManager *mgr, const OnlineModelConfig &config) + Manager *mgr, const OnlineModelConfig &config) : env_(ORT_LOGGING_LEVEL_ERROR), config_(config), encoder_sess_opts_(GetSessionOptions(config)), @@ -78,7 +82,6 @@ OnlineZipformer2TransducerModel::OnlineZipformer2TransducerModel( InitJoiner(buf.data(), buf.size()); } } -#endif void OnlineZipformer2TransducerModel::InitEncoder(void *model_data, size_t model_data_length) { @@ -97,7 +100,11 @@ void OnlineZipformer2TransducerModel::InitEncoder(void *model_data, std::ostringstream os; os << "---encoder---\n"; PrintModelMetadata(os, meta_data); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; // used in the macro below @@ -474,4 +481,14 @@ Ort::Value OnlineZipformer2TransducerModel::RunJoiner(Ort::Value encoder_out, return std::move(logit[0]); } +#if __ANDROID_API__ >= 9 +template OnlineZipformer2TransducerModel::OnlineZipformer2TransducerModel( + AAssetManager *mgr, const OnlineModelConfig &config); +#endif + +#if __OHOS__ +template OnlineZipformer2TransducerModel::OnlineZipformer2TransducerModel( + NativeResourceManager *mgr, const OnlineModelConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/online-zipformer2-transducer-model.h b/sherpa-onnx/csrc/online-zipformer2-transducer-model.h index aa0f46f81..93b124f61 100644 --- a/sherpa-onnx/csrc/online-zipformer2-transducer-model.h +++ b/sherpa-onnx/csrc/online-zipformer2-transducer-model.h @@ -9,11 +9,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/online-model-config.h" #include "sherpa-onnx/csrc/online-transducer-model.h" @@ -24,10 +19,9 @@ class OnlineZipformer2TransducerModel : public OnlineTransducerModel { public: explicit OnlineZipformer2TransducerModel(const OnlineModelConfig &config); -#if __ANDROID_API__ >= 9 - OnlineZipformer2TransducerModel(AAssetManager *mgr, + template + OnlineZipformer2TransducerModel(Manager *mgr, const OnlineModelConfig &config); -#endif std::vector StackStates( const std::vector> &states) const override; diff --git a/sherpa-onnx/csrc/symbol-table.cc b/sherpa-onnx/csrc/symbol-table.cc index 723fec68b..3455d2007 100644 --- a/sherpa-onnx/csrc/symbol-table.cc +++ b/sherpa-onnx/csrc/symbol-table.cc @@ -8,16 +8,16 @@ #include #include #include +#include #include #if __ANDROID_API__ >= 9 -#include #include "android/asset_manager.h" #include "android/asset_manager_jni.h" -#elif __OHOS__ -#include +#endif +#if __OHOS__ #include "rawfile/raw_file_manager.h" #endif diff --git a/sherpa-onnx/python/csrc/vad-model.cc b/sherpa-onnx/python/csrc/vad-model.cc index f304fd0ae..743628d38 100644 --- a/sherpa-onnx/python/csrc/vad-model.cc +++ b/sherpa-onnx/python/csrc/vad-model.cc @@ -4,6 +4,7 @@ #include "sherpa-onnx/python/csrc/vad-model.h" +#include #include #include "sherpa-onnx/csrc/vad-model.h" @@ -13,8 +14,10 @@ namespace sherpa_onnx { void PybindVadModel(py::module *m) { using PyClass = VadModel; py::class_(*m, "VadModel") - .def_static("create", &PyClass::Create, py::arg("config"), - py::call_guard()) + .def_static("create", + (std::unique_ptr(*)(const VadModelConfig &))( + &PyClass::Create), + py::arg("config"), py::call_guard()) .def("reset", &PyClass::Reset, py::call_guard()) .def( "is_speech", From 109fb799cafa8178054a227ecb637d41036270e3 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 27 Nov 2024 10:36:16 +0800 Subject: [PATCH 081/183] fix building for Android (#1568) --- sherpa-onnx/csrc/online-transducer-model.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sherpa-onnx/csrc/online-transducer-model.cc b/sherpa-onnx/csrc/online-transducer-model.cc index a38b5893f..9ebe40372 100644 --- a/sherpa-onnx/csrc/online-transducer-model.cc +++ b/sherpa-onnx/csrc/online-transducer-model.cc @@ -206,7 +206,7 @@ std::unique_ptr OnlineTransducerModel::Create( #if __ANDROID_API__ >= 9 template std::unique_ptr OnlineTransducerModel::Create( - Manager *mgr, const OnlineModelConfig &config); + AAssetManager *mgr, const OnlineModelConfig &config); #endif #if __OHOS__ From 315d8e2a478ab86de0d11347e6dbc423d809deb1 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 28 Nov 2024 17:30:16 +0800 Subject: [PATCH 082/183] Publish `sherpa_onnx.har` for HarmonyOS (#1572) --- .github/workflows/har.yaml | 181 ++++ README.md | 16 +- build-ohos-arm64-v8a.sh | 6 + build-ohos-x86-64.sh | 6 + harmony-os/.gitignore | 1 + harmony-os/SherpaOnnxHar/.gitignore | 12 + harmony-os/SherpaOnnxHar/AppScope/app.json5 | 10 + .../resources/base/element/string.json | 8 + .../resources/base/media/app_icon.png | Bin 0 -> 2777 bytes harmony-os/SherpaOnnxHar/README.md | 18 + harmony-os/SherpaOnnxHar/build-profile.json5 | 44 + harmony-os/SherpaOnnxHar/code-linter.json5 | 20 + harmony-os/SherpaOnnxHar/entry/.gitignore | 6 + .../SherpaOnnxHar/entry/build-profile.json5 | 28 + harmony-os/SherpaOnnxHar/entry/hvigorfile.ts | 6 + .../SherpaOnnxHar/entry/obfuscation-rules.txt | 23 + .../SherpaOnnxHar/entry/oh-package.json5 | 10 + .../main/ets/entryability/EntryAbility.ets | 43 + .../entrybackupability/EntryBackupAbility.ets | 12 + .../entry/src/main/ets/pages/Index.ets | 17 + .../SherpaOnnxHar/entry/src/main/module.json5 | 52 ++ .../main/resources/base/element/color.json | 8 + .../main/resources/base/element/string.json | 16 + .../main/resources/base/media/background.png | Bin 0 -> 57364 bytes .../main/resources/base/media/foreground.png | Bin 0 -> 12430 bytes .../resources/base/media/layered_image.json | 7 + .../main/resources/base/media/startIcon.png | Bin 0 -> 20093 bytes .../resources/base/profile/backup_config.json | 3 + .../resources/base/profile/main_pages.json | 5 + .../main/resources/en_US/element/string.json | 16 + .../main/resources/zh_CN/element/string.json | 16 + .../src/ohosTest/ets/test/Ability.test.ets | 35 + .../entry/src/ohosTest/ets/test/List.test.ets | 5 + .../entry/src/ohosTest/module.json5 | 13 + .../entry/src/test/List.test.ets | 5 + .../entry/src/test/LocalUnit.test.ets | 33 + .../SherpaOnnxHar/hvigor/hvigor-config.json5 | 22 + harmony-os/SherpaOnnxHar/hvigorfile.ts | 6 + harmony-os/SherpaOnnxHar/notes.md | 13 + .../SherpaOnnxHar/oh-package-lock.json5 | 19 + harmony-os/SherpaOnnxHar/oh-package.json5 | 9 + .../SherpaOnnxHar/sherpa_onnx/.gitignore | 6 + .../sherpa_onnx/BuildProfile.ets | 17 + .../SherpaOnnxHar/sherpa_onnx/Index.ets | 40 + .../SherpaOnnxHar/sherpa_onnx/README.md | 12 + .../sherpa_onnx/build-profile.json5 | 46 + .../sherpa_onnx/consumer-rules.txt | 0 .../SherpaOnnxHar/sherpa_onnx/hvigorfile.ts | 6 + .../sherpa_onnx/obfuscation-rules.txt | 23 + .../sherpa_onnx/oh-package-lock.json5 | 18 + .../sherpa_onnx/oh-package.json5 | 25 + .../sherpa_onnx/src/main/cpp/CMakeLists.txt | 69 ++ .../sherpa_onnx/src/main/cpp/audio-tagging.cc | 227 +++++ .../cpp/include/sherpa-onnx/c-api/README.md | 8 + .../cpp/include/sherpa-onnx/c-api/c-api.h | 1 + .../src/main/cpp/keyword-spotting.cc | 266 ++++++ .../sherpa_onnx/src/main/cpp/libs/.gitignore | 1 + .../sherpa_onnx/src/main/cpp/libs/README.md | 17 + .../src/main/cpp/libs/arm64-v8a/.gitkeep | 0 .../src/main/cpp/libs/x86_64/.gitkeep | 0 .../sherpa_onnx/src/main/cpp/macros.h | 63 ++ .../sherpa_onnx/src/main/cpp/my-patch.diff | 14 + .../src/main/cpp/non-streaming-asr.cc | 487 +++++++++++ .../cpp/non-streaming-speaker-diarization.cc | 310 +++++++ .../src/main/cpp/non-streaming-tts.cc | 329 +++++++ .../sherpa_onnx/src/main/cpp/punctuation.cc | 135 +++ .../main/cpp/sherpa-onnx-node-addon-api.cc | 47 + .../src/main/cpp/speaker-identification.cc | 808 +++++++++++++++++ .../cpp/spoken-language-identification.cc | 188 ++++ .../sherpa_onnx/src/main/cpp/streaming-asr.cc | 731 ++++++++++++++++ .../main/cpp/types/libsherpa_onnx/Index.d.ts | 35 + .../cpp/types/libsherpa_onnx/oh-package.json5 | 6 + .../sherpa_onnx/src/main/cpp/vad.cc | 700 +++++++++++++++ .../sherpa_onnx/src/main/cpp/wave-reader.cc | 171 ++++ .../sherpa_onnx/src/main/cpp/wave-writer.cc | 81 ++ .../src/main/ets/components/MainPage.ets | 21 + .../main/ets/components/NonStreamingAsr.ets | 162 ++++ .../src/main/ets/components/StreamingAsr.ets | 141 +++ .../src/main/ets/components/Vad.ets | 133 +++ .../sherpa_onnx/src/main/module.json5 | 11 + .../main/resources/base/element/string.json | 8 + .../main/resources/en_US/element/string.json | 8 + .../main/resources/zh_CN/element/string.json | 8 + .../src/ohosTest/ets/test/Ability.test.ets | 35 + .../src/ohosTest/ets/test/List.test.ets | 5 + .../sherpa_onnx/src/ohosTest/module.json5 | 13 + .../sherpa_onnx/src/test/List.test.ets | 5 + .../sherpa_onnx/src/test/LocalUnit.test.ets | 33 + scripts/node-addon-api/src/audio-tagging.cc | 228 +---- .../node-addon-api/src/keyword-spotting.cc | 267 +----- scripts/node-addon-api/src/macros.h | 52 +- .../node-addon-api/src/non-streaming-asr.cc | 462 +--------- .../src/non-streaming-speaker-diarization.cc | 311 +------ .../node-addon-api/src/non-streaming-tts.cc | 330 +------ scripts/node-addon-api/src/punctuation.cc | 136 +-- .../src/sherpa-onnx-node-addon-api.cc | 48 +- .../src/speaker-identification.cc | 809 +----------------- .../src/spoken-language-identification.cc | 189 +--- scripts/node-addon-api/src/streaming-asr.cc | 709 +-------------- scripts/node-addon-api/src/vad.cc | 669 +-------------- scripts/node-addon-api/src/wave-reader.cc | 92 +- scripts/node-addon-api/src/wave-writer.cc | 82 +- sherpa-onnx/c-api/c-api.cc | 23 + sherpa-onnx/c-api/c-api.h | 8 + 104 files changed, 6257 insertions(+), 4378 deletions(-) create mode 100644 .github/workflows/har.yaml create mode 100644 harmony-os/.gitignore create mode 100644 harmony-os/SherpaOnnxHar/.gitignore create mode 100644 harmony-os/SherpaOnnxHar/AppScope/app.json5 create mode 100644 harmony-os/SherpaOnnxHar/AppScope/resources/base/element/string.json create mode 100644 harmony-os/SherpaOnnxHar/AppScope/resources/base/media/app_icon.png create mode 100644 harmony-os/SherpaOnnxHar/README.md create mode 100644 harmony-os/SherpaOnnxHar/build-profile.json5 create mode 100644 harmony-os/SherpaOnnxHar/code-linter.json5 create mode 100644 harmony-os/SherpaOnnxHar/entry/.gitignore create mode 100644 harmony-os/SherpaOnnxHar/entry/build-profile.json5 create mode 100644 harmony-os/SherpaOnnxHar/entry/hvigorfile.ts create mode 100644 harmony-os/SherpaOnnxHar/entry/obfuscation-rules.txt create mode 100644 harmony-os/SherpaOnnxHar/entry/oh-package.json5 create mode 100644 harmony-os/SherpaOnnxHar/entry/src/main/ets/entryability/EntryAbility.ets create mode 100644 harmony-os/SherpaOnnxHar/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets create mode 100644 harmony-os/SherpaOnnxHar/entry/src/main/ets/pages/Index.ets create mode 100644 harmony-os/SherpaOnnxHar/entry/src/main/module.json5 create mode 100644 harmony-os/SherpaOnnxHar/entry/src/main/resources/base/element/color.json create mode 100644 harmony-os/SherpaOnnxHar/entry/src/main/resources/base/element/string.json create mode 100644 harmony-os/SherpaOnnxHar/entry/src/main/resources/base/media/background.png create mode 100644 harmony-os/SherpaOnnxHar/entry/src/main/resources/base/media/foreground.png create mode 100644 harmony-os/SherpaOnnxHar/entry/src/main/resources/base/media/layered_image.json create mode 100644 harmony-os/SherpaOnnxHar/entry/src/main/resources/base/media/startIcon.png create mode 100644 harmony-os/SherpaOnnxHar/entry/src/main/resources/base/profile/backup_config.json create mode 100644 harmony-os/SherpaOnnxHar/entry/src/main/resources/base/profile/main_pages.json create mode 100644 harmony-os/SherpaOnnxHar/entry/src/main/resources/en_US/element/string.json create mode 100644 harmony-os/SherpaOnnxHar/entry/src/main/resources/zh_CN/element/string.json create mode 100644 harmony-os/SherpaOnnxHar/entry/src/ohosTest/ets/test/Ability.test.ets create mode 100644 harmony-os/SherpaOnnxHar/entry/src/ohosTest/ets/test/List.test.ets create mode 100644 harmony-os/SherpaOnnxHar/entry/src/ohosTest/module.json5 create mode 100644 harmony-os/SherpaOnnxHar/entry/src/test/List.test.ets create mode 100644 harmony-os/SherpaOnnxHar/entry/src/test/LocalUnit.test.ets create mode 100644 harmony-os/SherpaOnnxHar/hvigor/hvigor-config.json5 create mode 100644 harmony-os/SherpaOnnxHar/hvigorfile.ts create mode 100644 harmony-os/SherpaOnnxHar/notes.md create mode 100644 harmony-os/SherpaOnnxHar/oh-package-lock.json5 create mode 100644 harmony-os/SherpaOnnxHar/oh-package.json5 create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/.gitignore create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/README.md create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/build-profile.json5 create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/consumer-rules.txt create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/hvigorfile.ts create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/obfuscation-rules.txt create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package-lock.json5 create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/CMakeLists.txt create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/audio-tagging.cc create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/include/sherpa-onnx/c-api/README.md create mode 120000 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/include/sherpa-onnx/c-api/c-api.h create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/keyword-spotting.cc create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/libs/.gitignore create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/libs/README.md create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/libs/arm64-v8a/.gitkeep create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/libs/x86_64/.gitkeep create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/macros.h create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/my-patch.diff create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-asr.cc create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-speaker-diarization.cc create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/punctuation.cc create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/sherpa-onnx-node-addon-api.cc create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/speaker-identification.cc create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/spoken-language-identification.cc create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/streaming-asr.cc create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/oh-package.json5 create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/vad.cc create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-reader.cc create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-writer.cc create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/MainPage.ets create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingAsr.ets create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/StreamingAsr.ets create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/Vad.ets create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/module.json5 create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/resources/base/element/string.json create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/resources/en_US/element/string.json create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/resources/zh_CN/element/string.json create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/ohosTest/ets/test/Ability.test.ets create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/ohosTest/ets/test/List.test.ets create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/ohosTest/module.json5 create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/test/List.test.ets create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/test/LocalUnit.test.ets mode change 100644 => 120000 scripts/node-addon-api/src/audio-tagging.cc mode change 100644 => 120000 scripts/node-addon-api/src/keyword-spotting.cc mode change 100644 => 120000 scripts/node-addon-api/src/macros.h mode change 100644 => 120000 scripts/node-addon-api/src/non-streaming-asr.cc mode change 100644 => 120000 scripts/node-addon-api/src/non-streaming-speaker-diarization.cc mode change 100644 => 120000 scripts/node-addon-api/src/non-streaming-tts.cc mode change 100644 => 120000 scripts/node-addon-api/src/punctuation.cc mode change 100644 => 120000 scripts/node-addon-api/src/sherpa-onnx-node-addon-api.cc mode change 100644 => 120000 scripts/node-addon-api/src/speaker-identification.cc mode change 100644 => 120000 scripts/node-addon-api/src/spoken-language-identification.cc mode change 100644 => 120000 scripts/node-addon-api/src/streaming-asr.cc mode change 100644 => 120000 scripts/node-addon-api/src/vad.cc mode change 100644 => 120000 scripts/node-addon-api/src/wave-reader.cc mode change 100644 => 120000 scripts/node-addon-api/src/wave-writer.cc diff --git a/.github/workflows/har.yaml b/.github/workflows/har.yaml new file mode 100644 index 000000000..afe99a1be --- /dev/null +++ b/.github/workflows/har.yaml @@ -0,0 +1,181 @@ +name: har + +on: + push: + branches: + - master + # - ohos-har + tags: + - 'v[0-9]+.[0-9]+.[0-9]+*' + + workflow_dispatch: + +concurrency: + group: har-${{ github.ref }} + cancel-in-progress: true + +jobs: + har: + name: Har + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: har-linux + + - name: cache-toolchain + id: cache-toolchain-ohos + uses: actions/cache@v4 + with: + path: command-line-tools + key: commandline-tools-linux-x64-5.0.5.200.zip + + - name: Download toolchain + if: steps.cache-toolchain-ohos.outputs.cache-hit != 'true' + shell: bash + run: | + curl -SL -O https://huggingface.co/csukuangfj/harmonyos-commandline-tools/resolve/main/commandline-tools-linux-x64-5.0.5.200.zip + unzip commandline-tools-linux-x64-5.0.5.200.zip + rm commandline-tools-linux-x64-5.0.5.200.zip + + - name: Set environment variable + shell: bash + run: | + echo "$GITHUB_WORKSPACE/command-line-tools/sdk/default/openharmony/native/build-tools/cmake/bin" >> "$GITHUB_PATH" + which cmake + + cmake --version + + ls -lh $GITHUB_WORKSPACE/command-line-tools/sdk/default/openharmony/native/build/cmake/ohos.toolchain.cmake + + echo "====" + cat $GITHUB_WORKSPACE/command-line-tools/sdk/default/openharmony/native/build/cmake/ohos.toolchain.cmake + echo "====" + + # echo "$GITHUB_WORKSPACE/command-line-tools/sdk/default/openharmony/native/llvm/bin" >> "$GITHUB_PATH" + + ls -lh $GITHUB_WORKSPACE/command-line-tools/sdk/default/openharmony/native/llvm/bin/ + echo "--" + ls -lh $GITHUB_WORKSPACE/command-line-tools/sdk/default/openharmony/native/llvm/bin/*unknown* + + cat $GITHUB_PATH + + # /home/runner/work/onnxruntime-libs/onnxruntime-libs/command-line-tools/sdk/default/openharmony/native/llvm/bin/aarch64-unknown-linux-ohos-clang -v || true + export PATH=$PWD/command-line-tools/sdk/default/openharmony/native/llvm/bin:$PATH + echo "path: $PATH" + + which aarch64-unknown-linux-ohos-clang++ || true + which aarch64-unknown-linux-ohos-clang || true + + aarch64-unknown-linux-ohos-clang++ --version || true + aarch64-unknown-linux-ohos-clang --version || true + + which armv7-unknown-linux-ohos-clang++ + which armv7-unknown-linux-ohos-clang + + armv7-unknown-linux-ohos-clang++ --version + armv7-unknown-linux-ohos-clang --version + + which x86_64-unknown-linux-ohos-clang++ + which x86_64-unknown-linux-ohos-clang + + x86_64-unknown-linux-ohos-clang++ --version + x86_64-unknown-linux-ohos-clang --version + + - name: Build libraries + shell: bash + run: | + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + cmake --version + + export OHOS_SDK_NATIVE_DIR="$GITHUB_WORKSPACE/command-line-tools/sdk/default/openharmony/native" + + ./build-ohos-arm64-v8a.sh + ./build-ohos-x86-64.sh + + - name: Build Har + shell: bash + run: | + export PATH="$GITHUB_WORKSPACE/command-line-tools/bin:$PATH" + + which hvigorw + + pushd harmony-os/SherpaOnnxHar + + hvigorw --mode module -p product=default -p module=sherpa_onnx@default assembleHar --analyze=normal --parallel --incremental --no-daemon + ls -lh ./sherpa_onnx/build/default/outputs/default/sherpa_onnx.har + cp -v ./sherpa_onnx/build/default/outputs/default/sherpa_onnx.har ../../ + + popd + + ls -lh *.har + + - name: Collect result + shell: bash + run: | + SHERPA_ONNX_VERSION=v$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + echo "SHERPA_ONNX_VERSION=$SHERPA_ONNX_VERSION" >> "$GITHUB_ENV" + + mv sherpa_onnx.har sherpa_onnx-$SHERPA_ONNX_VERSION.har + + - uses: actions/upload-artifact@v4 + with: + name: sherpa-onnx-har + path: ./sherpa_onnx*.har + + - name: Release jar + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + overwrite: true + file: ./*.har + # repo_name: k2-fsa/sherpa-onnx + # repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} + # tag: v1.10.32 + + - name: Publish to huggingface + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v3 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + + rm -rf huggingface + export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false + + SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" + + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-harmony-os huggingface + cd huggingface + git fetch + git pull + git merge -m "merge remote" --ff origin main + + d=har + mkdir -p $d + cp -v ../*.har $d/ + git status + git lfs track "*.har" + git add . + git commit -m "add more hars" + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-harmony-os main diff --git a/README.md b/README.md index 20882d636..96c7885c6 100644 --- a/README.md +++ b/README.md @@ -18,14 +18,13 @@ ### Supported platforms -|Architecture| Android | iOS | Windows | macOS | linux | -|------------|---------|---------|------------|-------|-------| -| x64 | ✔️ | | ✔️ | ✔️ | ✔️ | -| x86 | ✔️ | | ✔️ | | | -| arm64 | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | -| arm32 | ✔️ | | | | ✔️ | -| riscv64 | | | | | ✔️ | - +|Architecture| Android | iOS | Windows | macOS | linux | HarmonyOS | +|------------|---------|---------|------------|-------|-------|-----------| +| x64 | ✔️ | | ✔️ | ✔️ | ✔️ | ✔️ | +| x86 | ✔️ | | ✔️ | | | | +| arm64 | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | +| arm32 | ✔️ | | | | ✔️ | ✔️ | +| riscv64 | | | | | ✔️ | | ### Supported programming languages @@ -65,6 +64,7 @@ on the following platforms and operating systems: - Linux, macOS, Windows, openKylin - Android, WearOS - iOS + - HarmonyOS - NodeJS - WebAssembly - [Raspberry Pi][Raspberry Pi] diff --git a/build-ohos-arm64-v8a.sh b/build-ohos-arm64-v8a.sh index 182575bc1..4e0ecbb29 100755 --- a/build-ohos-arm64-v8a.sh +++ b/build-ohos-arm64-v8a.sh @@ -134,3 +134,9 @@ cp -fv $onnxruntime_dir/lib/libonnxruntime.so install/lib rm -rf install/share rm -rf install/lib/pkgconfig + +d=../harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/libs/arm64-v8a +if [ -d $d ]; then + cp -v install/lib/libsherpa-onnx-c-api.so $d/ + cp -v install/lib/libonnxruntime.so $d/ +fi diff --git a/build-ohos-x86-64.sh b/build-ohos-x86-64.sh index 58ff5da5a..9584edafc 100755 --- a/build-ohos-x86-64.sh +++ b/build-ohos-x86-64.sh @@ -134,3 +134,9 @@ cp -fv $onnxruntime_dir/lib/libonnxruntime.so install/lib rm -rf install/share rm -rf install/lib/pkgconfig + +d=../harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/libs/x86_64 +if [ -d $d ]; then + cp -v install/lib/libsherpa-onnx-c-api.so $d/ + cp -v install/lib/libonnxruntime.so $d/ +fi diff --git a/harmony-os/.gitignore b/harmony-os/.gitignore new file mode 100644 index 000000000..ddb010f66 --- /dev/null +++ b/harmony-os/.gitignore @@ -0,0 +1 @@ +!build-profile.json5 diff --git a/harmony-os/SherpaOnnxHar/.gitignore b/harmony-os/SherpaOnnxHar/.gitignore new file mode 100644 index 000000000..d2ff20141 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/.gitignore @@ -0,0 +1,12 @@ +/node_modules +/oh_modules +/local.properties +/.idea +**/build +/.hvigor +.cxx +/.clangd +/.clang-format +/.clang-tidy +**/.test +/.appanalyzer \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/AppScope/app.json5 b/harmony-os/SherpaOnnxHar/AppScope/app.json5 new file mode 100644 index 000000000..8f5c08b90 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/AppScope/app.json5 @@ -0,0 +1,10 @@ +{ + "app": { + "bundleName": "com.k2fsa.sherpa.onnx", + "vendor": "example", + "versionCode": 1000000, + "versionName": "1.0.0", + "icon": "$media:app_icon", + "label": "$string:app_name" + } +} diff --git a/harmony-os/SherpaOnnxHar/AppScope/resources/base/element/string.json b/harmony-os/SherpaOnnxHar/AppScope/resources/base/element/string.json new file mode 100644 index 000000000..a0fa21ba7 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/AppScope/resources/base/element/string.json @@ -0,0 +1,8 @@ +{ + "string": [ + { + "name": "app_name", + "value": "SherpaOnnxHar" + } + ] +} diff --git a/harmony-os/SherpaOnnxHar/AppScope/resources/base/media/app_icon.png b/harmony-os/SherpaOnnxHar/AppScope/resources/base/media/app_icon.png new file mode 100644 index 0000000000000000000000000000000000000000..a39445dc87828b76fed6d2ec470dd455c45319e3 GIT binary patch literal 2777 zcmV;~3MTc5P)9*YHQQH znh@I(s7WDIN`nJ+5@|<)iZcg=qN74U#DNnD1Se7u4fs(|1ivr?9ayP|B3iYCD$mfQ zCQ{S1n2)}^yxe#1J=_0pt-a1UPwQ^Z*?X_`Uu*sM+8<}X+baE^a`3seUF}?bEaiMO zrD`Qrd5@qw^epHZ>Df|p-qKBUEB%*?!m0{PHC6j|RplEgR~PkM5a^}N)Sfwi>W;Uz zdhwo_4HXBU%kRl^w@&7iKPx$e-n9%#IU!&oMI~iNsw0n19qSX;dS>I`G_G=WdcN9r z;_Rtv9XC<7kbL+HHxJ782T~pg05t)tf^>2vNJqfYt{YmqQDoBxkv+ra*BxxhcuK2v zm5%@Y)biQz)R8O%e=o%n${;ojY;EUP>`Qj6Cq)7GHm)C%2%^+hI;Z4T#a|oKIvshv z5H%!I+|I4PEXaXj04%ybsVolr%vhKnW7AEhC?eP!o1{y;8m2R#;}{6VZPc!+)ou0C zVWz$|1#2(|L5z%EYRxOzP+uLB>qYGuajX-<#^u;Kw&2uh&93)h>nHaFA%{&2PW=Nn zr?*a;gk3xvRhQIRa1de-!r(ss&?tRmZ=L2FMkhxI3lK6Jn<>5c*ID|@KU#^MCIo6> zpFA{|R(4fsBwHIW z9v!7G|7enadv4}~*8q_h%tD^j$7=PCnn0=dR0GKA(fgb9`2IRg6ksBIo+Gdw#|-3eSe=3tmDe zIqVN)tScM`0W#Z>2wc>~2Uv=3L)~D4gXqZtPQ8rifbYJqwkG>bv}95G7+};9Br?hF zWSa3b)X}z#79W9kukM%6-b_54WDJm~Ub=gsrJ0lz-8&lrQ7zfK1qzuZQkZvcE3|~S zZWmk0ETaNIHnMALn>akuvHLf5c4`y%!f+u>ZGp%@q_;T!`76_snc_?K;Wx%YpF;5K zw^F+BCYUPy`fpRif@5O@Im5cf?evD$>KlAgX;D0*HiO0`Yg3j;R4jT(9h(L_TsY6yxk*@ZBe%+dMqY=cB5oGs{D$QwOFbH)G$iVf<3Olcd7^#fr- zM{!ILWt#coT)s9ySkwDCPHv0oww8g8K%Yr{aR}msELVX(}JQr%F4Q8=KKn*OjSO*uSp;JK%GwhRF_K??vGC$ZqmJX z@+}8sQ)9Z}3*DiWl+L_7OXn_^{SW~2&C*b^;%IP!j$lkre7H&bMR1}7aTT*G8P}|G zHM1)hZDe{r_E3{{Y=d}}_PxJO_w4MaE4)$<<3JwzPdwPzfNemK(-X;{UCzmVr0zu5 zEnT}fzx)oVd!*W77`1Ig`DFcZ6TkPaI$hO1+`cGb$({ukz&{p4Ic-Xnwrg-KEkDqW zW3l$7Q`V$!1T(=QL1jgjIachdr75>-8>1A^h+;rTrD^nnwf?bw(Rang!*16Odj$Pn z@)JN5&5w~}ae6d};oa|&G>sT!)ixE#5;QW(u(=bqYHXcOflE%@t4A?n5fTUm0F~8_ zwpoz9rrU`@G=vsNjDRY(CrF(jIjqg8bd|CP02>eFag7T?u;C^ir+Z7YKmBYw;%%XdT2T}a$X4yR7EI;zaof3a)5Z;`OwVi%D?gbkBj!{;z2tOBSFk&E1DeiZXD**uvNqL}+|pO{ ztO$}2NMRit2ddU?)7Prq&*&H3X>&=E{-+j4iUz zrvL;?0$^@lyl=LHz9G^$SJV6ID__@7z->Bh>Vm=6AK&5bP%@heveHja5F@agGgUsY z@L@W2+^*NVoId0!kS~4XkWb%y;f}XBf>S+NIw9aHK;vN+4mJ|em)_QjIVfb2$;bwv zDKmoq6AThgKydS6Hs+UpKPWq|UA}s=UOEBZNM3oNT5qTAabY)X>L6jxfGDuu7&GD_ z=@@m?sJ-o2GS}&hNRW}-zHkr>o4&138@a8IC-FjSBxzjx?(*3@YmdmWGAd%0QvXzS zJ53JpX%Fp!=>v&`Hd7F@+Atw2vx9%^2M-APg0Jd|ePsRn3*B$#9Z5hCou4fo7W#SN z#}-@-N=##yQDh26pNzr9f*Q88krhI5@DHcf{dU-~PLSs}MvI4s1i|<=qxD~9`7>*~ znlw5lr$_6mTG4XbBNF_79BzvZ!TeIP)exdk3)kSHjYdW1P10ZJ_NCJSlrCuIU#gqw f88(SSw!Z%ZUzhC#9QlKF00000NkvXXu0mjfG$}gK literal 0 HcmV?d00001 diff --git a/harmony-os/SherpaOnnxHar/README.md b/harmony-os/SherpaOnnxHar/README.md new file mode 100644 index 000000000..936315564 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/README.md @@ -0,0 +1,18 @@ +# Introduction + +How to build `sherpa_onnx.har` from the command line: + +```bash +git clone https://github.com/k2-fsa/sherpa-onnx +cd sherpa-onnx +./build-ohos-arm64-v8a.sh +./build-ohos-x86-64.sh + +cd harmony-os/SherpaOnnxHar + +hvigorw clean --no-daemon + +hvigorw --mode module -p product=default -p module=sherpa_onnx@default assembleHar --analyze=normal --parallel --incremental --no-daemon + +ls -lh ./sherpa_onnx/build/default/outputs/default/sherpa_onnx.har +``` diff --git a/harmony-os/SherpaOnnxHar/build-profile.json5 b/harmony-os/SherpaOnnxHar/build-profile.json5 new file mode 100644 index 000000000..2b12adad0 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/build-profile.json5 @@ -0,0 +1,44 @@ +{ + "app": { + "signingConfigs": [], + "products": [ + { + "name": "default", + "signingConfig": "default", + "compatibleSdkVersion": "4.0.0(10)", + "runtimeOS": "HarmonyOS", + "buildOption": { + "strictMode": { + "caseSensitiveCheck": true, + } + } + } + ], + "buildModeSet": [ + { + "name": "debug", + }, + { + "name": "release" + } + ] + }, + "modules": [ + { + "name": "entry", + "srcPath": "./entry", + "targets": [ + { + "name": "default", + "applyToProducts": [ + "default" + ] + } + ] + }, + { + "name": "sherpa_onnx", + "srcPath": "./sherpa_onnx", + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/code-linter.json5 b/harmony-os/SherpaOnnxHar/code-linter.json5 new file mode 100644 index 000000000..77b31b517 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/code-linter.json5 @@ -0,0 +1,20 @@ +{ + "files": [ + "**/*.ets" + ], + "ignore": [ + "**/src/ohosTest/**/*", + "**/src/test/**/*", + "**/src/mock/**/*", + "**/node_modules/**/*", + "**/oh_modules/**/*", + "**/build/**/*", + "**/.preview/**/*" + ], + "ruleSet": [ + "plugin:@performance/recommended", + "plugin:@typescript-eslint/recommended" + ], + "rules": { + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/entry/.gitignore b/harmony-os/SherpaOnnxHar/entry/.gitignore new file mode 100644 index 000000000..e2713a277 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/entry/.gitignore @@ -0,0 +1,6 @@ +/node_modules +/oh_modules +/.preview +/build +/.cxx +/.test \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/entry/build-profile.json5 b/harmony-os/SherpaOnnxHar/entry/build-profile.json5 new file mode 100644 index 000000000..4d611879c --- /dev/null +++ b/harmony-os/SherpaOnnxHar/entry/build-profile.json5 @@ -0,0 +1,28 @@ +{ + "apiType": "stageMode", + "buildOption": { + }, + "buildOptionSet": [ + { + "name": "release", + "arkOptions": { + "obfuscation": { + "ruleOptions": { + "enable": false, + "files": [ + "./obfuscation-rules.txt" + ] + } + } + } + }, + ], + "targets": [ + { + "name": "default" + }, + { + "name": "ohosTest", + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/entry/hvigorfile.ts b/harmony-os/SherpaOnnxHar/entry/hvigorfile.ts new file mode 100644 index 000000000..c6edcd904 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/entry/hvigorfile.ts @@ -0,0 +1,6 @@ +import { hapTasks } from '@ohos/hvigor-ohos-plugin'; + +export default { + system: hapTasks, /* Built-in plugin of Hvigor. It cannot be modified. */ + plugins:[] /* Custom plugin to extend the functionality of Hvigor. */ +} diff --git a/harmony-os/SherpaOnnxHar/entry/obfuscation-rules.txt b/harmony-os/SherpaOnnxHar/entry/obfuscation-rules.txt new file mode 100644 index 000000000..272efb6ca --- /dev/null +++ b/harmony-os/SherpaOnnxHar/entry/obfuscation-rules.txt @@ -0,0 +1,23 @@ +# Define project specific obfuscation rules here. +# You can include the obfuscation configuration files in the current module's build-profile.json5. +# +# For more details, see +# https://developer.huawei.com/consumer/cn/doc/harmonyos-guides-V5/source-obfuscation-V5 + +# Obfuscation options: +# -disable-obfuscation: disable all obfuscations +# -enable-property-obfuscation: obfuscate the property names +# -enable-toplevel-obfuscation: obfuscate the names in the global scope +# -compact: remove unnecessary blank spaces and all line feeds +# -remove-log: remove all console.* statements +# -print-namecache: print the name cache that contains the mapping from the old names to new names +# -apply-namecache: reuse the given cache file + +# Keep options: +# -keep-property-name: specifies property names that you want to keep +# -keep-global-name: specifies names that you want to keep in the global scope + +-enable-property-obfuscation +-enable-toplevel-obfuscation +-enable-filename-obfuscation +-enable-export-obfuscation \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/entry/oh-package.json5 b/harmony-os/SherpaOnnxHar/entry/oh-package.json5 new file mode 100644 index 000000000..248c3b754 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/entry/oh-package.json5 @@ -0,0 +1,10 @@ +{ + "name": "entry", + "version": "1.0.0", + "description": "Please describe the basic information.", + "main": "", + "author": "", + "license": "", + "dependencies": {} +} + diff --git a/harmony-os/SherpaOnnxHar/entry/src/main/ets/entryability/EntryAbility.ets b/harmony-os/SherpaOnnxHar/entry/src/main/ets/entryability/EntryAbility.ets new file mode 100644 index 000000000..679d91453 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/entry/src/main/ets/entryability/EntryAbility.ets @@ -0,0 +1,43 @@ +import AbilityConstant from '@ohos.app.ability.AbilityConstant'; +import hilog from '@ohos.hilog'; +import UIAbility from '@ohos.app.ability.UIAbility'; +import Want from '@ohos.app.ability.Want'; +import window from '@ohos.window'; + +export default class EntryAbility extends UIAbility { + onCreate(want: Want, launchParam: AbilityConstant.LaunchParam): void { + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onCreate'); + } + + onDestroy(): void { + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onDestroy'); + } + + onWindowStageCreate(windowStage: window.WindowStage): void { + // Main window is created, set main page for this ability + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onWindowStageCreate'); + + windowStage.loadContent('pages/Index', (err) => { + if (err.code) { + hilog.error(0x0000, 'testTag', 'Failed to load the content. Cause: %{public}s', JSON.stringify(err) ?? ''); + return; + } + hilog.info(0x0000, 'testTag', 'Succeeded in loading the content.'); + }); + } + + onWindowStageDestroy(): void { + // Main window is destroyed, release UI related resources + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onWindowStageDestroy'); + } + + onForeground(): void { + // Ability has brought to foreground + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onForeground'); + } + + onBackground(): void { + // Ability has back to background + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onBackground'); + } +} diff --git a/harmony-os/SherpaOnnxHar/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets b/harmony-os/SherpaOnnxHar/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets new file mode 100644 index 000000000..d2c48b421 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets @@ -0,0 +1,12 @@ +import hilog from '@ohos.hilog'; +import BackupExtensionAbility, { BundleVersion } from '@ohos.application.BackupExtensionAbility'; + +export default class EntryBackupAbility extends BackupExtensionAbility { + async onBackup() { + hilog.info(0x0000, 'testTag', 'onBackup ok'); + } + + async onRestore(bundleVersion: BundleVersion) { + hilog.info(0x0000, 'testTag', 'onRestore ok %{public}s', JSON.stringify(bundleVersion)); + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/entry/src/main/ets/pages/Index.ets b/harmony-os/SherpaOnnxHar/entry/src/main/ets/pages/Index.ets new file mode 100644 index 000000000..423b4276e --- /dev/null +++ b/harmony-os/SherpaOnnxHar/entry/src/main/ets/pages/Index.ets @@ -0,0 +1,17 @@ +@Entry +@Component +struct Index { + @State message: string = 'Hello World'; + + build() { + Row() { + Column() { + Text(this.message) + .fontSize(50) + .fontWeight(FontWeight.Bold) + } + .width('100%') + } + .height('100%') + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/entry/src/main/module.json5 b/harmony-os/SherpaOnnxHar/entry/src/main/module.json5 new file mode 100644 index 000000000..a1cea8b6a --- /dev/null +++ b/harmony-os/SherpaOnnxHar/entry/src/main/module.json5 @@ -0,0 +1,52 @@ +{ + "module": { + "name": "entry", + "type": "entry", + "description": "$string:module_desc", + "mainElement": "EntryAbility", + "deviceTypes": [ + "phone", + "tablet", + "2in1" + ], + "deliveryWithInstall": true, + "installationFree": false, + "pages": "$profile:main_pages", + "abilities": [ + { + "name": "EntryAbility", + "srcEntry": "./ets/entryability/EntryAbility.ets", + "description": "$string:EntryAbility_desc", + "icon": "$media:layered_image", + "label": "$string:EntryAbility_label", + "startWindowIcon": "$media:startIcon", + "startWindowBackground": "$color:start_window_background", + "exported": true, + "skills": [ + { + "entities": [ + "entity.system.home" + ], + "actions": [ + "action.system.home" + ] + } + ] + } + ], + "extensionAbilities": [ + { + "name": "EntryBackupAbility", + "srcEntry": "./ets/entrybackupability/EntryBackupAbility.ets", + "type": "backup", + "exported": false, + "metadata": [ + { + "name": "ohos.extension.backup", + "resource": "$profile:backup_config" + } + ], + } + ] + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/entry/src/main/resources/base/element/color.json b/harmony-os/SherpaOnnxHar/entry/src/main/resources/base/element/color.json new file mode 100644 index 000000000..3c712962d --- /dev/null +++ b/harmony-os/SherpaOnnxHar/entry/src/main/resources/base/element/color.json @@ -0,0 +1,8 @@ +{ + "color": [ + { + "name": "start_window_background", + "value": "#FFFFFF" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/entry/src/main/resources/base/element/string.json b/harmony-os/SherpaOnnxHar/entry/src/main/resources/base/element/string.json new file mode 100644 index 000000000..f94595515 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/entry/src/main/resources/base/element/string.json @@ -0,0 +1,16 @@ +{ + "string": [ + { + "name": "module_desc", + "value": "module description" + }, + { + "name": "EntryAbility_desc", + "value": "description" + }, + { + "name": "EntryAbility_label", + "value": "label" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/entry/src/main/resources/base/media/background.png b/harmony-os/SherpaOnnxHar/entry/src/main/resources/base/media/background.png new file mode 100644 index 0000000000000000000000000000000000000000..f939c9fa8cc8914832e602198745f592a0dfa34d GIT binary patch literal 57364 zcmYIuc|6qL_rIk#Su&MMQlYU)cz{|$Qc0x~A^BEf( z`{n=HaSk>%wsfNM*uUkN^8dI{qxxW z*@b_`#>VlLWSG9 z0>QdPQ-&i_RCVdp2s$-u%S362^SHV0`EO6;@n(xK));G>#qwhPWrDXGk@OBMV}H!J za!48&`xhWJKj{_+f3ir<>Jg6Ax<&Xgn;)U7UJyAw{(u?zlf{oLsJTS-_o1?+lSg-j z8fcZj1*Ad(!X>WuuxM!H5t@V3*8vLL6`QnC!q!BwQjI{yk*;~@|3;B)`p$WYcDmnZ zt`R zr=oS6o-D$WZsYKh1PiOdhhX&YWGOzpc<6ITKzr^zi-#>z){t;yz3tu_a!>)(tTU9d zd}COuy~Tb}UIRNX@aVGJqEKUa)1#E-u}pl!sY)z4cu+Hu9==`6=0Ob#x-%q}t@jBp zmoiZDcfF1WL{PB0ZO**8yZ+%;LF6K*JDUoHrJkl0Wzak+Y%E( znUmuA^p@Jv6{%Y;MsiZ4O?#ID2b2ssEq6_KGL z8T%zdA3YhMnkBu19bNsa_$$_1^16jadx`0ZzPx`M%T>qZpYyNYOeDdmqLTNWpR5T% zOlRrW_xNCD+*3_WSxvt4P-@qQ9g_$aedDk-hcV~t>Oxw;UaAk1V?9m5<2k4%VrM$- z?{KH{)m_U~yJcBbX+vqVfq&4)Vf+FvAHd|s{V34=f#uJM!Tp?b32THmfzNn1unwY& zPNtaE{ZZ=OkZFh*xW2FT&fDF?64Q%l>dwdZ#Bg;^v;dAbU*QLEQG@_|ucNXFyx~H( z#h?kJKeI3jD^U~`e`*^zcm?PlIWj|tL_a8NC?HVl*gX%;5PW5Y%ZZ*G=jPn5#o+Sh zhnE>D@Wb!f*O>cZ0}ZT=HlEdoWVWk}5H1S;$vxe#Rv~;l5rJ=w--wPl621jCW}B|gxECKzT9 z3FKlD{=OfN5$J3?Ag0g4F5t8_D(RvO8W!*~?#q{Dhx(Sj=)^9ZlE|LyI?p1NXMWr| zGGbzFN^3)5?P^vfnD7XZo*8yf&O&>7XULUUvhJT@rHcF>PmjodH~u4RSmX4TH?v`IKg2cy7 z(T@e4&pPRHRczikEvwvO?jbblSVp z2qpyT+LHUFhHwcunP(^h)G#uA95vF`Gd&1k%F@wuCk3DnjNjw;b}*;dY{F5{7tNsg zLf4y|)RTV`PjQ^!NoWB3YA@S@Cw zUAr?mUcn7g)n!3J`D7*H`y{%TuT$wNY;))rP(y@kdFdPH#h|rjcW2#oRybxTchXlQ zwMW{bVcqRRc_2r^tI)Zav_+qLwdd|Bw=*pM!|pflbT%K&Eof^{6+|k{2_;HcrWd3? z#z;>@Y3dp#B^R5c9RhH8lT5MRr*;>xd<%C3sV2Y}>{On@a*oump`g#H<6V&DKeZ-?Zic$S$>ulEiZvJG8kHMeSzVE(R|E-<}cEG^n2E*Cp z-25-DQv_Mf+&WhT3r?23Phid$q`Z3HE($RgC{EJA0Yc1SP6(a(oZ4RU2L1~H6k0Q< zHY1Mj{)b(ll3Wr=HakbiEk13zYKN&f#9*}tMZiQ7h@Us+N(Jk`aWQHf)r!ObZAT>_STJuzjuO{qHMlTjN9^hPZ8sZBMl zl&MX}xk{d5VUEInRK9r^Tnx#HE2;hFoa7?NDufAxZV6Mj9B^NaAt4;oStAtWfVg8< zjQAfLPj#u>Xp*sALAi;M(f1>la|_-k(E*-1Sa_Vdt$KsCNAwAbm8CmvpDbwL$`Cx8 zkBC0&3#@q@7E3LVtGQcrGS=s-uh6FHuC)WTtU_@t5c_T~`Wv+F0Jd$a9s(?ucd&l{ zWThjQ*u4YqU6Wq{+^0sC%S;vXx~qO|+s%Am1m-N}zkd84>GT;5u}a1*p9&!g%3wk2 zl=rj+H9g>!z4_zdU1iItL}Zox?lwK^ykQ+_#Ym~p>s8CgcLQYV4wezL^K-_HzM$r! z1m$U&G13HqDckgHschNcoe73o=)$P$j46Y)SnaZK(U|F7d#{AGb%>@b+jX#5*Rf5x zq}@ejPTyyn&&@n|dDGl-o-=XF%6dndW+}@7JDd?6b}Mt-SX_GV^3{!3Yz5a~X@$Fw zyDIkaWq*rtn{8knumG6=yF(6lzQnq)&M@%8RzdC%{%-0Ey{v&0pp-aIPP$bTrF|=~!MvLftx2pd=0-86i#@A;;(b^r-TzBJn~W4d42|-V*)} zt}h95!TwDQ%xWD9TFS{BwGO@d9P>kia=+LQ@r>0>5VvEV8(&tEuw%+YP*Qm6KzZs9 z#qL6SPwl9DtPZ{0`)Vph`^ryNV|=I7r2Vf@LrX3<=$f6zv1^z*!<6j{f$|6Jw=%s2 zb)|d{?()1m_Xoab$B5r9#&taTI^E@0yTQ$UB1_f0nc<oQhFOi;b@!o=y6w&Tsrw|K5XXEJEA>@Eb?8hi( zlT-*bXZd6g*C+W9V6B5iF$2f(_-ek(ko^JW%$@}`#GJVV0S8A~FwzM(JdY)c1B&ls(qJ=bvy&S10cqD8@1Clbooq|3kmbD_she z@O#tu^ibgYfM#HD%WIF%%uf7+)sc&Dejs@WRQE+Q1jXlN2z>9dB;X9e>Y3a-&-A;T z>||D+o$j^$E>F`4y02DTELRMYH*biv(5+ed(cQq&82Gu z2~UNnOcNc&MwT3lD@S}nPJMsvOT%0L{`dN}DU&^Z#6?2^aE!5ulUV_Zct}2~K6R!_ z4ReuaX$@AP?M!XMpi&ZJwsY2up5F-xe0{ym`9#@pr%63v->d&@UoFthcC1`k$L=ze zYX1{xl49Q=z953h>NzyMc3UuH96t7)-k|lRw-P=T%Q`;dC7@r`uCOq8Eqi7gKC)~7 zb(*Q>H|T2(e>5DVf9nswM~C%V2G2 z#B|VOitZm{FlV>EydvsFF|Ue~ium0%0KOaFiMOLk(X}jHq@dI@*AM2G6XzCU zSpFR?#U4MPz~VZR>RA@a!CZu45#f<)^f#kJ+ULtRLJKzSj=cf+NxQ}Kw)Yme6wJz; zu3W=Jz<}rEm$g7sNy>yr-Z|OiI>qQ4m37~);`_~Xgr~N4wOAssk(HTh5er1XtFm+! zb`5FT&FoKA{ADaUP!Y#o^sGPb?mT2wBY9ZfQ}ujLk`C_dyTvT&)34sj!RXJcZ%lCzF?kE~i-xCSGh{ zy%iUR0+S_JP(#%W9!Npk=RL(8tFB7(up1ms-Q#8 z$-{dva97!EQB<5#@0KgW&2S|ddKN*<(?}37-=O@1bF668sG)3(D61=Ech&sJ;j|An zqu1a;`}bcMj;#tF3l~&Ue9ES7GRw~kIPKK&q&^No_3M#yjp?ygI;To&wcXbe%ju*z zpMI!gbi8@{AJVkgXR+py{dMSfko}H`^q^elZQ-5<2bG-K8tYq8Q@*4t)`Blvz!#v> zE;3kk_e^|Kew4?}eU;3n)q48yWgAm)d+F(;W(>jPB_glQLiH|IE=EDVFI*j_FBebS0vXyh5@x9LS?RNi7vXf?RckfXjvy^Pifki$9G zzwp&k7S+aNOI8%DUON~#xxv+a5rJDE+^6;@RcjnwKZ|%#%Ukq~@&vL#Pts;`f?jwYL)Y zDOROB^T8hlFfA@(=$bFYKWy{F^5$#{h*A1FG5GZZ1?>Y+!}UULap(oEekfHZCJkpC zppRS@+Uvrs>_Df!YT#HWpuaEwRq)V49)TgZ7Jf{A6@tpv&>tG)c9F&eZWo)(tDPDB z4Fkl6@ov*S4!gboeokhZ>My7@q%!Z93-zy>Y(_9axnH2W2Ie&#X2Z->o1A6ZoV(OgY z@PpdL`E%U}QN-vzdLCdkVX)Vp-z|CGg)^e06LvMfbj%1)ZdXNB>r>{Jk&ApwTkkLr z-2C5e31{3c{*xsm?)EItQ%pSW(%723B}AHgke#M{7KJW6TT*>9^+`FIe4;VHRwSF$ z9rBta7_>vwCuV;vFY=|NZ2KlX$A`EUk*phH=Pd~I8Kkr|v!j3sBAD^fPD!FoPpnHf zqP&jc&^s{jm0M&oBNXjUol2${7|G^u7UtOd2kxA0b?japS#xlwo_TaY+jh-`+$sfO zFLgfqb~kaemX{ErUn7}?_tb>g?G@UyT99HoY^;BG(5|gh>F3J!9J* zvrz6TP+;XdE$<41%Vony^Y}i*aCz@+4v^38p)5?Nhw`m%Cbg5Lpz%VOxaBnlA9P;N z9D=#{(>`$N_!?&CKf9eJGzIc>dhWes8XtpX`{gOhP;HMklZ8~@Yu~YT1bZZ{VwrAffDNiZ6Mh5vEzpq z=5A;0ff@>1MG@vbwRU!?7ZFD-SYng>JN(=>uwrkrl@4u6M^n6jl1shsk;DM`t#|F? z(H9W(@&~b(mmUR)30H=vAZdIrX%9iR7rMruZ_I4$Eq7YnBI4Z8T zj5;RTUu8?(ZsW>30%Hk#$^zfAtgZ&y!|p@5%e_4oe7)3{Y6c^x>zv=o_XPiF*wI1y zNe5L3p=L;8_D7-+5I+LfNgDYrOIUD_Iu_VJQD^=4v=Gd z_u%h$8{Lobyu6%VkeZI%T_vssgc#J4yD+&6pVkdLYl@3@NdcQbwl!J%4{RC80oF1z z`ksIXyrZT=Apq3kOR#m795+y}-8NizKBNESZCmBS#jqG`n4kCydp-4DZ^BF-zWD2# z1@F?p*^9m)EPrkd^E&cimk<1mN+iwSCVTHpqz^#`_Dj;-5xURqxK*!kp5asE##*=< zc{bFC-`m;q4VL3=| zKN6@)%XIu=yS*-K-9Bw`jN+-lWBttd77x>|g)~$UgPB_qH0h&bm}j3#sdLfV&xcR^ zQFk=d3;U8~YLqm@^61C zmaLbHw=dJ0oLP?>eyJ&=wgtZm!2mS9V!i~62x+n`%jyesf0bKruxRDH-)c2uF;&qT z4Z0drBbHg-G#ueH1vVaEJFTw$U))8mlUjFz?!PDqNpcIqZ%B6$Ju$CzrK@_na@?na5LpJODS}`)`8j7i#>C z0RNEb>nnQ8v$qXrgh)-(=VVRFwj4 zZKH}5T4rlZ$PiI2z3_*{`av5A0jPJY!Y*RQ?XbKPZmNdwp6ufAH4m~1%r{gYeOJBR zai+gl7I{Z35P0Q7EoGmkkLGHe5rR^{bdxWyMiC1~&kI@I-bYJrdGv{=O7!p&kKxN3 ztOoyzWj_asX!zA>`fa~&>#$n@3{c@VVcl3(1m5=dCI-~1uR+4s;@87ozKCU|Z(EhE z7~Csbr}e|&-zPK~*W}WcKqB+rv-rNRzvqfY299AvP zA5u^Rs->xN6b@MzP_f(M+}|~RxUHs#zO%D772V@B$F;5<%Jx|0#Oh_?#%yrHfV>}I z!Lfe59_VCjJ!pEQOWyUr;CdyL z-RzERMQjU_j%}N!Av?++44uVMc#r_KCTZxxSZL>4`xbm)#)*?4I#nFDOZLv10s^{6 zAyo6zfA)w8n^jk|KBb4J;|Gbx9)grFflY-Nyl_v8_@}gizDNn(Y2l6TqM&aN(+9Qg zTBo#J4N$h%f!;K&2NqBlT~J6aqHGy6HI`Xn*)UV$w2>iLk~P=l)VTdah9Ab`z%}dg zxIvG$xPG=H0NRw|6_-~Bzh+BPv9&C;z)58?`7t~$HupdHcF!F5dirrGrn3d}wAHr! z^@&!aoW@3sENjl#i@LzRYOZ4b#v|Jk_Mo$-VYlgbE3LQVKniS1mH)uO`90X{bc~{1 z*%Wm4$E_2-W__`4`mDu;Ld(wv8e147=mMu!AKSC=mw*4n^8S>~fm9mJgf4~8t(bb> z^_3WSK>aAZ6lK3OZ#_7g@)?z1#pZ zoR2>rm%_enbG!+Y34#Jmal)V9@-s8li+_Le^~z8cxHeF5vR%p~{93TJv%YmeTB|@^ zc=}q4Gofbju_Z#%Iv9|44|pawNvh^mFGBA_KZ5C^rx-l~Ytqf4;%SxezE8%O)aJh& z>2it7b`epB=P&=s^y`mJMjMq&9Jvpdhn}6sFHk)q%d zE_RV6%-}?H)w7yAW9TA)&7XbMyu=N}tRA-JTl2iG6u8;@?;!BW;ykyof{i+alo zJu1v~ITow6y^)5crWdi)&;yNs0d)3*vN+aSszJ%`1`(%9X-Hi}3gH#iRg@{Svm?cP zM}T*)U{A8FTQ7b@oc$7vr_EeTIj6N%Cr}VI5VcfZk+@1UFc>zpJkm3S%cb<~=~`BV ztbyjzOPJuDkTJJ!hL^nLk}*=2EXd?->%+3NWrq&5a$%1G{r2~cLQT2W>8!pd$9G;K ziQIDUErsVk$XQPRm)pDFYVuLFlx&eiFlnoixT|jvAoB)ryM_}euaYFXrdKLqi|4AL zG`rnvWi4Qa>Wvo=;Y+t@ecMjl{#37K;?VkYdoSbT(2m}8!k~RT{yv0l8cPp{jtiXr z$7KAJAvM_g4ak}0Yo*q!sO%PN_CK)Pv>lC7xoB~vG1hs?Wv>^kpOBU0WV@$|oL!cE z1FV3%^4Pjr5Fqc)|Sv+upxx8BCM z9*cYQYi3jY(^pUL8`I|3rHf+5>sq98e!hkPsfNMQ1@y7Tnf4{F2p zx9AO&@zYO;WpCQUf4G@!d<{t43@&RHh2Ukg^D-8_;De`dc{hz?yPS_7BzU!x^P-tj zBWt_uk{g94M1uo_&0l?m1qh!Q>=dKy5cx zRa7mv(}`xYKJOm)h3s8goQ*XK1OT<#&Ozf35uTB^VD8m)Z6Bnlal5r-bkso}J^TcM zo)ZSc#2@`h0Si}lrnCFt67JFa*e&}2avKCL|IIk<$R2*5sILkv4P( zesTX_tP#NqXN#>Q{4oe!N=G{SZ_I#~%^kq5ilGc=Q63_5uRt!D^j$k=&$`Ha&bGlAjZ2&hWa=M};Cw|5onME2e;8le z)-hK+mgNbGw-4puLN6g_q5p6T?0XM^dMo810rSBSw7Rrl(jt2JNVBwhB0o3``lZ1y zBr`Dy8LdVilxv`X5b0N8#{#(y<2vQrLj;qv`XA#RZ+@Q~*aYa^UY~;#F>6BL>75+E zeH2(L#HhLeI=Mz1#%^96zY$Se;@N)biYOvM6H1p6-4LcvA=&GP()#?u=_WXgAoZl* z+bR{6BA52?12Rex)v?(LMRsKvf9{KzP<^4&NISV{2!a;wEhr&E)EloHqSR9%ezb)? zl9X;qQSTg@es%UevGs9-KQk6RqJ;Ui(v@S0=JpkXQVYgXlRKQcfFLT2A%*#c?7(b} zjki==Q^Y#Qf}ZVpFtF6<4SbGKkkU>I6wY*Ps*EAzemS5Z0r!-oD>~r!<<+c~fHK+{ z`u4nWcW&4!()0%2>r>@zr$F6$;5*IAuq5bc>cn-IEZ+B|hkO&NPeBi&47YiU-<$w0 zq-j9aGH~K;Y%0{D&e90RZ(J_@o*`(e0TgqWM zz>V1_2|7MMg_6zbeK`A2oW6>`dUuDIll*?4hKaK{^>2t!B*N9o7_!iC51?A=hss#S zTOD48mGM}}JkMLeB>f0zNw|zPj8Efyx1Qh?QyT7Bp*PsC1%+$kgboSqDR=rTEs%8X z-t2|68n3XC`A-sBYO9tXuQqE7{}pE3mRASQTvScN7(%JH0{M|k4t%rE7xh`qUf4A- zgEE3f#zcuMyMYyiu;w=#PFC-_W0rb;u#{l@E}K0uMy~Ec1MBz-KglT}I_AG%m9nb!XAkpoW-`_85Umy)5g0j(3(>`;o1;w;CKp zLKdGc@@LrE*Y6B#H>jMeTcD6nZx;FZw zZ?8nd;T;sv#~t>9Stu`V2=$pLBHrDq3VNw9{KZU-50LlNLK@?o*hLF?1Kjl3op`;u z=nFLXc(CuUKp%gcxwwBm08`iDki>51cyobB9Eypc5@0Uv%$x+m$P}vtzJ@yXv2Y(6 z%G|Dfw#*GyPhoZ)9Obc;u$h*k0~W zv)EW8ChYvHNP~Ws5(MQk4JSGnG!l*4I-odrw$8E;u9uTN)1sDTSK-9%H|jqRi1XpO z_RLbdR5?V7FZiM9a@_RLzrIa?o8u(&ct}&dJFEmRO#py=1J(LW)$S@B$xLi6T)SOw|;fa7Myzv z?MOZ*b$o!rCg?J9&v6SsP#m&goHWvlC%0`IUKT~X&=s1cU$O`0Ea`_f|aU@(<=bXW{`6+7W#cu@H9t zagx-Usc&&vez&!Mjqpdk+Ol(}Uo_B;A&JhUaOe-iG9|*Z<)SYRZ;!m{$5X=V;9Cl+ zs(#H}WR`823f+9`wmRKF;(;wyt*?b3@Y`H^;&@1GipUF_{Gb_RzIV!3$qMq++{iyr8Th+msVi*eA69cY1K|TmaXNA-rCXT%k z%$21aDiQY_-+BI`52BI$rv}FI)tg7-CaaD7_O`l9ngVYH9#Xu44ly2flHy-xuzEyCWC^6c-^K*QrZW zNG1PL`B#xfh_CD57q**Q+=Ty9EEolHUwT`)Z`SWJPvsxa-f8_iHO;AQOj^^?v$Pd6 zy~3pjahT&?UwB@2zW1)s8+UfK$SFAL~tHHx3whuvPyW4mh3w z`_Q5~nHOsoDT0sx@+N~J<-Y&TvqV4MCkgXgo^ntecjdoSopR%@?wkEfAuHDOIVHQe z|K0}d$IAWT3jC{=QJCD$*L3=%k#f)T)tT7R=nTHqn)i5$Q)sm)53ZV1w&{swK_X#n zpD3;2Eb$r)$CDg__L8tv=0-5U5hB))B~SI2(6`QM95phAkktAVs0hU305vOGT{|^t zH`?>)3!24y5TBnjRfAJG|J9jjj_JYwB?gujfD3QwPf@~K(A2Z4KynC|m! zMt!}`yx4=~u?@-#ab5-T?In;dGAUlGajcN(yFF%ypy(av6(B6-=d(A}}k7wcgUJ%c_TA&p~<@ZA~EU-mvx5S_ykM?O8{R|mH|RE75BR5QQ#CTpy{;f{(N zFpFjUOJ}EEwov(%eX6wm&~H5dD|PO&*VQvG&6Br6eo1I>i7L)sk`T?{8}`lQfCB2R z@nDF(51Rl?^;uv9K%Wz-qpmyIoZjoO+tGhY)P>lU7U1Rpv;b{^)mu_I7=1e%POI7M zneWYe`!E(sG!D4Pm@9XD2!jhItDw15w=Vl)ioN}tjFK(3~fxy=!h!`6@!cQuCP6#aH;{{dyV2@O1#ZX{Zl4pLmD z7*{Ip)`V*gV-QVaE+>|4R`><5Z1*;n%pfkb3AiZ1s39)5f5khONJ{XZ5dEj{AwE^i zj6G1{WVlyMNlC3!_Nyk^Z0DjKo$ha)xbx}7UO*rnNj8he_fyO?v!so#$d4^uhxAXf zZNG(a)^5wM7^{-xB|`JITdre*!q^0$>^GMLKm@oauH?5G^;l>0Hp)xxzomAmYTE02 z+c%CPd*0$Be%v~(u%mMywt>EgIlKPOZH{Q%Y5c6=;F0usNLUPph9Xez1H1>s1YOPG zz|s4D9}W5qUuupaM_2#&;|@Jl=mK~Bc0i~OYb643=Gzqz>i%czm6IJ}e-nj~`8ZFe zGWf#c?5=VP0hlqMCIlRJj0p>6ob8O5e(*AYuP~QI>C$d^Yi`)_a|r1LwH(~NZ9P?Y ze?ts^N2upq=Br??YX8%HZ%xopU$9Z$(sjX zPFNIynnhW{IRi^L#G9#+Ew!gHJ%T1dibisJk2~6dM4r$&WR1@Yh3+PZbrp7G519Z>UKXw(mZMT+M-ozzkggshV_x`b zthj%~?f*E&m2-P{17aTUsk&fyuduoa3w}G`Ii-fByRE*XlORaY&Ax;2q^Y}9DeUiq zyMK?>G}eX;GkTjbS%GZr z5T&~;Y#yW|>Ep#W|B^P_r=X1$4uFNPGyw?zjr2lT?F6>ZQaaY;=%~?w4R^35Z=yWu z?(pW}`Hbg{7^L5u3abb48R>Wz-8&e~ld& zG34mkg*Nsz8LkANRe$e1~y0OAYcFkLVXfFw#0X3 z=EB)RkCjS-zhk?;_Eww$ZWCeYf2AIt@_v0+O&5H%+nUcKQQZ*-D#Mj9~nh zx&c!=`cApy)!}O~mTV6{@dbum`*7{`e8wKXQ$qf(L_&%pEl%&9Hz4Ua`%w=5(|{Fe zG=KtAxRHvVR%isJiC+qS)RMDX`xiqORyFg!x&NkABWs5}rYfi3W6r|&5P*I>{#$0n zSspPdl-FAPCWDVqU+`hp5SJ)}U4;QxQ*A|gM$`7-D_HnBBw1Px+%y8Fr*ZBkK&P(5 zLO)g}sM)3#vqJr|zOLiUYMzC)Ip0^+BMHE(YMU_d9|WolPeKCgmx*JYTE6;S>Wa~2 z4x7~9yMFQiL85QHvJtCUi;sWX->6#j?bP;4-B$$B=t*-7v~dwa7d_l5=?cxUgm6Cd zaZr_|B^X5;{k6{%BEZY5G{tgIXaw~PMvhi$_PDnHbyno3v;_gj5-=Qm12)lz+O@kglm5{q;M_RZxMCq-* znMrLfk)rYkS^lo@-6`Sd+^FUeRw9NYH^+}naYE(H+Zh38KI`SA9vUIYM`w7n(({Fc z<0<5oW06nE*}@UB$5AV7a^dI2srSJRcWrClmn7EQdBmJ6?_NrBl@wo_%pe-;K3ph= zN1j@y%^Bw-|7I#-OsQL<1zRV2i1N8h%Jz zJwR0GxN$z5cL7T2`h@=Nn-d!(GsG9!?+6zh=pQ$E{l5S3TiBHQ1&Bvy(*8{} z3j>EOJw+p*2|#VfcRh@u)N+@NMx-@QrQhRg>Tr5cY}aHl3CA*moGLkK0}rdRVR=E^ z{#;gyR7l*RccCrEo*H}%3X|@5YPQ+FM>u|=k#sp-M{J+EGRGl7LH4Z8UIUZqJ%O1S$-a-TXZC__K^ zV}HQ($I)a#fHDGwtEVN4+}*Rszq5|ewZGZEuA5Iw2OpA6%g^thr!`g2lSe?v{V!Zs zZR|Oezz_e)(WIs7nejBn3Q;m~{el(T15QaA3slu+pDiHa->pWfN1C6rVtf%}cuYmO zgKLKj2iNqdxC5nzUkN5bWkY7QyW{~Jm`(yqq=456x~COUo&to>DhmwrE0T1u8eLBX zmGKaO;crc6pm6&VjM@?bZCAXTbba*pRUvkbglVZYwEkF8YfO`T(Y8Hj5McaI z|C{H>yx3qKlRMuy-lc?Sc1!2)CVr8jr{HCfqbxH-_?m>w0h)fl`U3oh{a{=<4u=GG zzB1dSG{rJNtgG}nPU<2q1UPrW{mUkc8)_`L7OAnol7dZB_a(SX@-|yK8Wwm(0F1NEm_aN1wVsURw>% zPcJ-K`1h9E5@B%#7Tn`q0}2)m8v1N;72R}2#~JeoV=z!u6nMx5Hh%7WcQf@>B}s}j zpX2a$CtQcsC3W?=6QyG8m#bS^7MwKolNJR0blaxwZnvS?S;Zd`$Td4sdlY4B=DpVj z;GB--4WcwwL>bZgwia+-FoH)nTd?9m$)`kWfURntsPevI9OkDUq}At_Fhr2*m>J<7 z|K^#22*1UDq{{(|XIx*ulqtAAdQ3OrRygED^IBKe*@i}bZ9_@AZve0qu;T?J2LZ}j zw%cP}y=TD%H^Z>GUW2*063o&E!US9==;FnvZpXFNHRbelmmD_~T)}7{w z&e;xBEsak%$=pypJ3t9=dtnbS!6w40@X`hEdjEiR%*$gfB`8X5t54B?{Y@k+{O-C( zyWn|kD&H^1e6{Z}+mjH!-{_d1n-62-&sj0eAIe`j`?O4m+Khn*F7;(ko`grc}wJs-Gcu{X=-q9>JtlE}duQ+wL-kpryH@ zy?9QcUQwlU%a{$3@vO{6uEg-;vQ6$i3UQK;nO(8qR*T1<;wvvr-5aev6Kzq@WY?yI z8CkJ-_v2o5#Cy<>1tkp7W+umyd18ce*OX=Fs(i}ooB^lb_(Z+B(#0c+peWSQ7vamb z`z_V8WZ6ITb0VsHVCX3uI!$aMYq+2H_VJv|H+xOae}8%g0Ho5T!|3N(fPIQlqqpY} zehINqo%!U~bwZHmWWWQHbG6yOu;gWGMqLHRHz7_bwPG8clq4AvuJY+yO|fZb!!O?8 zu}-gsTJ7>_YGOwb9ZP{7Y~E_-54t0uZ3t;;kkys%#n||9@a5P2V=teS?-R*n9l4LU zX`b4bjK#bVZd&U8y01tpmu%od$DMxAMMv9l&MoL=#mqz@UrVGR_l0_DR1(?*60e1Gde-2*c+IsqkdsUBQplCu zbAh}kVEU~E+wWc#ljwacB1;-}=6;qO#+T9U6+R*7gTqwax52TW8BT?9baXZbe&3!{KI_6)y4?e%W{LkWI2jCl?{Trz8L**uH#O^Q>E0F; zvZVDQPmj+y3P_#pP5&8F;btP7L{R3-N@^b&z}P6C*IselB-bHG;@9&O))tmx7<0R@ zq~8V%kqZ)eZcoE~O~sQ8B8+i&1Ue*r4H|9dY8S&zqWooS;5LT2)V0emG9SEr9t7AM z08Kh_ER&MkZz||l>!~yU@mi`?QQ4AitwkZp6F1DCU$U*G8x922-bf6%3pYrD#i2*< zwpz(G$kV;(&?c|bI?kVkWtK(xu`&B#;UTMoJn+{-FXYMJH&~sfC%3D^A2%%pYB~Fx zYFb@KR!L)a;xpqnrzd^@O_;-5c!|es9)R%NkQ;Y{;h&+Ck8^jTn&jZ}P=M)n>!7A9 zbI=`ms%#Cn4 zcD|SP<@REH*!8~greM*drUAx|97aK~i?nk84xe+fW zZ{VZUt^WcR{^_IyCA?BgZ6gdxVu5?G1|-aEz1&EUsaWP+cJ~=7?fk17Km5W&X3{&= zr6*juZl+Xa>izM!qk7^<2X1*30KepqIdjyV2i+e+zNXSxbK7Tpa}Fm~tK0+5Cmz|g zd=qVePKdNVx^>DVw^plZ?2M6Lxb`!8Ti#RkyDG;^w5l=4mTJ7GuF?>G>j?|lQi82< zNSi&Ar21!4wJGm%haIm3(&qHRaalgKQ+Zo*VUmdvO3d*r$tQiZdevGg?sUI{@hBMB z#c4dG%$ziRt^bWNf~3wy9fsIN_Xz#^hwnqZ)3n%{%nU9mIShVGJbF@_aV%R@{2`Bd zRRV1z;iLf8vnhQhV!*)}h_XFiU+=HG5zruPk-I(^EEW2+SP43iUg88Ktt+fn{a3`C zxH5^rzt^)}NibifBptLnWW>O$q<;o81Ytp^|JHO2c^)R9nQizz@=pua-L?WcDwzsk zqLYg1NS}l0EoS1SEwfU_n>3wtIkq4r(>>1vzP9Z)u* z7!cFZk(y94Ta9;@KGI}VuVTz%OclFRP84+NBUYBAN9)j18h-Dk(N_YxRc+#$@;E!G zk3>;{dx`$+A4-y+OCDz=U?O~&oq10pF2=@SEP`h*hn*uC*BdqRBV;NUWL%7GQwvf+ zy^@Jg8oV=aF&&>FIZfBUhPx!`mVdKBuW_kcOjuX6o{4h~GUS(Oc#=*IhjnUUK6V>q z3|r^NJ1i%lyLPs-RMaW{5i$=F!>FC4M0Pj0<<@G%muXC?eGi&&ai*KS|^#9Ba>V z1r&49PJmi&clkkAhrws5!q)&@Ng2>63rG~VPQPfM6P3_7JQhw!k2;x7`97!rb;o&f zj*N+5e^fk>D^vzYxcBT!!vc`_!+5f!_>XV3z@oz}r2l;7v?ybOOoLg1yQEm1p==et z8!M{V&DaVz@Xg1^2sOzN<|B~4p!Qqom;IvMJuhY^iq(pcg1vcJBD)9j$F|MVwyRM%Pf=l_jD+NyPHL%YE6 z$(-O5y>IX=Oj2(?JA*YBgFzC#Ok z9`8k0Tqim&9(eUu$uOl3X@wSOFmmcm0q`1mIA64Ve_<%3$nNID@10j(FXICMN0-)z_1h!Y(wFt@%rzn&KWkzAN|(aV{DA=J;-G z#?ZdfVo{uhmv0)tmnXPt7NlYVPN%)+Ps(HATs zB#a;EeCAVi=f9W$o`(OvXpJzf;CLh}-04ibR;6BeF3%HSpb7P|@BS;Ns&;?bSOo4F z4DlH!B~h1(AX80$!u6fC-}OET`Dlw`(|?}OMDd~ z>qFr8tnPYIjcmoZtVUn^-ei%&OQA5Tc=Z`Iz9m6b8v)SNDYgGI z&ufpuaQSeQ_2BtH5K)eKXd4pr>O-P(?zf3-LUZVNwLsusL-~7SqM_*WS%%V#M4_TG z{P&M5x)q1sQS4zgx}C=u@Q?t@YU*P&n!}ih@#Hx{2kRN*I*QhP*keYtJ=k?c?y9!B$5bcgrQql3d(MDOE& z$&4)a62X+@f)63w)4wmU=x5`h3F6ai?c0HhJ~iZLYXK!aa#)hyA>2 z|mZaulq=2%a+*w}~-#`f<0;rmBC$8kUReVyk83I8Vz z9h*!SORnHE+X=(t1767g6#NDfz8iGC>whkQKj)G}l@4r;Kv22N_b&h+TX2u|j7#Oj z(K3uiNL1XY*yk@SMq0V^nF^C4tY7F%Xkl1!XVbIhi9k&fR@zT?lM-aSH@RdqE*fzT z0x=nU5YhN`oe2_Me7X&Slwrh-emZTam}o^KV=~utowP0%qBQVdeF^BBD(JrsnqT=g z0Kw~8J^_6p*PaLgV@w0$mjgf4%j*&bCxW;?u04g`wLQC{3<iiFFlUUNQ@-0`3U0PTr^* zMu`6+{ji*^jscj}HzT-Ix^mFBSE+}Zet434IpXr-z;GbHM|<6Z$ud>QLOHm$q>Yj? zi=X^?XVKh5dmh63E6q?c-(MkM>f(9y>kJ)X*W=($$*zh%V_IowxHcM_Px=q^tBS~D z^CNokYN*qIzqTFLw@*J|W1E6Y93dEjFM7bVH;omm!&C=Z%kF zDZ!5^rmEV)HlD6O6Tr*st_e4;^fb1cMxb2+e*K7{dMXd+lY~LT*&%qoG(^LQ;xu2U zlX&3i8OG86!Vntf_USh9iF4*U|J`}Z=mVM)PeAs{D4WZ*4$7P zB%t)P&$2Kr04o8Xy;J`g@KPzWe`1T}m6IZ9MOy`GPfato?=$ik(>JsouPv<{^B1k$GpotiH# zAFc}^jX-(p!24l8(M&7@pUe|Pfm=;J8d^`&7M`y}lC2ikiklLO3&7s(v`TZM_wLvp z)BGvu*V#(5myOg0-#f?hZM~gOm)pbI4r6l2`c;x+BoKN zlf8pTUa5LIE_z>y*IP(5Wwu|3hR`D}LJe2Z{OO%LwF75itx_bm2;*V*L_d!<^U`113iZ?AUR2fo{~@G!O7S z8ry*a+L@ya1s~1tUwKIw=9Y$~W4(^vWXYd@p8Pzd41rg5Et!ZFn)0i|BZzsFQS{Ma z45FpX$A2OpdxJDya+vhWuRX!EMr)~=G60EB#(9=Cm{yUH#1~9tH^>Jf<0R6m#c}G< zi(K*ezx7%l*|KrLE}7Nbi?ghND_o~9`pZ1q-*}Q*Q`{_{6rWZ;i3So3-$FI8e&&NC zWaY{pZS>)b>-mE2`c_1^jB#|!C|63e+q*hQFKyk1RQ#UTkJI!M6}>*G=VmpY(8bq{tn;^1f`?7^Zc-BLmxn4n zI7ms3JW&2@wCq%Iun#b{=0FF4fUU|6)~D`fAdrMDf-%qb7}(_}O-Q%nk`;V~i0&E` znTDr*@a5IOZ9_&vz`~lLmNpX8``JG1kxEJD;}0!4K|3<0TVqBa%r23*zlrBZWH4U0 z5PQ(DoTHN$fb7YEFYgjdU<)3`W~2TCFZR=#A)q&Z+nJ$iP35--s`>pS@B(Z1_+$t{8(iqnGXFSA(Eez$U z(rAcMIv(%#M&j7W?q4q*k#Rn$E zuip+NtT*wwH#{;4u5GD8u}hZ<6@&20Q`j4GxWAW}!MyTY;KIYKaj~9lLj|ADb-{w> zXQV5^!qH%Z=(nxMKm85L9tLs3cFQNel6fR6KmK|2x@yy>gzqqyx%l2?3(eDsLCocG zdslQ2dcLqbO%Nc`$|v^)KCTKql8YQ&?l90WQGtlNjj$*dWc`kau){M=;cMhq|fFjQ_6$TE)+((=L zN}9jU#9gO~MwryIRsj`Atd^e}?`()lD^;B%s>2xr9u$3Ux0maqBQ-M>|74?_%Xg7K z!Rj9hvpde``3walaYgh+!5Q07qw5!{qQ@py4<7ToKiaHbesEVf#mwc)!Ha{sUwaYR zYil{4w$X?jszTm52%aZddax+>6ZVji-I*L2fukc8YS$2F;Fp7qW|#QMx9#UKh&WC@ z@b|j|WKkGzxI%6W_|)$N(vBy^<2S&=M}T&+nZ~}8nxXRO<)lH7nb=UnCA)@o7GYXG zo3mta!~WY5Dh@By(QrLSG!7x6di% zS9=>}2G(da?F-j0X5}QM<)9<2P^&l*D$0iYCMgnRBFhgP;FHiQ{{xc#7njIn&F46G z?iOCDCSZ+j2-Bt2p^J`aBdnQ2?1U{L4m?WeF)8Z<2czjUtR`T$m;{Z_29g z>0R-hEnP?RcHD}C;UCvlJW`!Q#=eH%5m;&(#~y)~Xxx)!XmTP*e;VXL8x+aO(;`p| z^Y7W=lRA)%A&Qg4Ci82P=5l54I9(e#7KD~f&prgcc-_0=Y$*(6kGR#%a+Hj=nMsHH z{nStbI?Mq~mcO0m3g4GMOW%!sg=~(F zHo*;$bSAPDVg*dJd-V~f&<4;QrUGPQ6G10(WzW(3hbT`A_0#Y>R2$q%MZMcYywII% z>aI2%Lsu?S5d6~Z&+thwjJ}cHCua1T#4KIVsE)J)J~nf3t4Di|CU2=n)FGexBvJ*U zcqjy-l@EC24Xf1KX1_uW^(#D5hrp2oIs)xY*_=Xl}7sic0DaxuVQ;Vj(H8jl6{ ztl@;=7&sO8d1Gy79NJS|g5yuZoY}H4{hxfL0oDiPGb?VB&s?rXwe~sbb+Sdvx96Mi zf7XvCdY<~>#8qEs6=adRIh)T#cly&iVqloGZYgq2DE$sBY(0R;w#HyO5m{Xi|j`ryzeJhFvObXi}zQ$^dkUa z8-=*j7t{_XJ~$Hv+WXY=obm2O&HfejylNDi~KEqaO>WLW#z~4D&S_4?L?|I7O zd9bOA>y97h8sWz}k$zJxC8agx00PU z=&q>}m9ckFl0H+8hHU7@QXQTDL?Q5QW~dH6U!?M-P2yvDhHyR=*S$jlFb&0tEg}In&YcQjdt18>ST2pa1*s+G_eQ z$i_(cvP~<#>q^Bp?-6%4Xz=QHw?E&1dQfBsGqE1{N7)PW@SLg91&af=IdJ<2o23%I z=B3MHDwg?zEY+b7?2pWuog5RCD;Ts$p6L=wk|sWaAE$aA+6Z*uB?%5v$opCbw9)s| zLe|cu36WL79#gea+kAOY86xuP@wbA8`P>mQkI<_463)vU;mhz}ev%wYe9GJV8DG zsI*WsdD7gNyjS4W75N&vocg7{z5xOXo$IkwyV2@+8uJ0z_5FJ|yr3t0HolQ8DNX*! z@UtBrYSwpRoJm))>Ui-&I|GfHtg}9}+AglmSHBzP+5p0(>?gKNG`pAQ!o9wA#@CUV?kk=n|xk;NAC7^On%cCA6GUg(8h74Mx zmW0D{fTc@BUs1k3M=8z#svN%Ei)~)D$!SRh)g|_VkdkQiW;lkt?N}oDiND=P-Idjx zkXC>GUNXXJwB{;*6!`ng08u+T37|1I=G#2R0wvra0A!Sc!<9r=?}l{$d_EW{5PB5< zwUrHoXWjP(om^Xc&*V*LNj~HwO;dHpPQq`eu13BY+nHVMI=pjOlsk;VH~8AK#p3E# z1Ayw~&8+%!P<)FVQz)NqdGfTyNTcPU!_)~5lQhDRYkp zC_%1KG3Srg*YlBCiN@6Rz58(IAeQR&A_FooBDOZM83P*b{nB%0neKaT#g$Y7rGmbH zHMCz_Yq+w?u72_rRDz6F4}2GfvaFfx80_zu;fIdvk1$FYLSXCbPQ#V%gzb)_Nq(}y zU3ZOC)Aq>!)bT44i|W`IwFgrG;@_%k*I%D4G6?l|eYRk%UGdM|8h^+cnFz~LymyV5 z5h^5j|4ieG`CvT0^v)hdx>x$4e6v^czfVQlAfgj#Fy_(pxneG?yXsOU8$@^>PX-We zw`wab$am3g+C&Uz4)|>7a*fvwKsEZ&?Ybqt9)qDXf}-cC5E22Loax}F)rj@7O7$(2 z?!By3nfztcBnGSUa1VZ)041(8iYs;m!`C^1Tiyg?|0l^IwgFc*BSY;i+Ru*Uh}%B( zpGlO&;XTgsH^=xdf>7^jmsz*4(_pfM?Wj~cXnBx z$yXh{O^XBq{@qVmy!3{Fe;!W@={=aK2j2UzP5%pMBJj0CeFX*AMz0*|e5> z0wrQ0n97T;j_W9N+s3LX;fTC8`{qy)IZ0K9riL!D!5uE5b9WPVf&!-Q=RVOjTSwBi z;k8~2s=sRnuy~C3mJ|d`StNjPSpD|gN1T; zzn|xTg~NK#smNy7NR@gBtcTMt3~%0kdbzV9%NPq6P)tbZzz0`C{C#mdv%>;Ao>|XF z9T!uW%f{;V^q70#wi`Y&^GyCG4UkW@$`FG>2r$|+R>cng%Ay@aip@1NWmZ1+gcN$V zGh=iq+^Iy7a|>y}@#KfqSDsgM>yr($WF&@~n1*KGhMF{vmm|Fakd5mo!~zM$Gew zn{T}s^aD5dq_;fJQ%))f`$5s3r1`G7tNu9Cv_YzL=G)n86=SkQN(esj_>Q{^f$Q0l zj$sILcM@Rv$kp*t$s4ktEp{iiV&b;eWR+O7^3?$9y^dc_N(V^%wbpl*ZmZW}s~61t zC)3`KlBcpmunVa)|J8NwWr3e`izfB^AQkzeKpWXQY){k@)2p5_!R@8GcPFT#3p_sS zU2P7<-pWbsgYLk%M&LUO#ycYKV59bKe8nkHyyH-9+I^Gtsekp|x9$Vh6x$K2JW4MH z?B97keW}HJL>CBgaJvcIuqZwH&v0t{zp6rmOjcJdt=5#U0gz%O;r5BPbli`~bn-B~x)jPcuX;Qa4p=fVKCY!AcXB)_9R@svcMQ3a+3Qf#anpAW6c zy`hp8b*Np5O#tA*6rhnIK0?8wYULw21)NewAS@DQyw=aryfmQb0zC~6F(8jHAmH%yD&YeYF3g2R$mBpYO8RPkdMs{f+{XJILUCPEi(lE9^uM}al?6z}`_pj_)mbUDDEc^i26 z^#|94ClCxrF#PNB6U=hBSP%DQzhg!rc^sg`bNY4$x@IgCJ_Sk>1Ce0sp47kZzXIY9 z|7!cT`@e6#M>bl%n(^E0X@sPdj`Wk)&2m9A|eG&Uv*S&;NUT2*W&tD|}H=7Wpy5$Op4C z;lrxxFPj050yU58a@~5snJrO;gF|XTcxBFwrycmk?zoNvu6Cu}Gr@DrqBwXLlharC zl1vBO)RIe=mBUAV+QtI_*stF9v3zwjExdyrp!b|Em z^Qi{xZ+SxKi*%CxJR`=belBN2@N*NRaj@ydsNK{UIK2gkP!gwG=z;sfD^oQzTA#La zO5vBp_e3}q=cE4-Kbqa{n-PV-zF=n@csZ2&dJ< zfPr0T)65}Y8PR7?#2yb`jv;P)6TsvSoOqenNdzgKy#1i7h!>dojt|V;PIc}Z;55sXdP=l9(^p|759HpLCBthH#}Aa`oZ`9GAO=*n{lX#bRAm^gh`ld{8~~gycM6iYEUB7zn&$9I}i%`)4W;V0V(Jht>^f zV!k8yO{{Cv1jw`yBk8d85UqHM5mK#FpJ3fnn2WQtrDy9`CEQO68Kxw??(_}4`m&iQ zn>(Hh5S=F6y#FT24V9j|Trq(4`!-UVkr>`Hu!LD=3vz0ks3PQsHSoStgeYXiK=vGzZpKaR8a6rQN!4etGo|kBLTOdJzt8YADqF*68=L zY+4i#i9+9$xs`EF*s$V5G6!#;J-EZDvfDh2F4xfkUa^ny{IpzpCqRC?vPY5~C+HEo zw2A<6CfR4qiAr<&J`>#S`=sNLi@g%rg=i@z|;p+JN}{J+d~3!bwR|1_p_WZ*zFg8JdY2H&$(=>qm|h~`0d88 zWfyZh%%J_j4Dq6hl=rxTCAnU4frH$_ytGsCU*D1mn`Z+sw9>F*#!002LkOF@J|RgG z&VYXmonzYG{uD{CvS4 z2zvgHZG^kGrEZme_YMX^>Jp5Ekly?SG)UqM2$JF;2kQZuO3HlZJBAWt5XB?QAtk6p z;PZBUYmLv}O4#vA`t8Ta9W!j|LYfuO*R{kX~Gkj&k=x{OR zgyuxc7eyW4QKwM~Y;XaJ4k9|Rj;;=@E%@FF)P+@9Wx#6|HcbPs9Er>v%et4vJrx)Y z3O+mlAgaHtAg>Nf|0Z2za?+B6+hfpony5lDAE$d(o?L1}N0%V|tJR#e1J<;%&1W}W z4sdoDCj#!=VGrjHHMfK~!Aastb2s_g)o|qjTPwpxh%bS!912Ze_R1@tsT?0hUX>l= z0g~f3qq>IyyT|fEsc3UU%%e9f@6tYuSbu!PUgly3^o}%#>ptxjwWfP1pM1AwR0`_Q z%ul*q5UsD$nLPe0@(4Nfp56?GD!KCH8Cq7Ut-*bUr}KB^_liJCg=aP&2w@$IA|4wz z09gyWU?8N!5TMlMU;(rK)zk;6jObF@{cH>4aH;$*7AvDf@#!;Um?R*(8&!b z5TAj!VC4&7_>dCm<;$(+T{TeoPk0>2{Bi?uVfbTXN!yb(S#~8f2){1p713Ty*{jc_ zRf2HseOZT8+!fPXa&@%N3i994vCh!EtP(;}!4)kKE%-$Ir&(6wqjxugE|6~v?;rNi z^h=ZRn^;Nzm0U~}M7eO*=BYA-tWFv8ZnP1qe?Ete!mwVw)ZOGc|2qNyR1{vBFqdt9 zt8xG7xKiWPD||`~g42zB1A?)^}Kb zHZN&k&5<=QopZ~J#!ma`OZ1?J|EfUB-SQyjl4>N4fd(x7L!Tv?k{Xl|Zi zj!2NPdK#Lr$aN7wpAeRyx5Er=tJ$^W!M|(Z|tTlIzdC>lf3BIlUt5Nq<^Tm~-|%FF_W;5qeHfl!yrS z9V6$z>|&Do^kuvZw?FH)k}b0zXk(QJeS<=)fX#LP&{-( zR1mXZ<8?!2fYl{@0Ezi8RS2-g=bTa3d*Q&5p}B_RA`OEM>K{D%u@0Na==gQGyV{eE z-kFU(OR^Kv7pt2ORs?Lq@qv7IXi2vKqKf33 zR~4e`{tcY0mG_o&UQI&*yPiUi5dRcXr0|&)XZQi&;?5gVlgjsGONiCF!slVgk!>pJ ztZJM|yhmK~(d5AOK36q1cB9m~^hW}b?T;y(@{Wy2Pli96zt0DS-1xLeo%g87+w+(p z>nEs|=n}0MPb;Eh_?gkGvf)rv3^I(x!*_Q~yK^$LoJi7p0jnH_?F3AMe?u6qKfACz zxBXJe>2EQe*q$tu`?_BD9)1(HV@WigmKpH)8qa8vN?apP0c^wh78>C_RjVEiq^C_M ziLc~F=qyRnDrNWFk00VNCHidqC;&lO-YJo^ilZH&&-2-nnG7s%+mw0h_s~!K*O8R3 zdXceMp|+2$u<*a4dybOy{rsWgc1HcLhxIs2qQ3&MoFc#~p7=ka}> zSXC^xPkO?8?qUqhJM_C!S!&(m8G3Jwc`Rc0Lv(=16$e0NUMq zg&0AcMq)4ca){?MH15c7r++038WzbRm^di@BInT7Q-|RVTyl#F$ zN#cH-@iNC$)^ouQ!q6}$)J3U?09q+e;jv%7R-)S-Tg~Fv-s)g$Za{wkkBTK+0U;hs zJXGJte6PM&iTX!8$oZr`sB{db{2cefDoJ1AZ*D#m-oYZdmG{q?_rL4IK4v0^_kBK= z-j#xDpZt3e8`$7C&CK}3T!m8lU>~eN6kQ*41SgS%V5hKZw=j)Y0#FP)dY2(Th|uUH z*sKv>v8vZVEx?Sto1+TzzFaFnv5g#17WrL9fQ9+6OXt`vpdPYF5qWs`#godJitEns zqdqueW_c6LUNyQ!6e)bV(zIh${I@c-qB98Qqq!2VR${EvJCyR!=6RF<@y{hl_Qyl2 zRdh>gWyr&rj-TmBVa~l0g-EWuk#WqPgx0ure2V|klh;4=KQV%yBZ<&=`Hd`3vbOwb zM`EK7C~{MW#PqMwf&TJ@9#J1^mA=^L?)=LLp?z4} zz^fRs$dnB19)LxSBwkz09b)2&L~W|Jf5_!{@4+(syl>;jtxMRO)@!;>_C* zf|Li*srkh>E${4jGP6<;xw<_rokHRO<7G2pVd?P#keF5p9sPK4xZ#+U7-rMwnLkG= zQp}}lGrZ!*cZq-z186@_t{%;RgXMksAD(?aQ)6-CqZ=`L_M!Oh1Io|y@hP=8=Z;nE6WMYM!8hA-?f{1$b8cd%+$!rUIY(C?#tyd?@}8%cbPu%fuV zHmJ?qK(RGCn^1^sz0*lppm$UUzNT_2bypgib!{*TbgoE-8kMliGrE|*OR;L`nD~#8B-YU(wWNs_(+5Un**Ep zff5*To$NlVS%x59R8Luue(S12jXGt_L*fDL?dgaseG8>+IdO-~L@F|zkWY>U^Dh1x z0rk7Qi)kd!8?2c~1Fy)kWslqI^)fQSdt)j@1z`Z2M)M41OCzTRx}ZKg!ot(XDZH5;arI>LD3nB^1q++cv|OT~`i z8ZoAX%GydeBvt!>ee56IT-VRx%(otrPQUJ(00XuH?IE}$Y?tClldCSub+=SuqEB+D zkt!~vrgb*u#_nbS1i$a3D{OkQhQ9C*_ovEATl&}ISmP<2KAlQ_-Grxw;okhm`w5qK z$_!LEkAFQ2I`dNsF(z*}iya2}T2Gyy!JHg6a?(VNYQ-;G6|4Wf_7F}vyw!Qmqj_bZ z4>QdG;vN z=^|&NU-I7b*sajdJc@(!q=!6FXSTadlX49Q)nc-2%~l9^p=1bvHRosomH4qXkdb@k zwK%z;z?zgB&4?-P8#|sLzsT z%{Y;tU%0KwHCb3~$ktLakPPO$8i3d~dkjW@-}c&{roA_Xy008E#BLYgH~|6E5d|T5 z1-=~Mav%F2rjId+NmKW#&3}4tNTnvK&2WU!&Nh^Zcj&P(k)yJceJO~@ zoS%KO6uItbmOcCzhD!{lYhWV4@#fZO*oy7o-8*q#kz1lxvw;y#OF@^7UpH9N5Gr9D zYX;BMkr2>|+2vZuzwSUhgC&IIbE^sZG9UEj@$y~S&z<4_c`&!!@pbI=$YmMMAVTzP z!hhUsnCf~c_FROUC;_J{ehp==1oXfm^pPqb?6%TBxJWN{YB}-$xNgnc47!yy?)4~9 zW6^M%8DbP(-}y*_8Fcpo(^}Ga9~-mB)pA8)~?JOV4olI{h0(@B+Q$xC5d~le-8b& zY#`>{j%RNi=Y+3Q8JeK8lqc~AWDpn6ABE0bo)xBW^l5+iByDp*_AG z{a+ch7yxnh2-*Dy0ou!wH}(i)Tdy_C+LlrjNC}H6oR&W~t|{>)!iqZ@y6F z{Z9uEMXfon-58Px??G!D5oo{xn_qE58U8r<{UL@3iFJ7md=6aaM45`lyZE<6eG8P0 zM+Mung>esC$yKLmsfO4+x7~jV3cjMTb@*iwBQd_KiT~bVMD7G_Fp-i#3Ag3VvwvgJ zeDa^SDwA}O33bLZdDOqk{PT2>}^ZuiwC z;D=h{g{AxG60UoTEx_=y8X}RY`67bD=rAHwZ~`vs`Cl9+)W^D#c=^|MK^l0IzPS41 z>RH|V-K#!>g^OjYfWDh6G?-KFP~=n8*#jfad4nU}&x-_VP)ifu|NZ2NXLv%`xe)Rm zaN2*^Is&#*_a^vh`05^UOnY*g&NH5O**!7oW}4H9xfyUZnHgZ~0K+~v_b!(td%2#s zA|rICEg_#ru(Op_*H7m-p+vt=$fN zl0Qxne}1|j#4)x@(su-^ZXsUZ&0`U>#&wsB4sdxCkP>pfg9q8I)PzY^z-%`J?NJ5B#wAUF*E2Sh8%o4VuZNg zhn+rNdZLtMTj=$|uiVd*tJpT=#8*~vliD`09q3=`vI~SPiE2whwhMl##D7H+MK?>c z9qx91xPZQD#cTSpLwZk5pbp&Wau1%yZ&}IM+_TuhJ}t1BDZ>aUr;y5D*_dLM_>Nhu zW{83uG!i$muzqsesr7=fVVV|SlyYf&jCFxqiSH+5-I=A@KglOh93TnIQ06WWwkHLi z`0(;_E#OI;>y-BS` zRm|I);;aH=hTh%rn;-wey*2XFe+YF-UJX&cX5d(H!3o{=vw*t1xcbYe_}x`48RXm( z2qznisI9=Rd#nlMm0S%6sVZoNE5d{J7WmoU2tT+%aICh?!;F{08 zghazF>D0pG24#JQ)Ma6K)cNP>Qr8}e3zM4XO&dkAwC6^+Tqz0GK((Yks9PR52Y)ee zaK?{9Fh z1OzF{6Z6zi=_B4F_4tM&(p6ufcX59*0K|pS-EFRos`0#BxB7L5LxZ5_UPTdAX^u+4 zk$9hZ+`{9j{Wzi@62z>L9lE~Nu3YmmKinE@mFXWlux76q1Ml#$2J zy~IT%@vm!(DmvUe<1z?0uks9UEt46=ExfsnMMi5nUL=8;h@pbhLh_fZRqa!_-VAAd zZ4kcH@p+K$r|y5suWeCLiF|VN$gz@cGdn9NDaOHVBs;=*wIW}drsdk;6KY3lo`2{AI5+U$BDWJUFm)aqj6;(x(Lbi7|Yf6yphgBoS@~ z@&3jP+jYo3-s7Jh6Ll86nw__T=~6!L{6`!G;#on#%J<>gaa>pc!8nirBEEOvD83b2DkFGe}n&vL_Vt7~BYWb7J?oTY5-bIK) zp$Wj)JV^Tv$30cGG-B}zio@Xc`g9iODv@tv5F<*T9f*EXNsILj(&5p#`)vj&LmKE@ zJYK=(vAM@6xoIfSeNoq*%i(xKmjsrk_OgAueO~k`*L~Z7e zG3nQs*XWS(`E4m7!$u$_u$@tYTjlC(IjL@S==w_alVmiyuJ(^(Bk{5D*_u!pd?>(} z^uz1f=n5YEtRF!919q7GvVTZ946bY&zn`pou#&sWCoFn+UqEnf?{`r&uIVIm^~=t0jOnZog6W`^$>?)m1L z2WWq_QHkKRuh>q}4<3bzfY;F?HpDLG%OYwa7>9-nN+Ul$mb z)}d>ObXR{(Il?cG)(n0iFAyZ)9h^xvS4GnJ9BiMuw#9}|PnZ4``H#`sEItn+NY_H$ zMv-g$J)?uqt%56~B=5pwGp^d|uO2)V^?gePPWIHo$*p{ z6+>TaHo3+CrpMqvE_U%n%+Vyhm-mR_ATK2a?1MwQ%*mg=@YteVRT%l&W=yGK4z;hMYLiI-d7jH45`uo~Q7q7}y zfK7gF5dWbfX3pw)gOG;zXTO37mt-de`NkO^)!O{6<{4L)>i%1|53+~T9A(i`akJ^c zVFDALp43U8v>D_o9SpxwQi_`DP?%B&Ku-1){GRrlX=HAikQD)Me2ovR&?D%ca(EBy zc=&6#_LtuIsY!%%sA6fY@p~ziWhoQ=OCt;>AmG}gWuKyRHw+T%Zbbhx{2bgE2x;5! zB)Z951iOh|T-)vNQ3|j7e*I<$-p-u(XT(}{B8#*cX%1cNXeg+HS=?>T`tI0~hTw>N zhzHIt z-wJuuWFu!DV+jd3l5|wjKaQ|98RQ;JOz;H4ncj#z+^U` zrh{^b3RJ;17r6k%*gQr2UScJ8CD{Z1z(^5DtkdW}FR`S0=iBIWdp-)hfq8OYqaLfU z1j)d>Q8r|9uSww}e2xa&1zfFBm|-k`-&=jWhFe5At#mxI%{ zxjnzZQw#Kz8CyxCor{W>(GN?%*p)0Xv_PMTs$O2ZtL9|Ug4sOdsva*IZz%yyz6G$* z;-;YwJo=@9yjDSv?qfC`PdR~rF{7Wd);QPDwHYZ!7!Y7Gm~U! zPTv^s34I*{I?#&xv?sFNk?XNy@n%dg#LZ~za)Xn18G{%qTRd_Op)?D{3rivId@I6w zWO>o~SO{H*=eR5;{Z(3$xo3UK!SZcP9P99=JicQ3&^^Dw^?L%;Fj+G>Xe>|_dx)<~~ZxS{*H1P97@Za9mlfgC*wjU)~yV?`)M#>TrI1Q(tWCw*OwNV6^i5qdA5vX?j-LrqYfo7yX$8s?i zB&WcgzHzMi`pM*atDU{M*6tg4=^GUi0(f9>GJ;sxPN-fqYe^WAM3x@MzT=A*ViVp~YzR!-_9svJmMlBU;YuI& zB7T*I{Ix8mee5wL*+JO8dUtdMBbwX!t(~x2fO~qFx(8f*9Neeg4#bHB=YUKSmdzEziS6~iVSC^u(*farDs5R(tY^Xw6_y%; z^E>>!^z6x7;=2R?S(xHg#>*bjZ>y12AMNW>=vUWb> z{bfD^cEU>vj`kl$t;6MidWc4%E?U$wc+7wgbwC7g>^gFH1o2o@d(9PE>al6T6J;pAt)TKLm zG5w}$NZ@v)%JyIY?_6iiObOg2t$}0#g|R3~p0~x^h4LjU-918XT5Vz;XmRa@&Ycu3 z)(0M;zK)$F*|@oUcs1eSgQp#Fq&9Ykc^C_x)1XTA82F*U+S-Oo?Gl)RDsMpc70trd zg3{VgqdG=0Xlem!%O1q5_Fj|y<8stHbqkYdB(dUj%{tB8qLLJj^v^mPDp^~H?Yw_~ zkM}I-*RTA&g+nbnt+uww4yo;%)&wz0L)F6@1q$e>4xDKg-+Bjx9RRI7H`SOGIGhxG zD$V_3JanT!yi%WTyM-NfD8m|uru{+MME}-aT@wny`_(~~bd+yN1DR4@833DS?Yqm-|<5+gF7u)C>4f?f}&Xc{@vbRpcB?YG2!*^m1M)UieMh zw~N)&APr53HF6MxBukt?E$KQC zB6A}^=jseIY#R|bC#fB9q)U-tfj;U+X^&&GiiY3hT${ym`!k$>pSFA(8+*`kFHK2q zAzFTtdV4^C+7<0JROnyM>u0C_Dqx*`=y-KKDM-PGzwiTFX!XdJu=tEBfkT!=(Tl@2 zz!_e0q8m8?nYo!t_k9D{N*svv7bn9Y-9Y^K|9x=S6m#G$rc(wM0aXw+(%A(J6C`6S z+jY@&Q3v8v$9>(}aL&d)Mz+jc8?^qi8FJ|+3TS_^d-=vx zKFR8FKAp!#ex_PL&W?_3Fw~_S;9jSiqaVR=65uVF2ImC3+dre!&uGe7NGn>-_jI%g zj1)1_#*OVA*!_CK(Ido zaR)cL>XJ5VK%w3MpW!cuVY9{^!l)JzJDwr6Wt#I@(nF-1rw-P0a_b2_`=<8rYuS%R zn@fUwb*pJhgylPNKPBuoI=lT3=wNYD@S8PXU>Ng(7z5dny=~6v-k$-tPIftYNyJ>U z?xgCCsQddaz=^zurlg+=_-(qqp4(*B$J19*IALzYuZaQ`@11i_r(kQ$$XLPN?V5ul ztIh)9K-#Qb2YiJJQQ=e?GR;ixB86K%-GlKjt=0`kRqn(XMeM=VLhc}^&#Nrh!uS!Z z%=x8p;9w~NqLaz$`v-5wrJWwMoZfd%!M#ExN&m;a5sYxy|6BkR&5lBpR{mTh@@O&V_ar;XKeAZ*~?F4PEGzjal z(F_R1QT?90Le7%LUCR^%S*B;lk?&Xf}{r(5{mwO-Y zdtT=}pA~+SSKH!J@e;dPI{T-7&!;Mo) zhWCtZ*wr{k8#RuE|LSgxnf`TL;vhKSL}Fe|-fQT_#Hv^@r}wor1OAm;t{17?V|QkK!+JqCehFni7@_sOh_S3HiwgNHRV6>J%EwIQdXB>rIBo^_yCT zUx(?^>NTtUQtkCi*6#=vlTx4KDH0{p%lDMb9ehT3K$6PS-39q>{<>NR zm;Q?W6vAX|ck2|BQDgYMp<*klK(QoAYGrbq4=m$~a^5f-DqP;d0LZwv)>vdBEqUwF z?B35U0^_!80O1I<#q$a!MkU*&>y`J=Xe70qdF45 zLGzB#Blk3N57~M-L{F*;N60obdO(5`~06DL?qHL$^kx= zZ&>@B(*8Qimsl>B)(;P+#*q84%;u=Ek}`aI!aucI3mFLhzspI#YoT0@i0}~-nO3_E zDiu&ZT^j5Nw_7~R0Uc8X{;+!2{NSTvIC|ETwaxem?A9u;`||VXmc*7E#)F&*ATbHv zj?(kR-LL>|!!}D=?QFPEMFY&xYl<>o-kl9bfhoN-f55_9j3*M>KMa%&U+A6Q==?T8*J;%dbIRf-;pYA&M@X;-D*1i z7wouNogBnKFJa&IvY1vA|Np5K0%Y}@FW<8GM&%{p(haA776W?f?_Mv${1}+&Q zwqiY{_>6{XZd(sSnX*69BnIb?zu+cD?|-WnbeUiUiP=Cb7RpQ7%e7+5?s6eMIPGjU zMc(O&B1N##BW-b~)1~Ec+1X2sfFAAk)10mHJw|})SYZD6SK$eyt{$9OJ5RosaMzLJ z@qN0pgrW5!b4zH;U{o#0Oxkph2JD)ao%=C$+BD)s}q-aJI zRv_?_7i8^a!G8}&9D*%hrhKzbbt~5$gZ}tty!?XPp?@Ohg+sdgud6Z$evIBSgEkXT zFr1qTb2_M+kCX*=cE4qSxQO0Am%3QRI=FZmSq1WSmxnWwXg9UZ0pewPh_EQq!vT$B zr>S6+p;SF961n^rFJk%>Kj-21{K4c)iIG$o^~lR*fyyIkfmj4G*VJ3y?UlA;T)-*a zp=(PXBLDCBos+S9)o-U49|Q;`3cK>Etz7xJ!nSU!y1itzR) zcpaG+%B%9lU;Vz;WQ^FyHr(GW*FsyJg463D9G~_TC+so+tAqkWkS-!KHj40C#{`l* z@5g&wi85gFTWcxhtDn3UdjRJ}c5X`dE&Yc1j-vS8=yex>-1SUo&?YGzuD55o#H zqu;vsdRpMw`G`-_89A+FfdAZcJ#8dhXy?z`q?WOEW2f^zGR>T^p?i$2tA|TIzp;O|ZwINSoEoHpO z^E$(+rz@ycjUiyXPQaOd?C_wNPj;M@oP$EzWCn~|6`|sxu74>Hp}A~W7KefshCT8b zZY3YJ-}z8ieFhH&N5sk1=sqV?ZB@rFo&V9j>vNdAyGs^Q74Y-L^v3&7USa)(Vqo1c z*5zUw$Za=yStsg^)izn$fK4x%YT71W=E>mxKY;sf4vwrkY(SY|Fjp_e{IVOMcoOc4 zBYBhHpj_^?LjFoa*>utBiIsMyQ@V}ACt~Wz&p*Z=u2;$4=%K9uhU=K}T6fqD3qnt6 z_Ex4S8z@F5T&vv?+}y$Pn2+97bMc2P!)8rU9w8Cxm-=O^ca2HiO^SPZ^kHQ^N3RZ3 zn+W1i7W+E(TVr>>r?uQoQ+&+)4>A`&%0+8##oi0TZ_aEC^L|Y{j6LF*@&GQ_?5jab zrX%chQIWK&3O!ckoBz6*12;xW2*!MMe)utN14?lyz_flV^mn2PeyuvTZ{Pz~mkkIT zr1h;iH3P;wql4n|Ul-NJdh5LF(CquRW$szN&1zH7&!q73bRHo4>4p z_O*+feaIKIZv$l?2Gf&nBNkyB^&~l@1^Q3dG@yj|SgBE~sQi*olYapT+1;qP(E>bwc?=sSAhQrrN8%ey; zNyxa1bNH2;zzrQCM0=>y?ZDv?KUsMKm%@$IezQbo_@!-LrzN8t3G=a3T@0a zB$-^g`m+gnEBCoI_3mL7Ge;chmf}$BJqKzRDc}&e3`-1tvp#zpbex7`E>-kQ&?V5D zkWlr)w}l|sG0r8O`?1v#OT6>NiuRwlNoE}v9m?EtsD539S1<-JyAHOvGW(MOqtivR zUB4Q;sFYMLIFAKT=UC1#c(OsEMdN4}N(^Zq&Z8jZFUuikG9>Ico@N`*let@10Tl(Y zbC$~O7v0(M5vm4Z+oCkt{#_J(M)qFM`u(zL!U213*Zz$$hVRCbb0cVg#W#mI6)wKqz$W>3pn>%45liDw^ETFqD7 z546xl)PqV8>K3nyXIzRANr|LDRv#!*t^i_!J?iea6g7O!@%edv&-;)sX=PAuebbj` zqEpWYQty;ciJrz*|Kr#seFjl)C~TS#4Ih^8k$!_A#CeVY@@!>jZ)W&*(%Tsr zj}x5JkSy%X3G|Zv3HdEXj6+p>{_qyd{MmjZ&}@cJp*ncyy`D~b>q7W5c~WvGCw9fM zNaFDRu#5~pGjbzF*2{1>A|n}^zn6s)%u+y$fIS8t{yUziuPEmB=+Wsbg3aB z7EG(0D^^&jBrb;}6|ftWg^pzVYVDc%nzm8BlQE}zQ|mCG>KU!47Otu}X*KH-1R`I= z)4z;tRejDuKHRN1*B1fL1VwgZ1>nmmpSO?Uj~`49|M#bIj)$#W9C*c>`Gehk?07k3 z(78ie-MDA#y(o2*M|;+BX}7$By<(i*_Xa##+seuG+HG=eH~@&fcYSN5-FIlu17Y*E z2_$t8*(BR_X4rhuvp+MTs9+YP{dyvo@iNGa-Mj0JtCoB-U%~-nIqt-xB?*}=> z!Q#P-xyS<}D9beLe4L>Zi=$P4<WAFo; z1Ik5R)Fjxf^$CpT&ueiU_YIUm`pf}vDZx(8A?rVxK4=Z%cKEL`0Jb!>PqtJYjIaDU zKhpWjZNCpjXWg}=86)5t8vLDqA>N$7%Sv93V{7^s47ba;MVFoI!dtYzOY4lLLHraP z{Y=_C2O5OG>}6~fQ);n(y!*!8gOq}HM&!ixtpb$Ui+17W2$zX+P@)YbqD7#Z7Uli@ zrBaXv_3QPT8-_iLxvgY&SSEYQfAa%5S=n{6$~%?4+)tzrzwZw zT9oli5B}_tx8nw}EAYME$%7l6^~*guhP7_*+|&J@9zd?Oovw*1$7qxG=RtGV6y%}b6qBb!V$-MA|P^@|a`8a$7bdCBCyi!vY_bmgYLMRl- zC%-38_HuR~B;;GTrED8rcYHy6*lTVa5=s}rBqW=k4$G%54}G`g`D$(!UGVeLts>`b zX&YhX&u!-8X@r_$1o}hKG^WKrW+{s6UTu_zk{_)}+9&ZZBNJcpnF>HJ+NF+zPVTLe zC`gtFHJvxE2sR`!ej2t$xyiSg@JRH|BE{jX_t8Q(xkFmFyo|;i9QMH#1m1AM)~i*d zTIk_OMO#hM`sjLjqTltyON}R#ZZvArA>`cua+RDPrn%e+5=P(<;Ah-3Vz4Lp4N&LH zxFthC3Pd#R>3@5}O64(uVZdIEBcGWk?Am*;&Z*F>usHRkvBd0*jQpX1?*)E^vjYY= zYkft|Zv{4_FmNj5&HkCEYsu$5J_r{A>k~PO_(1dJ=7$%DC%FOgM1$sU>8Zo<+Fu~p z*Q=UeemyYo&W}*W8z@1xM?C8KxauaW<-h`Pe60YT8g1atirF9wY4CVa97`{%{wv=; z+1u@n&6OWdOYmOgoto`9nd0RuKd&>1RD4LX^hNVT`OKcfM`ZyXMh-4fLu=X}QIxi>8fhws)z>zwT2V&}Dp=ov zjwy#+!j2DK(OvKeb9YW=MOyD` zHn>&8`!8^(u#|n@{FCd6DQuAQf@-&t->L#BaUzQUxV@5`cr*+w1yMhf)*=x zoV}dHfw3C!V@7Bp$F7vZWsJ)HjZfH!C*S(Kb*aS}>Lp!YXOK!kJ0i_y`faDq(0{xD z2nKPgCy!f>tS;~fHvM>m#5OGT3{UYbx{Fk>IQ7+)$Du0qsu}JQUG(tfXy{piOu5-Z zkz?7d-zLm-Kx4tYk?-DXIZ15C5PGD`+vJw90ZrWZxLXgDeIEVWy`@oi_L45W?ta$< zBh=UUHB$jU0?W}v{okg+(3ZlKg*x%X zHC`?fE9u5v?B)a`JCmh5_IysX;t>_gig{wKP81wYO9{SBx$nUv9T}2xaDa9k!ka?4 z&DbUi4gv@;bRiJWVL>8jdxUYU;8Pfn1~cVN`R_?Xi*sJGfqsoCbiK(uHypUK1>z!A zzcac|az+3kG3G|YIh~iHUwuMQs#il7Q@XDR(`(c~9Ou#QwU7A)c>#D{mj$BI^UsQB z7xL;e-g|u2fw^<$3=5!k}S?Xg7AhdpF^JUM^F zOR=@eQ?P3G^fD@hAATp$c>}y|;(kFo=|N_TZQM!K*wUvt|5;ABU))UOa{#8T8=p!D_~U8%ME>V2Irm^m$HnxvYMmNC$e1*MOmbXBYvJt*bW`1 zZl%R~Z_QFf%3Y7re)wrsQgiulGeY6N<00;VjPvB;e+PpC|KLiUb1}b z`5L?bC0VV^IW?ALoblV0#V?F57jW(KJ=;y%-;bb&k6> z!0N^Gqu>83e#7WZ`$k6l-^*%8ft&a@uz!c;G_D;OsdUPuZW_44LXBQ__Q(5^QL|z` zWp=nMwRRArI5a*G1PRzqnKU?jGy=MOA_knp2fEImd2qC8-M1(B+qU9O?5FO@g~`q@ ziUEPRl!rvLu5hd`=J|ojU?xJ=48cAEcC|Hf09TKV^Gf?R((Vw{{i)&#Swe1@dF_ z8bF7y|FPH!Ep$bKrghtD#m02`dBkvBzdsx(W*XooPL!RJ!_^jDZTs&a*I7Gb9M)hs z+C!(PgGdydXSb=V;dd#1YTSeYb~XavtesuF`G()j_UAli_Q-qbh5glUxc|&{6hQ3r ziu39m5)Z6t@7`?stYxs<7WY~pqtLi#@IPZcv(q0}=kfO9b4hyKeyJRERpi3jWuj3Nkcbl$TzOQTl|+a_wH&*%phVtk^V1ad--#iLN77V8e-0e?YT^! zf-HP+q75i=@h@uR7aS)VE_}KBaxahk+X!O%uYwB^P94otejug)@7Z3Smk0BMn*B6v zpMV354hSh?c~e8_r?@Ejo{6}9f-5|!J>mlv-R*u)`J4n;0UmEd++l+HQ;B>mZ~mNFY%`>JuCWKvbnPFLrOAxRE)+Xt}yt4YA&DG`lK z`7y57u`AO?yx_);#vn&)v1!MO&1;9o=l0aOqYy5ZZ z1?$>YqV;%#ds``o!_hVxyXpE4JEWHC@kz#hhZ=;tt3%0+z@_d?|A=NJD&79wGWo%P z(%wYTgS3r(0p#bZS{*x`8XR_0`thirMoGNqs4H`L`5)xT!q;>7s9dL4xF;iAC0TT1 zfP|s#-gv}OAEIj?N;S^BZe_oQ_h$_6gddG{ndaFJ z{3p4o5Z?DIu-fPK8|mU4dE{&pq&$9x}{~okfwzMlJ+Tjnua5nC<(Ge85&_ z`64SI==z}c8cueu@#f|oSyG^N3$Z*1>-~;V3o7|LKNe0MKe6>STsPbFOuZRb!R}zz zcFz@_i*lB(^B|J6rrT@Ya8V-vq)2Z8opKVK%SxV@4qOB$aU7e~1|>Mrq)Wa2dn^4Y zm8tFab)!=tG_x3jYhEmbe+(G`QT}dF#Ib_W=%M`wM5y2}$XWzOR+r=3xSscSDy1VS zDMimsiD~n%qigf;X+yE6@gt_V4=(f55_A4Rmnnmf8;gu<3acYF1ky+6-Zngk4|cA2 zgyChD{@&=f@4)6atG(O8+w0Nk_yQW>Y0+t2cJu`UT%6RxzSLN`UK+No{D8}$MLe%5Z7xd$z7+H zq_va|EGiLjYcUH9xi5511H5|1&kfa(>s0t#1^eMm5GKyaD+bCw4xax^0m9a%1R|Dx zEd1+sv_CkVrIy+^Txtd5L(1wNn=$)c>tu4w8r|#J3dQK0&F{aK#t1+sat2(mH(;1Q z=zOg*e?=Bf-e6@4YPMFKD-$^Q3b89UL9_R&L9YmcuLzdv53gQJm9)qglViHSw&l#z+UO)(6kwwhneyUv$=c z4&H zwY{VMxu?@_;7*V#@Hh=vZCQaooPCl(v||t{?w>40S2k&S{SArw1YqczbymV#lKXp8 zO;TC^Am-wvjQs0`V5sUl1pWa6(N9_h5cXaCl0X|bH7VOGLpBu|aOXcb^mQZ7+-+O+ zWwZi4gZ&cX_w_olH|F?d*Hb|E#Gy?T0);5%b}ajZwBJS>ncnpO_Q~0L=a0qLSy%}6 zKkc>Y?byWMqTL(ATr`x@r>T2un1M1cX%EEnEFjYmBdkmmS(^Cx>j7!31XiitqVsOB znK0ILnxm(VD?VS(^6KJ7L{&UuPOlF8B2Xc6>l@8>FfMw~Uvb2lCe{AqC!Ooh5t5rw z?6#CBZdJhUx)B7p}ImJCvuH2<%YgQ3N zo3;Os4HJxYYtnS|nqq`9$%vK@+m|f!u`nE@_!nRDk6{iE<4Lln_nH_&dUJLNe^ zL;DS3P(xnN@w+W))Rb{=^V2_Wgn*P`Oc{ynf1NPseSdg(lk&Cq$u16Z{C6B}4U>3=a)uaH0tg_D4~#r!ql5;4_VtN_)sb_o6B0(t)Ip)X7Ov6~Dq6e|Fw zpYm&PP(C)k9UHm7pwz`QsMse}gOYyTPDS!=-)-zNft-h!2S@euiZm86!15SCeRqgi zAkLdX*>8Wb!fFq$uU!IE!FYLRwmBJy)UGoQI=ueX`R!K!#1H?To*UY^Ik_oELCR`bWUXv9zn_v)e@D^=;u0Ms9Y|P7MD&>*TsBrGq4f5OL)4i# za<~Qos`b*53M0X?HI$NQ_)#qByNegESw(?*Z%Redvh~ZU7g0#cDI!|kO^U&R=LX*= zTG+}T_B%aW@NOrL+x2`Bh@`rX5OjKM>X*evOD7%q`z6eZQ`95xMZO+mvc%^?7s2=+ z!->Ust<%q(IyNmoj7YCjk~I&ry+cA|ZVL@7r9>(`^UeL`qbxT7^y2LSD}RQfMNO`c z#C=y1FC}eK%I}%m?JBhm3KObP#m0}uF*F}I1WFWN=XPH!e-FF!W+ep-7Dv!#0PjVC zT><#uJsSup`*_0S$2BCogeM{au9gl!9Zx)o1ml%hpa0lQN{4Ix+Vz0K0`Mz6?3avC z>ly^H6DRA1-NqUA$~IB@9Y~D1zN!^nS|QBkxz*K$P5IuM>yqotF(dxh8LY3k$P~GC zJNQa~_+Jv;ALsBCMv{41_o~bJr1kzKu<+UsY#7$3PuDaIX$ljg1TP?&c8dun`b6f+fPmOfc3*voorAuD8!)ALz z9zmE=$M(#ucTl0&f)2S$r7i%;8K-AK7e{pAhX6C}_7JKR!Q>=*E zI>zmtr1{dOf&z64lKZJ(FOABJ;)6a+3FP~I1>%;DVV~|x*b@YHBXHT8xY8#0=_2|4#`FMq=gy>8??~k+8Sri<=(^<)lp~ z(x7CwP&6=LW~EkW(uA;#Ip)W4GFVCdNL+Q3??o6xP~>Ize#cgUbMRg&d~VEgZ>@8D zV(L#8Bhc`&8jhMSpM1rQNcvVm<^fNn(c$ZFC-Z^v6>d@A48ne63-!K&@ezQI0NjcM zIm4fR4GVL52{XdHDj*+Mi0hq&PoJWMUGxj7HFZVAh2mzd*24onvm)(=CwVs;vtHb! z8(Nivy(f5J`3QNSY_l+kQvB7(G}iQ}XWJw{Rh!dbV;UeCP(eyS67`9(AOJmjvm&>$ zlAFXdqog{#Zg&OlxK}*-bZC9|lgrsqFXM(dbfl$&EaITOcg2A1wRA9|>s;nH7B-A;3h7$0;GOCM$ke znTned0rm$g0EK;N zDLIeIf4j~~dU|lsmuP;r(3G|gn)sT}*`Ie{1`H*kkBYZo{Da0SjiJl}@#nQ4HCTB1 z*ev>vS@?e*4;J6$pUL4-F`U>sXSMh%;F!^83$qK*nu*H!Spn#m2K?M`f4VidAc z964PLdw}u+G{J)IihQ#->zC5Cz&0Sm4}6}{*YPi3uh?S!^rTi>QJdLk4=~-7{QmA} z4usypjbj8c)}WgdJTLz({aR44rW)!b=(}?l55%NpA?+XY-4xE%MgFjYyi~y_UIw_H z5f;U*%QgQZ#-w8p;=|WtO{BNd)`}++rUNwaSKbG&Uq?iAq6rm37QfK3Hf8u1>9F_H zlYwaAtw6VV1n%)D_54O9xasz%W13G#^IPnDh4W)$^XK&(Ev6=yoqx86hIr{(YcPjqnS0dIglTK*jWdpr!eLkr;J&p5gns&Hb zc`F#s{4_L?{o>36d(v#65)*xDXY-LoHT7<3=vBza)TTL!wa1d^=By(Cz%w;b;g1@kCc95U9Rn zzI~K%GFGB(eMqj~a2Qcv3U@wx$6heU2BCF-EJyNxnruGA;cvtJbL!tlfVM=#lN{#) z4NK}~@~oVa?IvH+2w=%!tB7+bc0Ee*R-HnwFCL5!!f)jKj##!_aB*J>ygA}LGXF%f zm=XTk={<~2?$JeLLi3HD@^Wr|%hso?!~gVcGA7=`l1|sItgZ>L3yXP8Nc+#4J6iXJ zsWA!cj3s*FHLRd{5VSdvK@CW8t@5YDi$txkKc5|{c6a>2`X01E~3MgRA3_ws31vt+DENJiEr8BW+} zv%`C)s0`sD&%b}}b6{5l48Ko^Zh%fS(lKeqLBrgy2^mt-T+2y*@(<3}+>2{?xG5DM zl;?E3zf_IlZYqD41VTr(;C)6-CQ6#s=#KRpn;D{z{zg3BuOx4NyF|>LU?^S$VXN>- zdX?KJMwNO6QJuj&m!|{tYVcod>XJWAmk%Qd<1UH3e z3yX0ru`B%}3b)_}wFbrGL}5hZ($ThKeV%>Ausf!PTlF-bto&kBN>u&Fn+@jK8Q`Bi zh>v(+Z<>M%m*Z3Mea=a?vKn_$s@RqKUf<~$?;eKRnQ9HnZ0sFa!>-JBuk4G?m90Ps zmS#h0s9c7=;?ab+m&LOS*PfgHK)>ZZrKfM|tgJ*70C&1t$SWOFxaPeaQZiW4^Ka8M zTEJtc2DL{C(F|^j5%Iss5ZM?>WSS1XfMRl7_RwT)BF8rWuaxl8t_;SO<7o*N-Q3X} zfEytr(d6EQpers`Lna?0+fgJ!GyPDmUu?q7{{@3EzvX(I)H{W9kwO+fW++hAtP7$`Y@-OyKm|JCJij8#Te4JE&w3oa+S1`XXN4^!2|7Wsq?~-;?vr=a7N|`_E-FE zEPE&={pK8g?mQ4v2GXJ{W&?+FOUA$Vj_rBh=H_%mg{v8p6!%D*2z3>!G*rJqni7A8z;wiCOhVZt;3!|9xfM-^RWFyi{)#7W_zr{q67dT1+DxI{BvNk%ok zo@Dd!DU`@dQZ}=Lr0kY3d;f{0EX&*+^g&uWFP%PCZJ1PlQ@G**JQmp`#Wh3Tu>ZwN zsXigqr9eOo7g?vBcP8B|Z22-m{hIlvsc-6xW4$@6{Fs z=eX>H3uwH*eUQjtLAm1cgY83?^BG#+@(*~RibD}UXfAp4(F4PvNukrBruIW22l-~v zd>6Bg56qE?YpbrcT%KPP%7Xz%WWjA;2O_ zzy0!a)Wkby1BaVnMdzVNz(TRWN9GO2E%WjB_8W|TxL|G(fjY<^1qm;4#Ci9(1a7}F z$qz(1QUUpOICJ_7R52-pMh6<93VAyj89U9(pc}4&nT?H~c#cy@ECDB_5||$G_#1L` z`{>zqRgXjx2+a!sQehS<8!*+oyt-=ESJU)=Xv_l{H-662Zj_NQfAV`Kmg?J*xPjXB z6ga{9RaE#UMt=Upy$J%3zq4<&r))&V=vd268jsvXDONCeRcq6{4k%0v>&7}vVvY8G zrvWEdqe^V9rEqzoiG%Z|1Rx}OsCtJL^u5-b8f}V4!P8EjDSpd-3-D_i`C4;P4pR7p zt4KrKxV^f#xB5dO!e>_%~x1xshps8f^f6`A1 zTP$J76FV&k@?A=>+lptg7~$S$;Mrzq?RJ+=nzCZ3rZwAtv>S7GQWA2m?tIcvk>WT_{TrDw+JD;PtZ$m!g7EYLiyx-oe z=3)h5oijW@*_^?OEaK!N=h~;WDdL9rviT=0aeU0oy-&fDO_Ol-!vOWFDpK-4KFHR6 z#Z;%K5Gn9ablk@?hF=p6Y7>TYFT~+}PG80Xu(hE6>)zt_H-B~&Q+&dPbeu=0McUr} z$ukJY2TB!Y+&+Ngh*a8R=j(J!rBt=cGIHTVi}xyHn9Iy#=yQj4-)8NxnMl?pP*%%| zCnc?1o9QvN`z4`zQ^r)`jb>JMRUX5=4y=zpl*Uq|TGZ17gu7oSa4_ql=LyWZB&{%i zV0|rDaygdKrEc*zDj6o8^W_nDyQ$uDBgKFd0SXY#{ZTDJ6M9loK!q~=z7T=Hx?dzh zm_#@H2s=}R>?8pu?3l+Ru5X&tVo<_0$cK>>7y$n|x=*F`Dr3SzeP0ZZ z(@N7Pw6(s}73u7Bz4l9;AC5kvUueD~vDG4!vZ5c9r^O)KN zAn0{r2(q$0=p2>DdGg_mOv-IT13Ev9cFsJx*$*fFb%#aw)XnVQbO#S=zy~*MhwY)jvcFvf|jPcZ%$FHf|o0N5lk7(0qZrGNHD?@@na2O-F zV>$x}+&H0tgn%LGbn4O&Iek@S^><|WIsoyx?#{11JnqKlIOm{_w_bl+G$A9IrUsiWgU3vh@d+TIWa}S(L+8$>>$^$Frv*N4q^1ZC^ zTY}4;1P?jawj$Z$KYzu&lub|2mcQ*gAz%sf5FWbJik5d^cI>>!ocPMp->1T>6PXZWh<7+ z%lLTajSwXwY5XvA+tCL28YY&^W7y~kWI-vjbHMYf(i zQ{4-7L=Wk$pbzGoefNMPmn2F+7QS6!lAID!LXO=$+YD6Z#G#1{Aid<-D_a9`xXMx4QI$7Q$r6eMcVaGxt!(Uv8QJcVl(dBX#_m%**6G=*M4z9ptE3%c=4X~fj?BfrFRI7fQ zXC2rX^LVjAySbJh!Ogh|z`L{ky^lH73F*n(7a4ot@Gq$z?+T_d!*d!u0<6YO$dawkN;1(go^0Fo2ffdmob*hx#)5N$(+N_T9 zKm`A&y^7Y+Mr|QqKG?I>KlaGw^6!7jCLx>aKWTfTMZ36kpq6p9jgGvsELP!AB#BF!)?Z6 ziHwYt!-vz0%dgb$6zDmHY>2`K`Y2sLjrfoDlSGkoVWq18JP^@X@DqX4?%`N@)bL*)5)V`W5u-@Ws6>w8h~w@iDAk~=Y&Dj+al}|F=3<~6 zf5izR$#$rhj`sE5YMGAnZt0Qg$#72BOt&JVl(LXYk@G&`kEZussaRJS3pms3_^lua zk}O7D5EdQN=0z1Vsu`En&P$sVZ&Z~ zuik`VN|eO&Db7)6YtB{?Ouh_2NaXCku*)j)jev!p7~a3(Z>g5I~{f4I?|d7 zWt>u6pM}H+J{Mc+8R=B~J%i?J(msew+X@XuD>f-qNv@B;`t{?upw5a#2Q_3xRbIo3 zL&y+sPi#q++PvA&MX2dwTX%6o>s$A%O-J@s&I+TIKDcwY-Si#JpyMnyE+d;ImUVjf z7oV~-0eXpPrfEzl}FPi=k8FEdXH|ARpw5J_+V_9vTtP#b35y z-F`r>nXm_b8S!_)(Z4xgP0`q3MV8oLJ%FFZNS#<$E#k3D%SIzeG&J5gk%ZZ4tbBcc z{S3a+vP(i!LVda6u=R2hX;_g`RLg5w6VX;eBB2!JyhFMNhj+7P^L>PcTAzebQG`=E zIGl~XzW5!1sf_+_>yi_%0bITNZ4#FlEbvKZsM~aq;m+o@z*@iM(bJdOdH0yZ>(|HW z{O{iqMm~`4u4hZ^5zxr>g<)URP_!;*&2~`4QPBNIG!5y~4Y@KHkOxO0^{TyqSZ&ri zh+m`#w!eUO*k2Nl6L4vpAP&X!U^Wf}(}Kz%>@{ge!}^~(-@!m_;;lID43G(S zmMc7-3+4RkO_d4+Gx5f#R-6^Sgg?BWo+#}z_!hmUY6y}~Bb|gE?`~)Ncj*lF zxm~F{8QZkI#ynizt0&GOr3J(}{8!NjeJFxG+nTDl{j&V%&?{!Y}a4 z-k=?%dL%~3X|3!Ujizd0W49PgiW@dx&<&#sMhU;gwznSSmAL~oaagI^4iJ_vZf^ZZ zsR0fNiWz>Db3GTbD&9y4I5pbR11{945~N_e8*j5t?oZva8-QS^LzL=H(f5#6=K}I2 ztzfJQ5;F7qR&6kT+_XISl_s1wWe`W!56|(zm_*%I@9z`)h5E=Nkn#DVYOdSj>~#@xg1do>VbZ3I&YPiX=G zsF3stE0q~1#!aADQwS@(`{X?%sFXa~U?8wU)0t)5N)?%+FT3YI9uz<^C?oak4+>pK zta-`Z!I7VJ6sgs_`A%m877UL*aw2|-BgADd8Ie@6qVTI&um?2X=y#4@YlUDj zNdUPKY@qT<86Qy2H?f){XVWtPDqj4Mk2STiQn>SRX5NzXpVV`uOR2Mv(A9vXiL9gKK&|P}GAM=|0^Aas_|a1xvpUdfwD!d|-FEB;lV|Fpu7>qR}qU$cKyILbUUp>{m5#j-_t zX!@`9!3)7e?1)FmT>xHZZ1KO560#`|moyt<&P5o}n_P8n=y)8xj+z&~H6iw$M+fzA zd(4!_%^U~?;a1v`KQX)tRl2PipwR<5lp}Rh*S7BtkZ4Hwp`uPKg^p9sdqtj zL(-LK9GOj7v+8(m3c*Kv`eXHq{Pw%}K6nY2SLxk3=<2rn;toGa&HB?Xqy0yveNuMd z`0^}zC`rQ*sAA`mNlEUT`BV8wF?3=$Ofh2<1@J--CF9(bjP4w8-39tdO=lK6;Zhtr zc+$o-)Nbzq&C^Or!x( z8A*)EpHX`0UDyRat$#0i{`QqD`Zv;4ix4$&O_J3OxABRpnF~06X=-K{Wc;)(bbR^K zzl}s1h+jIw9~_r}u_}l4+IBC)hNh;9V~$%S)6F;~iUV=&{M4g>9+@bf!G?uf*(^w0 zhGN=>#};(&jw>mE;1q$5z-7^^DCpeZ+tMPPDy!4&pMTmERlA_#U~|M#0S#tZPD$qz z6BrvLt@%(Y1&05;su^M?G7)l&p|KS?6w&Etwkz7{N^7Ti>3scv6`hGc6aF8^UBx#_ zCCa&!tCF))WGh1CsN99g8Oa>EXH#TuIYx+8lB-C`S(|(A$z6`wm}_E(W7Ce`exJYL z^LTtd@AvC?uC}?z!xkmbYed%L7^70p18+^m_q(UM#nKW%-OT>n+Bb+l zSqH8|`QAur+(M-);uX>tGc|kis&JCVLCiFTcIM*wLY%(W#b3b1A(PkVD65)K756nZ zU!1QDD_T(#ojel4xaZ=|lnA2wdcIZqO_-UrL~QZFOjIuJ=a4CWL+<4QMr#Lb=G>r} za}UK&8?CNGz1K^f!ekRokg5?WhAa*EQLe@kU$}BRBle zl~PIZkT17oV7f;I@M%24qOn&T#%ZhjPw0jl$xH3&1x5sALWow&=#7V%$|iVNEQO5p z4LqBiwQ&839J^6njLC@)M&JB)*hQr1dF<4ckKyN~1foa7T)D+A&o$9&94Y+h*=~x@ z%Hks#N{-F*wd0&ON;QE|2u(KiE8yby>4YE5&N$D|BXF_KlYo55o*(+2bx2|I4LB~^ z?5FKhc*p7S1e)v6Uy3V~x&nX&>BuW0ARwK5fJL9vPRPjbRbE|Ra*&*Ts-Ylh8sI^X zr9a8Sjk^6c^+DjZt=6CSeiMAPb}$oR6K{YWK2Q-qOU-;B4YhktnZHXPgXvpBeN^)^5%}xrU_rdc%d33*q;Y20HZM&X0bm zJO(=|)FlC&4kyHGrYO&qQ%GkcSR^c`9UIE@a&8g&rXT?Mm70nBFOpIC4Ila78t!Lrq{E!Q#_v*6R__?`ZP-ZeUz8`VfE{dGtsw#QMg;-0?0H%LxEK6Nt`L@w4?%v%Y=A~fpKd# zF@^&oS2_Jc#&&4l{aSvq-Yq({;}!Vx^8NV;pkgF#kiD8YREuKq*yTFv_#>$uRW=pU zjs6ku^j~5Z2{|^MN+M$%cg{<&9V`Gw60eyyf>9JT0q{M?J44f}8|zzX2BOWQU#jjZ zB|5_0pjSU-kG*~F#e#VC+6^e^FkE`V45_yi3TkvcnDI|#e4*6e*=pr$npT26OV;; zGS?{NSCyn1Zh!e;`expBc6$a~E;o63zh|YEaX{ixwL5FU_#t}BhAE>7bSv29=Dj6t z#O$Y|?9BgL2aqJR{Z~TWnY*W5sv;Rr4=TSMHuwnM;ST5jsN-2%ddJWIu+8{Bk$6S^ z5_Y#~rQQcf)|MCnZ{8HVUtRBU*uDLrdr@Skvl<@YL9;w=DwlVJ#;CqnPrzc2NtsoP zH=GQacFI{CS`dc6i8?w`Z2B3h_r=R=Z7eD8Umwa?I^W0M(72{;AX9NroIOx$J-avr z3D}0M39HmE%>&R&Mc|d$V{B3QMxV$WQPtcb`ZMSJ7MmfF18xNsRAHPfp3b*p7&*Ro zMN}7QMXfURQxwV$TNL>GLRc?+i3~Smjo99t80Ffn=MMKZ?9VnWTd&dYhy66ayIFY) z+=%5P4WG-Q<=}k^1N;BAtI|${GL#rSkb4uTFedDTJp78JN;b}Xy?!$ z_8rsf9Kt?ghHm#EMGY=|eHL8EIYn*925V#!w_+K(KezLZrq>}Svl%M|e_ z+2yZ3ak4Z&d?KjQzauYB0|ef0?|ty<4moc5Tf|7N(zpN9SdDl8@N!qF90VGQ8|yzK zd5hPFE@AOHJZ|{*q-aV$)O3-j2}|31_uf75-w$4bQpzvzCbi4iMtC^7Cn=>Gy!^#G z4^aK8RPL=auT;#@St{gdl%cUWXl^4!VG*@5_VMXn?=@RJ$zl=xNH4wcovlDccc#*8 zb=#*nMKzMh(w=y?!DqN7uR^Wp8S7;63ZEIv+S6(ZO{IQ8DV^D}jwueTTtE$N;LufxV^OO+#+psO~ocX-5I93%G6mctSgcFPGgxBzwLYI5NM1w_~nX{A%- zQ~=hgA4ezp@&>B)N8%dXPMo`!EA+VX8YxrY?LyLm5k|R7Q;J&c%a8+He}}Y*d+7ot z3jm=ZNO5QRf+MK_3&U9h!ZqQu;(&A7wl}{Fe^n91bm|caHnK^A4akvWjmIw- zR>sehuo(GwESIH_SFPuRA`b^K7W5VJZ6cUi4e!X-WiK9hBCHFF|Gk=*bQOK?{Dr{p#W(XqZOk*8qrS>u z=a;5ZQ9DH_5r&de032c*a?-p7T6f`b9elxdonok5a6mu#RJd4)vgSlZ`Td=nHyxP6 z*_#KuQqrJ9kiH}ES)RHw@yeYEJ7g!A+;4LN%5mv9^=Z?Qv+d7V7Q-ABzB_zFrRR$XL;n*&xnB?%ty0QwqX8=6`=H97Add5 zgEhoA+cZXOo_Rr4E#}}EZGF>C2PRo{4Zu~+J1M_6 z+B|+8Jhpp248{tsGq3Y>pI)@V>; zn&kyfS7nZdJPeDd1v%9~SaTIr=2<`o!O@uM!(F0RBCM#=>0R=5Nm;rzvuj5^YidNF zR``BOU+00>{Eb!e!mcB5>#Gp68Od{|L5Z^aqVUT<8SabV_M>tJuJE)WP7dbDL1ONc zVrhMivCHag8PMlW$Tz(z4(CqBszunvuvkSD?%TVrM2XFYhbQI!`?&Yd(^WH7>d)!< z{nN-d#(qJd$V1mT9cFja#ZgNe&LIl$?+Nu#BM8v!;>SfU5iv=uhBI!-aZ>>^(A&U$ zHh&XKymV0>zYo?0R)&CSuY~j#cxv) zI9T@!Jw=tz?c=Szwvt53?o_uPjImq+t2~L48}ewuEXCV%0ZgRBE|^l}vZI2)d7pXt z9%rO;7gnwd%f3oGaOd1+fcc5Zrpv-tC#><20gn{Or+$3Vv9rF|j1_?Aeg#6WO!RUd z>+nUWHMda35L=2@S%G)_nl!mh|FWTrHisA%6RK}J9SMXYVkR`s?l1D*oumUChlgSr z87&u&&8+F6UA5d9`kmOKK4Fxd^77`nwmOcJN2~vKy6J}4bbl4Q!#8;XVdJMp1;!H= zlbbX&P^%=tQ4^8*7-?N+G<}NRJyp>=+Yxm8r}NQ1cdRf-kaajIMtE*W9u%mj1bZCV58=2k zE_ORNGYs`vC#>wgbSV_ZlOPO&UMj~%5e<1LsXu|*=|qfOymXIPRHu7kQn?H?J*Fo6 zmF2{h2I}8NlEo4;4THSQ}dFv3UkI?<)NqdlxK@_#9ti2PrKLi%2 zaO*zEQiWN>(O=fO{uF#=(YIAyJrwNVslH3hQFi<*pKE7?MU1TBV%)U$E=R=V#n_m; z$i7*Vo}QqVOJ&#Mqk0TY7cUxfzg6OyLa*}UQc+A{e2C*w$h}KiFY)>QB#VSZ0wrgG z;>i+3J!SO(9#C%Qsi1E0A@JdR1W^P17T2A|*;3Fq=H1s52*~M|OZ(}ydlZ}ZUZn!` z5F5&xsid-4*m*Dz*lieL8WJg{6>kIlYlr4|@DMluPQzK2;5~`H8=nWtH&5}3OYWSj zXc4BFp+z&`D-p&{s;a*Z=rnB`IFBnk*MjD0FDg4@aQrdWGAYjj9$1Xu#pNiawx%+) z72r+Tv>&Yk$i)z9x(hlQ#QY&iLNk$Yy8Sn(l3m!Q(sqC6`s=g>beQXeXvB+Hbrdoc zyhm8{^D5Oj=PN^d=DrcE*LJDq&uc=fKJI(oYW`r{fJ=>s2MR9uZlp^l4#0C(w0qF<3R$nCK;ldd{ zlP=_V)gQ@d$EF&IRls|+6<}&70V>5YYmGBL32tu#`!&IjD+D-&05g~7bGQ$KOJfDc zz8}HR6%D6Wr-G<6Uwokb@(9NkYE%+;wik0!TSQdQ#MhSg8)WcVvb-kZgMR+EvtTx1 z=rU{5g=y$Us(m=sX>%UkT1^6TY(_HB6u~&HRp5ma;R4gfg9}kWj_h{A;>E+bznO;% z#LOz0{rRc%?ug%?91W~E6kU59#om^aM_;y)&mEXhS=KEZn{TaP?0=ZA`9y2flXk#B zWqmjV&|1>$Z?#XbEEF{V#h&B~BzQm0J!{M5PC!fX(0X_6UZ^IDa#t}F;4Zx5N;GQ` z-sXCBVR*&*N}_rZ$^}e|GWszC51zdRwJF`z9yDVT=^BEni%HT(76@%nv`2lO>kn=a z$tBk=3=Xx|XfnSCEK?Q*b+x^=j#{i?E|>c6NQhvHwRZ`)%&WcK{l0~<6CZL_ zBDeE#$JH3kt2Tpk;HpLYj%ui78J$s@f|>wxB; zV!n?%v@;e4kNmEKwod3BDn)&KN^wls}WE98?}`ogG~W7%*AbR-Xt7jhfh z#SZhfOyVPYs*AqSg?BQvajV2uHQmw_{XMbau*^&<$fJ#GM&Gowk*KWJdT3@}`F$qY zcOShO9^A252-M?~mBO|gXFI1FPtUyP5C={U zr9)lL_vbJvs)8-94qU%-fy3#QN2&nm3n$?cc0y&!gBLDfXy(T+|FG1R`FXi%WAxnH z-aknn@`?cS^&nt4KM}uRBU7;Fgr;uyJwXAIKY9HzOt^lVi;7`_E{&aB;uZgUdwm>}*NAV4eKUxa}N8$*BzCE}DS3MX>>eMm>eeYEy}#QXlt zX#Y-;I-odap3l4-13llvCJ6FP44l!i>s?B~Xxth_72%pV(}+y!p$8nGsyIz>sXE`2 zsbL=P%ssO1GLXRL!nVO7BZ;|V{eENNehua4>#T#1Y}!^B29^U%9z1yvkl#LhMGTZa z&rz0ARdx~F6zstom)bLkc4{6DbXh85}FxVEdkLi z$&Z_E!$W6Nxa})i>;>^%qF}fFbfT6#5720~gTxR{yR|%7m?!hX+T4Sf1Kb1Lvzc>& zfKX6;q)Bgq!#E9#{s2!dhkM7NyedKEh~fb~Y;y2Jx5a?)h*+zb_a6hV*c)x`;Q1#w z3xJ56(Thc9qEygNA%C!{`z+OlzSo;v0G3r3-5A8zt)@26_A}r>sl1)8n1%x_X+x?CwjqDxeM_(>kwQ?t zckV}7=1c^~J^588R}Yp}4M4jApk6l1qYv;FWwW93p6V})%ixtad8WyhYqet~1Gze~ z-tyxnHlIp#r#^oN1g}D_%%=DS%RY)@-3r~NPw+$kWIO+!f&R0I?>bH;3d468s({1B zXr@3jzvZZlCd}va-txmQ#mS?*+%=J;8yQy+ODkHXNTM4f38%IZ)hKKzkGPv^6r~^`$$~7=Cv38mE@XnbOb-2psK<3!<4&L|O{_KdwXGc%4-3eqSPFI>e zbKSrNYy76<*wnj%8JhrK%_RWj$LnccB>%+M*IQ(rY37Dw&lvoZNQ}~|Fkps(^Ouy- zc0*+%G#^z<8yYAdf?f6s@t#^S=KAKrhoZQ5GEN}DC%iOuZX*XDXp}u@u0xsYxW_ouBxwM}`0H_=wyA| zE8)_i>OKbmw$;eho9to8`su9p#>P@i{m>v!HYrMx`by5{s2fgqV%IN2u``G2{;S#} z7(C_JHL#g4!TVKzH-;cqyTWYUbYJYD51;o&OW{neeF^8u{&=>3MOrA~?FdpJV zSYd`@e7yIF=r>t}q62JMgr{OifCEZ+OqL@U0qnPCM~vzAVAWSinbTGsoAj%8aAv*o zuWD3^SdZJGJp`)nD#ZmjSqj)I^?gr($f>AJ$#J))lJ(;mu}!}FFX04CDff;uyZT$@ z44yzaWcc(;REg2B-keS7+|){0hao1Ky6u~P!(lZL$EGcIp3i^I>#mUn%_C6l5a^P! z>!#Rsp#cEt6KG$x)xQV)s9bQ9Udl5Q!j2ysPa78L&HdLqdHuyUL@dr}NJnn_or0#u z)ho3h3FLS-gf8mRizhfvtzM0;@IyPk-^a6h9oP}I+0o=6~N{Rb6BX3y4 z5iV4cW^ZW|en}IQMT+TnetP+OC=>YD9ENf2e>0Cg{8J!oHPOl6dW}=^aM*Unss)1+rbRF+Sba7% zS^dsY{r8^f?G9m8-(u)oUlX_hU>wvBfuHDZcJ$scFzxx_sGe>&>$_MnNuJCsS&yi* z?S#{Ys<=ZKzX4zFL(&!$TFy;eGq<}lHtC1pKHZ{AsJ|Suh|q}G&Hj5`YQ6kg>-TLH z@Kyi8(;^duC=6+%3mPF4l)6`@ir!|39??Zz7I ztV%vhgYW=#7VO2Wemv>Gq}*g@;q;+w3>`V;kYxK;6FPKtq`3YYe^ONz(}&E_>Aq4d zi=*$Z4@FD3K~IDg#yC21E&p50#uK=4t=!6S^zF}6jtF|OY2C#@@z}oC8anXk#M0LC zd+<`)JID$k59QE^GI&PGf^LN=Mk)-?G zAp#plve>m9P|9#iZEcyjfDFB2Y_A!F^9a*j3Pm!I-(LKYNI0 A4*&oF literal 0 HcmV?d00001 diff --git a/harmony-os/SherpaOnnxHar/entry/src/main/resources/base/media/foreground.png b/harmony-os/SherpaOnnxHar/entry/src/main/resources/base/media/foreground.png new file mode 100644 index 0000000000000000000000000000000000000000..4483ddad1f079e1089d685bd204ee1cfe1d01902 GIT binary patch literal 12430 zcmeHuS6EX)+pUO#NL3(IK|}&d7YKwF5CM@UBE5tjTBw4Q5KwvxB2pw25vBJIB27p@ zOaSQt5eZd#CxmkF|4+F-=Q)?(#XNgvmzlk1)~tDFz3+~Fs;5bRo%8yoOPA=i9zS|^ z=@P~5f9V?4rAwDs!Yjfq4p(5Rx~i8hRVUG&*j~LT%Q>2AIqB+Nx_^yhg70E+c&i!%2~zqE0}mxIX= zz1$7|sWj&3yL#7D|4uLjQqV+x(Rz4WC{A9|^m@1A6`BNi38Cf3B^aJyqxF{TjS&2q=3$BC zB1Fu04C;%o9V_Yg;Ed;xpmge>%b<|5q52W_pTd9o;Qty2mQ+-Peu)^(K)RH^d5byH z>AGB-I7$|~9l)J0H_LPDsUUL#brIHpjO1>dJ9@_5&W zLV)s!AVn7*Hy{o<1zLA_Ky-TWzJ_^1=W=Gfyc#1ssqeY_2ww>;ANX%JT)(9uNHOtU zeqU2_{Wu6pLvCMBLgy+dx=13ZG-+cMrBf;#8KezD^}_F2x>_Nob0^iXEv>aML;8RQ@@sN(#bq~VsOa>) zW9RDe#_!zLkj)PyQ<05AjbPk5yJ^|B6q=sMX2L0JE|(P%=v2$6+4QL)cu$c*yt`EC z?)p#@xE12zK?QF2u^(xb0>KieYWS%DH`?=eOiFd!6)WRmCo6Joq6}7e=Nl_;oNJ{1 zu&szm^c0s*wAxfHSlk^+hb)aB<&B?9+_YvxC1LEy$(dDJ8J)d!>rwz?q zGTpJ5&uVwR#t4%B`T{*~RAd_Unnf&`*9c^zbZfsVc;v*@=BHOCX7VbyhnS5G*Pik} z@`U!W&dq$A-&GCYAWg@rG3W6ANL_2a)|;&HJSig{zyfyO87W{;ej&@-)yx~eu|G6S zO)U5U?QD)!ey@XcxEKX?m{R4VZN!*V9gT}6_lv@YD^}}y4OM(*#%kMMBij<9x4*by zCkGRQ3vqoZ)HvQ4oY~=kh{c09u`@Lzqk8)3R+$+hcYuhqajQqgq8qWy8X_QMy@1+T z0&yU)D$XzuW+GZpAB%%|^3*{x!r`8nOWhu6>t(2mvERH# zwD(@F(UyHL)A@d0q#?|SOaIrK7`~^_KhtD69y6E{G70hSpvkOuvhEmR1(|2efAmi@Xw9*}m%vZb>kVqe?t6*aL%179k2-;CD<(T2&{-rQ;%g&4b= zStwf@&UH8&T6lBt>jybuLy}~>HTF7(kmQuR6(8*l&xSQq79o~y=t@1Z0aSiA&-LWp z0NQ{@*q$n1m#1Z}?sFj0=6jxX!@eHh_D<=qD}vOG`kCQ^44In=iDu`srXYt8{4c&) z7G9;S9(*ydG({X#u#N%3l}&Yaq*lzrY-E%htNRQTrjCrX1NMi~a!soU$|=0*dXokbDxSFnm6OHLV@%5(K&ZQB%e+ZFne-TrP|veCOrVj;0pG zdbMMl{Z%MBfVA6b>SKLi zXyRQXFc}Krl(owbvDh?Um&9l0#P)rbdiZxK)8=RY8XvSG1@0=@vGxtW|3E{`T&9Zk zC0==A6=d?8`t>?}z3d12SZ$YU4KZHQPf~|w zJD7n^6bjSS+&0Kq6nxhj*9}9qDZC~A`nzEz{<+9lxx)v#qaCsGWko<{ahFVncU-R|715> z33|Jp;8Iq?Z)NXe;h$K{z8#lRB#JC*XUod!9+#hCfkg#-^FD5Jq@>Dt!SzYr@q0(& z;I!1>qg(PU*HMX7>G-#T5V;IOw~4L@XQ&5le>B4Va!sx0P1pm1PMa!%L##WB{CukUKwQLR#mw_r{d1DneIIJT(j#O#-det^FD zbdwZ-8R%84+Bo+g5iyd(a6x;*5F0xuclibP*ff{7PNPESiBNJu^Q2?h!4}38?XKcb z1cb%?RlBpM10D9~`7(D`#uzQxY}K)shcU_}%#WJZ`~FU)C1j&^b5i=Wc7uJW8^-NB z(rs3^Wms@#S~)+us~_(~uocjV^vU^euJHB^upc~CY%6gqBXHR3{FJ}D^V0uB8xrdo z%j>^}CvVUV6jaGJf5i$e;gXng&>{)uK?nWhEUaVrv+x8njtfCz>cqP8uUTn1`McQ;CD+jm zGle#Cefq~0!!v@W2XnNsA~8j@Gaaj+fT)QzP<&gR$L=bGEJ8^z*tHxS)sZ=vZPV!4 zw*)4rK3To_7<;de8PvEPu4Q5d;D=g00$bPnaG|sEP6(kDsxwc2+y=l@=8Gy3^DW?X z$=3@Y|B6^8mUadWxX-6z(Oh@9|3%Nv*Hz=bA3)}AiK3MrA@eOvp)YSd(Nf|v;6dz-v zI5xYnKImXz)PTM}jxK=GJh_OrE2HXqKgh*KB!U~;4W!DpXN6A98^kNt%~i7+I+`g5 zW}~Qod0A;Lw*Q@m73+!Rfuir!WXqcTd5mXE^DWV3AUSVk>5EA&b6Svd&!yh*!z+6( zh^>CvoV~2?y`UJ#Jho<+PlUEw=Y?Hyd8C#Oj$c!5d!Du*w4OQ9G&OxhDmQ=)tzD()srM-?#=f>aw-$x}3Z?qLOIJ{gnZu zd`Y3Pu@-6CD7)$*a6189&`vfy%c7^DmCj90Mw>5FgU_yh15-*dsMPOLpn%G&Gbq@c z)NN;i4jF!g3-}@w-}i(YUbp4WY;xYi8`sa3ep2V_UXf_!7A{;Fhp25CGF=6{xLd&d z!Mvrklt74KI=0hsCRMYBXM0Z?v1sDfN=Y&W2dW!hUyqiiU@A}R-XCxbIudes32?<&DQ!Hr>qn`aYQ?jSq?4X|x(CCDAB;b=wcWVCH1CfwqU1di z!|LlwpE@R5*{9XlM;`OM$(VZBN$c{`%$ZT3S3aYJwVO}kw)@4_EyP4SXgXkd)Q z7PtWeexnE98(N{TMKt-aG+YpQs`a~e_Y;}upm;CRXlTWI->sMI?cj%D`$7K@mQ<-e z6c3=23v>}kQ!+Z{G2&KQ99s+el!e053~lQJc`8%`$;xt_RQ&16M-jjl$HK)VZG-0esPL)%m(*xgTxhvj>YKkE?dOv3G%g-W9;dgR&pG1FoW|wrm7v|b_Y-VU zKV&S7NcSkHSjm4nrPIy#Wvwp8(lbN>^x7o60ICQ5m?QwOuUY9q(q~<6`0+a7 z_`Zhdli4>YUiT%XT1&z74m|S7pZ;||I*2@$Zd5=|9{V~xFLGS|sAE`ZQ=toXwPUzSz%(Ar!@#M}4%I2r*Ca<9 ze?7@cjo0^QC6zocYls~PXjm{I-w|^|?Hpmvl_!6;&?vERiS^(A2e-)2qxQ#IfuJ_M zgEhyUo8K;fE}w8OE$6nq26w$M-YgMyeYnhwguXF-@5ca=0xYn%I)Rl=_lZaUn5tgl zq{GPw`_E=ilA8s)Jy=%ks{*^ijmr0SqHYg5D%zYfzlqy~#fp6GHI7wm_SN!mo*B=(4jED535Cy$0WQgpMk_!VjQ zhjwgVnse1csNUVP_rkF)3q*bk`=D| zRm=kyT3qxBA7a}d4b433h)JR1r_zBVy6)DMRyM?5%=@^}YMnjurETi?w8)8Y2lox+B2Mc9(WcW709kmg&QO^PydT;QZ_K7tmYO8aA8M?Y);N zSn^>S4^jpy!tF}ZAn_;hcCNY$eyakky`&>*Nh{Yf8H17GR#{9&%f^ps6IAlo`0a7| z-5WT~hwWze!uONxb4D$Was0UyM#f|Al`@rMWg(+oyWOL{(2>P6$`ht&d;q3uD6W+D zQQKN!nzWpx$Ya8CUKa3dgn={(ad!Lm7qDcu`SB#dKHvAM#GW}Z>EZmS6yG22dWcVi zef}3H%>*xQE6XidovM|h{PD;~31ijm0ia9g=-tnlFk!0PDn12luSSt7gWP{nbUK-G z_;*xp66cFpR2OkYg+1wGZF$3SCHuNOh~T{QxmE}&DI?a%s+Q&BqRkJ^37TgbKmAKA z-lXW9)FAv@J#Z=C2lSk4@W5q7S0~BpAs>m(p{^)b2MCFka=_0~yTtPvSKJEH%6&GW zKv;f{iTBYXA0^wmTAmssRXI(3556s-FYRfgXSs2F7D?)Muw3X(n96>Fe~#_y!;5dQ zdOQ?Kp<{m8r8ee4PPIETr3Sr=L{BgNp=Hl~>nSiYS!vY-rs7>zJE&K9>k00!&bs>P zD`CMT*(GNFuh#^fdZE?R`V};&3K^rq3z5UT^^KE~V+Yq@nxU<{+Ug^t(FEIk@f~5* zgnEN(6_Zcdmg55!i|T1Xn2NBcinnnFghvgYxT5oG<#r&$ky|k5SaFs(+Vr@W6W!wc zhr8=;xACvw0kVQ6m+uK@w0M_|3*`l1D1SbQ1B%k-HMIa!=~kGkCfuQ8^C^ZQ&7xn%?zUs@ zJv~f?$}gE-(aEgrt|vKx z;}Q@0S-w8jTszP4_+Em>MvCg@+IT%eNk_MIr)gA`;*lhuP%vm}{=>pIah-$r^3{Da zp;l8BZIY#N3v`sN%POMh>Q=e-o^BM2OK_7-ztamrbZ{m49XWXIgg1Gqa+C!XfX?gxVvl@Yc z?lm`jKKariU3($HdVP4LPtp4+4mV=+tw*rjI~_q%R6DfIW|6`<`}My)W_VK!6c^i* zIvi5RI=c%+#{fOc1^%pnKBkmGk{n2 zC<)woa7^dmGd|$2v77jNVg{v9cP;?R<5Hz&w)i1YTrbpNc6%p0{Khx8hi!J94klTx zC9LuDS+2u)()U%ug}~voR<>Cq}#OQfXF2)TCm)4nk4dkJK<{Ji<% zcP30SBMi`eN&Lves%5zi8b`z0j<83Tc~cBqc7F%;N9zZcNAe!JR3!n;@j1h z1lCS;R&Xw6EFbwYNCw_`r4_DiPb}ogRDYy^watxfz7Xy(zQ=RKaRMV#RY}`WgLrrF zVY?S>T2T_0_gmfEc1P>euBpQk$h-TAw(GijhS$+YK=Tg$zQ6?>D}F1vFkHMoukc{a zEy_ED8Uf0r#&yr0HH7|2|B-{vV9-6x6%+AEp3Hd}4fvb`f5|t#1a^r!L``xWv0pYp zK_sWYo?M7Ka~?Ti?_2#VSWzD;+NOTq_0`+=>-+<27aH>r;wtxc2mAJdsVzr(62hGT z)&mW2D1I;#ot)2O9iIWid6J}Na=-qm<@K(sk9ppYVwcO*IkP(P8P9ER7!PsMfNBn& za^K3zdtRPHN^c^l9lmBs5m>rjxgOV7Io|5p!v}X)j;Ax&u7K?;q%XjX_~o%@lPr_8 z*9Uqq$6~D2?gL>l^=mP&+~8z3yT!99Io|+z9QCQwYR2S? z(t}t86UG(B`86l3E&Y`O1p($K!sj_~Szh|(peg0h(+?ymZ?)sk6C*iUD89q@SVAIS z4_&>H|FtF3pZ<_*-;w|rv%!y93`xISUXVWp-T~!8n*#@16?Q}v>{P^~9I69_ z%n*6qXY%Yy!%fWkW5OADjlkEKjP5d$8>`wRrhp=ra6@iEL)prjHQ=o3@+N$WN7maZarII1Zz-rqUrBVRY znukG8!4Q$))$$`IcgoPA;izr~)m2%Wl&%&EHeRmOXUJsiSwge{CQ5;l6K*f{(Y$dK zr+Ms$jZr918R?`Rysv0Z+#6wT~L%t0b;+Q^{rT$Y_J%=|3^Wd zt6$*epNax{<>cRLLyEm2t&MjM8j1U)pYxwc-MDWDwN~$V|G#;ney}e?-YB~f0-n-M zw?G0{JBvufZPvKoY*5O85X8y3)1IFwLkMFr+5G1knQdDje8Y{BGoelP12*9EUN%KY zxk|^L1xHs)rNCp_@p0*`=#9{%r)_7IsX3T&x{b&X;mgnjUOMtgKs#ylC}%kSdtkjl z8!FE;zg-elNMzzYzDjZ0)^Ieq?HW_G)|Sg=4mBA1EloCGZTG(+tr)OPwRZ{J7OY5O z-u^rg$|QACu3Cq*Al+><3gPrW!35XM#YAriTfXw+!m_NkpMN$HY+wKfNr4L9PYUX6 zzlS_jplR*TFaNt8ide7lbsipOGdSE!+zhi$@D8y%FCwjQ$r9L{z>FOk9`c^?Kjmj` zMuYzJ3lU=4n6Q;tr@a$L?%8~af{fraE2*s=hn>Cp;YCQ#>re~C6xoCO7}(mj#Xh*k zba*^&l5yo%qnHQd!W*<-IXZ+8vnMb>c^cM={07F5{v1ulw!aVecf>C42Ir44Vz);s zT-%=b<-{YEZ*nD{U;m4uIi#wyf4G^ggB0@5%#DRIbN7hz&!Bb!hl?A6#(~|dZ%%iN z%o^Sc0oq?wn5_;1HQ*s%km5+`HK!Bq9^dL$ZL7!o2j@&piKs-)bi>dGD9BCC4PSIk zrGJIk0P-Fv?{`4G0`eU>*i`V_XN2xXw%*xTUlVENh%_|iZDkl5p@Y866#=@Xg{cbE zjZtS75AB(^xEogv2B)1x^m!0XZdCqOZ~=~2%7kuI!6E74!u_j2iau*{do^aD^2Vk^O2eW~KSv(BzRD>xw` z&*Gb6ksujl^_Fg<9{Nxn%B8jSv6jcmU+Kw5-Q&psk7EU|G|_)%rogKwNzemwy6QX^ z@ujX`ZkT$alQ%3oWJ2VOJGz{G(ukN|LF&Ga)nKml$M>IY@1F)}2mL&m6~?A)CN|YS zLi^lZj;aN$DQnmlc~AgqcDB7)?<<0=D*JMD zM3%;`BX_AsO%3+;YjwAbOnkT+m^;*q5X>@S2hO@Aa1J zJCCx~6B|ewT}HQECVls)>JqY95!(x8tJTl^D9t}c_G8p6;&167Z{2*+*qbjZdPBKR zwYTwFdQwnL?Q_fZ1S5+O2`Bi&@(s_P_cQY7?>NOU&FL}U5YmlM6yw@TASK}~;pon& z&{?aE)kw+rf)rVR1R!KIA&R@6^&5tt+oJ8h+P)7GWpbZ0xhG1hCCSz8pFjdYT5mJUum4y`e6ST z&@%+@8U+Bx-^#X6vpu~G2`=~;;97zryltTvX_;q&`r%A)oV7(xhxX1-Obw!r%_aBq zXumue@LLi`iFY=9t~-zHYJC&!zW;W6TKK3YgAe-4E5@wu_HwjtlH4Ep5vqLS-2C5$ zSxHdkc#a7g$_vSgCJ_dxxPL&~SeaPflc=j>z18KsBxhHfhSRvim6wzyuJBI@*m2g@ zc2$Hh#1|Nide`x;s zFEY{lfS)AO1(&M2`md$eil6mNBxu2_M(#la)vUt>ub2uO+!3=jb#6Ic2xq$*jBF`n z%L9sP{NK&^17myQl!*yca`I%e*{%{^D5ld#5&5Dbmw2He%xl{Z?Bv@+UmIbjXEHB5 zH5Sh@UPidw19)2ZMmXkn`O@)IsF`Fbj+RLtb$qTJ#B-vXrZ?7??}cA6N56t|TzFj4 z=rAukcL+Zk?vE$J3_QP=HeaZiJ>sPUrar&8Ao}%X-FpDz+o?UsRbtr6!(ES)@vCo94^P>R%u%q(-9wy%Duenrn)jXuW z+2hV;WWLbrH-awRI4^BBwkb{USY=a|U+=L6IJbHc+!%aSb|KB}H$ z?;wmaMfCf`2o^LLsVRHayM++C2aVlLWRbMjawRSh!|`u4I8tjLx>H>?ZR&ba(LJXj z?DRP5gyUNUnznwc)C%qsQ!aTlw6i(@viQ+~|0fLN?FR=&Mz z!m?8%ms9Zm`@?A{S+a>p-JQ}TICnZa{gktp_;s>#3Wv_=7#GC;f$M! z&TRADKS2F7Grq42P=N2(^g3PHSv9Sr5khe~OZap~yE3UUWM-{Fh{H-BGK9MOV3L#y zw*TZQX^enrYRj7iXkEaCLTZF5z%T)MU*{_RxA-*;G{sl{7ry_e1h+X~HM>NyBnnV6 zzcFEEZvv5PId&nY^VG0nqu!l%4Ln9L8OVmkfQi1}=-j_u=t%I1_~|`SZ_zv+SV@2>e1;w+Y$vY75F((`NKQU2vax&tTw!~HE>c2M3z3d>g zk@W;ee$-qtx3IgJ&cQ;-5AmGPIIdtV0YQvcV7G)N!(PWkx#qq=;AiOzb$C@x+Z zu##CR=Q`hVF-LGTr?w9-umq+&6PrkTr)T1CJ!@XV9i+em9sS#E=UO}BNMwuBrCayH zAub{V#`%5ecrycz1$eSV8<2Ikv6CQ5E=h^K%3m6h74APzqFYP{oejD^Y7o_E2b3p| zeA*LbkS?zNs8`f>wX`CuZF=Vcnc?D9l|P;QF8KedIQiHkm!f>Y3}# zl9AL|w=FC#e&CG1Vj1SX@K&6z&wEdwI}i+9}=0 zD)hP8t2qSqGq-zz1>nRbHpsOX+Ou&rc&B>1K5Z`l|60?OVRG!%y@dyXhC`Y)1x&pBnbuTa%|7f^nM;OIHu%(W6&Ci`84e(2e5z z*ThM)rgG_sjP#cQ+Xs8;_5jS%p3?)1Cd0epUI+qH6)RAoaWyIr#O{wWN#wI+_de=e zPHAv`+(8DcYwZezvF?o<#{{xGw05-!dGx*J-i6B-YsG?>W6ke;g4Hg#P+$=@?s0UEI-*Bw6RE<{1I7> zjBlz61z%K{w(Fbs@*+5i`|zyRlh@qP_iu#(*1Wcpz$is&$q|YHc+dRFT7N)#@B@znBGn$2wXOi+ggc5BJ<+2( zlI3ksg*I$2(gaUp4h9pJY${1?hgh6#mU-3e=N{4cTb2V_4R`HbSASd)X&1AJD{hd8 z^}36_R=S?hhh>k{b|Q{V4g^$!<)__{4ZCIAOzE}*nn%8FpA_Bmaub%88)q94qdSj& zU&K}EwoAH(N;V`V{ZfKgP}7P8xX{2STb>)D)y3#SF&&=+6Jz=_o8pqGbBI1lUdL(1 zD2L567hm`YXfrYLV3fz4yv?7yE!3uaicqZ7ufRny<0U&B6qh8bcqsL`r9)-JOxkXy z+l@a1(ptpJ`{M2l$g!g@DX;KZcoPP93JT=vi}|dQ!tn5*k@U)brT5a*!NEAJ2Apj0 z3jNsKvYjiiy-sUG06+A3T)f+N_X|`ZAX$1+M8W1ZaK3Nm6Dd}Xw#CnL+A?Xi*n>}B z+g^J-yeBCQ;(6yjA1~5bLwIzXXp>6syw2d^&DXBrf$G@}~y*QOne;u_UdZD^Cl zXxza$QKpgXzp22W4GZI|8N{0M2?78Z`$wi+S>waN@uSr9`u5+ghvrjfhcjQNuoDp; zk9szfi0j_VBAd2M+55}LBoF!BASF5?QV6q5zf94lQ$2goh8#I@&N4tiMK&5WOgt0H zRiGPL-7G)N zj%2#teK$kweDwBL1+DK?B#>r?tjR02JIr zUq=)|zME?3CA9?-DRGfqM+;h7w&xgGmLjhTAOdy`b%#?iM;>=l7v)^GADOA64 zy}x#1eDIpJ^iQ-mHzp5#R2_{6(~wo;npi>z4tuCy@Z6Ovw1EGFOaCWi{Qog*{?+*F cSLciz6AsI{U0tD9;7S&f z3`9H(<`G*WCN>bN493AFOi{!!!L|afI7%o`6&6lXK&2`L1YumJiZTQ+5doQ^Fu|gz zI6Nvw1cME>!8`;4iI*N+z3;u_gZtzG5&vyF~^*1 z?S1yyXYbweAFzGO*PdLxe&gE9j&{c{J=rY}9i1#6cCzdq+ASx~UzXhiC(H6orN{Ar zj;qq$yDTU7NWP@ws1J2_*G}Ykx7%{iE$G@-7-eF^Y3#}`(v#ySiIZdTj}`y+a>=Im9Vq=f1W5yxR*!@kj+Rxz&v=+4_?qb>2v z^P8^zTt$BB=j8B|JpIS7`QY>Jz4z#w<>ZT>lB09T6nS2-t-LNa`Yg!ixr}^gvZsB` z{B;rQ@uVEqwOt7oA8%Sn=e2VBs;^`dNc~|xx$^LKH+*6BuO8<1`K9&UDuw8t_%!FY zoV0NZ!^eH~qhBH?uakr4K4~ZC5VHnAA|L9#J5r^|-)7;Y zUl$mM>pDMqeipwr+7#N+YO&F-3t!twD#tH9_S*S{wQ+C`@f*(uNuw}s=xXMh&DI;Q z;_u$0c(3`5*FEq(O?pz@6#ee_pZMDAFS)(D{hdnlGw+UhHaZ&vMC3y~_HorR=oT!) zD&Jv0*w5!@vBS?MX~$>r(d*!xjZ=9%U3__Gl0?W|%cDAF&TIVSk@)+3cqc!3boGhhYzil=`)k_5%wL2pqQz`Ju@50G)sNfVj zoXGZ|Q(f3+@xx0`O2~K<`L6lJ-SXStp$#*Nk@$Du%RKJ9@n>4_fX zCq4RXG{SB86?4nquk-Hy-E#B;AN86?zpBs|J16`d(I5ZXNB^!~KL7eV0uKN-_1L$Q zfhXMkzP+y=*8|%=cJL*vJ8JS$i*h!V@e z?gp)OZL3q^qPRQ$mTS*l z!1Lo9sgwA)pzOQd7ry0nSAP)8dF^z>J#;@|{wb*sK5UU+HV4!!`0VEJLKou6^E1;q z{-F(t{g8gMTs+F%4CL8B(dE++Be1u} zQa1d_@^?2B{4?(K#G2gBZ2YKxYj^wS1vv8wb2h-K`rtLS+C4j5oS5zZQT6pjk(( zJ4B5)x)C<~DS-Jn#3lX27u>p0yp_M+jn)mGYaUy>+T%Nnb1#0!>tbyAQ%)nklRSgJ z&7=Ic?ks-hoA@5fJ^x~JiY`PYkDmW0C(plGd!Q$Ex;t|N@d~qieC9rdJUa(Jbmg%% zxJoLcUW^RY7oUugb$iXkOVyLI8AJG+ zNchYly!4G7Y^6~5nrXo&e$8p}lUVB0m<1UOEOBY-ht5+)-??6hPx|GZjRV(b``>-$ zM|{PjUt-09)0*964ZWy4qG3A!iZuCL5J4vSq$?ol?wO2=1e&!;9t z{HK#&d2T{`aKZSSV$8nw`5IF+b?d?_&_RB2Nn@S=KEJHRZ&{wfFD-HANt+d!8=g@V${FeVy<@Q=p|RCl}k1iW;RIY+rXYw+ro1J ztScYrS3bq4R+FlcH(!!*-yB2t`NcV#59x0CP?FiqC-VdG1vMIuAg3o=Td=#P|3Z0B%|-@17rLGk-6p<6~!$6~POh1kU3(XXZO`=|>$d z!lw$=5_RyEi#Jr~RP#^%iC^4A^2m;K+VClBHe2;z6Z14*Mk&|$%X0f<_lmdugY8>E zPThfcKaZ0b)2b2Pn1`Dkmvb_pUZ*zC08jjo)ep|hccB`;;R{6kL;Ts-DL%Zk@M}Ec zYe??S-~5VIlRb~$9A!25WQb$>P5#6re$4=RZ7!m^$ICJHQwLq8^3qO zSIW*0ziJfhY2#Np#+5qaD29V6USiSHHu0r%dVQte1>d!Te30L9h<8T(gM1~;2HMmK zAIaG=K2h~u$+A`Ao#yL~^C@rnmi3*Dn>*0%_Q|VFij#Is9D-CUfq|-t52LPSO>Mf;|h8QzG9r>i*kxj)D&%wf12-@hxpQE(boL;`OLW% z&4ra*97R9KXL{m{MVR>LH~jeO-Z?hkb&`yq#K-O6lT$@0DD?-g)^Uzc7T&5n8gw__ z0DpXP`45D@vQE5>CYLA9MXJba02$ioVhjTWVS5bZ6(4zN`ENe`p5>!H^k})NKh(Lb zKhik@lUA-Xx~smjY)TJqEB4J>%kshNC(AGX&hhfC|NQ3id+))>f~iYr%eBS5L6diS z0c(T7VNUk2yzB*+mM{H`dzO#=6GzJf`m=$1G@nblG}%hD(09V$W~@UCQLSS;5BqEV zWae*vfSYo>EH@?Gc;aOFp#GTWmw)f}@_j#ZYkBJ*Le`;RxE%9>G%3oHFxKHSfF_;E zFF&fw_1jO}dg1SWTfI@g(_fZ9_1ee&mj2x4J1a|pX>wLqgaW;Whu>GnNZR9Y^4s;%W zx4i1NzvUU8TZ6Uq$a?oX>%J5^9jAU9em|0;-_C;e(1}uEYG}e zr$t+qTP`-spu!U-M~AgevS79|o^g>`wAc>y@e7Vk`?z91a^qxq>GOBXzxbc8ET8gX z-7Xxv6CigTGJZUUv*`9=vmA1gzg4h49N+Y^ODZ8#@KI9`q-_X zaPu5;fuSS!*@le$mhP;#HK&jK(B1NbUvXvmPhY0_kiYDk{5AHRoIkT@vw@Z8z;F1q z7l7fCCi(MA@@nf@5q}|i{jv8-IsM&M6%o3LI{BfEQREKp4HG$@wUJ1eYx}Q!%BAIh z`K$LWk8838tEq&7|H$p$UeKq__MwZg*U!9Rnw3=(J#1>imzU))z3%$*uKvrZuZ{Wd>ES!5dgNmrfBPTZ zSl;rks&UNFhD?$g9J)KT33%MPXFTyAfBeSP=e+&fch`Iedi2_(FPHhgB&G`tFhZFY^iGZTPO8%A6S;JedWE&6Z7VgKJMLTtbV@Au;oe}a$|fo@8QFpeTE;~ z=(!{4cwATZ_x+vv)3p?oK6COMai}`b-FNw9`G;R}pRW2^Ajgt*_)SjojgA<};ZV-D zH)q&q4iEL*eWU|BFmM=S?>NY;&)5I;`<6?(5sl{jyXGx}^8>dxQX%Vtv5PEo8w6JK zToHH6efQkYp6Q3Mqvhz+s$i(tXF7XpLn?CV%Z6Oqu_p_+nw!5{zT;K*3%heMNzF;f zzun5oTzGVll(CU?9of+U+nP1y(OpU zvv~w9Sr;nLG5?3p<|70ueyyDbUY}Yd!E0=`V+1F2S@%7DUU z!+3G5v_Yp@FhhD(9o{OXys6YM@?dLP0LotS!( zZ~o{ThY!62s*m!Sg&e-XdU0#<$S=0*Pb|w{eYqaXoLkS+K6Rp~Y^EN+{G*Qi6P;tq z8XuKI#YV0>%Nz^2?6yhv9fh2b=evx?JV#`6&=bQOMZM+dz(~P{OOO4g=JV%2_LA3t zIWdLGe~6_L*6U?ZoidN$t=;E~mp$XEY0L*5)a)#9%C_**_ejXj1}SaGL~lF&7ro-L z5_Il{V)fCw*fu?YZqYMj%cgB7z3S~eAahn{_@cQMlFic3)%3UY#Noj!JH4cEvRr#S z^9EDCiHH1&FTSjo9Q4r{^K&2ha-QnFK^=vKuFYqvdxW=7K2uz)M)&XO4}*2S)oU;32*?s`tzhPoNdy zMK~{~T*=4;PVlC()T`0MfB8pTs;kbv+GgKHr(Rq!;3+S|5(B&y+n5*@z^5dLrcGjDVs3` zF=w9B8T=Q$;LA>~9`X4+qVFJ-liI=f8qb5;adlP9$i*t%;M>z~dBL;M7jh(|v1O@a za}jzx7Y{1+b#a=fVe#WfJ$C)~F&^GD!hg8&3xD97hwY{wLOxnA2;wJqo|?br07>n| zdc9}P-SQkmio~mhtX%z&MJycY7!O^|^}~~L*w+vLY!DscBm0>6jPaAr#6u#lPtl}a zn^g8A4RF_SY<9BpclX?P?PZtsH(oFGD^X@u>A2cxb^Xba#{f#>E7Bp? ztFxkR`P@dmpq)Vyx9`@uFnA8e#&tpr-DGb_G^IYIlqLQGW*i-bW1&6e29O6Y4AR#5 zvw3QcRQo|aIrZklmvExE$M4X$oUyA07_9mhM=sXuWE_~5;nT=?xmN7c}VZTZ(}?rL~jVuDCHDd zW0I>4RkJL)P{rpZ{mdS{51lA{3Pf+T`jPlbs|k>vbZN6ZbRkPI+fmPp0DeI6t7Nc~ z$NhZ%nT)>k;6(Zz50&~yf1iG^fs4sKviK#}-Dl{r>Bu~hY2DR;F}T*pmL9|4wUTbw z@xnlPQdFhr&E%R&<~6QfTI+#VgCJrYF+`(acGqTfD_@rASLH)IiT<#`a<+xCqjpL` z>#D>_%Q%UnL=``~nBcrnhfBLfp$0UGM~}`pY-%%xL2Su?1!0>O+=jhV^Q|SHHsi~S zD~0ov1zlYjfNIlt^GFNNb-;qpg1EPAM(ME^ps)?4i@M~QXic5q&!wGA8~zyJ#}kr& z^`4JJ%2R4dCKVL9!V%6$c5)Gv^*q_xt7|K06))bGDUPP7^FtSfX;?h<0|XKb062A zIY|b0!pj0C)Y$7;i^P=d-~9Mh&zQKh^`h&1%>hsw!5hUsnpx4t z<}nU3;cAnu{B7X&Vn5^sgN95?k&<*Nw-dMSz$p_Pc^$xvIFk*X^*T}DEO_*uml7(B z&nEcAJ#m?Xu}#P#5u(vuOElFSM`G;J(?_?d0s0skGYz4+p=0BMwY@=f?C04B`6n16 z7Y+?9wH$J zAxS-==YiY@80*`{n1+s)KEk056AV77g?$%2H0xq(Q))9XS&VWbRL_G=l_J9>UJl0D zL}N3`NDj2QCw^L+J)AKpGPZ04N*&EdoH2o<_uVvg5ExqK?h8cD!pAn(v{$fP*#~QU zh>wrmGmlPAjvv4qPUcCCWLhX|Ka2&~1>W*WY1;yK(tBoXnGCEf#s(&kaR8=O7&`Rb z4)NokexjR!kF~8MOFmU5aQ$lW3aOlWOo#8pn)8ot^lQLVQZO5XoZ}x``u%x;$Cmjs zwt{}jE1RV@QuzczTVvNF(%{QMY#aX3$pievr_W(l1ZA{3C6z9Llh!WOKW`#3*AYhq z-tucRhL5MYjUq^yq;P4yz(j=;Uhu<*6tg}0;12PFp$~4~hxPm_+Zg8Ct>f7*BneZNsSb8?%&Jh@KlZTTrOg zc*d4a&)A=--&QSt^&=aCKtMfi2RM(tjY0_3lN)$zC%(pMOo(G{xaW#VQD)ml*8}*( zn%f398D{+~2NGYgRbLr0gOY-ta%{uQ8}bVGoMs=E!xb*`2zR1d+}H1qgGY~B`-@YJ z>*a;j$od&444i_t&M>U#WibY2>CmtI+6%Qc>JFq&fKMxFac!J|LFhSyp@oAfvh|$Q!ky#K zhS(4BtuuI=bE{5uez>A2b4!3M+hm`g$1$&w|CB6iS~rUj(~}eO8bJK3dJ?_67ebx{ zSHS|R%y8%`=YQMnAR>?_}JgGOix59Mum~lwBBOj7l{Dr%(^B9~CeuB#Ukb0`^qvuU*Y(62BICR)&Tg!A&&-M+!2eTcS zQp|kcb?_I5@TRuW`$zm0SeN?*o>tHfJx!tLIT3p}glz!EcCx$YvH;wLhF24aiOPLh zoyM4vMhXD7pn%KA%I|SJ3pjFVbc&HshPKa%R-zM#w$p3fhA+q*C$x=DN^`o8SMD%{ zlYy6XyKVf(AvWYbX0=U|B7A&%L$qy^lSpgCbq?mNVK#inCYah3&VIO?=1DXw=#`qC zbt3TAho;;JwjNhLV1kW_T;f+5&f5zw$zb{>8{!V`+%h~%KVy-DqlO+=H=VZ=FkY%TPJGOKbO-eUMZb@k`Qw5*kXQI4 zNn-VY-V}k{dvi=NgDj)aFv2b;9&Lhj62jH0Xgt5%4NV`a$nS9VFeZ8jwL3ZT-35mn zvUwAUQ9a=cgBJ%U^%9B`*>UXEt~NPJ9a#K=jILPgIq5_LF4);`bivL2J}%hVmz_pI z&(zfWn4ASNsVrtA?CTky6@SLgnCP>dnQ&s$k2bCduV@v=0M<$2v&?X_w&f?0 zdVL4q!ob4O|06wo;ixOrj>l#y;~Gg=-=WAx*pV-hTSqte=+)3!U&FCJJ(R7IGj_tH zSk_m_@)csRD}7KQl3@|As*N?`C_c!U@vo=O(oUUM9HYTXr$fev>%5uanu%NzjR zCb4pse%58Ff_FbT99ZTs=22SCWBp8Il>D>{j4u>gKeWxhWg0&$HJ{gkdPXCf61P@& ztiI#OvjYd~D)hvhL4pdPanYqKH?T(AS0xsJjcpoa4(T1TJw`VIoTCqRpI?P*;>dsN z5f0BOf=znyxkaZ2tJWn8N$N>lK}c;lWS?W5vOBR=JKko}KC|$3Z%PH$J5|jKJ-NqE z_ZknrZ7W~D$^f(y8P~onU3Oty2J4NY*@llDx%i|JpU9&wHDK(xtG@VU#^kYat*h>i zdSLC^jL7(-#cz$a=M=p%&kPDtW4)wR`B-^()-G4{E(m^LY+5LRq%6%7l<6vOPNhVCyvY=4yUI zIx&MxLE28(nmXlm7viLOLSs$b4|GCD7I{^>sJ)bo<7qB^r=YAS^^JFY6;xwEh zZpDM~;ZEeb0~BvkTQTEG0U3VZL5j9H_mXvxdHwoPMGk8H%GZ$DSUoG};o!Bp*+kXX z`qy7&0LlzDGC5UnIv&!hC5g%LKEG*AaEI$`J|`zF9*~_UC6v2ef%Yt=w?iGS=`x{m`*tc1v}Pz zf~slY{K=p-7He#u7L@_cNMwKhd*f^(-Vaneam*r{gTf>LelwEqaEL>^IXTI3UTi}^ zZkltHCYX)!fRgkGlZFWF0F?CZ*bebcbNh5(fov2_4=P{4lkUMPb=`l~2uhFxu>7&DseW}mFpI(L7m<98w3m<&s^gYwzKLS`@ ziH2UU5yjHI=Sa0E5;z6n)mm>R$Iaaa0HpF2H=cyKrST)6aY5j>Y2EFa4KyaOJpi`Y z0cR0NFVNX;eH&s&2RLs_Wk`!X1Ktl5EXMuVY^M5^Na4ay{PgzMr(hU*GqwVm<`|tx zHqpMHc}$IYj}CnPhO8RSa9ryZ-xY7p0CWe2u`wOua|f#J0CPySsjO015zUoj^|=$R z&P!8a>m2?Q`plg2TfXWox!mch;lqB)b!%4}(i&%-8hjt^C)?8v8krgXwGp&JSbXUmUuKNKj;seLQ@+i{*gD4%I@RALNg?5Nv zHQN3d?-dcg{ZuEQo!};N-E}JHlr|#Z=D+=Y^?ah~?(8cL)5{VsbD?G)a@Zyct*NHxP>~FNNVt39Nz-u{udkt;$vC~g<^Q~(o z@!$ErW946qkAsrqYR=YH5b{$F!kam>41*1>C($G?Qu;QuA8=!KcHIVdWNDr-8-7uK zNuNiULdrZEx{d!~v71dXW?a|C=vhDe#uyuYWb4hW)6k0ypF8ER{BAwTAx;YE-wb!) zU;16Was^(;$OUp5dXvkJY0hDAS|8fn=gyP6&xSuan8cZ0vW)z(=x@DiJPDG%HphC= z- zpYdSh-(EFF=R=BYI@>x#_%jYWdLEjhM|USaBzVpNLG3+y_(R$BD_RmMas$MWs~oG^0ClV~+&9ED$w?cD|Yz+=nu2k$xd2U}uu6PP0V zCo+iBf#`{lqWxs#{-;()(J&9)cV& z*MIxg+j{>(@hd`~jcXbH;1z zth?n%0u(-3tD58KJI#tQPuPp_{T#@NnLsv#(utmIWON>=r)G}FN{F5lNBD@6U;Bn9 z>MqnKn+0+&Jbe!0Sg#XY1|IL>WT_VXUT;oA+Kv6ir{@DlMjpC8`1rDX*N^ifn3Oa- zP>v=r{|3wSjsMrp<+?rvZ1#&IQ%o*?Q%fUy9{OfIvd7w82leqs-`IVe19y5!^8?p+ z%lE(O);9mymq@O`lr{MH-Gap%a!lvK(+9_5!wv_d}s`<0wzR2F;-6sG^f)1 zfAhBE<$Hhn)^a}|--)B-fGBwkg|A}DfUPxB;ADB-k7x(+!4Wu(Z^V|l+qB6&n>1q*9dcD_jHBlT z*vR|+hTp{?KmT(AyX9Nn__#hpI{B~9Yw%ik6(uW2wP}cuI}>`1H0k-6=fBTqX`C$v zyXpzH+GeRX%|8xjW>_S<&=S+Pnr``~H$Jia)W5&2PruNUE@20Cie;tIvIjt59r&b0 zjV=c|+__#ALk??qI+k=+1B_gv^QeSsUl&j? z;p|tZ|KgJ`FMscq_bfcG=0&dhz{tYj7c4!e`8Av9+C(?nNM0J_+A`~hL2+5Y%lGV- zcj`{^cVGXwo}+cX;<;dQvT7u2?0R+qYFq{XM198e*L=}E%d_>lL3~zo=0om&Voy%^ z%h9>f^lD0ytPpr zg~{1jZAiO~^T97J@yeh09w`1xwSh24F`NSEhCjRLSXJn`%mH@4#+$x@;up2ebwIl&_3snm%EJ(YEoj{-clclgY{Q#$UL- z{G^^VuQM1Gu)n(U2vif97a;}2J2D&cm4Ei0<mZtf?9#n|`tkjxXn6KX&EI1=R@*$+Kyw>;|^ zN6TfsKa#H^pu#R*_}$O*#n-X_6q!ggu8IzGT!q@a0d4&GoYsxW{s08 zxcb6`!zl91*VjDiv#}r4pKJ1goci!UFDRc`2%OJ$tT_0@2dCnL<$j-qr9L&M`lL5D z(Jg%h*(2AFmk(S^Onhux>cB?H;>YJE=cKZwR~3}pmJcYob}zo~KupBx=(Nh~M4*nz zFreXsw&7fy?>G)Rb7uLh_>fd0az4fHf;q3Jlg~yVw=Ucr;=5V{Uqw2b-#L3OowL9U z9j+Ix`1q<;8v}WtQ-xXig+I)9(3;nXc|pGNB1^pvR0~0A$kl-?YrweTR}h1GVi

c)ijgxDm}8EsRXFt3h@+Ufr7@DN z^55r2UpdZvo*$)c`MJ_3zXBARbH%T}ifygzYy6g*WBtspGU<*Ccb`wpyW!Ui$gZ}y zo>MwK`K>f-62KfvO2{S zXF|ni6T=gB=C>=mF~5ojWS?I%DBt!ouB^&}v*S8G>5&(6>bM<0W9)PIeSXbv;v2lq zgZx&0)nJZqzUPEz=3RZouldy~VSciFe9|fxrs_KoD#u$hYz3BTu8Twxs@yt>*lp{< zm_XbpVEfL5#v}%x;+@AY<0*cV$ZF-248A&7CXCUG-9e@z7Va=V8J*&{q4I$n{~M-~K{qUmg-Y{N~tC__Y!6wZ`uS zAN=8SKnb`wARia}P{>}4q*mFJ2rt$xz9z}40>2@prKgMpJ4y?1MK zsu;8LLY(s8tNKp-L`??i35r}^567PuI=u8S&*EdFoy9Nf;48%{S#m8d=h|q*N!*Hw zE&QzCc2jn4u4(uar*pTPKCQ7DC)&Cs49?>3$7+X~)XJA`!=HT>p7`~r%@S~FvIWT% zL)t28t$h|BY!xpHnSQNXihG*>p${(0U;hi2mrwZcOUrZh0ee^UiT1oYO{3$5Hop*u zLXEN0l1qM=vD`rN)XOLJdon_5oHz3`AzpsrE1f=|*Mk1={U^)6{EcJ3kodUYZmX=p z&l4~2a)h&L*mG4|<3d+3_?Prr)`vgu$Y1U7EWIl2?@iUEd5K>;n9zxxlFNU^0vTLl zH@o9AcfQkuuVr{d?>6N1tv`70$?|*eKGqA1!uC8^rS(s+P1LOQ9lYFac+7nk_^^=}_9|LQHrRm;gm z#jgtmwd-2xd;fSm;rGSZd-@wbDeXS|)%sP&lv@b1qs`Sf43!0V?3qvsHeeF4^Q(*h z^}o7zxuRcU@`@_U0N4FIMxo}rPTLvJc{K#}XhYWmowJJ2$Yjbl`u)zkPnNIv?#GvR zeQ>x@oZ)FOm|m&l>_ivC(ek;URCk@4f5BINBIPcJedSknv#$7sL09O4r%@qb_M zz2et2d?)PSD|vhJv?jf^coe^7;*5D_(i{GoNjc@GFgNZjMJ5=HK91L-#6s_k5ZsDS zGS%RQ&sF+5eNE*3{W~3);ByDsjH9O)4$S@$?yR>?gy?){V`EPI$n>{$7kZJt&E|jq z@9tl&>KhB0wjiX?fvux_ph<@^P`xU#l~@YcVmvoP|52 zFCDST=db-|m-UT`(xE24+%n&4gZ%FnLi&Yo)!)!<`8*?XqEn@~PlG4oI{hPQc|SBA-3UqQo@Ok7n} zIAZ21l@78Rn`X^sw|ukiJP&AnypS?sjm)BYgRrvd_2vm*-zj>cKd@`Ab&91Yp=>6{)F%4)7auKu@lUJhnvWozKNZb^uG+`E@Y3=U zeK~|@uUf1nf;jWRpXQgYuqA_|MTZQJmcB;TNR^GlS{T8}iC6rO{IH|tWqO{uY5h}C zK^05FmfvX7IMk$1hE*ehH{+tKyHIa1DdB;;rJvHi z@XysN8q8vy7k-&z&tLr~zqICPT-#vO+|kk)bI{UP%}!$rHS^6TDD1uXt~a|@W*~+c z8vo^wJW;Rw34f4ZJkG`2_D~Yj%WRNd2O^Mwn=s<$0*s{9@EYCPT5v)bA~e(n|~6M0EUxGtnrcN&$s(s zzN8S(XWAcol9+ za@NCPqQw`HsBTqo#8>DWj&U^~+CTP~&69^IHqX$ty#E|%_>m7|XO7~asM|V+|Xy_l(fh&fm#RNST>VcoN?=6S_DPi%0~BG=sQt4-78)-@|b)lahBHa~PL<9jHj zNE~dl9PG02qUPM@QPu+cEDu-Af8%z}zB%Ihfge*{9Wd$&G+)E(=&9+o!^CjO`cwNdjVRH+WU`h_MXAOitJp5x3ifW{$igPf9iBj$(b=HI#x==`-hy-E&gI#->XR(BW&pMdcoR19-nNcPkY4s2bR7uK27u z;T-wi{Jv$d3tg^Khr|3zu!D-f$3GV1rd-BjB{h8+psmB&uHFO}3e<>-KnIym}P_oSC zslstp61Dm&1NiV|^pEbaNt}ZX!rh1GA<@OoA~K`yhAgd{@foOROsg!`F}gM(u1!jB zP-&PeM7Vk8W1#d^)-p1e`o(13g|c~w?dj`;4_bZu^_E|g3d=E{cLES;rdxmDH283uG=7WUKG<2~ea{IxU4q0( zBCeM((XD0e;O571>R|^u&Ev*jpsQGwzvm-2(K$^ICifY)?_e`E(umG-isbY(H;sFS z_TV{-u;uIR9OWMt?$V=eCxZbQ9k$3lC>2^A@xz~@XvD&(_uWN31AO=Zpf(=jB!lHh zOT3|j8)NsuFr00(J`~5*Aa@-yCcZDeY#2MK^7+byjE?yuYo4B|14zoWZPTeh8BIOF zi#LZ9-0pPpQq1&2arSg`YF@vQoGhb26RLwnlb*1L_^M-Vlx>giHItHpV-y+pt6ZEK z556G7lZ4?GS?qbNp_S;OAM&IlDs9+mIL@;^vinA)D6z3H9OHAVWxzHP_n^luSJ#<< zbsIty2lS^g(Tp%sL>_Jx%DMrbLPR&IRuN*2au@Mv3b3wQaDyVnmOp4Ma3Q*l1@}l- z7!@6xqcC>X;&3#^WC@2>d~Pt-WCFI;DSS*he8-yHfN>hl!&k7gZRoJWX*}IU_<3Dv zFh%O=_d;$wPTu#$88_QzeaYlJH`gOD^~u}%0AtVi0{v!P<5awgzdH2uJ`V|wUL*2lawezA2~fq&{P;mfB?8T6HUC*4h6A&Uoa8O-j$RT~z$aZBVg6 zzF?cyl6N zdHw?sJ7Tp$XXHMr#>SS7hWS(q4Vv|F6FxR`qoAKa__u1W&%AQI4T^VKan^IyU>zfs zE|$R$NQPNwnbWKcmi{dLjG5%b9r@2i8f!K??SvY4H+*lPY@EblJRiC1P#E;CqroIW z@amJ2xy(A56v{9|GuaTpMMj+DK>H#%Xah4-!k=}#^ zneQH-ALI49-brtya+(0Rs?MoH;W4xa=7q~HKFb7Z1nBuy5&@vrkTKXDY=saRII;oP z3R%&P2^nF-NYearIVR*J3O2Ys934KH3%!qF8Ezacu`vg0S*Oab^yt!p+xLq-xy5gM z#Kw5jI=`XA!CkZ&zAqE&VEj1=NFmPhl*4MSO=PEas`~e2-T71-1sApc|fu*Q}= zsYFnC_DZcy+zSDb@&j)&>t^-n;oK7;%>Y=GI zf;q6^#lf=W>#ky4S#ll)lVVQT_DO*_|C(c%5cIB9nT$1w zdZdwu#x~{=-+@S!Al?*`YqRX_$W)w|mL<42l`iKk-%cwYqIN?eH8`i)kL=}d1?JZx ztLCs2KGwvGug#(X==ud4yo;s5T!B+uNNV9YMyc!;d~C+efEeaJa{IVw7aDzJFOkR6 zSlJt<<>?A3vyx@)YW!;#RD~3cJ<+yt$FWi*K*_8K6|i@y5t3Ja zJ+H|ads>I+vjj95MRGK=^x>=qv2joEMXBp_IFN4`AdHaye#ZCSN+T3ki zEEWhGJ-%>&Q^eAnKgqhuJba{|Jl+AxddOr{Cxi+(@50!IbHi4?hjyY5LQ=XVPTEpb zyqVjwx1@vOf~d3GC@cCi=V6PSGqd|Ua>`SZ|JP5mkUUL?=|EPi{@-nlH?JLkAw z*sMbLgtgvL+o_1?*wJfZjcXpC5>GR~M4yu?y`l7N54Pg1hB01ME2+8Z!14qfU-Yz@ zpP&@C_lf&Q^@(4j;1EbkPV$`KhCay2t@XoalE&DO(HG;)bGsV$(1$|8a365@r{WKw zNW$FkEp^Sm<|7b9uV3Ad{N#D~L@0goVuYqx6L^T_<{Zg#=0otZT7J0Sg93< zJ_mX2IquB#Bm6s#^rsweb>du#$y5q2icb}=oNpi;{UA7T{^iK)*yGw5d6=pq_?*D>mRC&iQRDaItw;A9 zUwyN}YMcO55)^&3H9%p>YklyFuHBgRqrZ5o{^}Fg-RyE2Q&BkPr4P7!;2dsBBY5kZ z6MOo=-HSke#!JD&S`O^!e_!8v^T8YV)+p1?{L!gB{K1puy1vT%sWe=-JBLXqC(&~o zh8QdS8g_rYT88wPo<6+$(H>5CKO8#&q^#c>*j4hprAvR9e{%Kyt8YGf`?u>?8Tz14 zS1k!Et{sV(!ehcu#U^0M9yMmukRS`=W<1D5*Xuj%0?f#3B#i1AuV%Dk0a#p(np`Z z@Ny<>{{ZDV5+@v)mOs>&&;9Vv>-)pHaOkS3YygE%;ePHnZ!h`bKx(H9HZuLnZ`piM z2ii=ClLN3rsu>=c{+jNjKd(=0rLpid^!u4*y(mWJPG6kjm0Yv8i=0jt@0q$c?3SO6 zo`T_+i0(Myt98b;JQvD(PJ8@c_^spR4R6xbATVp;gA^fWJoolt6Viy=aHkR(bL6>a z0*u#QIOR-CHs#1eI_@gp{LgMJH~1i?ZcMM{ufkCb2He+@V%l*Br$@ccN`(OGk)9u)8Cl^IS$70>cnNtJOD;^adIv1mfzOH@{j*A zpUGT+)Iu&-&YD8$81J|E-`Afpo?Sod(=~-f1KG?W4N<>A4H|trX(W)6k{Oa&+m(#9NV~FpO<-jgq5FpLo=R80h%`t-tc094&kfl2?<-(g>J|r?=r^r}OA> zmp&f(`pX~wSI3@L@|*kMoPV!t)up3lQ3afNHGkNJ?ukAA%&S+P!*d|=aQo0Nz5YfK zKR4s_UId|>uzYyqbjJt5=GTt(Ez-yS$U9G{Cqm(9+ajN> zgT~ide(a0*RMefm>R_qQXttNTKUJiWa#G(o>gibbxL(-&eO>l^>-4Yw{;}#f=Ndog zTpjgwLr5GKkp=Bm^VjU9%39U~*@|iCk3RCfSN<|`f4G7d?}tSDTy`AIwQL?;#$97+ ztSvnwvYK=4p}Io0?fv>@g@5oyeJpBc$rtZF^xS26hCWZ4#Yok->p2VeHu^YSPUGG2k^A|XtmgmW>+a9E=9)4OCk5TSW^(Rd;pI_JfySLre zQLOv*sbCN46V?6wuS}=FN|eBT_p(bFq*`MXpIA`Vg(EMp(umI{;a4t?=!xmyYV?&H2P7PMKv=d+vjRBWh(As6Lj0Qcn$#3?!%y6`&&<3aj!!;n$@xk0 z*`QFf2~yb7*ZgYBR84)J;s=KZ&x_vE!tWtII60`G5(@|IFyHPr=5zVG<@(X_<1hTc z_kGCwAo)o&!Uw+XL*A!{f;S*LxN;y5=0e-ZrK)pdNED2liw(!iVbw-%n7!XMpG8kA zGUJMmr0RBj5-MyJddQOpL{O*s7%s{`6u+WXrgQwlI?smCIg$&Q{AYgqCt0wKb7$_% zm%{TugWsEv_{Fa|uJO;}cZ_9uLpG0)>jq*Vhu`WPlbLjiH(IU~Fm-o{X+n|rIebs+ zBK*FBMohVN%r4@=_@qH>4)KXqe5CL#cK)Tu;+Dei@z-rsKEYOe;uO{W-~*^lGv{e} zg4af91r84J?WZul<4pXy&Q9bMAD7uEiayKu@j6WtFdw~+#;%<5b$dDfR;X#?4us;} z-~EhV6zs>~=Rof`?o~=VM~9%M_?8J+n!&AcCV)?AP=;fE71{~UeEA>#S{QucDki=r zzHybu$j{hvT>Nr&n2+r=zY;+&dlw*cHh$KbFJ$UN=-6jIG7AR2vDH_c$iN1FmhpRt z?{%2s!?BZglURd~-k|DP8~&9Flv)o?mLI$Jz3h>-Z8i{UeJRS<(K9vL#!-~$F*1Sp z9>4-|wb7EC2gB>kF9$2`EI#_O(HBeOdGZy+=Ze2BPH_+Mi?qgP47=j(>kB=mJ%oMS z9r<0iE@an9F`Z)KGra&4x%#2EIrCiSSMf=2pI?~4w>$UPbpC{gT;8zlrl=Bb2 zc!MuoiVfHWSDf^|NDlF(^ZW;&*`LSHX6X1EeyW$cIeN{P*pA<}=H;OUB#~>P2l%!Y z!u69#KlsSz*U2UJ{M*;+{q-Mwz4pdlJGFtZ-+TGiS1Ql<#B&y|xO2F8BP#-G95X!= zS3AtF&0v5*jT?Lk8~!j1%0_T}otooBko6is#Sgz&6@Aj7$ONp`$^7Ks*zOGN$=Vl+ z!3WfQyRB%BY(65Ff(S*v1=yWtyJ{I0gB$4W-~OP!g>&~BlI$ss{JeWJ0Y~lvE4La}LgwmJ{B^=-^LrxrR*K+!NY34Y z%M z<9FfUS32e(gAJbEtbl5ub8iasSIo+HYW6cI2(;PPCVrX9hj6>)HIID%gYPzH@6^%v zv^{*@-@5)2n!;y#NN$bBu|)+fn^0}89(_q=8AGE|lG!A3qm}-*G$sPd@g2 zSN`*ry_F8$fdaX8yu3>5_^=Mm3a>SxDq|(W496V3gthog+!l-+gI^0x3>K~U0B9_I z@g1v9#%%cbQY(J<)|7{e%NhR$c6@0R)3;{wt|Y5hT-qAn?23((Ie*Is_;P_4Gx3j1 z3^!RMCcZ=O#~*wM_}}BBm6H6+W|(D1K9`SA_)O&v{7zZehxLm7tBQH}eC`H%|3AL+ zwv$WC=ZSiwBbOHn*aasRMW->jDp-wcQfvqt$sDPv&GGOq`KuGkd^o;c>O`@?JJE_` zdU788%6;TNa;;()znFK!uf=i(n|UXb!}$}T5F5S&N6!Fu`(`Au^2Zij=Z|V?HNBZ# z{Jg_J&>P3Qlh3>HhAVHIXs5)?*?J{TB9TPPY-Gp32p`^F3!lv=`TY2MT!#Dn_EX5YDwXjm4@%zo zyA%j0dpPZ8aUi>rp!dHqyG~d+l6Q>+x9T-*oC&4dQmFv;TYcH~Spj>DJ0esIt zzWNO+#A`{>E5i(Xk;Z0`sjgNLsQM^ePYfMu`tZTDpWqGSgiZetwnduxeT7P8ynTsi zel~9SC}kpn5&t6m<~Z?*-@e9Xw_7%@1cxGiwOUv!*ZAgV{^YpI;WyoHSsAi`#H6j9 zt$aSe;%xY&tQ7Q@%CCLw|GfH*c7B0V=63;TLHuy07aBFXpK@e@kz6>#YSGcv3{ghz zzVXF3=^Q@()T&z5KP7&Q>i!XZTNu&$kfkNQnO!8-_aDL+?R~C8sjF4t! z6x@c9tB)3F@nK85F<=By?G&Gi4}X@LiXJ2XmM&tvDMDVeZJcH{s6W+y1bgFn`9~ZXTFjEjziZ(}(o3vn z`%X>ZGshK%2W48h%Jnqix>9=bSGbGC-{Va~Hp{r_k-l2)R5e=9GXJFTue#GuTPtHLO_kpoE;{;<|N8ou=yCIP zN<{A~WY5T@7mLhsKlK)EER*b9LF?v{dT-&+=Hpvd_~PVB{13->Hs|DD_AU++MKR^? zVbs#s_)ceV^X6!`7vaB08NBAP@4xarcZzYI{jMLv_MN@||G4r!x9+?3(b^}k&qm0m zIJo%3!Mf<)XVROminu6NX7e>E)#+h2O$}L)eu$)~=3}XaGUgyZ_V8KMnK#)7zjPHp z_Ts=j%wK(OAJ%4maf|Pa51wLAKZDR6(r+-k<@J}An;-pDHxE9y+0Rj)g#6$aUwirP zX!kYxQ0mVy-QN2yL-92;)+QS*i|kvrv|fAPK+-?Jmin%y1ZS6N0LGw(w2!|y(vgZ*y#F}>^b>-1db)Nj=f;xC|Ft8@YI zMIq1nn~#0+?)d1{!hey9e+8a5izk@{Oplez2GHqrSUlSN&@^wrvVyP!giSlmuO%9r zW`jOGD83?gYTjdlCEZT%G_f_YKb`yp!)N?Qcc8y6-5c~LFW-9YpKRX@b^v?Vs?#fW z*DlT`JnOH$|Jl3C_q|fP=kqnu&(d`7^YSrkS5(VraZMu&zIv_2t3qXyto_-1d=_pk z^vbJk!~$p|XLVszAW2V_Pv+Y=r{jaEb~--#@C&o@YkYyT{(x!uak=@SdyXFer}KN5 zFTlMk$hvZOMZ0@2f4q3@#*LTjFKs?eK|fUioJEMtmjUO-<02&yOE|p|V-%X=6Xv@X(oCxjr1jf2;npdQ$tQM<2QW z=azp~pZ|S`@O0`r&8O4l#eLPLy7n@?{`u15<>(>(HP?sj)ax^gp0C0^Q@=iWK*f2c zD)fL#sXs~F-K&MVM;neWi6M8@tERwteOT%%cv{JMqtu2a&-F?ld~arKwAH@y=LKKw z#h-2EA?L&VSjQ(K-_mq$Dl8u&b4}hKRXUGo8jtD{dqj15STlZy(C<7sI)2CQ_~fnE k9@EG3{4s5ok?kb>|H;3ubeVRY^#A|>07*qoM6N<$f~C=$asU7T literal 0 HcmV?d00001 diff --git a/harmony-os/SherpaOnnxHar/entry/src/main/resources/base/profile/backup_config.json b/harmony-os/SherpaOnnxHar/entry/src/main/resources/base/profile/backup_config.json new file mode 100644 index 000000000..78f40ae7c --- /dev/null +++ b/harmony-os/SherpaOnnxHar/entry/src/main/resources/base/profile/backup_config.json @@ -0,0 +1,3 @@ +{ + "allowToBackupRestore": true +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/entry/src/main/resources/base/profile/main_pages.json b/harmony-os/SherpaOnnxHar/entry/src/main/resources/base/profile/main_pages.json new file mode 100644 index 000000000..1898d94f5 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/entry/src/main/resources/base/profile/main_pages.json @@ -0,0 +1,5 @@ +{ + "src": [ + "pages/Index" + ] +} diff --git a/harmony-os/SherpaOnnxHar/entry/src/main/resources/en_US/element/string.json b/harmony-os/SherpaOnnxHar/entry/src/main/resources/en_US/element/string.json new file mode 100644 index 000000000..f94595515 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/entry/src/main/resources/en_US/element/string.json @@ -0,0 +1,16 @@ +{ + "string": [ + { + "name": "module_desc", + "value": "module description" + }, + { + "name": "EntryAbility_desc", + "value": "description" + }, + { + "name": "EntryAbility_label", + "value": "label" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/entry/src/main/resources/zh_CN/element/string.json b/harmony-os/SherpaOnnxHar/entry/src/main/resources/zh_CN/element/string.json new file mode 100644 index 000000000..597ecf95e --- /dev/null +++ b/harmony-os/SherpaOnnxHar/entry/src/main/resources/zh_CN/element/string.json @@ -0,0 +1,16 @@ +{ + "string": [ + { + "name": "module_desc", + "value": "模块描述" + }, + { + "name": "EntryAbility_desc", + "value": "description" + }, + { + "name": "EntryAbility_label", + "value": "label" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/entry/src/ohosTest/ets/test/Ability.test.ets b/harmony-os/SherpaOnnxHar/entry/src/ohosTest/ets/test/Ability.test.ets new file mode 100644 index 000000000..8aa374977 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/entry/src/ohosTest/ets/test/Ability.test.ets @@ -0,0 +1,35 @@ +import hilog from '@ohos.hilog'; +import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from '@ohos/hypium'; + +export default function abilityTest() { + describe('ActsAbilityTest', () => { + // Defines a test suite. Two parameters are supported: test suite name and test suite function. + beforeAll(() => { + // Presets an action, which is performed only once before all test cases of the test suite start. + // This API supports only one parameter: preset action function. + }) + beforeEach(() => { + // Presets an action, which is performed before each unit test case starts. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: preset action function. + }) + afterEach(() => { + // Presets a clear action, which is performed after each unit test case ends. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: clear action function. + }) + afterAll(() => { + // Presets a clear action, which is performed after all test cases of the test suite end. + // This API supports only one parameter: clear action function. + }) + it('assertContain', 0, () => { + // Defines a test case. This API supports three parameters: test case name, filter parameter, and test case function. + hilog.info(0x0000, 'testTag', '%{public}s', 'it begin'); + let a = 'abc'; + let b = 'b'; + // Defines a variety of assertion methods, which are used to declare expected boolean conditions. + expect(a).assertContain(b); + expect(a).assertEqual(a); + }) + }) +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/entry/src/ohosTest/ets/test/List.test.ets b/harmony-os/SherpaOnnxHar/entry/src/ohosTest/ets/test/List.test.ets new file mode 100644 index 000000000..794c7dc4e --- /dev/null +++ b/harmony-os/SherpaOnnxHar/entry/src/ohosTest/ets/test/List.test.ets @@ -0,0 +1,5 @@ +import abilityTest from './Ability.test'; + +export default function testsuite() { + abilityTest(); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/entry/src/ohosTest/module.json5 b/harmony-os/SherpaOnnxHar/entry/src/ohosTest/module.json5 new file mode 100644 index 000000000..55725a929 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/entry/src/ohosTest/module.json5 @@ -0,0 +1,13 @@ +{ + "module": { + "name": "entry_test", + "type": "feature", + "deviceTypes": [ + "phone", + "tablet", + "2in1" + ], + "deliveryWithInstall": true, + "installationFree": false + } +} diff --git a/harmony-os/SherpaOnnxHar/entry/src/test/List.test.ets b/harmony-os/SherpaOnnxHar/entry/src/test/List.test.ets new file mode 100644 index 000000000..bb5b5c373 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/entry/src/test/List.test.ets @@ -0,0 +1,5 @@ +import localUnitTest from './LocalUnit.test'; + +export default function testsuite() { + localUnitTest(); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/entry/src/test/LocalUnit.test.ets b/harmony-os/SherpaOnnxHar/entry/src/test/LocalUnit.test.ets new file mode 100644 index 000000000..165fc1615 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/entry/src/test/LocalUnit.test.ets @@ -0,0 +1,33 @@ +import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from '@ohos/hypium'; + +export default function localUnitTest() { + describe('localUnitTest', () => { + // Defines a test suite. Two parameters are supported: test suite name and test suite function. + beforeAll(() => { + // Presets an action, which is performed only once before all test cases of the test suite start. + // This API supports only one parameter: preset action function. + }); + beforeEach(() => { + // Presets an action, which is performed before each unit test case starts. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: preset action function. + }); + afterEach(() => { + // Presets a clear action, which is performed after each unit test case ends. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: clear action function. + }); + afterAll(() => { + // Presets a clear action, which is performed after all test cases of the test suite end. + // This API supports only one parameter: clear action function. + }); + it('assertContain', 0, () => { + // Defines a test case. This API supports three parameters: test case name, filter parameter, and test case function. + let a = 'abc'; + let b = 'b'; + // Defines a variety of assertion methods, which are used to declare expected boolean conditions. + expect(a).assertContain(b); + expect(a).assertEqual(a); + }); + }); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/hvigor/hvigor-config.json5 b/harmony-os/SherpaOnnxHar/hvigor/hvigor-config.json5 new file mode 100644 index 000000000..06b278367 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/hvigor/hvigor-config.json5 @@ -0,0 +1,22 @@ +{ + "modelVersion": "5.0.0", + "dependencies": { + }, + "execution": { + // "analyze": "normal", /* Define the build analyze mode. Value: [ "normal" | "advanced" | false ]. Default: "normal" */ + // "daemon": true, /* Enable daemon compilation. Value: [ true | false ]. Default: true */ + // "incremental": true, /* Enable incremental compilation. Value: [ true | false ]. Default: true */ + // "parallel": true, /* Enable parallel compilation. Value: [ true | false ]. Default: true */ + // "typeCheck": false, /* Enable typeCheck. Value: [ true | false ]. Default: false */ + }, + "logging": { + // "level": "info" /* Define the log level. Value: [ "debug" | "info" | "warn" | "error" ]. Default: "info" */ + }, + "debugging": { + // "stacktrace": false /* Disable stacktrace compilation. Value: [ true | false ]. Default: false */ + }, + "nodeOptions": { + // "maxOldSpaceSize": 8192 /* Enable nodeOptions maxOldSpaceSize compilation. Unit M. Used for the daemon process. Default: 8192*/ + // "exposeGC": true /* Enable to trigger garbage collection explicitly. Default: true*/ + } +} diff --git a/harmony-os/SherpaOnnxHar/hvigorfile.ts b/harmony-os/SherpaOnnxHar/hvigorfile.ts new file mode 100644 index 000000000..f3cb9f1a8 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/hvigorfile.ts @@ -0,0 +1,6 @@ +import { appTasks } from '@ohos/hvigor-ohos-plugin'; + +export default { + system: appTasks, /* Built-in plugin of Hvigor. It cannot be modified. */ + plugins:[] /* Custom plugin to extend the functionality of Hvigor. */ +} diff --git a/harmony-os/SherpaOnnxHar/notes.md b/harmony-os/SherpaOnnxHar/notes.md new file mode 100644 index 000000000..6926a7bb6 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/notes.md @@ -0,0 +1,13 @@ +# Notes + +## How to publish a package + +Please see + - + - + - + +## How to sign the HAP file from commandline + +Please see + diff --git a/harmony-os/SherpaOnnxHar/oh-package-lock.json5 b/harmony-os/SherpaOnnxHar/oh-package-lock.json5 new file mode 100644 index 000000000..f538ae290 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/oh-package-lock.json5 @@ -0,0 +1,19 @@ +{ + "meta": { + "stableOrder": true + }, + "lockfileVersion": 3, + "ATTENTION": "THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.", + "specifiers": { + "@ohos/hypium@1.0.19": "@ohos/hypium@1.0.19" + }, + "packages": { + "@ohos/hypium@1.0.19": { + "name": "@ohos/hypium", + "version": "1.0.19", + "integrity": "sha512-cEjDgLFCm3cWZDeRXk7agBUkPqjWxUo6AQeiu0gEkb3J8ESqlduQLSIXeo3cCsm8U/asL7iKjF85ZyOuufAGSQ==", + "resolved": "https://ohpm.openharmony.cn/ohpm/@ohos/hypium/-/hypium-1.0.19.har", + "registryType": "ohpm" + } + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/oh-package.json5 b/harmony-os/SherpaOnnxHar/oh-package.json5 new file mode 100644 index 000000000..a79d5300e --- /dev/null +++ b/harmony-os/SherpaOnnxHar/oh-package.json5 @@ -0,0 +1,9 @@ +{ + "modelVersion": "5.0.0", + "description": "Please describe the basic information.", + "dependencies": { + }, + "devDependencies": { + "@ohos/hypium": "1.0.19" + } +} diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/.gitignore b/harmony-os/SherpaOnnxHar/sherpa_onnx/.gitignore new file mode 100644 index 000000000..e2713a277 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/.gitignore @@ -0,0 +1,6 @@ +/node_modules +/oh_modules +/.preview +/build +/.cxx +/.test \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets new file mode 100644 index 000000000..3a501e5dd --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets @@ -0,0 +1,17 @@ +/** + * Use these variables when you tailor your ArkTS code. They must be of the const type. + */ +export const HAR_VERSION = '1.0.0'; +export const BUILD_MODE_NAME = 'debug'; +export const DEBUG = true; +export const TARGET_NAME = 'default'; + +/** + * BuildProfile Class is used only for compatibility purposes. + */ +export default class BuildProfile { + static readonly HAR_VERSION = HAR_VERSION; + static readonly BUILD_MODE_NAME = BUILD_MODE_NAME; + static readonly DEBUG = DEBUG; + static readonly TARGET_NAME = TARGET_NAME; +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets new file mode 100644 index 000000000..185aa51fd --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets @@ -0,0 +1,40 @@ +export { readWave, readWaveFromBinary } from "libsherpa_onnx.so"; + +export { + CircularBuffer, + SileroVadConfig, + SpeechSegment, + Vad, + VadConfig, +} from './src/main/ets/components/Vad'; + + +export { + Samples, + OfflineStream, + FeatureConfig, + OfflineTransducerModelConfig, + OfflineParaformerModelConfig, + OfflineNemoEncDecCtcModelConfig, + OfflineWhisperModelConfig, + OfflineTdnnModelConfig, + OfflineSenseVoiceModelConfig, + OfflineMoonshineModelConfig, + OfflineModelConfig, + OfflineLMConfig, + OfflineRecognizerConfig, + OfflineRecognizerResult, + OfflineRecognizer, +} from './src/main/ets/components/NonStreamingAsr'; + +export { + OnlineStream, + OnlineTransducerModelConfig, + OnlineParaformerModelConfig, + OnlineZipformer2CtcModelConfig, + OnlineModelConfig, + OnlineCtcFstDecoderConfig, + OnlineRecognizerConfig, + OnlineRecognizerResult, + OnlineRecognizer, +} from './src/main/ets/components/StreamingAsr'; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md b/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md new file mode 100644 index 000000000..2690bf624 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md @@ -0,0 +1,12 @@ +# Introduction + +[sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx) is one of the deployment +frameworks of [Next-gen Kaldi](https://github.com/k2-fsa). + +It supports speech-to-text, text-to-speech, speaker diarization, and VAD using +onnxruntime without Internet connection. + +It also supports embedded systems, Android, iOS, HarmonyOS, +Raspberry Pi, RISC-V, x86_64 servers, websocket server/client, +C/C++, Python, Kotlin, C#, Go, NodeJS, Java, Swift, Dart, JavaScript, +Flutter, Object Pascal, Lazarus, Rust, etc. diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/build-profile.json5 b/harmony-os/SherpaOnnxHar/sherpa_onnx/build-profile.json5 new file mode 100644 index 000000000..8f789fb2a --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/build-profile.json5 @@ -0,0 +1,46 @@ +{ + "apiType": "stageMode", + "buildOption": { + "externalNativeOptions": { + "path": "./src/main/cpp/CMakeLists.txt", + "arguments": "", + "cppFlags": "", + "abiFilters": [ + "arm64-v8a", + "x86_64", + ], + }, + }, + "buildOptionSet": [ + { + "name": "release", + "arkOptions": { + "obfuscation": { + "ruleOptions": { + "enable": false, + "files": [ + "./obfuscation-rules.txt" + ] + }, + "consumerFiles": [ + "./consumer-rules.txt" + ] + } + }, + "nativeLib": { + "debugSymbol": { + "strip": true, + "exclude": [] + } + } + }, + ], + "targets": [ + { + "name": "default" + }, + { + "name": "ohosTest" + } + ] +} diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/consumer-rules.txt b/harmony-os/SherpaOnnxHar/sherpa_onnx/consumer-rules.txt new file mode 100644 index 000000000..e69de29bb diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/hvigorfile.ts b/harmony-os/SherpaOnnxHar/sherpa_onnx/hvigorfile.ts new file mode 100644 index 000000000..421870714 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/hvigorfile.ts @@ -0,0 +1,6 @@ +import { harTasks } from '@ohos/hvigor-ohos-plugin'; + +export default { + system: harTasks, /* Built-in plugin of Hvigor. It cannot be modified. */ + plugins:[] /* Custom plugin to extend the functionality of Hvigor. */ +} diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/obfuscation-rules.txt b/harmony-os/SherpaOnnxHar/sherpa_onnx/obfuscation-rules.txt new file mode 100644 index 000000000..272efb6ca --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/obfuscation-rules.txt @@ -0,0 +1,23 @@ +# Define project specific obfuscation rules here. +# You can include the obfuscation configuration files in the current module's build-profile.json5. +# +# For more details, see +# https://developer.huawei.com/consumer/cn/doc/harmonyos-guides-V5/source-obfuscation-V5 + +# Obfuscation options: +# -disable-obfuscation: disable all obfuscations +# -enable-property-obfuscation: obfuscate the property names +# -enable-toplevel-obfuscation: obfuscate the names in the global scope +# -compact: remove unnecessary blank spaces and all line feeds +# -remove-log: remove all console.* statements +# -print-namecache: print the name cache that contains the mapping from the old names to new names +# -apply-namecache: reuse the given cache file + +# Keep options: +# -keep-property-name: specifies property names that you want to keep +# -keep-global-name: specifies names that you want to keep in the global scope + +-enable-property-obfuscation +-enable-toplevel-obfuscation +-enable-filename-obfuscation +-enable-export-obfuscation \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package-lock.json5 b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package-lock.json5 new file mode 100644 index 000000000..2585b2e83 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package-lock.json5 @@ -0,0 +1,18 @@ +{ + "meta": { + "stableOrder": true + }, + "lockfileVersion": 3, + "ATTENTION": "THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.", + "specifiers": { + "libsherpa_onnx.so@src/main/cpp/types/libsherpa_onnx": "libsherpa_onnx.so@src/main/cpp/types/libsherpa_onnx" + }, + "packages": { + "libsherpa_onnx.so@src/main/cpp/types/libsherpa_onnx": { + "name": "libsherpa_onnx.so", + "version": "1.0.0", + "resolved": "src/main/cpp/types/libsherpa_onnx", + "registryType": "local" + } + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 new file mode 100644 index 000000000..cc2260a23 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 @@ -0,0 +1,25 @@ +{ + "name": "sherpa_onnx", + "version": "1.0.0", + "description": "Speech-to-text, text-to-speech, and speaker diarization using Next-gen Kaldi without internet connection", + "main": "Index.ets", + "author": "The next-gen Kaldi team", + "license": "Apache-2.0", + "homepage": "https://github.com/k2-fsa/sherpa-onnx", + "repository": "https://github.com/k2-fsa/sherpa-onnx/tree/master/harmonyos-SherpaOnnxHar", + "dependencies": { + "libsherpa_onnx.so": "file:./src/main/cpp/types/libsherpa_onnx" + }, + "keywords": [ + "tts", + "asr", + "locally", + "diarization", + "privacy", + "open-source", + "speaker", + ], + "bugs": { + "url": "https://github.com/k2-fsa/sherpa-onnx/issues" + }, +} diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/CMakeLists.txt b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/CMakeLists.txt new file mode 100644 index 000000000..e131b21da --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/CMakeLists.txt @@ -0,0 +1,69 @@ +# the minimum version of CMake. +cmake_minimum_required(VERSION 3.13.0) +project(myNpmLib) + +# Disable warning about +# +# "The DOWNLOAD_EXTRACT_TIMESTAMP option was not given and policy CMP0135 is +# not set. +if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0") + cmake_policy(SET CMP0135 NEW) +endif() + +set(NATIVERENDER_ROOT_PATH ${CMAKE_CURRENT_SOURCE_DIR}) + +if(DEFINED PACKAGE_FIND_FILE) + include(${PACKAGE_FIND_FILE}) +endif() + +include_directories(${NATIVERENDER_ROOT_PATH} + ${NATIVERENDER_ROOT_PATH}/include) + +include(FetchContent) +FetchContent_Declare(node_addon_api + GIT_REPOSITORY "https://github.com/nodejs/node-addon-api.git" + GIT_TAG c679f6f4c9dc6bf9fc0d99cbe5982bd24a5e2c7b + PATCH_COMMAND git checkout . && git apply --ignore-whitespace "${CMAKE_CURRENT_LIST_DIR}/my-patch.diff" +) +FetchContent_MakeAvailable(node_addon_api) +FetchContent_GetProperties(node_addon_api) +if(NOT node_addon_api_POPULATED) + message(STATUS "Downloading node-addon-api from") + FetchContent_Populate(node_addon_api) +endif() + +message(STATUS "node-addon-api is downloaded to ${node_addon_api_SOURCE_DIR}") +include_directories(${node_addon_api_SOURCE_DIR}) + +add_library(sherpa_onnx SHARED + audio-tagging.cc + keyword-spotting.cc + non-streaming-asr.cc + non-streaming-speaker-diarization.cc + non-streaming-tts.cc + punctuation.cc + sherpa-onnx-node-addon-api.cc + speaker-identification.cc + spoken-language-identification.cc + streaming-asr.cc + vad.cc + wave-reader.cc + wave-writer.cc +) + +add_library(sherpa_onnx_c_api SHARED IMPORTED) +set_target_properties(sherpa_onnx_c_api + PROPERTIES + IMPORTED_LOCATION ${CMAKE_CURRENT_SOURCE_DIR}/libs/${OHOS_ARCH}/libsherpa-onnx-c-api.so) + +add_library(onnxruntime SHARED IMPORTED) +set_target_properties(onnxruntime + PROPERTIES + IMPORTED_LOCATION ${CMAKE_CURRENT_SOURCE_DIR}/libs/${OHOS_ARCH}/libonnxruntime.so) + + +target_link_libraries(sherpa_onnx PUBLIC libace_napi.z.so + libhilog_ndk.z.so # for hilog + librawfile.z.so + sherpa_onnx_c_api onnxruntime +) diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/audio-tagging.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/audio-tagging.cc new file mode 100644 index 000000000..bed4e48a2 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/audio-tagging.cc @@ -0,0 +1,227 @@ +// scripts/node-addon-api/src/audio-tagging.cc +// +// Copyright (c) 2024 Xiaomi Corporation +#include + +#include "macros.h" // NOLINT +#include "napi.h" // NOLINT +#include "sherpa-onnx/c-api/c-api.h" + +static SherpaOnnxOfflineZipformerAudioTaggingModelConfig +GetAudioTaggingZipformerModelConfig(Napi::Object obj) { + SherpaOnnxOfflineZipformerAudioTaggingModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("zipformer") || !obj.Get("zipformer").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("zipformer").As(); + + SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); + + return c; +} + +static SherpaOnnxAudioTaggingModelConfig GetAudioTaggingModelConfig( + Napi::Object obj) { + SherpaOnnxAudioTaggingModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("model") || !obj.Get("model").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("model").As(); + c.zipformer = GetAudioTaggingZipformerModelConfig(o); + + SHERPA_ONNX_ASSIGN_ATTR_STR(ced, ced); + + SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); + + if (o.Has("debug") && + (o.Get("debug").IsNumber() || o.Get("debug").IsBoolean())) { + if (o.Get("debug").IsBoolean()) { + c.debug = o.Get("debug").As().Value(); + } else { + c.debug = o.Get("debug").As().Int32Value(); + } + } + SHERPA_ONNX_ASSIGN_ATTR_STR(provider, provider); + + return c; +} + +static Napi::External CreateAudioTaggingWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsObject()) { + Napi::TypeError::New(env, "You should pass an object as the only argument.") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Object o = info[0].As(); + + SherpaOnnxAudioTaggingConfig c; + memset(&c, 0, sizeof(c)); + c.model = GetAudioTaggingModelConfig(o); + + SHERPA_ONNX_ASSIGN_ATTR_STR(labels, labels); + SHERPA_ONNX_ASSIGN_ATTR_INT32(top_k, topK); + + const SherpaOnnxAudioTagging *at = SherpaOnnxCreateAudioTagging(&c); + + if (c.model.zipformer.model) { + delete[] c.model.zipformer.model; + } + + if (c.model.ced) { + delete[] c.model.ced; + } + + if (c.model.provider) { + delete[] c.model.provider; + } + + if (c.labels) { + delete[] c.labels; + } + + if (!at) { + Napi::TypeError::New(env, "Please check your config!") + .ThrowAsJavaScriptException(); + + return {}; + } + + return Napi::External::New( + env, const_cast(at), + [](Napi::Env env, SherpaOnnxAudioTagging *at) { + SherpaOnnxDestroyAudioTagging(at); + }); +} + +static Napi::External +AudioTaggingCreateOfflineStreamWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New( + env, "You should pass an audio tagging pointer as the only argument") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxAudioTagging *at = + info[0].As>().Data(); + + const SherpaOnnxOfflineStream *stream = + SherpaOnnxAudioTaggingCreateOfflineStream(at); + + return Napi::External::New( + env, const_cast(stream), + [](Napi::Env env, SherpaOnnxOfflineStream *stream) { + SherpaOnnxDestroyOfflineStream(stream); + }); +} + +static Napi::Object AudioTaggingComputeWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 3) { + std::ostringstream os; + os << "Expect only 3 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New( + env, "You should pass an audio tagging pointer as the first argument") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[1].IsExternal()) { + Napi::TypeError::New( + env, "You should pass an offline stream pointer as the second argument") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[2].IsNumber()) { + Napi::TypeError::New(env, + "You should pass an integer as the third argument") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxAudioTagging *at = + info[0].As>().Data(); + + SherpaOnnxOfflineStream *stream = + info[1].As>().Data(); + + int32_t top_k = info[2].As().Int32Value(); + + const SherpaOnnxAudioEvent *const *events = + SherpaOnnxAudioTaggingCompute(at, stream, top_k); + + auto p = events; + int32_t k = 0; + while (p && *p) { + ++k; + ++p; + } + + Napi::Array ans = Napi::Array::New(env, k); + for (uint32_t i = 0; i != k; ++i) { + Napi::Object obj = Napi::Object::New(env); + obj.Set(Napi::String::New(env, "name"), + Napi::String::New(env, events[i]->name)); + obj.Set(Napi::String::New(env, "index"), + Napi::Number::New(env, events[i]->index)); + obj.Set(Napi::String::New(env, "prob"), + Napi::Number::New(env, events[i]->prob)); + ans[i] = obj; + } + + SherpaOnnxAudioTaggingFreeResults(events); + + return ans; +} + +void InitAudioTagging(Napi::Env env, Napi::Object exports) { + exports.Set(Napi::String::New(env, "createAudioTagging"), + Napi::Function::New(env, CreateAudioTaggingWrapper)); + + exports.Set(Napi::String::New(env, "audioTaggingCreateOfflineStream"), + Napi::Function::New(env, AudioTaggingCreateOfflineStreamWrapper)); + + exports.Set(Napi::String::New(env, "audioTaggingCompute"), + Napi::Function::New(env, AudioTaggingComputeWrapper)); +} diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/include/sherpa-onnx/c-api/README.md b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/include/sherpa-onnx/c-api/README.md new file mode 100644 index 000000000..95744c221 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/include/sherpa-onnx/c-api/README.md @@ -0,0 +1,8 @@ +# Node + +[./c-api.h](./c-api.h) is a symbolic link to +https://github.com/k2-fsa/sherpa-onnx/blob/master/sherpa-onnx/c-api/c-api.h + +If you are using Windows, then you need to manually replace this file with +https://github.com/k2-fsa/sherpa-onnx/blob/master/sherpa-onnx/c-api/c-api.h +since Windows does not support symbolic links. diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/include/sherpa-onnx/c-api/c-api.h b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/include/sherpa-onnx/c-api/c-api.h new file mode 120000 index 000000000..d9c1b82e1 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/include/sherpa-onnx/c-api/c-api.h @@ -0,0 +1 @@ +../../../../../../../../../sherpa-onnx/c-api/c-api.h \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/keyword-spotting.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/keyword-spotting.cc new file mode 100644 index 000000000..2b5a24100 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/keyword-spotting.cc @@ -0,0 +1,266 @@ +// scripts/node-addon-api/src/keyword-spotting.cc +// +// Copyright (c) 2024 Xiaomi Corporation +#include + +#include "macros.h" // NOLINT +#include "napi.h" // NOLINT +#include "sherpa-onnx/c-api/c-api.h" + +// defined ./streaming-asr.cc +SherpaOnnxFeatureConfig GetFeatureConfig(Napi::Object obj); + +// defined ./streaming-asr.cc +SherpaOnnxOnlineModelConfig GetOnlineModelConfig(Napi::Object obj); + +static Napi::External CreateKeywordSpotterWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsObject()) { + Napi::TypeError::New(env, "Expect an object as the argument") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Object o = info[0].As(); + SherpaOnnxKeywordSpotterConfig c; + memset(&c, 0, sizeof(c)); + c.feat_config = GetFeatureConfig(o); + c.model_config = GetOnlineModelConfig(o); + + SHERPA_ONNX_ASSIGN_ATTR_INT32(max_active_paths, maxActivePaths); + SHERPA_ONNX_ASSIGN_ATTR_INT32(num_trailing_blanks, numTrailingBlanks); + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(keywords_score, keywordsScore); + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(keywords_threshold, keywordsThreshold); + SHERPA_ONNX_ASSIGN_ATTR_STR(keywords_file, keywordsFile); + SHERPA_ONNX_ASSIGN_ATTR_STR(keywords_buf, keywordsBuf); + SHERPA_ONNX_ASSIGN_ATTR_INT32(keywords_buf_size, keywordsBufSize); + + SherpaOnnxKeywordSpotter *kws = SherpaOnnxCreateKeywordSpotter(&c); + + if (c.model_config.transducer.encoder) { + delete[] c.model_config.transducer.encoder; + } + + if (c.model_config.transducer.decoder) { + delete[] c.model_config.transducer.decoder; + } + + if (c.model_config.transducer.joiner) { + delete[] c.model_config.transducer.joiner; + } + + if (c.model_config.paraformer.encoder) { + delete[] c.model_config.paraformer.encoder; + } + + if (c.model_config.paraformer.decoder) { + delete[] c.model_config.paraformer.decoder; + } + + if (c.model_config.zipformer2_ctc.model) { + delete[] c.model_config.zipformer2_ctc.model; + } + + if (c.model_config.tokens) { + delete[] c.model_config.tokens; + } + + if (c.model_config.provider) { + delete[] c.model_config.provider; + } + + if (c.model_config.model_type) { + delete[] c.model_config.model_type; + } + + if (c.keywords_file) { + delete[] c.keywords_file; + } + + if (c.keywords_buf) { + delete[] c.keywords_buf; + } + + if (!kws) { + Napi::TypeError::New(env, "Please check your config!") + .ThrowAsJavaScriptException(); + + return {}; + } + + return Napi::External::New( + env, kws, [](Napi::Env env, SherpaOnnxKeywordSpotter *kws) { + SherpaOnnxDestroyKeywordSpotter(kws); + }); +} + +static Napi::External CreateKeywordStreamWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New( + env, "You should pass a keyword spotter pointer as the only argument") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxKeywordSpotter *kws = + info[0].As>().Data(); + + SherpaOnnxOnlineStream *stream = SherpaOnnxCreateKeywordStream(kws); + + return Napi::External::New( + env, stream, [](Napi::Env env, SherpaOnnxOnlineStream *stream) { + SherpaOnnxDestroyOnlineStream(stream); + }); +} + +static Napi::Boolean IsKeywordStreamReadyWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be a keyword spotter pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[1].IsExternal()) { + Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxKeywordSpotter *kws = + info[0].As>().Data(); + + SherpaOnnxOnlineStream *stream = + info[1].As>().Data(); + + int32_t is_ready = SherpaOnnxIsKeywordStreamReady(kws, stream); + + return Napi::Boolean::New(env, is_ready); +} + +static void DecodeKeywordStreamWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be a keyword spotter pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + if (!info[1].IsExternal()) { + Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + SherpaOnnxKeywordSpotter *kws = + info[0].As>().Data(); + + SherpaOnnxOnlineStream *stream = + info[1].As>().Data(); + + SherpaOnnxDecodeKeywordStream(kws, stream); +} + +static Napi::String GetKeywordResultAsJsonWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be a keyword spotter pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[1].IsExternal()) { + Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxKeywordSpotter *kws = + info[0].As>().Data(); + + SherpaOnnxOnlineStream *stream = + info[1].As>().Data(); + + const char *json = SherpaOnnxGetKeywordResultAsJson(kws, stream); + + Napi::String s = Napi::String::New(env, json); + + SherpaOnnxFreeKeywordResultJson(json); + + return s; +} + +void InitKeywordSpotting(Napi::Env env, Napi::Object exports) { + exports.Set(Napi::String::New(env, "createKeywordSpotter"), + Napi::Function::New(env, CreateKeywordSpotterWrapper)); + + exports.Set(Napi::String::New(env, "createKeywordStream"), + Napi::Function::New(env, CreateKeywordStreamWrapper)); + + exports.Set(Napi::String::New(env, "isKeywordStreamReady"), + Napi::Function::New(env, IsKeywordStreamReadyWrapper)); + + exports.Set(Napi::String::New(env, "decodeKeywordStream"), + Napi::Function::New(env, DecodeKeywordStreamWrapper)); + + exports.Set(Napi::String::New(env, "getKeywordResultAsJson"), + Napi::Function::New(env, GetKeywordResultAsJsonWrapper)); +} diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/libs/.gitignore b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/libs/.gitignore new file mode 100644 index 000000000..140f8cf80 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/libs/.gitignore @@ -0,0 +1 @@ +*.so diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/libs/README.md b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/libs/README.md new file mode 100644 index 000000000..0094a8def --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/libs/README.md @@ -0,0 +1,17 @@ +# Introduction + +You need to get the following four `.so` files using + + - [build-ohos-arm64-v8a.sh](https://github.com/k2-fsa/sherpa-onnx/blob/master/build-ohos-arm64-v8a.sh) + - [build-ohos-x86-64.sh](https://github.com/k2-fsa/sherpa-onnx/blob/master/build-ohos-x86-64.sh) + +``` +. +├── README.md +├── arm64-v8a +│   ├── libonnxruntime.so +│   └── libsherpa-onnx-c-api.so +└── x86_64 + ├── libonnxruntime.so + └── libsherpa-onnx-c-api.so +``` diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/libs/arm64-v8a/.gitkeep b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/libs/arm64-v8a/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/libs/x86_64/.gitkeep b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/libs/x86_64/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/macros.h b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/macros.h new file mode 100644 index 000000000..7a6cc93e6 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/macros.h @@ -0,0 +1,63 @@ +// scripts/node-addon-api/src/macros.h +// +// Copyright (c) 2024 Xiaomi Corporation +#ifndef SCRIPTS_NODE_ADDON_API_SRC_MACROS_H_ +#define SCRIPTS_NODE_ADDON_API_SRC_MACROS_H_ + +#include +#include + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#include "hilog/log.h" + +#undef LOG_DOMAIN +#undef LOG_TAG + +// https://gitee.com/openharmony/docs/blob/145a084f0b742e4325915e32f8184817927d1251/en/contribute/OpenHarmony-Log-guide.md#hilog-api-usage-specifications +#define LOG_DOMAIN 0x6666 +#define LOG_TAG "sherpa_onnx" +#endif + +#define SHERPA_ONNX_ASSIGN_ATTR_STR(c_name, js_name) \ + do { \ + if (o.Has(#js_name) && o.Get(#js_name).IsString()) { \ + Napi::String _str = o.Get(#js_name).As(); \ + std::string s = _str.Utf8Value(); \ + char *p = new char[s.size() + 1]; \ + std::copy(s.begin(), s.end(), p); \ + p[s.size()] = 0; \ + \ + c.c_name = p; \ + } else if (o.Has(#js_name) && o.Get(#js_name).IsTypedArray()) { \ + Napi::Uint8Array _array = o.Get(#js_name).As(); \ + char *p = new char[_array.ElementLength() + 1]; \ + std::copy(_array.Data(), _array.Data() + _array.ElementLength(), p); \ + p[_array.ElementLength()] = '\0'; \ + \ + c.c_name = p; \ + } \ + } while (0) + +#define SHERPA_ONNX_ASSIGN_ATTR_INT32(c_name, js_name) \ + do { \ + if (o.Has(#js_name) && o.Get(#js_name).IsNumber()) { \ + c.c_name = o.Get(#js_name).As().Int32Value(); \ + } \ + } while (0) + +#define SHERPA_ONNX_ASSIGN_ATTR_FLOAT(c_name, js_name) \ + do { \ + if (o.Has(#js_name) && o.Get(#js_name).IsNumber()) { \ + c.c_name = o.Get(#js_name).As().FloatValue(); \ + } \ + } while (0) + +#define SHERPA_ONNX_DELETE_C_STR(p) \ + do { \ + if (p) { \ + delete[] p; \ + } \ + } while (0) + +#endif // SCRIPTS_NODE_ADDON_API_SRC_MACROS_H_ diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/my-patch.diff b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/my-patch.diff new file mode 100644 index 000000000..535dc38d1 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/my-patch.diff @@ -0,0 +1,14 @@ +diff --git a/napi-inl.h b/napi-inl.h +index e7141c0..0fd90d8 100644 +--- a/napi-inl.h ++++ b/napi-inl.h +@@ -2156,7 +2156,8 @@ inline ArrayBuffer::ArrayBuffer(napi_env env, napi_value value) + + inline void* ArrayBuffer::Data() { + void* data; +- napi_status status = napi_get_arraybuffer_info(_env, _value, &data, nullptr); ++ size_t byte_length; ++ napi_status status = napi_get_arraybuffer_info(_env, _value, &data, &byte_length); + NAPI_THROW_IF_FAILED(_env, status, nullptr); + return data; + } diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-asr.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-asr.cc new file mode 100644 index 000000000..c7d9560a2 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-asr.cc @@ -0,0 +1,487 @@ +// scripts/node-addon-api/src/non-streaming-asr.cc +// +// Copyright (c) 2024 Xiaomi Corporation +#include + +#include "macros.h" // NOLINT +#include "napi.h" // NOLINT +#include "sherpa-onnx/c-api/c-api.h" + +// defined in ./streaming-asr.cc +SherpaOnnxFeatureConfig GetFeatureConfig(Napi::Object obj); + +static SherpaOnnxOfflineTransducerModelConfig GetOfflineTransducerModelConfig( + Napi::Object obj) { + SherpaOnnxOfflineTransducerModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("transducer") || !obj.Get("transducer").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("transducer").As(); + + SHERPA_ONNX_ASSIGN_ATTR_STR(encoder, encoder); + SHERPA_ONNX_ASSIGN_ATTR_STR(decoder, decoder); + SHERPA_ONNX_ASSIGN_ATTR_STR(joiner, joiner); + + return c; +} + +static SherpaOnnxOfflineParaformerModelConfig GetOfflineParaformerModelConfig( + Napi::Object obj) { + SherpaOnnxOfflineParaformerModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("paraformer") || !obj.Get("paraformer").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("paraformer").As(); + + SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); + + return c; +} + +static SherpaOnnxOfflineNemoEncDecCtcModelConfig GetOfflineNeMoCtcModelConfig( + Napi::Object obj) { + SherpaOnnxOfflineNemoEncDecCtcModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("nemoCtc") || !obj.Get("nemoCtc").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("nemoCtc").As(); + + SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); + + return c; +} + +static SherpaOnnxOfflineWhisperModelConfig GetOfflineWhisperModelConfig( + Napi::Object obj) { + SherpaOnnxOfflineWhisperModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("whisper") || !obj.Get("whisper").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("whisper").As(); + + SHERPA_ONNX_ASSIGN_ATTR_STR(encoder, encoder); + SHERPA_ONNX_ASSIGN_ATTR_STR(decoder, decoder); + SHERPA_ONNX_ASSIGN_ATTR_STR(language, language); + SHERPA_ONNX_ASSIGN_ATTR_STR(task, task); + SHERPA_ONNX_ASSIGN_ATTR_INT32(tail_paddings, tailPaddings); + + return c; +} + +static SherpaOnnxOfflineMoonshineModelConfig GetOfflineMoonshineModelConfig( + Napi::Object obj) { + SherpaOnnxOfflineMoonshineModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("moonshine") || !obj.Get("moonshine").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("moonshine").As(); + + SHERPA_ONNX_ASSIGN_ATTR_STR(preprocessor, preprocessor); + SHERPA_ONNX_ASSIGN_ATTR_STR(encoder, encoder); + SHERPA_ONNX_ASSIGN_ATTR_STR(uncached_decoder, uncachedDecoder); + SHERPA_ONNX_ASSIGN_ATTR_STR(cached_decoder, cachedDecoder); + + return c; +} + +static SherpaOnnxOfflineTdnnModelConfig GetOfflineTdnnModelConfig( + Napi::Object obj) { + SherpaOnnxOfflineTdnnModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("tdnn") || !obj.Get("tdnn").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("tdnn").As(); + + SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); + + return c; +} + +static SherpaOnnxOfflineSenseVoiceModelConfig GetOfflineSenseVoiceModelConfig( + Napi::Object obj) { + SherpaOnnxOfflineSenseVoiceModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("senseVoice") || !obj.Get("senseVoice").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("senseVoice").As(); + + SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); + SHERPA_ONNX_ASSIGN_ATTR_STR(language, language); + SHERPA_ONNX_ASSIGN_ATTR_INT32(use_itn, useInverseTextNormalization); + + return c; +} + +static SherpaOnnxOfflineModelConfig GetOfflineModelConfig(Napi::Object obj) { + SherpaOnnxOfflineModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("modelConfig") || !obj.Get("modelConfig").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("modelConfig").As(); + + c.transducer = GetOfflineTransducerModelConfig(o); + c.paraformer = GetOfflineParaformerModelConfig(o); + c.nemo_ctc = GetOfflineNeMoCtcModelConfig(o); + c.whisper = GetOfflineWhisperModelConfig(o); + c.tdnn = GetOfflineTdnnModelConfig(o); + c.sense_voice = GetOfflineSenseVoiceModelConfig(o); + c.moonshine = GetOfflineMoonshineModelConfig(o); + + SHERPA_ONNX_ASSIGN_ATTR_STR(tokens, tokens); + SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); + + if (o.Has("debug") && + (o.Get("debug").IsNumber() || o.Get("debug").IsBoolean())) { + if (o.Get("debug").IsBoolean()) { + c.debug = o.Get("debug").As().Value(); + } else { + c.debug = o.Get("debug").As().Int32Value(); + } + } + + SHERPA_ONNX_ASSIGN_ATTR_STR(provider, provider); + SHERPA_ONNX_ASSIGN_ATTR_STR(model_type, modelType); + SHERPA_ONNX_ASSIGN_ATTR_STR(modeling_unit, modelingUnit); + SHERPA_ONNX_ASSIGN_ATTR_STR(bpe_vocab, bpeVocab); + SHERPA_ONNX_ASSIGN_ATTR_STR(telespeech_ctc, teleSpeechCtc); + + return c; +} + +static SherpaOnnxOfflineLMConfig GetOfflineLMConfig(Napi::Object obj) { + SherpaOnnxOfflineLMConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("lmConfig") || !obj.Get("lmConfig").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("lmConfig").As(); + + SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(scale, scale); + + return c; +} + +static Napi::External +CreateOfflineRecognizerWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); +#if __OHOS__ + // the last argument is the NativeResourceManager + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } +#else + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } +#endif + + if (!info[0].IsObject()) { + Napi::TypeError::New(env, "Expect an object as the argument") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Object o = info[0].As(); + + SherpaOnnxOfflineRecognizerConfig c; + memset(&c, 0, sizeof(c)); + c.feat_config = GetFeatureConfig(o); + c.model_config = GetOfflineModelConfig(o); + c.lm_config = GetOfflineLMConfig(o); + + SHERPA_ONNX_ASSIGN_ATTR_STR(decoding_method, decodingMethod); + SHERPA_ONNX_ASSIGN_ATTR_INT32(max_active_paths, maxActivePaths); + SHERPA_ONNX_ASSIGN_ATTR_STR(hotwords_file, hotwordsFile); + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(hotwords_score, hotwordsScore); + SHERPA_ONNX_ASSIGN_ATTR_STR(rule_fsts, ruleFsts); + SHERPA_ONNX_ASSIGN_ATTR_STR(rule_fars, ruleFars); + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(blank_penalty, blankPenalty); + +#if __OHOS__ + std::unique_ptr mgr (OH_ResourceManager_InitNativeResourceManager(env, info[1]), &OH_ResourceManager_ReleaseNativeResourceManager); + + const SherpaOnnxOfflineRecognizer *recognizer = + SherpaOnnxCreateOfflineRecognizerOHOS(&c, mgr.get()); +#else + const SherpaOnnxOfflineRecognizer *recognizer = + SherpaOnnxCreateOfflineRecognizer(&c); +#endif + + SHERPA_ONNX_DELETE_C_STR(c.model_config.transducer.encoder); + SHERPA_ONNX_DELETE_C_STR(c.model_config.transducer.decoder); + SHERPA_ONNX_DELETE_C_STR(c.model_config.transducer.joiner); + + SHERPA_ONNX_DELETE_C_STR(c.model_config.paraformer.model); + + SHERPA_ONNX_DELETE_C_STR(c.model_config.nemo_ctc.model); + + SHERPA_ONNX_DELETE_C_STR(c.model_config.whisper.encoder); + SHERPA_ONNX_DELETE_C_STR(c.model_config.whisper.decoder); + SHERPA_ONNX_DELETE_C_STR(c.model_config.whisper.language); + SHERPA_ONNX_DELETE_C_STR(c.model_config.whisper.task); + + SHERPA_ONNX_DELETE_C_STR(c.model_config.tdnn.model); + + SHERPA_ONNX_DELETE_C_STR(c.model_config.sense_voice.model); + SHERPA_ONNX_DELETE_C_STR(c.model_config.sense_voice.language); + + SHERPA_ONNX_DELETE_C_STR(c.model_config.moonshine.preprocessor); + SHERPA_ONNX_DELETE_C_STR(c.model_config.moonshine.encoder); + SHERPA_ONNX_DELETE_C_STR(c.model_config.moonshine.uncached_decoder); + SHERPA_ONNX_DELETE_C_STR(c.model_config.moonshine.cached_decoder); + + SHERPA_ONNX_DELETE_C_STR(c.model_config.tokens); + SHERPA_ONNX_DELETE_C_STR(c.model_config.provider); + SHERPA_ONNX_DELETE_C_STR(c.model_config.model_type); + SHERPA_ONNX_DELETE_C_STR(c.model_config.modeling_unit); + SHERPA_ONNX_DELETE_C_STR(c.model_config.bpe_vocab); + SHERPA_ONNX_DELETE_C_STR(c.model_config.telespeech_ctc); + + SHERPA_ONNX_DELETE_C_STR(c.lm_config.model); + + SHERPA_ONNX_DELETE_C_STR(c.decoding_method); + SHERPA_ONNX_DELETE_C_STR(c.hotwords_file); + SHERPA_ONNX_DELETE_C_STR(c.rule_fsts); + SHERPA_ONNX_DELETE_C_STR(c.rule_fars); + + if (!recognizer) { + Napi::TypeError::New(env, "Please check your config!") + .ThrowAsJavaScriptException(); + + return {}; + } + + return Napi::External::New( + env, const_cast(recognizer), + [](Napi::Env env, SherpaOnnxOfflineRecognizer *recognizer) { + SherpaOnnxDestroyOfflineRecognizer(recognizer); + }); +} + +static Napi::External CreateOfflineStreamWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New( + env, + "You should pass an offline recognizer pointer as the only argument") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxOfflineRecognizer *recognizer = + info[0].As>().Data(); + + const SherpaOnnxOfflineStream *stream = + SherpaOnnxCreateOfflineStream(recognizer); + + return Napi::External::New( + env, const_cast(stream), + [](Napi::Env env, SherpaOnnxOfflineStream *stream) { + SherpaOnnxDestroyOfflineStream(stream); + }); +} + +static void AcceptWaveformOfflineWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be an online stream pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + SherpaOnnxOfflineStream *stream = + info[0].As>().Data(); + + if (!info[1].IsObject()) { + Napi::TypeError::New(env, "Argument 1 should be an object") + .ThrowAsJavaScriptException(); + + return; + } + + Napi::Object obj = info[1].As(); + + if (!obj.Has("samples")) { + Napi::TypeError::New(env, "The argument object should have a field samples") + .ThrowAsJavaScriptException(); + + return; + } + + if (!obj.Get("samples").IsTypedArray()) { + Napi::TypeError::New(env, "The object['samples'] should be a typed array") + .ThrowAsJavaScriptException(); + + return; + } + + if (!obj.Has("sampleRate")) { + Napi::TypeError::New(env, + "The argument object should have a field sampleRate") + .ThrowAsJavaScriptException(); + + return; + } + + if (!obj.Get("sampleRate").IsNumber()) { + Napi::TypeError::New(env, "The object['samples'] should be a number") + .ThrowAsJavaScriptException(); + + return; + } + + Napi::Float32Array samples = obj.Get("samples").As(); + int32_t sample_rate = obj.Get("sampleRate").As().Int32Value(); + +#if __OHOS__ + // Note(fangjun): For unknown reasons on HarmonyOS, we need to divide it by + // sizeof(float) here + SherpaOnnxAcceptWaveformOffline(stream, sample_rate, samples.Data(), + samples.ElementLength() / sizeof(float)); +#else + SherpaOnnxAcceptWaveformOffline(stream, sample_rate, samples.Data(), + samples.ElementLength()); +#endif +} + +static void DecodeOfflineStreamWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, + "Argument 0 should be an offline recognizer pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + if (!info[1].IsExternal()) { + Napi::TypeError::New(env, "Argument 1 should be an offline stream pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + SherpaOnnxOfflineRecognizer *recognizer = + info[0].As>().Data(); + + SherpaOnnxOfflineStream *stream = + info[1].As>().Data(); + + SherpaOnnxDecodeOfflineStream(recognizer, stream); +} + +static Napi::String GetOfflineStreamResultAsJsonWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be an online stream pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxOfflineStream *stream = + info[0].As>().Data(); + + const char *json = SherpaOnnxGetOfflineStreamResultAsJson(stream); + Napi::String s = Napi::String::New(env, json); + + SherpaOnnxDestroyOfflineStreamResultJson(json); + + return s; +} + +void InitNonStreamingAsr(Napi::Env env, Napi::Object exports) { + exports.Set(Napi::String::New(env, "createOfflineRecognizer"), + Napi::Function::New(env, CreateOfflineRecognizerWrapper)); + + exports.Set(Napi::String::New(env, "createOfflineStream"), + Napi::Function::New(env, CreateOfflineStreamWrapper)); + + exports.Set(Napi::String::New(env, "acceptWaveformOffline"), + Napi::Function::New(env, AcceptWaveformOfflineWrapper)); + + exports.Set(Napi::String::New(env, "decodeOfflineStream"), + Napi::Function::New(env, DecodeOfflineStreamWrapper)); + + exports.Set(Napi::String::New(env, "getOfflineStreamResultAsJson"), + Napi::Function::New(env, GetOfflineStreamResultAsJsonWrapper)); +} diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-speaker-diarization.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-speaker-diarization.cc new file mode 100644 index 000000000..a35f7924a --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-speaker-diarization.cc @@ -0,0 +1,310 @@ +// scripts/node-addon-api/src/non-streaming-speaker-diarization.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include +#include + +#include "macros.h" // NOLINT +#include "napi.h" // NOLINT +#include "sherpa-onnx/c-api/c-api.h" + +static SherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig +GetOfflineSpeakerSegmentationPyannoteModelConfig(Napi::Object obj) { + SherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("pyannote") || !obj.Get("pyannote").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("pyannote").As(); + SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); + + return c; +} + +static SherpaOnnxOfflineSpeakerSegmentationModelConfig +GetOfflineSpeakerSegmentationModelConfig(Napi::Object obj) { + SherpaOnnxOfflineSpeakerSegmentationModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("segmentation") || !obj.Get("segmentation").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("segmentation").As(); + + c.pyannote = GetOfflineSpeakerSegmentationPyannoteModelConfig(o); + + SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); + + if (o.Has("debug") && + (o.Get("debug").IsNumber() || o.Get("debug").IsBoolean())) { + if (o.Get("debug").IsBoolean()) { + c.debug = o.Get("debug").As().Value(); + } else { + c.debug = o.Get("debug").As().Int32Value(); + } + } + + SHERPA_ONNX_ASSIGN_ATTR_STR(provider, provider); + + return c; +} + +static SherpaOnnxSpeakerEmbeddingExtractorConfig +GetSpeakerEmbeddingExtractorConfig(Napi::Object obj) { + SherpaOnnxSpeakerEmbeddingExtractorConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("embedding") || !obj.Get("embedding").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("embedding").As(); + + SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); + SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); + + if (o.Has("debug") && + (o.Get("debug").IsNumber() || o.Get("debug").IsBoolean())) { + if (o.Get("debug").IsBoolean()) { + c.debug = o.Get("debug").As().Value(); + } else { + c.debug = o.Get("debug").As().Int32Value(); + } + } + + SHERPA_ONNX_ASSIGN_ATTR_STR(provider, provider); + + return c; +} + +static SherpaOnnxFastClusteringConfig GetFastClusteringConfig( + Napi::Object obj) { + SherpaOnnxFastClusteringConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("clustering") || !obj.Get("clustering").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("clustering").As(); + + SHERPA_ONNX_ASSIGN_ATTR_INT32(num_clusters, numClusters); + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(threshold, threshold); + + return c; +} + +static Napi::External +CreateOfflineSpeakerDiarizationWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsObject()) { + Napi::TypeError::New(env, "Expect an object as the argument") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Object o = info[0].As(); + + SherpaOnnxOfflineSpeakerDiarizationConfig c; + memset(&c, 0, sizeof(c)); + + c.segmentation = GetOfflineSpeakerSegmentationModelConfig(o); + c.embedding = GetSpeakerEmbeddingExtractorConfig(o); + c.clustering = GetFastClusteringConfig(o); + + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(min_duration_on, minDurationOn); + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(min_duration_off, minDurationOff); + + const SherpaOnnxOfflineSpeakerDiarization *sd = + SherpaOnnxCreateOfflineSpeakerDiarization(&c); + + if (c.segmentation.pyannote.model) { + delete[] c.segmentation.pyannote.model; + } + + if (c.segmentation.provider) { + delete[] c.segmentation.provider; + } + + if (c.embedding.model) { + delete[] c.embedding.model; + } + + if (c.embedding.provider) { + delete[] c.embedding.provider; + } + + if (!sd) { + Napi::TypeError::New(env, "Please check your config!") + .ThrowAsJavaScriptException(); + + return {}; + } + + return Napi::External::New( + env, const_cast(sd), + [](Napi::Env env, SherpaOnnxOfflineSpeakerDiarization *sd) { + SherpaOnnxDestroyOfflineSpeakerDiarization(sd); + }); +} + +static Napi::Number OfflineSpeakerDiarizationGetSampleRateWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New( + env, "Argument 0 should be an offline speaker diarization pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + const SherpaOnnxOfflineSpeakerDiarization *sd = + info[0].As>().Data(); + + int32_t sample_rate = SherpaOnnxOfflineSpeakerDiarizationGetSampleRate(sd); + + return Napi::Number::New(env, sample_rate); +} + +static Napi::Array OfflineSpeakerDiarizationProcessWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New( + env, "Argument 0 should be an offline speaker diarization pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + const SherpaOnnxOfflineSpeakerDiarization *sd = + info[0].As>().Data(); + + if (!info[1].IsTypedArray()) { + Napi::TypeError::New(env, "Argument 1 should be a typed array") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Float32Array samples = info[1].As(); + + const SherpaOnnxOfflineSpeakerDiarizationResult *r = + SherpaOnnxOfflineSpeakerDiarizationProcess(sd, samples.Data(), + samples.ElementLength()); + + int32_t num_segments = + SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments(r); + + const SherpaOnnxOfflineSpeakerDiarizationSegment *segments = + SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime(r); + + Napi::Array ans = Napi::Array::New(env, num_segments); + + for (int32_t i = 0; i != num_segments; ++i) { + Napi::Object obj = Napi::Object::New(env); + + obj.Set(Napi::String::New(env, "start"), segments[i].start); + obj.Set(Napi::String::New(env, "end"), segments[i].end); + obj.Set(Napi::String::New(env, "speaker"), segments[i].speaker); + + ans.Set(i, obj); + } + + SherpaOnnxOfflineSpeakerDiarizationDestroySegment(segments); + SherpaOnnxOfflineSpeakerDiarizationDestroyResult(r); + + return ans; +} + +static void OfflineSpeakerDiarizationSetConfigWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New( + env, "Argument 0 should be an offline speaker diarization pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + const SherpaOnnxOfflineSpeakerDiarization *sd = + info[0].As>().Data(); + + if (!info[1].IsObject()) { + Napi::TypeError::New(env, "Expect an object as the argument") + .ThrowAsJavaScriptException(); + + return; + } + + Napi::Object o = info[0].As(); + + SherpaOnnxOfflineSpeakerDiarizationConfig c; + memset(&c, 0, sizeof(c)); + + c.clustering = GetFastClusteringConfig(o); + SherpaOnnxOfflineSpeakerDiarizationSetConfig(sd, &c); +} + +void InitNonStreamingSpeakerDiarization(Napi::Env env, Napi::Object exports) { + exports.Set(Napi::String::New(env, "createOfflineSpeakerDiarization"), + Napi::Function::New(env, CreateOfflineSpeakerDiarizationWrapper)); + + exports.Set( + Napi::String::New(env, "getOfflineSpeakerDiarizationSampleRate"), + Napi::Function::New(env, OfflineSpeakerDiarizationGetSampleRateWrapper)); + + exports.Set( + Napi::String::New(env, "offlineSpeakerDiarizationProcess"), + Napi::Function::New(env, OfflineSpeakerDiarizationProcessWrapper)); + + exports.Set( + Napi::String::New(env, "offlineSpeakerDiarizationSetConfig"), + Napi::Function::New(env, OfflineSpeakerDiarizationSetConfigWrapper)); +} diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc new file mode 100644 index 000000000..70d97cddb --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc @@ -0,0 +1,329 @@ +// scripts/node-addon-api/src/non-streaming-tts.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include +#include + +#include "macros.h" // NOLINT +#include "napi.h" // NOLINT +#include "sherpa-onnx/c-api/c-api.h" + +static SherpaOnnxOfflineTtsVitsModelConfig GetOfflineTtsVitsModelConfig( + Napi::Object obj) { + SherpaOnnxOfflineTtsVitsModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("vits") || !obj.Get("vits").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("vits").As(); + SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); + SHERPA_ONNX_ASSIGN_ATTR_STR(lexicon, lexicon); + SHERPA_ONNX_ASSIGN_ATTR_STR(tokens, tokens); + SHERPA_ONNX_ASSIGN_ATTR_STR(data_dir, dataDir); + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(noise_scale, noiseScale); + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(noise_scale_w, noiseScaleW); + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(length_scale, lengthScale); + SHERPA_ONNX_ASSIGN_ATTR_STR(dict_dir, dictDir); + + return c; +} + +static SherpaOnnxOfflineTtsModelConfig GetOfflineTtsModelConfig( + Napi::Object obj) { + SherpaOnnxOfflineTtsModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("model") || !obj.Get("model").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("model").As(); + + c.vits = GetOfflineTtsVitsModelConfig(o); + + SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); + + if (o.Has("debug") && + (o.Get("debug").IsNumber() || o.Get("debug").IsBoolean())) { + if (o.Get("debug").IsBoolean()) { + c.debug = o.Get("debug").As().Value(); + } else { + c.debug = o.Get("debug").As().Int32Value(); + } + } + + SHERPA_ONNX_ASSIGN_ATTR_STR(provider, provider); + + return c; +} + +static Napi::External CreateOfflineTtsWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsObject()) { + Napi::TypeError::New(env, "Expect an object as the argument") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Object o = info[0].As(); + + SherpaOnnxOfflineTtsConfig c; + memset(&c, 0, sizeof(c)); + + c.model = GetOfflineTtsModelConfig(o); + + SHERPA_ONNX_ASSIGN_ATTR_STR(rule_fsts, ruleFsts); + SHERPA_ONNX_ASSIGN_ATTR_INT32(max_num_sentences, maxNumSentences); + SHERPA_ONNX_ASSIGN_ATTR_STR(rule_fars, ruleFars); + + SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTts(&c); + + if (c.model.vits.model) { + delete[] c.model.vits.model; + } + + if (c.model.vits.lexicon) { + delete[] c.model.vits.lexicon; + } + + if (c.model.vits.tokens) { + delete[] c.model.vits.tokens; + } + + if (c.model.vits.data_dir) { + delete[] c.model.vits.data_dir; + } + + if (c.model.vits.dict_dir) { + delete[] c.model.vits.dict_dir; + } + + if (c.model.provider) { + delete[] c.model.provider; + } + + if (c.rule_fsts) { + delete[] c.rule_fsts; + } + + if (c.rule_fars) { + delete[] c.rule_fars; + } + + if (!tts) { + Napi::TypeError::New(env, "Please check your config!") + .ThrowAsJavaScriptException(); + + return {}; + } + + return Napi::External::New( + env, tts, [](Napi::Env env, SherpaOnnxOfflineTts *tts) { + SherpaOnnxDestroyOfflineTts(tts); + }); +} + +static Napi::Number OfflineTtsSampleRateWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be an offline tts pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxOfflineTts *tts = + info[0].As>().Data(); + + int32_t sample_rate = SherpaOnnxOfflineTtsSampleRate(tts); + + return Napi::Number::New(env, sample_rate); +} + +static Napi::Number OfflineTtsNumSpeakersWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be an offline tts pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxOfflineTts *tts = + info[0].As>().Data(); + + int32_t num_speakers = SherpaOnnxOfflineTtsNumSpeakers(tts); + + return Napi::Number::New(env, num_speakers); +} + +static Napi::Object OfflineTtsGenerateWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be an offline tts pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxOfflineTts *tts = + info[0].As>().Data(); + + if (!info[1].IsObject()) { + Napi::TypeError::New(env, "Argument 1 should be an object") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Object obj = info[1].As(); + + if (!obj.Has("text")) { + Napi::TypeError::New(env, "The argument object should have a field text") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Get("text").IsString()) { + Napi::TypeError::New(env, "The object['text'] should be a string") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Has("sid")) { + Napi::TypeError::New(env, "The argument object should have a field sid") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Get("sid").IsNumber()) { + Napi::TypeError::New(env, "The object['sid'] should be a number") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Has("speed")) { + Napi::TypeError::New(env, "The argument object should have a field speed") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Get("speed").IsNumber()) { + Napi::TypeError::New(env, "The object['speed'] should be a number") + .ThrowAsJavaScriptException(); + + return {}; + } + + bool enable_external_buffer = true; + if (obj.Has("enableExternalBuffer") && + obj.Get("enableExternalBuffer").IsBoolean()) { + enable_external_buffer = + obj.Get("enableExternalBuffer").As().Value(); + } + + Napi::String _text = obj.Get("text").As(); + std::string text = _text.Utf8Value(); + int32_t sid = obj.Get("sid").As().Int32Value(); + float speed = obj.Get("speed").As().FloatValue(); + + const SherpaOnnxGeneratedAudio *audio = + SherpaOnnxOfflineTtsGenerate(tts, text.c_str(), sid, speed); + + if (enable_external_buffer) { + Napi::ArrayBuffer arrayBuffer = Napi::ArrayBuffer::New( + env, const_cast(audio->samples), sizeof(float) * audio->n, + [](Napi::Env /*env*/, void * /*data*/, + const SherpaOnnxGeneratedAudio *hint) { + SherpaOnnxDestroyOfflineTtsGeneratedAudio(hint); + }, + audio); + Napi::Float32Array float32Array = + Napi::Float32Array::New(env, audio->n, arrayBuffer, 0); + + Napi::Object ans = Napi::Object::New(env); + ans.Set(Napi::String::New(env, "samples"), float32Array); + ans.Set(Napi::String::New(env, "sampleRate"), audio->sample_rate); + return ans; + } else { + // don't use external buffer + Napi::ArrayBuffer arrayBuffer = + Napi::ArrayBuffer::New(env, sizeof(float) * audio->n); + + Napi::Float32Array float32Array = + Napi::Float32Array::New(env, audio->n, arrayBuffer, 0); + + std::copy(audio->samples, audio->samples + audio->n, float32Array.Data()); + + Napi::Object ans = Napi::Object::New(env); + ans.Set(Napi::String::New(env, "samples"), float32Array); + ans.Set(Napi::String::New(env, "sampleRate"), audio->sample_rate); + SherpaOnnxDestroyOfflineTtsGeneratedAudio(audio); + return ans; + } +} + +void InitNonStreamingTts(Napi::Env env, Napi::Object exports) { + exports.Set(Napi::String::New(env, "createOfflineTts"), + Napi::Function::New(env, CreateOfflineTtsWrapper)); + + exports.Set(Napi::String::New(env, "getOfflineTtsSampleRate"), + Napi::Function::New(env, OfflineTtsSampleRateWrapper)); + + exports.Set(Napi::String::New(env, "getOfflineTtsNumSpeakers"), + Napi::Function::New(env, OfflineTtsNumSpeakersWrapper)); + + exports.Set(Napi::String::New(env, "offlineTtsGenerate"), + Napi::Function::New(env, OfflineTtsGenerateWrapper)); +} diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/punctuation.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/punctuation.cc new file mode 100644 index 000000000..df079b96d --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/punctuation.cc @@ -0,0 +1,135 @@ +// scripts/node-addon-api/src/punctuation.cc +// +// Copyright (c) 2024 Xiaomi Corporation +#include + +#include "macros.h" // NOLINT +#include "napi.h" // NOLINT +#include "sherpa-onnx/c-api/c-api.h" + +static SherpaOnnxOfflinePunctuationModelConfig GetOfflinePunctuationModelConfig( + Napi::Object obj) { + SherpaOnnxOfflinePunctuationModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("model") || !obj.Get("model").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("model").As(); + + SHERPA_ONNX_ASSIGN_ATTR_STR(ct_transformer, ctTransformer); + + SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); + + if (o.Has("debug") && + (o.Get("debug").IsNumber() || o.Get("debug").IsBoolean())) { + if (o.Get("debug").IsBoolean()) { + c.debug = o.Get("debug").As().Value(); + } else { + c.debug = o.Get("debug").As().Int32Value(); + } + } + SHERPA_ONNX_ASSIGN_ATTR_STR(provider, provider); + + return c; +} + +static Napi::External +CreateOfflinePunctuationWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsObject()) { + Napi::TypeError::New(env, "You should pass an object as the only argument.") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Object o = info[0].As(); + + SherpaOnnxOfflinePunctuationConfig c; + memset(&c, 0, sizeof(c)); + c.model = GetOfflinePunctuationModelConfig(o); + + const SherpaOnnxOfflinePunctuation *punct = + SherpaOnnxCreateOfflinePunctuation(&c); + + if (c.model.ct_transformer) { + delete[] c.model.ct_transformer; + } + + if (c.model.provider) { + delete[] c.model.provider; + } + + if (!punct) { + Napi::TypeError::New(env, "Please check your config!") + .ThrowAsJavaScriptException(); + + return {}; + } + + return Napi::External::New( + env, const_cast(punct), + [](Napi::Env env, SherpaOnnxOfflinePunctuation *punct) { + SherpaOnnxDestroyOfflinePunctuation(punct); + }); +} + +static Napi::String OfflinePunctuationAddPunctWraper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New( + env, + "You should pass an offline punctuation pointer as the first argument") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[1].IsString()) { + Napi::TypeError::New(env, "You should pass a string as the second argument") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxOfflinePunctuation *punct = + info[0].As>().Data(); + Napi::String js_text = info[1].As(); + std::string text = js_text.Utf8Value(); + + const char *punct_text = + SherpaOfflinePunctuationAddPunct(punct, text.c_str()); + + Napi::String ans = Napi::String::New(env, punct_text); + SherpaOfflinePunctuationFreeText(punct_text); + return ans; +} + +void InitPunctuation(Napi::Env env, Napi::Object exports) { + exports.Set(Napi::String::New(env, "createOfflinePunctuation"), + Napi::Function::New(env, CreateOfflinePunctuationWrapper)); + + exports.Set(Napi::String::New(env, "offlinePunctuationAddPunct"), + Napi::Function::New(env, OfflinePunctuationAddPunctWraper)); +} diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/sherpa-onnx-node-addon-api.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/sherpa-onnx-node-addon-api.cc new file mode 100644 index 000000000..3f0affd79 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/sherpa-onnx-node-addon-api.cc @@ -0,0 +1,47 @@ +// scripts/node-addon-api/src/sherpa-onnx-node-addon-api.cc +// +// Copyright (c) 2024 Xiaomi Corporation +#include "napi.h" // NOLINT + +void InitStreamingAsr(Napi::Env env, Napi::Object exports); + +void InitNonStreamingAsr(Napi::Env env, Napi::Object exports); + +void InitNonStreamingTts(Napi::Env env, Napi::Object exports); + +void InitVad(Napi::Env env, Napi::Object exports); + +void InitWaveReader(Napi::Env env, Napi::Object exports); + +void InitWaveWriter(Napi::Env env, Napi::Object exports); + +void InitSpokenLanguageID(Napi::Env env, Napi::Object exports); + +void InitSpeakerID(Napi::Env env, Napi::Object exports); + +void InitAudioTagging(Napi::Env env, Napi::Object exports); + +void InitPunctuation(Napi::Env env, Napi::Object exports); + +void InitKeywordSpotting(Napi::Env env, Napi::Object exports); + +void InitNonStreamingSpeakerDiarization(Napi::Env env, Napi::Object exports); + +Napi::Object Init(Napi::Env env, Napi::Object exports) { + InitStreamingAsr(env, exports); + InitNonStreamingAsr(env, exports); + InitNonStreamingTts(env, exports); + InitVad(env, exports); + InitWaveReader(env, exports); + InitWaveWriter(env, exports); + InitSpokenLanguageID(env, exports); + InitSpeakerID(env, exports); + InitAudioTagging(env, exports); + InitPunctuation(env, exports); + InitKeywordSpotting(env, exports); + InitNonStreamingSpeakerDiarization(env, exports); + + return exports; +} + +NODE_API_MODULE(addon, Init) diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/speaker-identification.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/speaker-identification.cc new file mode 100644 index 000000000..a08a6ed66 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/speaker-identification.cc @@ -0,0 +1,808 @@ +// scripts/node-addon-api/src/speaker-identification.cc +// +// Copyright (c) 2024 Xiaomi Corporation +#include +#include + +#include "macros.h" // NOLINT +#include "napi.h" // NOLINT +#include "sherpa-onnx/c-api/c-api.h" + +static Napi::External +CreateSpeakerEmbeddingExtractorWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsObject()) { + Napi::TypeError::New(env, "You should pass an object as the only argument.") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Object o = info[0].As(); + + SherpaOnnxSpeakerEmbeddingExtractorConfig c; + memset(&c, 0, sizeof(c)); + + SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); + SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); + + if (o.Has("debug") && + (o.Get("debug").IsNumber() || o.Get("debug").IsBoolean())) { + if (o.Get("debug").IsBoolean()) { + c.debug = o.Get("debug").As().Value(); + } else { + c.debug = o.Get("debug").As().Int32Value(); + } + } + + SHERPA_ONNX_ASSIGN_ATTR_STR(provider, provider); + + const SherpaOnnxSpeakerEmbeddingExtractor *extractor = + SherpaOnnxCreateSpeakerEmbeddingExtractor(&c); + + if (c.model) { + delete[] c.model; + } + + if (c.provider) { + delete[] c.provider; + } + + if (!extractor) { + Napi::TypeError::New(env, "Please check your config!") + .ThrowAsJavaScriptException(); + + return {}; + } + + return Napi::External::New( + env, const_cast(extractor), + [](Napi::Env env, SherpaOnnxSpeakerEmbeddingExtractor *extractor) { + SherpaOnnxDestroySpeakerEmbeddingExtractor(extractor); + }); +} + +static Napi::Number SpeakerEmbeddingExtractorDimWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New( + env, "Argument 0 should be a speaker embedding extractor pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxSpeakerEmbeddingExtractor *extractor = + info[0].As>().Data(); + + int32_t dim = SherpaOnnxSpeakerEmbeddingExtractorDim(extractor); + + return Napi::Number::New(env, dim); +} + +static Napi::External +SpeakerEmbeddingExtractorCreateStreamWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, + "You should pass a speaker embedding extractor " + "pointer as the only argument") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxSpeakerEmbeddingExtractor *extractor = + info[0].As>().Data(); + + const SherpaOnnxOnlineStream *stream = + SherpaOnnxSpeakerEmbeddingExtractorCreateStream(extractor); + + return Napi::External::New( + env, const_cast(stream), + [](Napi::Env env, SherpaOnnxOnlineStream *stream) { + SherpaOnnxDestroyOnlineStream(stream); + }); +} + +static Napi::Boolean SpeakerEmbeddingExtractorIsReadyWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New( + env, "Argument 0 should be a speaker embedding extractor pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[1].IsExternal()) { + Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxSpeakerEmbeddingExtractor *extractor = + info[0].As>().Data(); + + SherpaOnnxOnlineStream *stream = + info[1].As>().Data(); + + int32_t is_ready = + SherpaOnnxSpeakerEmbeddingExtractorIsReady(extractor, stream); + + return Napi::Boolean::New(env, is_ready); +} + +static Napi::Float32Array SpeakerEmbeddingExtractorComputeEmbeddingWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 2 && info.Length() != 3) { + std::ostringstream os; + os << "Expect only 2 or 3 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New( + env, "Argument 0 should be a speaker embedding extractor pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[1].IsExternal()) { + Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + bool enable_external_buffer = true; + if (info.Length() == 3) { + if (info[2].IsBoolean()) { + enable_external_buffer = info[2].As().Value(); + } else { + Napi::TypeError::New(env, "Argument 2 should be a boolean.") + .ThrowAsJavaScriptException(); + } + } + + SherpaOnnxSpeakerEmbeddingExtractor *extractor = + info[0].As>().Data(); + + SherpaOnnxOnlineStream *stream = + info[1].As>().Data(); + + const float *v = + SherpaOnnxSpeakerEmbeddingExtractorComputeEmbedding(extractor, stream); + + int32_t dim = SherpaOnnxSpeakerEmbeddingExtractorDim(extractor); + + if (enable_external_buffer) { + Napi::ArrayBuffer arrayBuffer = Napi::ArrayBuffer::New( + env, const_cast(v), sizeof(float) * dim, + [](Napi::Env /*env*/, void *data) { + SherpaOnnxSpeakerEmbeddingExtractorDestroyEmbedding( + reinterpret_cast(data)); + }); + + return Napi::Float32Array::New(env, dim, arrayBuffer, 0); + } else { + // don't use external buffer + Napi::ArrayBuffer arrayBuffer = + Napi::ArrayBuffer::New(env, sizeof(float) * dim); + + Napi::Float32Array float32Array = + Napi::Float32Array::New(env, dim, arrayBuffer, 0); + + std::copy(v, v + dim, float32Array.Data()); + + SherpaOnnxSpeakerEmbeddingExtractorDestroyEmbedding(v); + + return float32Array; + } +} + +static Napi::External +CreateSpeakerEmbeddingManagerWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsNumber()) { + Napi::TypeError::New(env, + "You should pass an integer as the only argument.") + .ThrowAsJavaScriptException(); + + return {}; + } + + int32_t dim = info[0].As().Int32Value(); + + const SherpaOnnxSpeakerEmbeddingManager *manager = + SherpaOnnxCreateSpeakerEmbeddingManager(dim); + + if (!manager) { + Napi::TypeError::New(env, "Please check your input dim!") + .ThrowAsJavaScriptException(); + + return {}; + } + + return Napi::External::New( + env, const_cast(manager), + [](Napi::Env env, SherpaOnnxSpeakerEmbeddingManager *manager) { + SherpaOnnxDestroySpeakerEmbeddingManager(manager); + }); +} + +static Napi::Boolean SpeakerEmbeddingManagerAddWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, + "You should pass a speaker embedding manager pointer " + "as the first argument.") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[1].IsObject()) { + Napi::TypeError::New(env, "Argument 1 should be an object") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxSpeakerEmbeddingManager *manager = + info[0].As>().Data(); + + Napi::Object obj = info[1].As(); + + if (!obj.Has("v")) { + Napi::TypeError::New(env, "The argument object should have a field v") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Get("v").IsTypedArray()) { + Napi::TypeError::New(env, "The object['v'] should be a typed array") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Has("name")) { + Napi::TypeError::New(env, "The argument object should have a field name") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Get("name").IsString()) { + Napi::TypeError::New(env, "The object['name'] should be a string") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Float32Array v = obj.Get("v").As(); + Napi::String js_name = obj.Get("name").As(); + std::string name = js_name.Utf8Value(); + + int32_t ok = + SherpaOnnxSpeakerEmbeddingManagerAdd(manager, name.c_str(), v.Data()); + return Napi::Boolean::New(env, ok); +} + +static Napi::Boolean SpeakerEmbeddingManagerAddListFlattenedWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, + "You should pass a speaker embedding manager pointer " + "as the first argument.") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[1].IsObject()) { + Napi::TypeError::New(env, "Argument 1 should be an object") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxSpeakerEmbeddingManager *manager = + info[0].As>().Data(); + + Napi::Object obj = info[1].As(); + + if (!obj.Has("vv")) { + Napi::TypeError::New(env, "The argument object should have a field vv") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Get("vv").IsTypedArray()) { + Napi::TypeError::New(env, "The object['vv'] should be a typed array") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Has("name")) { + Napi::TypeError::New(env, "The argument object should have a field name") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Get("name").IsString()) { + Napi::TypeError::New(env, "The object['name'] should be a string") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Has("n")) { + Napi::TypeError::New(env, "The argument object should have a field n") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Get("n").IsNumber()) { + Napi::TypeError::New(env, "The object['n'] should be an integer") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Float32Array v = obj.Get("vv").As(); + Napi::String js_name = obj.Get("name").As(); + int32_t n = obj.Get("n").As().Int32Value(); + + std::string name = js_name.Utf8Value(); + + int32_t ok = SherpaOnnxSpeakerEmbeddingManagerAddListFlattened( + manager, name.c_str(), v.Data(), n); + + return Napi::Boolean::New(env, ok); +} + +static Napi::Boolean SpeakerEmbeddingManagerRemoveWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, + "You should pass a speaker embedding manager pointer " + "as the first argument.") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[1].IsString()) { + Napi::TypeError::New(env, "Argument 1 should be string") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxSpeakerEmbeddingManager *manager = + info[0].As>().Data(); + + Napi::String js_name = info[1].As(); + std::string name = js_name.Utf8Value(); + + int32_t ok = SherpaOnnxSpeakerEmbeddingManagerRemove(manager, name.c_str()); + + return Napi::Boolean::New(env, ok); +} + +static Napi::String SpeakerEmbeddingManagerSearchWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, + "You should pass a speaker embedding manager pointer " + "as the first argument.") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[1].IsObject()) { + Napi::TypeError::New(env, "Argument 1 should be an object") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxSpeakerEmbeddingManager *manager = + info[0].As>().Data(); + + Napi::Object obj = info[1].As(); + + if (!obj.Has("v")) { + Napi::TypeError::New(env, "The argument object should have a field v") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Get("v").IsTypedArray()) { + Napi::TypeError::New(env, "The object['v'] should be a typed array") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Has("threshold")) { + Napi::TypeError::New(env, + "The argument object should have a field threshold") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Get("threshold").IsNumber()) { + Napi::TypeError::New(env, "The object['threshold'] should be a float") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Float32Array v = obj.Get("v").As(); + float threshold = obj.Get("threshold").As().FloatValue(); + + const char *name = + SherpaOnnxSpeakerEmbeddingManagerSearch(manager, v.Data(), threshold); + const char *p = name; + if (!p) { + p = ""; + } + + Napi::String js_name = Napi::String::New(env, p); + SherpaOnnxSpeakerEmbeddingManagerFreeSearch(name); + + return js_name; +} + +static Napi::Boolean SpeakerEmbeddingManagerVerifyWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, + "You should pass a speaker embedding manager pointer " + "as the first argument.") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[1].IsObject()) { + Napi::TypeError::New(env, "Argument 1 should be an object") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxSpeakerEmbeddingManager *manager = + info[0].As>().Data(); + + Napi::Object obj = info[1].As(); + + if (!obj.Has("v")) { + Napi::TypeError::New(env, "The argument object should have a field v") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Get("v").IsTypedArray()) { + Napi::TypeError::New(env, "The object['v'] should be a typed array") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Has("threshold")) { + Napi::TypeError::New(env, + "The argument object should have a field threshold") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Get("threshold").IsNumber()) { + Napi::TypeError::New(env, "The object['threshold'] should be a float") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Has("name")) { + Napi::TypeError::New(env, "The argument object should have a field name") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Get("name").IsString()) { + Napi::TypeError::New(env, "The object['name'] should be a string") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Float32Array v = obj.Get("v").As(); + float threshold = obj.Get("threshold").As().FloatValue(); + + Napi::String js_name = obj.Get("name").As(); + std::string name = js_name.Utf8Value(); + + int32_t found = SherpaOnnxSpeakerEmbeddingManagerVerify(manager, name.c_str(), + v.Data(), threshold); + + return Napi::Boolean::New(env, found); +} + +static Napi::Boolean SpeakerEmbeddingManagerContainsWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, + "You should pass a speaker embedding manager pointer " + "as the first argument.") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[1].IsString()) { + Napi::TypeError::New(env, "Argument 1 should be a string") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxSpeakerEmbeddingManager *manager = + info[0].As>().Data(); + + Napi::String js_name = info[1].As(); + std::string name = js_name.Utf8Value(); + + int32_t exists = + SherpaOnnxSpeakerEmbeddingManagerContains(manager, name.c_str()); + + return Napi::Boolean::New(env, exists); +} + +static Napi::Number SpeakerEmbeddingManagerNumSpeakersWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, + "You should pass a speaker embedding manager pointer " + "as the first argument.") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxSpeakerEmbeddingManager *manager = + info[0].As>().Data(); + + int32_t num_speakers = SherpaOnnxSpeakerEmbeddingManagerNumSpeakers(manager); + + return Napi::Number::New(env, num_speakers); +} + +static Napi::Array SpeakerEmbeddingManagerGetAllSpeakersWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, + "You should pass a speaker embedding manager pointer " + "as the first argument.") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxSpeakerEmbeddingManager *manager = + info[0].As>().Data(); + + int32_t num_speakers = SherpaOnnxSpeakerEmbeddingManagerNumSpeakers(manager); + if (num_speakers == 0) { + return {}; + } + + const char *const *all_speaker_names = + SherpaOnnxSpeakerEmbeddingManagerGetAllSpeakers(manager); + + Napi::Array ans = Napi::Array::New(env, num_speakers); + for (uint32_t i = 0; i != num_speakers; ++i) { + ans[i] = Napi::String::New(env, all_speaker_names[i]); + } + SherpaOnnxSpeakerEmbeddingManagerFreeAllSpeakers(all_speaker_names); + return ans; +} + +void InitSpeakerID(Napi::Env env, Napi::Object exports) { + exports.Set(Napi::String::New(env, "createSpeakerEmbeddingExtractor"), + Napi::Function::New(env, CreateSpeakerEmbeddingExtractorWrapper)); + + exports.Set(Napi::String::New(env, "speakerEmbeddingExtractorDim"), + Napi::Function::New(env, SpeakerEmbeddingExtractorDimWrapper)); + + exports.Set( + Napi::String::New(env, "speakerEmbeddingExtractorCreateStream"), + Napi::Function::New(env, SpeakerEmbeddingExtractorCreateStreamWrapper)); + + exports.Set( + Napi::String::New(env, "speakerEmbeddingExtractorIsReady"), + Napi::Function::New(env, SpeakerEmbeddingExtractorIsReadyWrapper)); + + exports.Set( + Napi::String::New(env, "speakerEmbeddingExtractorComputeEmbedding"), + Napi::Function::New(env, + SpeakerEmbeddingExtractorComputeEmbeddingWrapper)); + + exports.Set(Napi::String::New(env, "createSpeakerEmbeddingManager"), + Napi::Function::New(env, CreateSpeakerEmbeddingManagerWrapper)); + + exports.Set(Napi::String::New(env, "speakerEmbeddingManagerAdd"), + Napi::Function::New(env, SpeakerEmbeddingManagerAddWrapper)); + + exports.Set( + Napi::String::New(env, "speakerEmbeddingManagerAddListFlattened"), + Napi::Function::New(env, SpeakerEmbeddingManagerAddListFlattenedWrapper)); + + exports.Set(Napi::String::New(env, "speakerEmbeddingManagerRemove"), + Napi::Function::New(env, SpeakerEmbeddingManagerRemoveWrapper)); + + exports.Set(Napi::String::New(env, "speakerEmbeddingManagerSearch"), + Napi::Function::New(env, SpeakerEmbeddingManagerSearchWrapper)); + + exports.Set(Napi::String::New(env, "speakerEmbeddingManagerVerify"), + Napi::Function::New(env, SpeakerEmbeddingManagerVerifyWrapper)); + + exports.Set(Napi::String::New(env, "speakerEmbeddingManagerContains"), + Napi::Function::New(env, SpeakerEmbeddingManagerContainsWrapper)); + + exports.Set( + Napi::String::New(env, "speakerEmbeddingManagerNumSpeakers"), + Napi::Function::New(env, SpeakerEmbeddingManagerNumSpeakersWrapper)); + + exports.Set( + Napi::String::New(env, "speakerEmbeddingManagerGetAllSpeakers"), + Napi::Function::New(env, SpeakerEmbeddingManagerGetAllSpeakersWrapper)); +} diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/spoken-language-identification.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/spoken-language-identification.cc new file mode 100644 index 000000000..35ade6541 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/spoken-language-identification.cc @@ -0,0 +1,188 @@ +// scripts/node-addon-api/src/spoken-language-identification.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include + +#include "macros.h" // NOLINT +#include "napi.h" // NOLINT +#include "sherpa-onnx/c-api/c-api.h" + +static SherpaOnnxSpokenLanguageIdentificationWhisperConfig +GetSpokenLanguageIdentificationWhisperConfig(Napi::Object obj) { + SherpaOnnxSpokenLanguageIdentificationWhisperConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("whisper") || !obj.Get("whisper").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("whisper").As(); + + SHERPA_ONNX_ASSIGN_ATTR_STR(encoder, encoder); + SHERPA_ONNX_ASSIGN_ATTR_STR(decoder, decoder); + SHERPA_ONNX_ASSIGN_ATTR_INT32(tail_paddings, tailPaddings); + + return c; +} + +static Napi::External +CreateSpokenLanguageIdentificationWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsObject()) { + Napi::TypeError::New(env, "You should pass an object as the only argument.") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Object o = info[0].As(); + + SherpaOnnxSpokenLanguageIdentificationConfig c; + memset(&c, 0, sizeof(c)); + c.whisper = GetSpokenLanguageIdentificationWhisperConfig(o); + + SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); + + if (o.Has("debug") && + (o.Get("debug").IsNumber() || o.Get("debug").IsBoolean())) { + if (o.Get("debug").IsBoolean()) { + c.debug = o.Get("debug").As().Value(); + } else { + c.debug = o.Get("debug").As().Int32Value(); + } + } + SHERPA_ONNX_ASSIGN_ATTR_STR(provider, provider); + + const SherpaOnnxSpokenLanguageIdentification *slid = + SherpaOnnxCreateSpokenLanguageIdentification(&c); + + if (c.whisper.encoder) { + delete[] c.whisper.encoder; + } + + if (c.whisper.decoder) { + delete[] c.whisper.decoder; + } + + if (c.provider) { + delete[] c.provider; + } + + if (!slid) { + Napi::TypeError::New(env, "Please check your config!") + .ThrowAsJavaScriptException(); + + return {}; + } + + return Napi::External::New( + env, const_cast(slid), + [](Napi::Env env, SherpaOnnxSpokenLanguageIdentification *slid) { + SherpaOnnxDestroySpokenLanguageIdentification(slid); + }); +} + +static Napi::External +SpokenLanguageIdentificationCreateOfflineStreamWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New( + env, + "You should pass an offline language ID pointer as the only argument") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxSpokenLanguageIdentification *slid = + info[0] + .As>() + .Data(); + + SherpaOnnxOfflineStream *stream = + SherpaOnnxSpokenLanguageIdentificationCreateOfflineStream(slid); + + return Napi::External::New( + env, stream, [](Napi::Env env, SherpaOnnxOfflineStream *stream) { + SherpaOnnxDestroyOfflineStream(stream); + }); +} + +static Napi::String SpokenLanguageIdentificationComputeWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New( + env, "Argument 0 should be an offline spoken language ID pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[1].IsExternal()) { + Napi::TypeError::New(env, "Argument 1 should be an offline stream pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxSpokenLanguageIdentification *slid = + info[0] + .As>() + .Data(); + + SherpaOnnxOfflineStream *stream = + info[1].As>().Data(); + + const SherpaOnnxSpokenLanguageIdentificationResult *r = + SherpaOnnxSpokenLanguageIdentificationCompute(slid, stream); + + std::string lang = r->lang; + SherpaOnnxDestroySpokenLanguageIdentificationResult(r); + + return Napi::String::New(env, lang); +} + +void InitSpokenLanguageID(Napi::Env env, Napi::Object exports) { + exports.Set( + Napi::String::New(env, "createSpokenLanguageIdentification"), + Napi::Function::New(env, CreateSpokenLanguageIdentificationWrapper)); + + exports.Set( + Napi::String::New(env, "createSpokenLanguageIdentificationOfflineStream"), + Napi::Function::New( + env, SpokenLanguageIdentificationCreateOfflineStreamWrapper)); + + exports.Set( + Napi::String::New(env, "spokenLanguageIdentificationCompute"), + Napi::Function::New(env, SpokenLanguageIdentificationComputeWrapper)); +} diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/streaming-asr.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/streaming-asr.cc new file mode 100644 index 000000000..ffe562d79 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/streaming-asr.cc @@ -0,0 +1,731 @@ +// scripts/node-addon-api/src/streaming-asr.cc +// +// Copyright (c) 2024 Xiaomi Corporation +#include + +#include "macros.h" // NOLINT +#include "napi.h" // NOLINT +#include "sherpa-onnx/c-api/c-api.h" +/* +{ + 'featConfig': { + 'sampleRate': 16000, + 'featureDim': 80, + } +}; + */ +SherpaOnnxFeatureConfig GetFeatureConfig(Napi::Object obj) { + SherpaOnnxFeatureConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("featConfig") || !obj.Get("featConfig").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("featConfig").As(); + + SHERPA_ONNX_ASSIGN_ATTR_INT32(sample_rate, sampleRate); + SHERPA_ONNX_ASSIGN_ATTR_INT32(feature_dim, featureDim); + + return c; +} +/* +{ + 'transducer': { + 'encoder': './encoder.onnx', + 'decoder': './decoder.onnx', + 'joiner': './joiner.onnx', + } +} + */ + +static SherpaOnnxOnlineTransducerModelConfig GetOnlineTransducerModelConfig( + Napi::Object obj) { + SherpaOnnxOnlineTransducerModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("transducer") || !obj.Get("transducer").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("transducer").As(); + + SHERPA_ONNX_ASSIGN_ATTR_STR(encoder, encoder); + SHERPA_ONNX_ASSIGN_ATTR_STR(decoder, decoder); + SHERPA_ONNX_ASSIGN_ATTR_STR(joiner, joiner); + + return c; +} + +static SherpaOnnxOnlineZipformer2CtcModelConfig +GetOnlineZipformer2CtcModelConfig(Napi::Object obj) { + SherpaOnnxOnlineZipformer2CtcModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("zipformer2Ctc") || !obj.Get("zipformer2Ctc").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("zipformer2Ctc").As(); + + SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); + + return c; +} + +static SherpaOnnxOnlineParaformerModelConfig GetOnlineParaformerModelConfig( + Napi::Object obj) { + SherpaOnnxOnlineParaformerModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("paraformer") || !obj.Get("paraformer").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("paraformer").As(); + + SHERPA_ONNX_ASSIGN_ATTR_STR(encoder, encoder); + SHERPA_ONNX_ASSIGN_ATTR_STR(decoder, decoder); + + return c; +} + +SherpaOnnxOnlineModelConfig GetOnlineModelConfig(Napi::Object obj) { + SherpaOnnxOnlineModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("modelConfig") || !obj.Get("modelConfig").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("modelConfig").As(); + + c.transducer = GetOnlineTransducerModelConfig(o); + c.paraformer = GetOnlineParaformerModelConfig(o); + c.zipformer2_ctc = GetOnlineZipformer2CtcModelConfig(o); + + SHERPA_ONNX_ASSIGN_ATTR_STR(tokens, tokens); + SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); + SHERPA_ONNX_ASSIGN_ATTR_STR(provider, provider); + + if (o.Has("debug") && + (o.Get("debug").IsNumber() || o.Get("debug").IsBoolean())) { + if (o.Get("debug").IsBoolean()) { + c.debug = o.Get("debug").As().Value(); + } else { + c.debug = o.Get("debug").As().Int32Value(); + } + } + + SHERPA_ONNX_ASSIGN_ATTR_STR(model_type, modelType); + SHERPA_ONNX_ASSIGN_ATTR_STR(modeling_unit, modelingUnit); + SHERPA_ONNX_ASSIGN_ATTR_STR(bpe_vocab, bpeVocab); + SHERPA_ONNX_ASSIGN_ATTR_STR(tokens_buf, tokensBuf); + SHERPA_ONNX_ASSIGN_ATTR_INT32(tokens_buf_size, tokensBufSize); + + return c; +} + +static SherpaOnnxOnlineCtcFstDecoderConfig GetCtcFstDecoderConfig( + Napi::Object obj) { + SherpaOnnxOnlineCtcFstDecoderConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("ctcFstDecoderConfig") || + !obj.Get("ctcFstDecoderConfig").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("ctcFstDecoderConfig").As(); + + SHERPA_ONNX_ASSIGN_ATTR_STR(graph, graph); + SHERPA_ONNX_ASSIGN_ATTR_INT32(max_active, maxActive); + + return c; +} + +static Napi::External CreateOnlineRecognizerWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); +#if __OHOS__ + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } +#else + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } +#endif + + if (!info[0].IsObject()) { + Napi::TypeError::New(env, "Expect an object as the argument") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Object o = info[0].As(); + SherpaOnnxOnlineRecognizerConfig c; + memset(&c, 0, sizeof(c)); + c.feat_config = GetFeatureConfig(o); + c.model_config = GetOnlineModelConfig(o); + + SHERPA_ONNX_ASSIGN_ATTR_STR(decoding_method, decodingMethod); + SHERPA_ONNX_ASSIGN_ATTR_INT32(max_active_paths, maxActivePaths); + + // enableEndpoint can be either a boolean or an integer + if (o.Has("enableEndpoint") && (o.Get("enableEndpoint").IsNumber() || + o.Get("enableEndpoint").IsBoolean())) { + if (o.Get("enableEndpoint").IsNumber()) { + c.enable_endpoint = + o.Get("enableEndpoint").As().Int32Value(); + } else { + c.enable_endpoint = o.Get("enableEndpoint").As().Value(); + } + } + + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(rule1_min_trailing_silence, + rule1MinTrailingSilence); + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(rule2_min_trailing_silence, + rule2MinTrailingSilence); + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(rule3_min_utterance_length, + rule3MinUtteranceLength); + SHERPA_ONNX_ASSIGN_ATTR_STR(hotwords_file, hotwordsFile); + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(hotwords_score, hotwordsScore); + SHERPA_ONNX_ASSIGN_ATTR_STR(rule_fsts, ruleFsts); + SHERPA_ONNX_ASSIGN_ATTR_STR(rule_fars, ruleFars); + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(blank_penalty, blankPenalty); + SHERPA_ONNX_ASSIGN_ATTR_STR(hotwords_buf, hotwordsBuf); + SHERPA_ONNX_ASSIGN_ATTR_INT32(hotwords_buf_size, hotwordsBufSize); + + c.ctc_fst_decoder_config = GetCtcFstDecoderConfig(o); + +#if __OHOS__ + std::unique_ptr mgr (OH_ResourceManager_InitNativeResourceManager(env, info[1]), &OH_ResourceManager_ReleaseNativeResourceManager); + + const SherpaOnnxOnlineRecognizer *recognizer = + SherpaOnnxCreateOnlineRecognizerOHOS(&c, mgr.get()); +#else + const SherpaOnnxOnlineRecognizer *recognizer = + SherpaOnnxCreateOnlineRecognizer(&c); +#endif + + if (c.model_config.transducer.encoder) { + delete[] c.model_config.transducer.encoder; + } + + if (c.model_config.transducer.decoder) { + delete[] c.model_config.transducer.decoder; + } + + if (c.model_config.transducer.joiner) { + delete[] c.model_config.transducer.joiner; + } + + if (c.model_config.paraformer.encoder) { + delete[] c.model_config.paraformer.encoder; + } + + if (c.model_config.paraformer.decoder) { + delete[] c.model_config.paraformer.decoder; + } + + if (c.model_config.zipformer2_ctc.model) { + delete[] c.model_config.zipformer2_ctc.model; + } + + if (c.model_config.tokens) { + delete[] c.model_config.tokens; + } + + if (c.model_config.provider) { + delete[] c.model_config.provider; + } + + if (c.model_config.model_type) { + delete[] c.model_config.model_type; + } + + if (c.model_config.modeling_unit) { + delete[] c.model_config.modeling_unit; + } + + if (c.model_config.bpe_vocab) { + delete[] c.model_config.bpe_vocab; + } + + if (c.model_config.tokens_buf) { + delete[] c.model_config.tokens_buf; + } + + if (c.decoding_method) { + delete[] c.decoding_method; + } + + if (c.hotwords_file) { + delete[] c.hotwords_file; + } + + if (c.rule_fsts) { + delete[] c.rule_fsts; + } + + if (c.rule_fars) { + delete[] c.rule_fars; + } + + if (c.hotwords_buf) { + delete[] c.hotwords_buf; + } + + if (c.ctc_fst_decoder_config.graph) { + delete[] c.ctc_fst_decoder_config.graph; + } + + if (!recognizer) { + Napi::TypeError::New(env, "Please check your config!") + .ThrowAsJavaScriptException(); + + return {}; + } + + return Napi::External::New( + env, const_cast(recognizer), + [](Napi::Env env, SherpaOnnxOnlineRecognizer *recognizer) { + SherpaOnnxDestroyOnlineRecognizer(recognizer); + }); +} + +static Napi::External CreateOnlineStreamWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New( + env, + "You should pass an online recognizer pointer as the only argument") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxOnlineRecognizer *recognizer = + info[0].As>().Data(); + + const SherpaOnnxOnlineStream *stream = + SherpaOnnxCreateOnlineStream(recognizer); + + return Napi::External::New( + env, const_cast(stream), + [](Napi::Env env, SherpaOnnxOnlineStream *stream) { + SherpaOnnxDestroyOnlineStream(stream); + }); +} + +static void AcceptWaveformWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be an online stream pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + SherpaOnnxOnlineStream *stream = + info[0].As>().Data(); + + if (!info[1].IsObject()) { + Napi::TypeError::New(env, "Argument 1 should be an object") + .ThrowAsJavaScriptException(); + + return; + } + + Napi::Object obj = info[1].As(); + + if (!obj.Has("samples")) { + Napi::TypeError::New(env, "The argument object should have a field samples") + .ThrowAsJavaScriptException(); + + return; + } + + if (!obj.Get("samples").IsTypedArray()) { + Napi::TypeError::New(env, "The object['samples'] should be a typed array") + .ThrowAsJavaScriptException(); + + return; + } + + if (!obj.Has("sampleRate")) { + Napi::TypeError::New(env, + "The argument object should have a field sampleRate") + .ThrowAsJavaScriptException(); + + return; + } + + if (!obj.Get("sampleRate").IsNumber()) { + Napi::TypeError::New(env, "The object['samples'] should be a number") + .ThrowAsJavaScriptException(); + + return; + } + + Napi::Float32Array samples = obj.Get("samples").As(); + int32_t sample_rate = obj.Get("sampleRate").As().Int32Value(); + +#if __OHOS__ + SherpaOnnxOnlineStreamAcceptWaveform(stream, sample_rate, samples.Data(), + samples.ElementLength() / sizeof(float)); +#else + SherpaOnnxOnlineStreamAcceptWaveform(stream, sample_rate, samples.Data(), + samples.ElementLength()); +#endif +} + +static Napi::Boolean IsOnlineStreamReadyWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, + "Argument 0 should be an online recognizer pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[1].IsExternal()) { + Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxOnlineRecognizer *recognizer = + info[0].As>().Data(); + + SherpaOnnxOnlineStream *stream = + info[1].As>().Data(); + + int32_t is_ready = SherpaOnnxIsOnlineStreamReady(recognizer, stream); + + return Napi::Boolean::New(env, is_ready); +} + +static void DecodeOnlineStreamWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, + "Argument 0 should be an online recognizer pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + if (!info[1].IsExternal()) { + Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + SherpaOnnxOnlineRecognizer *recognizer = + info[0].As>().Data(); + + SherpaOnnxOnlineStream *stream = + info[1].As>().Data(); + + SherpaOnnxDecodeOnlineStream(recognizer, stream); +} + +static Napi::String GetOnlineStreamResultAsJsonWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, + "Argument 0 should be an online recognizer pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[1].IsExternal()) { + Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxOnlineRecognizer *recognizer = + info[0].As>().Data(); + + SherpaOnnxOnlineStream *stream = + info[1].As>().Data(); + + const char *json = SherpaOnnxGetOnlineStreamResultAsJson(recognizer, stream); + Napi::String s = Napi::String::New(env, json); + + SherpaOnnxDestroyOnlineStreamResultJson(json); + + return s; +} + +static void InputFinishedWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be an online stream pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + SherpaOnnxOnlineStream *stream = + info[0].As>().Data(); + + SherpaOnnxOnlineStreamInputFinished(stream); +} + +static void ResetOnlineStreamWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, + "Argument 0 should be an online recognizer pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + if (!info[1].IsExternal()) { + Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + SherpaOnnxOnlineRecognizer *recognizer = + info[0].As>().Data(); + + SherpaOnnxOnlineStream *stream = + info[1].As>().Data(); + + SherpaOnnxOnlineStreamReset(recognizer, stream); +} + +static Napi::Boolean IsEndpointWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, + "Argument 0 should be an online recognizer pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[1].IsExternal()) { + Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxOnlineRecognizer *recognizer = + info[0].As>().Data(); + + SherpaOnnxOnlineStream *stream = + info[1].As>().Data(); + + int32_t is_endpoint = SherpaOnnxOnlineStreamIsEndpoint(recognizer, stream); + + return Napi::Boolean::New(env, is_endpoint); +} + +static Napi::External CreateDisplayWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsNumber()) { + Napi::TypeError::New(env, "Expect a number as the argument") + .ThrowAsJavaScriptException(); + + return {}; + } + int32_t max_word_per_line = info[0].As().Int32Value(); + + const SherpaOnnxDisplay *display = SherpaOnnxCreateDisplay(max_word_per_line); + + return Napi::External::New( + env, const_cast(display), + [](Napi::Env env, SherpaOnnxDisplay *display) { + SherpaOnnxDestroyDisplay(display); + }); +} + +static void PrintWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 3) { + std::ostringstream os; + os << "Expect only 3 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be an online stream pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + if (!info[1].IsNumber()) { + Napi::TypeError::New(env, "Argument 1 should be a number.") + .ThrowAsJavaScriptException(); + + return; + } + + if (!info[2].IsString()) { + Napi::TypeError::New(env, "Argument 2 should be a string.") + .ThrowAsJavaScriptException(); + + return; + } + + SherpaOnnxDisplay *display = + info[0].As>().Data(); + + int32_t idx = info[1].As().Int32Value(); + + Napi::String text = info[2].As(); + std::string s = text.Utf8Value(); + SherpaOnnxPrint(display, idx, s.c_str()); +} + +void InitStreamingAsr(Napi::Env env, Napi::Object exports) { + exports.Set(Napi::String::New(env, "createOnlineRecognizer"), + Napi::Function::New(env, CreateOnlineRecognizerWrapper)); + + exports.Set(Napi::String::New(env, "createOnlineStream"), + Napi::Function::New(env, CreateOnlineStreamWrapper)); + + exports.Set(Napi::String::New(env, "acceptWaveformOnline"), + Napi::Function::New(env, AcceptWaveformWrapper)); + + exports.Set(Napi::String::New(env, "isOnlineStreamReady"), + Napi::Function::New(env, IsOnlineStreamReadyWrapper)); + + exports.Set(Napi::String::New(env, "decodeOnlineStream"), + Napi::Function::New(env, DecodeOnlineStreamWrapper)); + + exports.Set(Napi::String::New(env, "getOnlineStreamResultAsJson"), + Napi::Function::New(env, GetOnlineStreamResultAsJsonWrapper)); + + exports.Set(Napi::String::New(env, "inputFinished"), + Napi::Function::New(env, InputFinishedWrapper)); + + exports.Set(Napi::String::New(env, "reset"), + Napi::Function::New(env, ResetOnlineStreamWrapper)); + + exports.Set(Napi::String::New(env, "isEndpoint"), + Napi::Function::New(env, IsEndpointWrapper)); + + exports.Set(Napi::String::New(env, "createDisplay"), + Napi::Function::New(env, CreateDisplayWrapper)); + + exports.Set(Napi::String::New(env, "print"), + Napi::Function::New(env, PrintWrapper)); +} diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts new file mode 100644 index 000000000..10ff7745c --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts @@ -0,0 +1,35 @@ +export const readWave: (filename: string, enableExternalBuffer: boolean = true) => {samples: Float32Array, sampleRate: number}; +export const readWaveFromBinary: (data: Uint8Array, enableExternalBuffer: boolean = true) => {samples: Float32Array, sampleRate: number}; +export const createCircularBuffer: (capacity: number) => object; +export const circularBufferPush: (handle: object, samples: Float32Array) => void; +export const circularBufferGet: (handle: object, index: number, n: number, enableExternalBuffer: boolean = true) => Float32Array; +export const circularBufferPop: (handle: object, n: number) => void; +export const circularBufferSize: (handle: object) => number; +export const circularBufferHead: (handle: object) => number; +export const circularBufferReset: (handle: object) => void; + +export const createVoiceActivityDetector: (config: object, bufferSizeInSeconds: number, mgr?: object) => object; +export const voiceActivityDetectorAcceptWaveform: (handle: object, samples: Float32Array) => void; +export const voiceActivityDetectorIsEmpty: (handle: object) => boolean; +export const voiceActivityDetectorIsDetected: (handle: object) => boolean; +export const voiceActivityDetectorPop: (handle: object) => void; +export const voiceActivityDetectorClear: (handle: object) => void; +export const voiceActivityDetectorFront: (handle: object, enableExternalBuffer: boolean = true) => {samples: Float32Array, start: number}; +export const voiceActivityDetectorReset: (handle: object) => void; +export const voiceActivityDetectorFlush: (handle: object) => void; + +export const createOfflineRecognizer: (config: object, mgr?: object) => object; +export const createOfflineStream: (handle: object) => object; +export const acceptWaveformOffline: (handle: object, audio: object) => void; +export const decodeOfflineStream: (handle: object, streamHandle: object) => void; +export const getOfflineStreamResultAsJson: (streamHandle: object) => string; + +export const createOnlineRecognizer: (config: object, mgr?: object) => object; +export const createOnlineStream: (handle: object) => object; +export const acceptWaveformOnline: (handle: object, audio: object) => void; +export const inputFinished: (streamHandle: object) => void; +export const isOnlineStreamReady: (handle: object, streamHandle: object) => boolean; +export const decodeOnlineStream: (handle: object, streamHandle: object) => void; +export const isEndpoint: (handle: object, streamHandle: object) => boolean; +export const reset: (handle: object, streamHandle: object) => void; +export const getOnlineStreamResultAsJson: (handle: object, streamHandle: object) => string; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/oh-package.json5 b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/oh-package.json5 new file mode 100644 index 000000000..09065d8d3 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/oh-package.json5 @@ -0,0 +1,6 @@ +{ + "name": "libsherpa_onnx.so", + "types": "./Index.d.ts", + "version": "1.0.0", + "description": "Please describe the basic information." +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/vad.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/vad.cc new file mode 100644 index 000000000..b505c53d2 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/vad.cc @@ -0,0 +1,700 @@ +// scripts/node-addon-api/src/vad.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include +#include + +#include "macros.h" // NOLINT +#include "napi.h" // NOLINT +#include "sherpa-onnx/c-api/c-api.h" + +static Napi::External CreateCircularBufferWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsNumber()) { + Napi::TypeError::New(env, "You should pass an integer as the argument.") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxCircularBuffer *buf = + SherpaOnnxCreateCircularBuffer(info[0].As().Int32Value()); + + return Napi::External::New( + env, buf, [](Napi::Env env, SherpaOnnxCircularBuffer *p) { + SherpaOnnxDestroyCircularBuffer(p); + }); +} + +static void CircularBufferPushWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be an CircularBuffer pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + SherpaOnnxCircularBuffer *buf = + info[0].As>().Data(); + + if (!info[1].IsTypedArray()) { + Napi::TypeError::New(env, "Argument 1 should be a Float32Array.") + .ThrowAsJavaScriptException(); + + return; + } + + Napi::Float32Array data = info[1].As(); + +#if __OHOS__ + // Note(fangjun): Normally, we don't need to divied it by sizeof(float). + // However, data.ElementLength() here returns number of bytes, not number of elements. + SherpaOnnxCircularBufferPush(buf, data.Data(), data.ElementLength() / sizeof(float)); +#else + SherpaOnnxCircularBufferPush(buf, data.Data(), data.ElementLength()); +#endif +} + +// see https://github.com/nodejs/node-addon-api/blob/main/doc/typed_array.md +// https://github.com/nodejs/node-addon-examples/blob/main/src/2-js-to-native-conversion/typed_array_to_native/node-addon-api/typed_array_to_native.cc +static Napi::Float32Array CircularBufferGetWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 3 && info.Length() != 4) { + std::ostringstream os; + os << "Expect only 3 or 4 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be an CircularBuffer pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxCircularBuffer *buf = + info[0].As>().Data(); + + if (!info[1].IsNumber()) { + Napi::TypeError::New(env, "Argument 1 should be an integer (startIndex).") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[2].IsNumber()) { + Napi::TypeError::New(env, "Argument 2 should be an integer (n).") + .ThrowAsJavaScriptException(); + + return {}; + } + + bool enable_external_buffer = true; + if (info.Length() == 4) { + if (info[3].IsBoolean()) { + enable_external_buffer = info[3].As().Value(); + } else { + Napi::TypeError::New(env, "Argument 3 should be a boolean.") + .ThrowAsJavaScriptException(); + } + } + + int32_t start_index = info[1].As().Int32Value(); + int32_t n = info[2].As().Int32Value(); + + const float *data = SherpaOnnxCircularBufferGet(buf, start_index, n); + + if (enable_external_buffer) { + Napi::ArrayBuffer arrayBuffer = Napi::ArrayBuffer::New( + env, const_cast(data), sizeof(float) * n, + [](Napi::Env /*env*/, void *p) { + SherpaOnnxCircularBufferFree(reinterpret_cast(p)); + }); + + Napi::Float32Array float32Array = + Napi::Float32Array::New(env, n, arrayBuffer, 0); + + return float32Array; + } else { + // don't use external buffer + Napi::ArrayBuffer arrayBuffer = + Napi::ArrayBuffer::New(env, sizeof(float) * n); + + Napi::Float32Array float32Array = + Napi::Float32Array::New(env, n, arrayBuffer, 0); + + std::copy(data, data + n, float32Array.Data()); + + SherpaOnnxCircularBufferFree(data); + + return float32Array; + } +} + +static void CircularBufferPopWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be an CircularBuffer pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + SherpaOnnxCircularBuffer *buf = + info[0].As>().Data(); + + if (!info[1].IsNumber()) { + Napi::TypeError::New(env, "Argument 1 should be an integer (n).") + .ThrowAsJavaScriptException(); + + return; + } + + int32_t n = info[1].As().Int32Value(); + + SherpaOnnxCircularBufferPop(buf, n); +} + +static Napi::Number CircularBufferSizeWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be an CircularBuffer pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxCircularBuffer *buf = + info[0].As>().Data(); + + int32_t size = SherpaOnnxCircularBufferSize(buf); + + return Napi::Number::New(env, size); +} + +static Napi::Number CircularBufferHeadWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be an CircularBuffer pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxCircularBuffer *buf = + info[0].As>().Data(); + + int32_t size = SherpaOnnxCircularBufferHead(buf); + + return Napi::Number::New(env, size); +} + +static void CircularBufferResetWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be an CircularBuffer pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + SherpaOnnxCircularBuffer *buf = + info[0].As>().Data(); + + SherpaOnnxCircularBufferReset(buf); +} + +static SherpaOnnxSileroVadModelConfig GetSileroVadConfig( + const Napi::Object &obj) { + SherpaOnnxSileroVadModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("sileroVad") || !obj.Get("sileroVad").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("sileroVad").As(); + SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(threshold, threshold); + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(min_silence_duration, minSilenceDuration); + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(min_speech_duration, minSpeechDuration); + SHERPA_ONNX_ASSIGN_ATTR_INT32(window_size, windowSize); + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(max_speech_duration, maxSpeechDuration); + + return c; +} + +static Napi::External +CreateVoiceActivityDetectorWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); +#if __OHOS__ + // the last argument is a NativeResourceManager + if (info.Length() != 3) { + std::ostringstream os; + os << "Expect only 3 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } +#else + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } +#endif + + if (!info[0].IsObject()) { + Napi::TypeError::New(env, + "You should pass an object as the first argument.") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[1].IsNumber()) { + Napi::TypeError::New(env, + "You should pass an integer as the second argument.") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Object o = info[0].As(); + + SherpaOnnxVadModelConfig c; + memset(&c, 0, sizeof(c)); + c.silero_vad = GetSileroVadConfig(o); + + SHERPA_ONNX_ASSIGN_ATTR_INT32(sample_rate, sampleRate); + SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); + SHERPA_ONNX_ASSIGN_ATTR_STR(provider, provider); + + if (o.Has("debug") && + (o.Get("debug").IsNumber() || o.Get("debug").IsBoolean())) { + if (o.Get("debug").IsBoolean()) { + c.debug = o.Get("debug").As().Value(); + } else { + c.debug = o.Get("debug").As().Int32Value(); + } + } + + float buffer_size_in_seconds = info[1].As().FloatValue(); + +#if __OHOS__ + std::unique_ptr mgr(OH_ResourceManager_InitNativeResourceManager(env, info[2]), &OH_ResourceManager_ReleaseNativeResourceManager); + + SherpaOnnxVoiceActivityDetector *vad = + SherpaOnnxCreateVoiceActivityDetectorOHOS(&c, buffer_size_in_seconds, mgr.get()); +#else + SherpaOnnxVoiceActivityDetector *vad = + SherpaOnnxCreateVoiceActivityDetector(&c, buffer_size_in_seconds); +#endif + + if (c.silero_vad.model) { + delete[] c.silero_vad.model; + } + + if (c.provider) { + delete[] c.provider; + } + + return Napi::External::New( + env, vad, [](Napi::Env env, SherpaOnnxVoiceActivityDetector *p) { + SherpaOnnxDestroyVoiceActivityDetector(p); + }); +} + +static void VoiceActivityDetectorAcceptWaveformWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be a VAD pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + SherpaOnnxVoiceActivityDetector *vad = + info[0].As>().Data(); + + if (!info[1].IsTypedArray()) { + Napi::TypeError::New( + env, "Argument 1 should be a Float32Array containing samples") + .ThrowAsJavaScriptException(); + + return; + } + + Napi::Float32Array samples = info[1].As(); + +#if __OHOS__ + // Note(fangjun): For unknown reasons, we need to use `/sizeof(float)` here for Huawei + SherpaOnnxVoiceActivityDetectorAcceptWaveform(vad, samples.Data(), + samples.ElementLength() / sizeof(float)); +#else + SherpaOnnxVoiceActivityDetectorAcceptWaveform(vad, samples.Data(), + samples.ElementLength()); +#endif +} + +static Napi::Boolean VoiceActivityDetectorEmptyWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be a VAD pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxVoiceActivityDetector *vad = + info[0].As>().Data(); + + int32_t is_empty = SherpaOnnxVoiceActivityDetectorEmpty(vad); + + return Napi::Boolean::New(env, is_empty); +} + +static Napi::Boolean VoiceActivityDetectorDetectedWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be a VAD pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxVoiceActivityDetector *vad = + info[0].As>().Data(); + + int32_t is_detected = SherpaOnnxVoiceActivityDetectorDetected(vad); + + return Napi::Boolean::New(env, is_detected); +} + +static void VoiceActivityDetectorPopWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be a VAD pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + SherpaOnnxVoiceActivityDetector *vad = + info[0].As>().Data(); + + SherpaOnnxVoiceActivityDetectorPop(vad); +} + +static void VoiceActivityDetectorClearWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be a VAD pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + SherpaOnnxVoiceActivityDetector *vad = + info[0].As>().Data(); + + SherpaOnnxVoiceActivityDetectorClear(vad); +} + +static Napi::Object VoiceActivityDetectorFrontWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 1 && info.Length() != 2) { + std::ostringstream os; + os << "Expect only 1 or 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be a VAD pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + bool enable_external_buffer = true; + if (info.Length() == 2) { + if (info[1].IsBoolean()) { + enable_external_buffer = info[1].As().Value(); + } else { + Napi::TypeError::New(env, "Argument 1 should be a boolean.") + .ThrowAsJavaScriptException(); + } + } + + SherpaOnnxVoiceActivityDetector *vad = + info[0].As>().Data(); + + const SherpaOnnxSpeechSegment *segment = + SherpaOnnxVoiceActivityDetectorFront(vad); + + if (enable_external_buffer) { + Napi::ArrayBuffer arrayBuffer = Napi::ArrayBuffer::New( + env, const_cast(segment->samples), sizeof(float) * segment->n, + [](Napi::Env /*env*/, void * /*data*/, + const SherpaOnnxSpeechSegment *hint) { + SherpaOnnxDestroySpeechSegment(hint); + }, + segment); + + Napi::Float32Array float32Array = + Napi::Float32Array::New(env, segment->n, arrayBuffer, 0); + + Napi::Object obj = Napi::Object::New(env); + obj.Set(Napi::String::New(env, "start"), segment->start); + obj.Set(Napi::String::New(env, "samples"), float32Array); + + return obj; + } else { + Napi::ArrayBuffer arrayBuffer = + Napi::ArrayBuffer::New(env, sizeof(float) * segment->n); + + Napi::Float32Array float32Array = + Napi::Float32Array::New(env, segment->n, arrayBuffer, 0); + + std::copy(segment->samples, segment->samples + segment->n, + float32Array.Data()); + + Napi::Object obj = Napi::Object::New(env); + obj.Set(Napi::String::New(env, "start"), segment->start); + obj.Set(Napi::String::New(env, "samples"), float32Array); + + SherpaOnnxDestroySpeechSegment(segment); + + return obj; + } +} + +static void VoiceActivityDetectorResetWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be a VAD pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + SherpaOnnxVoiceActivityDetector *vad = + info[0].As>().Data(); + + SherpaOnnxVoiceActivityDetectorReset(vad); +} + +static void VoiceActivityDetectorFlushWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be a VAD pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + SherpaOnnxVoiceActivityDetector *vad = + info[0].As>().Data(); + + SherpaOnnxVoiceActivityDetectorFlush(vad); +} + +void InitVad(Napi::Env env, Napi::Object exports) { + exports.Set(Napi::String::New(env, "createCircularBuffer"), + Napi::Function::New(env, CreateCircularBufferWrapper)); + + exports.Set(Napi::String::New(env, "circularBufferPush"), + Napi::Function::New(env, CircularBufferPushWrapper)); + + exports.Set(Napi::String::New(env, "circularBufferGet"), + Napi::Function::New(env, CircularBufferGetWrapper)); + + exports.Set(Napi::String::New(env, "circularBufferPop"), + Napi::Function::New(env, CircularBufferPopWrapper)); + + exports.Set(Napi::String::New(env, "circularBufferSize"), + Napi::Function::New(env, CircularBufferSizeWrapper)); + + exports.Set(Napi::String::New(env, "circularBufferHead"), + Napi::Function::New(env, CircularBufferHeadWrapper)); + + exports.Set(Napi::String::New(env, "circularBufferReset"), + Napi::Function::New(env, CircularBufferResetWrapper)); + + exports.Set(Napi::String::New(env, "createVoiceActivityDetector"), + Napi::Function::New(env, CreateVoiceActivityDetectorWrapper)); + + exports.Set( + Napi::String::New(env, "voiceActivityDetectorAcceptWaveform"), + Napi::Function::New(env, VoiceActivityDetectorAcceptWaveformWrapper)); + + exports.Set(Napi::String::New(env, "voiceActivityDetectorIsEmpty"), + Napi::Function::New(env, VoiceActivityDetectorEmptyWrapper)); + + exports.Set(Napi::String::New(env, "voiceActivityDetectorIsDetected"), + Napi::Function::New(env, VoiceActivityDetectorDetectedWrapper)); + + exports.Set(Napi::String::New(env, "voiceActivityDetectorPop"), + Napi::Function::New(env, VoiceActivityDetectorPopWrapper)); + + exports.Set(Napi::String::New(env, "voiceActivityDetectorClear"), + Napi::Function::New(env, VoiceActivityDetectorClearWrapper)); + + exports.Set(Napi::String::New(env, "voiceActivityDetectorFront"), + Napi::Function::New(env, VoiceActivityDetectorFrontWrapper)); + + exports.Set(Napi::String::New(env, "voiceActivityDetectorReset"), + Napi::Function::New(env, VoiceActivityDetectorResetWrapper)); + + exports.Set(Napi::String::New(env, "voiceActivityDetectorFlush"), + Napi::Function::New(env, VoiceActivityDetectorFlushWrapper)); +} diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-reader.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-reader.cc new file mode 100644 index 000000000..2973c6169 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-reader.cc @@ -0,0 +1,171 @@ +// scripts/node-addon-api/src/wave-reader.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include +#include + +#include "napi.h" // NOLINT +#include "sherpa-onnx/c-api/c-api.h" + +static Napi::Object ReadWaveWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() > 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsString()) { + Napi::TypeError::New(env, "Argument 0 should be a string") + .ThrowAsJavaScriptException(); + + return {}; + } + + std::string filename = info[0].As().Utf8Value(); + + bool enable_external_buffer = true; + if (info.Length() == 2) { + if (info[1].IsBoolean()) { + enable_external_buffer = info[1].As().Value(); + } else { + Napi::TypeError::New(env, "Argument 1 should be a boolean") + .ThrowAsJavaScriptException(); + + return {}; + } + } + + const SherpaOnnxWave *wave = SherpaOnnxReadWave(filename.c_str()); + if (!wave) { + std::ostringstream os; + os << "Failed to read '" << filename << "'"; + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (enable_external_buffer) { + Napi::ArrayBuffer arrayBuffer = Napi::ArrayBuffer::New( + env, const_cast(wave->samples), + sizeof(float) * wave->num_samples, + [](Napi::Env /*env*/, void * /*data*/, const SherpaOnnxWave *hint) { + SherpaOnnxFreeWave(hint); + }, + wave); + Napi::Float32Array float32Array = + Napi::Float32Array::New(env, wave->num_samples, arrayBuffer, 0); + + Napi::Object obj = Napi::Object::New(env); + obj.Set(Napi::String::New(env, "samples"), float32Array); + obj.Set(Napi::String::New(env, "sampleRate"), wave->sample_rate); + return obj; + } else { + // don't use external buffer + Napi::ArrayBuffer arrayBuffer = + Napi::ArrayBuffer::New(env, sizeof(float) * wave->num_samples); + + Napi::Float32Array float32Array = + Napi::Float32Array::New(env, wave->num_samples, arrayBuffer, 0); + + std::copy(wave->samples, wave->samples + wave->num_samples, + float32Array.Data()); + + Napi::Object obj = Napi::Object::New(env); + obj.Set(Napi::String::New(env, "samples"), float32Array); + obj.Set(Napi::String::New(env, "sampleRate"), wave->sample_rate); + + SherpaOnnxFreeWave(wave); + + return obj; + } +} + +static Napi::Object ReadWaveFromBinaryWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() > 2) { + std::ostringstream os; + os << "Expect only 1 or 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsTypedArray()) { + Napi::TypeError::New(env, "Argument 0 should be a float32 array") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Uint8Array data = info[0].As(); + int32_t n = data.ElementLength(); + const SherpaOnnxWave *wave = SherpaOnnxReadWaveFromBinaryData(reinterpret_cast(data.Data()), n); + if (!wave) { + std::ostringstream os; + os << "Failed to read wave"; + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + bool enable_external_buffer = true; + if (info.Length() == 2) { + if (info[1].IsBoolean()) { + enable_external_buffer = info[1].As().Value(); + } else { + Napi::TypeError::New(env, "Argument 1 should be a boolean") + .ThrowAsJavaScriptException(); + + return {}; + } + } + + if (enable_external_buffer) { + Napi::ArrayBuffer arrayBuffer = Napi::ArrayBuffer::New( + env, const_cast(wave->samples), + sizeof(float) * wave->num_samples, + [](Napi::Env /*env*/, void * /*data*/, const SherpaOnnxWave *hint) { + SherpaOnnxFreeWave(hint); + }, + wave); + Napi::Float32Array float32Array = + Napi::Float32Array::New(env, wave->num_samples, arrayBuffer, 0); + + Napi::Object obj = Napi::Object::New(env); + obj.Set(Napi::String::New(env, "samples"), float32Array); + obj.Set(Napi::String::New(env, "sampleRate"), wave->sample_rate); + return obj; + } else { + // don't use external buffer + Napi::ArrayBuffer arrayBuffer = + Napi::ArrayBuffer::New(env, sizeof(float) * wave->num_samples); + + Napi::Float32Array float32Array = + Napi::Float32Array::New(env, wave->num_samples, arrayBuffer, 0); + + std::copy(wave->samples, wave->samples + wave->num_samples, + float32Array.Data()); + + Napi::Object obj = Napi::Object::New(env); + obj.Set(Napi::String::New(env, "samples"), float32Array); + obj.Set(Napi::String::New(env, "sampleRate"), wave->sample_rate); + + SherpaOnnxFreeWave(wave); + + return obj; + } +} + +void InitWaveReader(Napi::Env env, Napi::Object exports) { + exports.Set(Napi::String::New(env, "readWave"), + Napi::Function::New(env, ReadWaveWrapper)); + + exports.Set(Napi::String::New(env, "readWaveFromBinary"), + Napi::Function::New(env, ReadWaveFromBinaryWrapper)); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-writer.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-writer.cc new file mode 100644 index 000000000..3ade695a0 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-writer.cc @@ -0,0 +1,81 @@ +// scripts/node-addon-api/src/wave-writer.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include + +#include "napi.h" // NOLINT +#include "sherpa-onnx/c-api/c-api.h" + +// (filename, {samples: samples, sampleRate: sampleRate} +static Napi::Boolean WriteWaveWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsString()) { + Napi::TypeError::New(env, "Argument 0 should be a string") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[1].IsObject()) { + Napi::TypeError::New(env, "Argument 1 should be an object") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Object obj = info[1].As(); + + if (!obj.Has("samples")) { + Napi::TypeError::New(env, "The argument object should have a field samples") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Get("samples").IsTypedArray()) { + Napi::TypeError::New(env, "The object['samples'] should be a typed array") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Has("sampleRate")) { + Napi::TypeError::New(env, + "The argument object should have a field sampleRate") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Get("sampleRate").IsNumber()) { + Napi::TypeError::New(env, "The object['samples'] should be a number") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Float32Array samples = obj.Get("samples").As(); + int32_t sample_rate = obj.Get("sampleRate").As().Int32Value(); + + int32_t ok = + SherpaOnnxWriteWave(samples.Data(), samples.ElementLength(), sample_rate, + info[0].As().Utf8Value().c_str()); + + return Napi::Boolean::New(env, ok); +} + +void InitWaveWriter(Napi::Env env, Napi::Object exports) { + exports.Set(Napi::String::New(env, "writeWave"), + Napi::Function::New(env, WriteWaveWrapper)); +} diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/MainPage.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/MainPage.ets new file mode 100644 index 000000000..34b4b2c88 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/MainPage.ets @@ -0,0 +1,21 @@ +import hilog from '@ohos.hilog'; +import testNapi from 'libsherpa_onnx.so'; + +@Component +export struct MainPage { + @State message: string = 'Hello World'; + + build() { + Row() { + Column() { + Text(this.message) + .fontSize(50) + .fontWeight(FontWeight.Bold) + .onClick(() => { + }) + } + .width('100%') + } + .height('100%') + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingAsr.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingAsr.ets new file mode 100644 index 000000000..0cc8466a9 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingAsr.ets @@ -0,0 +1,162 @@ +import { + acceptWaveformOffline, + createOfflineRecognizer, + createOfflineStream, + decodeOfflineStream, + getOfflineStreamResultAsJson, +} from 'libsherpa_onnx.so'; + +export interface Samples { + samples: Float32Array; + sampleRate: number; +} + +export class OfflineStream { + public handle: object; + + constructor(handle: object) { + this.handle = handle; + } + + // obj is {samples: samples, sampleRate: sampleRate} + // samples is a float32 array containing samples in the range [-1, 1] + // sampleRate is a number + acceptWaveform(obj: Samples) { + acceptWaveformOffline(this.handle, obj) + } +} + +export class FeatureConfig { + public sampleRate: number = 16000; + public featureDim: number = 80; +} + +export class OfflineTransducerModelConfig { + public encoder: string = ''; + public decoder: string = ''; + public joiner: string = ''; +} + +export class OfflineParaformerModelConfig { + public model: string = ''; +} + +export class OfflineNemoEncDecCtcModelConfig { + public model: string = ''; +} + +export class OfflineWhisperModelConfig { + public encoder: string = ''; + public decoder: string = ''; + public language: string = ''; + public task: string = 'transcribe'; + public tailPaddings: number = -1; +} + +export class OfflineTdnnModelConfig { + public model: string = ''; +} + +export class OfflineSenseVoiceModelConfig { + public model: string = ''; + public language: string = ''; + public useItn: boolean = false; +} + +export class OfflineMoonshineModelConfig { + public preprocessor: string = ''; + public encoder: string = ''; + public uncachedDecoder: string = ''; + public cachedDecoder: string = ''; +} + +export class OfflineModelConfig { + public transducer: OfflineTransducerModelConfig = new OfflineTransducerModelConfig(); + public paraformer: OfflineParaformerModelConfig = new OfflineParaformerModelConfig(); + public nemoCtc: OfflineNemoEncDecCtcModelConfig = new OfflineNemoEncDecCtcModelConfig(); + public whisper: OfflineWhisperModelConfig = new OfflineWhisperModelConfig(); + public tdnn: OfflineTdnnModelConfig = new OfflineTdnnModelConfig(); + public tokens: string = ''; + public numThreads: number = 1; + public debug: boolean = false; + public provider: string = "cpu"; + public modelType: string = ''; + public modelingUnit: string = "cjkchar"; + public bpeVocab: string = ''; + public telespeechCtc: string = ''; + public senseVoice: OfflineSenseVoiceModelConfig = new OfflineSenseVoiceModelConfig(); + public moonshine: OfflineMoonshineModelConfig = new OfflineMoonshineModelConfig(); +} + +export class OfflineLMConfig { + public model: string = ''; + public scale: number = 1.0; +} + +export class OfflineRecognizerConfig { + public featConfig: FeatureConfig = new FeatureConfig(); + public modelConfig: OfflineModelConfig = new OfflineModelConfig(); + public lmConfig: OfflineLMConfig = new OfflineLMConfig(); + public decodingMethod: string = "greedy_search"; + public maxActivePaths: number = 4; + public hotwordsFfile: string = ''; + public hotwordsScore: number = 1.5; + public ruleFsts: string = ''; + public ruleFars: string = ''; + public blankPenalty: number = 0; +} + +export class OfflineRecognizerResult { + public text: string = ''; + public timestamps: number[] = []; + public tokens: string[] = []; + public json = ''; + public lang: string = ''; + public emotion: string = ''; + public event: string = ''; +} + +interface OfflineRecognizerResultJson { + text: string; + timestamps: number[]; + tokens: string[]; + lang: string; + emotion: string; + event: string; +} + +export class OfflineRecognizer { + public handle: object; + public config: OfflineRecognizerConfig; + + constructor(config: OfflineRecognizerConfig, mgr?: object) { + this.handle = createOfflineRecognizer(config, mgr); + this.config = config + } + + createStream(): OfflineStream { + const handle: object = createOfflineStream(this.handle); + return new OfflineStream(handle); + } + + decode(stream: OfflineStream) { + decodeOfflineStream(this.handle, stream.handle); + } + + getResult(stream: OfflineStream): OfflineRecognizerResult { + const jsonStr: string = getOfflineStreamResultAsJson(stream.handle); + + let o = JSON.parse(jsonStr) as OfflineRecognizerResultJson; + + const r = new OfflineRecognizerResult() + r.text = o.text + r.timestamps = o.timestamps; + r.tokens = o.tokens; + r.json = jsonStr; + r.lang = o.lang; + r.emotion = o.emotion; + r.event = o.event; + + return r; + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/StreamingAsr.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/StreamingAsr.ets new file mode 100644 index 000000000..7ecc552ca --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/StreamingAsr.ets @@ -0,0 +1,141 @@ +import { + acceptWaveformOnline, + createOnlineRecognizer, + createOnlineStream, + decodeOnlineStream, + getOnlineStreamResultAsJson, + inputFinished, + isEndpoint, + isOnlineStreamReady, + reset, +} from 'libsherpa_onnx.so'; + +import { FeatureConfig, Samples } from './NonStreamingAsr'; + +export class OnlineStream { + public handle: object; + + constructor(handle: object) { + this.handle = handle; + } + + // obj is {samples: samples, sampleRate: sampleRate} + // samples is a float32 array containing samples in the range [-1, 1] + // sampleRate is a number + acceptWaveform(obj: Samples) { + acceptWaveformOnline(this.handle, obj) + } + + inputFinished() { + inputFinished(this.handle) + } +} + +export class OnlineTransducerModelConfig { + public encoder: string = ''; + public decoder: string = ''; + public joiner: string = ''; +} + +export class OnlineParaformerModelConfig { + public encoder: string = ''; + public decoder: string = ''; +} + +export class OnlineZipformer2CtcModelConfig { + public model: string = ''; +} + +export class OnlineModelConfig { + public transducer: OnlineTransducerModelConfig = new OnlineTransducerModelConfig(); + public paraformer: OnlineParaformerModelConfig = new OnlineParaformerModelConfig(); + public zipformer2_ctc: OnlineZipformer2CtcModelConfig = new OnlineZipformer2CtcModelConfig(); + public tokens: string = ''; + public numThreads: number = 1; + public provider: string = "cpu"; + public debug: boolean = false; + public modelType: string = ''; + public modelingUnit: string = "cjkchar"; + public bpeVocab: string = ''; +} + +export class OnlineCtcFstDecoderConfig { + public graph: string = ''; + public maxActive: number = 3000; +} + +export class OnlineRecognizerConfig { + public featConfig: FeatureConfig = new FeatureConfig(); + public modelConfig: OnlineModelConfig = new OnlineModelConfig(); + public decodingMethod: string = "greedy_search"; + public maxActivePaths: number = 4; + public enableEndpoint: boolean = false; + public rule1MinTrailingSilence: number = 2.4; + public rule2MinTrailingSilence: number = 1.2; + public rule3MinUtteranceLength: number = 20; + public hotwordsFile: string = ''; + public hotwordsScore: number = 1.5; + public ctcFstDecoderConfig: OnlineCtcFstDecoderConfig = new OnlineCtcFstDecoderConfig(); + public ruleFsts: string = ''; + public ruleFars: string = ''; + public blankPenalty: number = 0; +} + +interface OnlineRecognizerResultJson { + text: string; + timestamps: number[]; + tokens: string[]; +} + +export class OnlineRecognizerResult { + public text: string = ''; + public tokens: string[] = []; + public timestamps: number[] = []; + public json: string = ''; +} + +export class OnlineRecognizer { + public handle: object; + public config: OnlineRecognizerConfig + + constructor(config: OnlineRecognizerConfig, mgr?: object) { + this.handle = createOnlineRecognizer(config, mgr); + this.config = config + } + + createStream(): OnlineStream { + const handle: object = createOnlineStream(this.handle); + return new OnlineStream(handle); + } + + isReady(stream: OnlineStream): boolean { + return isOnlineStreamReady(this.handle, stream.handle); + } + + decode(stream: OnlineStream) { + decodeOnlineStream(this.handle, stream.handle); + } + + isEndpoint(stream: OnlineStream): boolean { + return isEndpoint(this.handle, stream.handle); + } + + reset(stream: OnlineStream) { + reset(this.handle, stream.handle); + } + + getResult(stream: OnlineStream): OnlineRecognizerResult { + const jsonStr: string = + getOnlineStreamResultAsJson(this.handle, stream.handle); + + let o = JSON.parse(jsonStr) as OnlineRecognizerResultJson; + + const r = new OnlineRecognizerResult() + r.text = o.text + r.timestamps = o.timestamps; + r.tokens = o.tokens; + r.json = jsonStr; + + return r; + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/Vad.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/Vad.ets new file mode 100644 index 000000000..155eac680 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/Vad.ets @@ -0,0 +1,133 @@ +import { + circularBufferGet, + circularBufferHead, + circularBufferPop, + circularBufferPush, + circularBufferReset, + circularBufferSize, + createCircularBuffer, + createVoiceActivityDetector, + voiceActivityDetectorAcceptWaveform, + voiceActivityDetectorClear, + voiceActivityDetectorFlush, + voiceActivityDetectorFront, + voiceActivityDetectorIsDetected, + voiceActivityDetectorIsEmpty, + voiceActivityDetectorPop, + voiceActivityDetectorReset, +} from 'libsherpa_onnx.so'; + +export class SileroVadConfig { + public model: string; + public threshold: number; + public minSpeechDuration: number; + public minSilenceDuration: number; + public windowSize: number; + + public constructor(model: string, threshold: number, minSpeechDuration: number, minSilenceDuration: number, + windowSize: number) { + this.model = model; + this.threshold = threshold; + this.minSpeechDuration = minSpeechDuration; + this.minSilenceDuration = minSilenceDuration; + this.windowSize = windowSize; + } +} + +export class VadConfig { + public sileroVad: SileroVadConfig; + public sampleRate: number; + public debug: boolean; + public numThreads: number; + + public constructor(sileroVad: SileroVadConfig, sampleRate: number, debug: boolean, numThreads: number) { + this.sileroVad = sileroVad; + this.sampleRate = sampleRate; + this.debug = debug; + this.numThreads = numThreads; + } +} + +export class CircularBuffer { + private handle: object; + + constructor(capacity: number) { + this.handle = createCircularBuffer(capacity); + } + + // samples is a float32 array + push(samples: Float32Array) { + console.log(`here samples: ${samples}`); + circularBufferPush(this.handle, samples); + } + + // return a float32 array + get(startIndex: number, n: number, enableExternalBuffer: boolean = true): Float32Array { + return circularBufferGet( + this.handle, startIndex, n, enableExternalBuffer); + } + + pop(n: number) { + circularBufferPop(this.handle, n); + } + + size(): number { + return circularBufferSize(this.handle); + } + + head(): number { + return circularBufferHead(this.handle); + } + + reset() { + circularBufferReset(this.handle); + } +} + +export interface SpeechSegment { + samples: Float32Array; + start: number; +} + +export class Vad { + public config: VadConfig; + private handle: object; + + constructor(config: VadConfig, bufferSizeInSeconds?: number, mgr?: object) { + this.handle = + createVoiceActivityDetector(config, bufferSizeInSeconds, mgr); + this.config = config; + } + + acceptWaveform(samples: Float32Array): void { + voiceActivityDetectorAcceptWaveform(this.handle, samples); + } + + isEmpty(): boolean { + return voiceActivityDetectorIsEmpty(this.handle); + } + + isDetected(): boolean { + return voiceActivityDetectorIsDetected(this.handle); + } + + pop(): void { + voiceActivityDetectorPop(this.handle); + } + + clear(): void { + voiceActivityDetectorClear(this.handle); + } + + front(enableExternalBuffer = true): SpeechSegment { + return voiceActivityDetectorFront(this.handle, enableExternalBuffer); + } + + reset(): void { + voiceActivityDetectorReset(this.handle); + } + + flush(): void { + voiceActivityDetectorFlush(this.handle); + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/module.json5 b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/module.json5 new file mode 100644 index 000000000..1db0eb692 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/module.json5 @@ -0,0 +1,11 @@ +{ + "module": { + "name": "sherpa_onnx", + "type": "har", + "deviceTypes": [ + "default", + "tablet", + "2in1" + ] + } +} diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/resources/base/element/string.json b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/resources/base/element/string.json new file mode 100644 index 000000000..f51a9c846 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/resources/base/element/string.json @@ -0,0 +1,8 @@ +{ + "string": [ + { + "name": "page_show", + "value": "page from package" + } + ] +} diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/resources/en_US/element/string.json b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/resources/en_US/element/string.json new file mode 100644 index 000000000..f51a9c846 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/resources/en_US/element/string.json @@ -0,0 +1,8 @@ +{ + "string": [ + { + "name": "page_show", + "value": "page from package" + } + ] +} diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/resources/zh_CN/element/string.json b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/resources/zh_CN/element/string.json new file mode 100644 index 000000000..f51a9c846 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/resources/zh_CN/element/string.json @@ -0,0 +1,8 @@ +{ + "string": [ + { + "name": "page_show", + "value": "page from package" + } + ] +} diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/ohosTest/ets/test/Ability.test.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/ohosTest/ets/test/Ability.test.ets new file mode 100644 index 000000000..8aa374977 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/ohosTest/ets/test/Ability.test.ets @@ -0,0 +1,35 @@ +import hilog from '@ohos.hilog'; +import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from '@ohos/hypium'; + +export default function abilityTest() { + describe('ActsAbilityTest', () => { + // Defines a test suite. Two parameters are supported: test suite name and test suite function. + beforeAll(() => { + // Presets an action, which is performed only once before all test cases of the test suite start. + // This API supports only one parameter: preset action function. + }) + beforeEach(() => { + // Presets an action, which is performed before each unit test case starts. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: preset action function. + }) + afterEach(() => { + // Presets a clear action, which is performed after each unit test case ends. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: clear action function. + }) + afterAll(() => { + // Presets a clear action, which is performed after all test cases of the test suite end. + // This API supports only one parameter: clear action function. + }) + it('assertContain', 0, () => { + // Defines a test case. This API supports three parameters: test case name, filter parameter, and test case function. + hilog.info(0x0000, 'testTag', '%{public}s', 'it begin'); + let a = 'abc'; + let b = 'b'; + // Defines a variety of assertion methods, which are used to declare expected boolean conditions. + expect(a).assertContain(b); + expect(a).assertEqual(a); + }) + }) +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/ohosTest/ets/test/List.test.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/ohosTest/ets/test/List.test.ets new file mode 100644 index 000000000..794c7dc4e --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/ohosTest/ets/test/List.test.ets @@ -0,0 +1,5 @@ +import abilityTest from './Ability.test'; + +export default function testsuite() { + abilityTest(); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/ohosTest/module.json5 b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/ohosTest/module.json5 new file mode 100644 index 000000000..b7a1665a7 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/ohosTest/module.json5 @@ -0,0 +1,13 @@ +{ + "module": { + "name": "sherpa_onnx_test", + "type": "feature", + "deviceTypes": [ + "default", + "tablet", + "2in1" + ], + "deliveryWithInstall": true, + "installationFree": false + } +} diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/test/List.test.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/test/List.test.ets new file mode 100644 index 000000000..bb5b5c373 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/test/List.test.ets @@ -0,0 +1,5 @@ +import localUnitTest from './LocalUnit.test'; + +export default function testsuite() { + localUnitTest(); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/test/LocalUnit.test.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/test/LocalUnit.test.ets new file mode 100644 index 000000000..165fc1615 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/test/LocalUnit.test.ets @@ -0,0 +1,33 @@ +import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from '@ohos/hypium'; + +export default function localUnitTest() { + describe('localUnitTest', () => { + // Defines a test suite. Two parameters are supported: test suite name and test suite function. + beforeAll(() => { + // Presets an action, which is performed only once before all test cases of the test suite start. + // This API supports only one parameter: preset action function. + }); + beforeEach(() => { + // Presets an action, which is performed before each unit test case starts. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: preset action function. + }); + afterEach(() => { + // Presets a clear action, which is performed after each unit test case ends. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: clear action function. + }); + afterAll(() => { + // Presets a clear action, which is performed after all test cases of the test suite end. + // This API supports only one parameter: clear action function. + }); + it('assertContain', 0, () => { + // Defines a test case. This API supports three parameters: test case name, filter parameter, and test case function. + let a = 'abc'; + let b = 'b'; + // Defines a variety of assertion methods, which are used to declare expected boolean conditions. + expect(a).assertContain(b); + expect(a).assertEqual(a); + }); + }); +} \ No newline at end of file diff --git a/scripts/node-addon-api/src/audio-tagging.cc b/scripts/node-addon-api/src/audio-tagging.cc deleted file mode 100644 index bed4e48a2..000000000 --- a/scripts/node-addon-api/src/audio-tagging.cc +++ /dev/null @@ -1,227 +0,0 @@ -// scripts/node-addon-api/src/audio-tagging.cc -// -// Copyright (c) 2024 Xiaomi Corporation -#include - -#include "macros.h" // NOLINT -#include "napi.h" // NOLINT -#include "sherpa-onnx/c-api/c-api.h" - -static SherpaOnnxOfflineZipformerAudioTaggingModelConfig -GetAudioTaggingZipformerModelConfig(Napi::Object obj) { - SherpaOnnxOfflineZipformerAudioTaggingModelConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("zipformer") || !obj.Get("zipformer").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("zipformer").As(); - - SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); - - return c; -} - -static SherpaOnnxAudioTaggingModelConfig GetAudioTaggingModelConfig( - Napi::Object obj) { - SherpaOnnxAudioTaggingModelConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("model") || !obj.Get("model").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("model").As(); - c.zipformer = GetAudioTaggingZipformerModelConfig(o); - - SHERPA_ONNX_ASSIGN_ATTR_STR(ced, ced); - - SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); - - if (o.Has("debug") && - (o.Get("debug").IsNumber() || o.Get("debug").IsBoolean())) { - if (o.Get("debug").IsBoolean()) { - c.debug = o.Get("debug").As().Value(); - } else { - c.debug = o.Get("debug").As().Int32Value(); - } - } - SHERPA_ONNX_ASSIGN_ATTR_STR(provider, provider); - - return c; -} - -static Napi::External CreateAudioTaggingWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsObject()) { - Napi::TypeError::New(env, "You should pass an object as the only argument.") - .ThrowAsJavaScriptException(); - - return {}; - } - - Napi::Object o = info[0].As(); - - SherpaOnnxAudioTaggingConfig c; - memset(&c, 0, sizeof(c)); - c.model = GetAudioTaggingModelConfig(o); - - SHERPA_ONNX_ASSIGN_ATTR_STR(labels, labels); - SHERPA_ONNX_ASSIGN_ATTR_INT32(top_k, topK); - - const SherpaOnnxAudioTagging *at = SherpaOnnxCreateAudioTagging(&c); - - if (c.model.zipformer.model) { - delete[] c.model.zipformer.model; - } - - if (c.model.ced) { - delete[] c.model.ced; - } - - if (c.model.provider) { - delete[] c.model.provider; - } - - if (c.labels) { - delete[] c.labels; - } - - if (!at) { - Napi::TypeError::New(env, "Please check your config!") - .ThrowAsJavaScriptException(); - - return {}; - } - - return Napi::External::New( - env, const_cast(at), - [](Napi::Env env, SherpaOnnxAudioTagging *at) { - SherpaOnnxDestroyAudioTagging(at); - }); -} - -static Napi::External -AudioTaggingCreateOfflineStreamWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New( - env, "You should pass an audio tagging pointer as the only argument") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxAudioTagging *at = - info[0].As>().Data(); - - const SherpaOnnxOfflineStream *stream = - SherpaOnnxAudioTaggingCreateOfflineStream(at); - - return Napi::External::New( - env, const_cast(stream), - [](Napi::Env env, SherpaOnnxOfflineStream *stream) { - SherpaOnnxDestroyOfflineStream(stream); - }); -} - -static Napi::Object AudioTaggingComputeWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 3) { - std::ostringstream os; - os << "Expect only 3 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New( - env, "You should pass an audio tagging pointer as the first argument") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[1].IsExternal()) { - Napi::TypeError::New( - env, "You should pass an offline stream pointer as the second argument") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[2].IsNumber()) { - Napi::TypeError::New(env, - "You should pass an integer as the third argument") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxAudioTagging *at = - info[0].As>().Data(); - - SherpaOnnxOfflineStream *stream = - info[1].As>().Data(); - - int32_t top_k = info[2].As().Int32Value(); - - const SherpaOnnxAudioEvent *const *events = - SherpaOnnxAudioTaggingCompute(at, stream, top_k); - - auto p = events; - int32_t k = 0; - while (p && *p) { - ++k; - ++p; - } - - Napi::Array ans = Napi::Array::New(env, k); - for (uint32_t i = 0; i != k; ++i) { - Napi::Object obj = Napi::Object::New(env); - obj.Set(Napi::String::New(env, "name"), - Napi::String::New(env, events[i]->name)); - obj.Set(Napi::String::New(env, "index"), - Napi::Number::New(env, events[i]->index)); - obj.Set(Napi::String::New(env, "prob"), - Napi::Number::New(env, events[i]->prob)); - ans[i] = obj; - } - - SherpaOnnxAudioTaggingFreeResults(events); - - return ans; -} - -void InitAudioTagging(Napi::Env env, Napi::Object exports) { - exports.Set(Napi::String::New(env, "createAudioTagging"), - Napi::Function::New(env, CreateAudioTaggingWrapper)); - - exports.Set(Napi::String::New(env, "audioTaggingCreateOfflineStream"), - Napi::Function::New(env, AudioTaggingCreateOfflineStreamWrapper)); - - exports.Set(Napi::String::New(env, "audioTaggingCompute"), - Napi::Function::New(env, AudioTaggingComputeWrapper)); -} diff --git a/scripts/node-addon-api/src/audio-tagging.cc b/scripts/node-addon-api/src/audio-tagging.cc new file mode 120000 index 000000000..42b052870 --- /dev/null +++ b/scripts/node-addon-api/src/audio-tagging.cc @@ -0,0 +1 @@ +../../../harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/audio-tagging.cc \ No newline at end of file diff --git a/scripts/node-addon-api/src/keyword-spotting.cc b/scripts/node-addon-api/src/keyword-spotting.cc deleted file mode 100644 index 2b5a24100..000000000 --- a/scripts/node-addon-api/src/keyword-spotting.cc +++ /dev/null @@ -1,266 +0,0 @@ -// scripts/node-addon-api/src/keyword-spotting.cc -// -// Copyright (c) 2024 Xiaomi Corporation -#include - -#include "macros.h" // NOLINT -#include "napi.h" // NOLINT -#include "sherpa-onnx/c-api/c-api.h" - -// defined ./streaming-asr.cc -SherpaOnnxFeatureConfig GetFeatureConfig(Napi::Object obj); - -// defined ./streaming-asr.cc -SherpaOnnxOnlineModelConfig GetOnlineModelConfig(Napi::Object obj); - -static Napi::External CreateKeywordSpotterWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsObject()) { - Napi::TypeError::New(env, "Expect an object as the argument") - .ThrowAsJavaScriptException(); - - return {}; - } - - Napi::Object o = info[0].As(); - SherpaOnnxKeywordSpotterConfig c; - memset(&c, 0, sizeof(c)); - c.feat_config = GetFeatureConfig(o); - c.model_config = GetOnlineModelConfig(o); - - SHERPA_ONNX_ASSIGN_ATTR_INT32(max_active_paths, maxActivePaths); - SHERPA_ONNX_ASSIGN_ATTR_INT32(num_trailing_blanks, numTrailingBlanks); - SHERPA_ONNX_ASSIGN_ATTR_FLOAT(keywords_score, keywordsScore); - SHERPA_ONNX_ASSIGN_ATTR_FLOAT(keywords_threshold, keywordsThreshold); - SHERPA_ONNX_ASSIGN_ATTR_STR(keywords_file, keywordsFile); - SHERPA_ONNX_ASSIGN_ATTR_STR(keywords_buf, keywordsBuf); - SHERPA_ONNX_ASSIGN_ATTR_INT32(keywords_buf_size, keywordsBufSize); - - SherpaOnnxKeywordSpotter *kws = SherpaOnnxCreateKeywordSpotter(&c); - - if (c.model_config.transducer.encoder) { - delete[] c.model_config.transducer.encoder; - } - - if (c.model_config.transducer.decoder) { - delete[] c.model_config.transducer.decoder; - } - - if (c.model_config.transducer.joiner) { - delete[] c.model_config.transducer.joiner; - } - - if (c.model_config.paraformer.encoder) { - delete[] c.model_config.paraformer.encoder; - } - - if (c.model_config.paraformer.decoder) { - delete[] c.model_config.paraformer.decoder; - } - - if (c.model_config.zipformer2_ctc.model) { - delete[] c.model_config.zipformer2_ctc.model; - } - - if (c.model_config.tokens) { - delete[] c.model_config.tokens; - } - - if (c.model_config.provider) { - delete[] c.model_config.provider; - } - - if (c.model_config.model_type) { - delete[] c.model_config.model_type; - } - - if (c.keywords_file) { - delete[] c.keywords_file; - } - - if (c.keywords_buf) { - delete[] c.keywords_buf; - } - - if (!kws) { - Napi::TypeError::New(env, "Please check your config!") - .ThrowAsJavaScriptException(); - - return {}; - } - - return Napi::External::New( - env, kws, [](Napi::Env env, SherpaOnnxKeywordSpotter *kws) { - SherpaOnnxDestroyKeywordSpotter(kws); - }); -} - -static Napi::External CreateKeywordStreamWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New( - env, "You should pass a keyword spotter pointer as the only argument") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxKeywordSpotter *kws = - info[0].As>().Data(); - - SherpaOnnxOnlineStream *stream = SherpaOnnxCreateKeywordStream(kws); - - return Napi::External::New( - env, stream, [](Napi::Env env, SherpaOnnxOnlineStream *stream) { - SherpaOnnxDestroyOnlineStream(stream); - }); -} - -static Napi::Boolean IsKeywordStreamReadyWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be a keyword spotter pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[1].IsExternal()) { - Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxKeywordSpotter *kws = - info[0].As>().Data(); - - SherpaOnnxOnlineStream *stream = - info[1].As>().Data(); - - int32_t is_ready = SherpaOnnxIsKeywordStreamReady(kws, stream); - - return Napi::Boolean::New(env, is_ready); -} - -static void DecodeKeywordStreamWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be a keyword spotter pointer.") - .ThrowAsJavaScriptException(); - - return; - } - - if (!info[1].IsExternal()) { - Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.") - .ThrowAsJavaScriptException(); - - return; - } - - SherpaOnnxKeywordSpotter *kws = - info[0].As>().Data(); - - SherpaOnnxOnlineStream *stream = - info[1].As>().Data(); - - SherpaOnnxDecodeKeywordStream(kws, stream); -} - -static Napi::String GetKeywordResultAsJsonWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be a keyword spotter pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[1].IsExternal()) { - Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxKeywordSpotter *kws = - info[0].As>().Data(); - - SherpaOnnxOnlineStream *stream = - info[1].As>().Data(); - - const char *json = SherpaOnnxGetKeywordResultAsJson(kws, stream); - - Napi::String s = Napi::String::New(env, json); - - SherpaOnnxFreeKeywordResultJson(json); - - return s; -} - -void InitKeywordSpotting(Napi::Env env, Napi::Object exports) { - exports.Set(Napi::String::New(env, "createKeywordSpotter"), - Napi::Function::New(env, CreateKeywordSpotterWrapper)); - - exports.Set(Napi::String::New(env, "createKeywordStream"), - Napi::Function::New(env, CreateKeywordStreamWrapper)); - - exports.Set(Napi::String::New(env, "isKeywordStreamReady"), - Napi::Function::New(env, IsKeywordStreamReadyWrapper)); - - exports.Set(Napi::String::New(env, "decodeKeywordStream"), - Napi::Function::New(env, DecodeKeywordStreamWrapper)); - - exports.Set(Napi::String::New(env, "getKeywordResultAsJson"), - Napi::Function::New(env, GetKeywordResultAsJsonWrapper)); -} diff --git a/scripts/node-addon-api/src/keyword-spotting.cc b/scripts/node-addon-api/src/keyword-spotting.cc new file mode 120000 index 000000000..66230e586 --- /dev/null +++ b/scripts/node-addon-api/src/keyword-spotting.cc @@ -0,0 +1 @@ +../../../harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/keyword-spotting.cc \ No newline at end of file diff --git a/scripts/node-addon-api/src/macros.h b/scripts/node-addon-api/src/macros.h deleted file mode 100644 index 1f4f401e3..000000000 --- a/scripts/node-addon-api/src/macros.h +++ /dev/null @@ -1,51 +0,0 @@ -// scripts/node-addon-api/src/macros.h -// -// Copyright (c) 2024 Xiaomi Corporation -#ifndef SCRIPTS_NODE_ADDON_API_SRC_MACROS_H_ -#define SCRIPTS_NODE_ADDON_API_SRC_MACROS_H_ - -#include -#include - -#define SHERPA_ONNX_ASSIGN_ATTR_STR(c_name, js_name) \ - do { \ - if (o.Has(#js_name) && o.Get(#js_name).IsString()) { \ - Napi::String _str = o.Get(#js_name).As(); \ - std::string s = _str.Utf8Value(); \ - char *p = new char[s.size() + 1]; \ - std::copy(s.begin(), s.end(), p); \ - p[s.size()] = 0; \ - \ - c.c_name = p; \ - } else if (o.Has(#js_name) && o.Get(#js_name).IsTypedArray()) { \ - Napi::Uint8Array _array = o.Get(#js_name).As(); \ - char *p = new char[_array.ElementLength() + 1]; \ - std::copy(_array.Data(), _array.Data() + _array.ElementLength(), p); \ - p[_array.ElementLength()] = '\0'; \ - \ - c.c_name = p; \ - } \ - } while (0) - -#define SHERPA_ONNX_ASSIGN_ATTR_INT32(c_name, js_name) \ - do { \ - if (o.Has(#js_name) && o.Get(#js_name).IsNumber()) { \ - c.c_name = o.Get(#js_name).As().Int32Value(); \ - } \ - } while (0) - -#define SHERPA_ONNX_ASSIGN_ATTR_FLOAT(c_name, js_name) \ - do { \ - if (o.Has(#js_name) && o.Get(#js_name).IsNumber()) { \ - c.c_name = o.Get(#js_name).As().FloatValue(); \ - } \ - } while (0) - -#define SHERPA_ONNX_DELETE_C_STR(p) \ - do { \ - if (p) { \ - delete[] p; \ - } \ - } while (0) - -#endif // SCRIPTS_NODE_ADDON_API_SRC_MACROS_H_ diff --git a/scripts/node-addon-api/src/macros.h b/scripts/node-addon-api/src/macros.h new file mode 120000 index 000000000..3d541e3da --- /dev/null +++ b/scripts/node-addon-api/src/macros.h @@ -0,0 +1 @@ +../../../harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/macros.h \ No newline at end of file diff --git a/scripts/node-addon-api/src/non-streaming-asr.cc b/scripts/node-addon-api/src/non-streaming-asr.cc deleted file mode 100644 index a95c892a8..000000000 --- a/scripts/node-addon-api/src/non-streaming-asr.cc +++ /dev/null @@ -1,461 +0,0 @@ -// scripts/node-addon-api/src/non-streaming-asr.cc -// -// Copyright (c) 2024 Xiaomi Corporation -#include - -#include "macros.h" // NOLINT -#include "napi.h" // NOLINT -#include "sherpa-onnx/c-api/c-api.h" - -// defined in ./streaming-asr.cc -SherpaOnnxFeatureConfig GetFeatureConfig(Napi::Object obj); - -static SherpaOnnxOfflineTransducerModelConfig GetOfflineTransducerModelConfig( - Napi::Object obj) { - SherpaOnnxOfflineTransducerModelConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("transducer") || !obj.Get("transducer").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("transducer").As(); - - SHERPA_ONNX_ASSIGN_ATTR_STR(encoder, encoder); - SHERPA_ONNX_ASSIGN_ATTR_STR(decoder, decoder); - SHERPA_ONNX_ASSIGN_ATTR_STR(joiner, joiner); - - return c; -} - -static SherpaOnnxOfflineParaformerModelConfig GetOfflineParaformerModelConfig( - Napi::Object obj) { - SherpaOnnxOfflineParaformerModelConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("paraformer") || !obj.Get("paraformer").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("paraformer").As(); - - SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); - - return c; -} - -static SherpaOnnxOfflineNemoEncDecCtcModelConfig GetOfflineNeMoCtcModelConfig( - Napi::Object obj) { - SherpaOnnxOfflineNemoEncDecCtcModelConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("nemoCtc") || !obj.Get("nemoCtc").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("nemoCtc").As(); - - SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); - - return c; -} - -static SherpaOnnxOfflineWhisperModelConfig GetOfflineWhisperModelConfig( - Napi::Object obj) { - SherpaOnnxOfflineWhisperModelConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("whisper") || !obj.Get("whisper").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("whisper").As(); - - SHERPA_ONNX_ASSIGN_ATTR_STR(encoder, encoder); - SHERPA_ONNX_ASSIGN_ATTR_STR(decoder, decoder); - SHERPA_ONNX_ASSIGN_ATTR_STR(language, language); - SHERPA_ONNX_ASSIGN_ATTR_STR(task, task); - SHERPA_ONNX_ASSIGN_ATTR_INT32(tail_paddings, tailPaddings); - - return c; -} - -static SherpaOnnxOfflineMoonshineModelConfig GetOfflineMoonshineModelConfig( - Napi::Object obj) { - SherpaOnnxOfflineMoonshineModelConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("moonshine") || !obj.Get("moonshine").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("moonshine").As(); - - SHERPA_ONNX_ASSIGN_ATTR_STR(preprocessor, preprocessor); - SHERPA_ONNX_ASSIGN_ATTR_STR(encoder, encoder); - SHERPA_ONNX_ASSIGN_ATTR_STR(uncached_decoder, uncachedDecoder); - SHERPA_ONNX_ASSIGN_ATTR_STR(cached_decoder, cachedDecoder); - - return c; -} - -static SherpaOnnxOfflineTdnnModelConfig GetOfflineTdnnModelConfig( - Napi::Object obj) { - SherpaOnnxOfflineTdnnModelConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("tdnn") || !obj.Get("tdnn").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("tdnn").As(); - - SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); - - return c; -} - -static SherpaOnnxOfflineSenseVoiceModelConfig GetOfflineSenseVoiceModelConfig( - Napi::Object obj) { - SherpaOnnxOfflineSenseVoiceModelConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("senseVoice") || !obj.Get("senseVoice").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("senseVoice").As(); - - SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); - SHERPA_ONNX_ASSIGN_ATTR_STR(language, language); - SHERPA_ONNX_ASSIGN_ATTR_INT32(use_itn, useInverseTextNormalization); - - return c; -} - -static SherpaOnnxOfflineModelConfig GetOfflineModelConfig(Napi::Object obj) { - SherpaOnnxOfflineModelConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("modelConfig") || !obj.Get("modelConfig").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("modelConfig").As(); - - c.transducer = GetOfflineTransducerModelConfig(o); - c.paraformer = GetOfflineParaformerModelConfig(o); - c.nemo_ctc = GetOfflineNeMoCtcModelConfig(o); - c.whisper = GetOfflineWhisperModelConfig(o); - c.tdnn = GetOfflineTdnnModelConfig(o); - c.sense_voice = GetOfflineSenseVoiceModelConfig(o); - c.moonshine = GetOfflineMoonshineModelConfig(o); - - SHERPA_ONNX_ASSIGN_ATTR_STR(tokens, tokens); - SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); - - if (o.Has("debug") && - (o.Get("debug").IsNumber() || o.Get("debug").IsBoolean())) { - if (o.Get("debug").IsBoolean()) { - c.debug = o.Get("debug").As().Value(); - } else { - c.debug = o.Get("debug").As().Int32Value(); - } - } - - SHERPA_ONNX_ASSIGN_ATTR_STR(provider, provider); - SHERPA_ONNX_ASSIGN_ATTR_STR(model_type, modelType); - SHERPA_ONNX_ASSIGN_ATTR_STR(modeling_unit, modelingUnit); - SHERPA_ONNX_ASSIGN_ATTR_STR(bpe_vocab, bpeVocab); - SHERPA_ONNX_ASSIGN_ATTR_STR(telespeech_ctc, teleSpeechCtc); - - return c; -} - -static SherpaOnnxOfflineLMConfig GetOfflineLMConfig(Napi::Object obj) { - SherpaOnnxOfflineLMConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("lmConfig") || !obj.Get("lmConfig").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("lmConfig").As(); - - SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); - SHERPA_ONNX_ASSIGN_ATTR_FLOAT(scale, scale); - - return c; -} - -static Napi::External -CreateOfflineRecognizerWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsObject()) { - Napi::TypeError::New(env, "Expect an object as the argument") - .ThrowAsJavaScriptException(); - - return {}; - } - - Napi::Object o = info[0].As(); - - SherpaOnnxOfflineRecognizerConfig c; - memset(&c, 0, sizeof(c)); - c.feat_config = GetFeatureConfig(o); - c.model_config = GetOfflineModelConfig(o); - c.lm_config = GetOfflineLMConfig(o); - - SHERPA_ONNX_ASSIGN_ATTR_STR(decoding_method, decodingMethod); - SHERPA_ONNX_ASSIGN_ATTR_INT32(max_active_paths, maxActivePaths); - SHERPA_ONNX_ASSIGN_ATTR_STR(hotwords_file, hotwordsFile); - SHERPA_ONNX_ASSIGN_ATTR_FLOAT(hotwords_score, hotwordsScore); - SHERPA_ONNX_ASSIGN_ATTR_STR(rule_fsts, ruleFsts); - SHERPA_ONNX_ASSIGN_ATTR_STR(rule_fars, ruleFars); - SHERPA_ONNX_ASSIGN_ATTR_FLOAT(blank_penalty, blankPenalty); - - const SherpaOnnxOfflineRecognizer *recognizer = - SherpaOnnxCreateOfflineRecognizer(&c); - - SHERPA_ONNX_DELETE_C_STR(c.model_config.transducer.encoder); - SHERPA_ONNX_DELETE_C_STR(c.model_config.transducer.decoder); - SHERPA_ONNX_DELETE_C_STR(c.model_config.transducer.joiner); - - SHERPA_ONNX_DELETE_C_STR(c.model_config.paraformer.model); - - SHERPA_ONNX_DELETE_C_STR(c.model_config.nemo_ctc.model); - - SHERPA_ONNX_DELETE_C_STR(c.model_config.whisper.encoder); - SHERPA_ONNX_DELETE_C_STR(c.model_config.whisper.decoder); - SHERPA_ONNX_DELETE_C_STR(c.model_config.whisper.language); - SHERPA_ONNX_DELETE_C_STR(c.model_config.whisper.task); - - SHERPA_ONNX_DELETE_C_STR(c.model_config.tdnn.model); - - SHERPA_ONNX_DELETE_C_STR(c.model_config.sense_voice.model); - SHERPA_ONNX_DELETE_C_STR(c.model_config.sense_voice.language); - - SHERPA_ONNX_DELETE_C_STR(c.model_config.moonshine.preprocessor); - SHERPA_ONNX_DELETE_C_STR(c.model_config.moonshine.encoder); - SHERPA_ONNX_DELETE_C_STR(c.model_config.moonshine.uncached_decoder); - SHERPA_ONNX_DELETE_C_STR(c.model_config.moonshine.cached_decoder); - - SHERPA_ONNX_DELETE_C_STR(c.model_config.tokens); - SHERPA_ONNX_DELETE_C_STR(c.model_config.provider); - SHERPA_ONNX_DELETE_C_STR(c.model_config.model_type); - SHERPA_ONNX_DELETE_C_STR(c.model_config.modeling_unit); - SHERPA_ONNX_DELETE_C_STR(c.model_config.bpe_vocab); - SHERPA_ONNX_DELETE_C_STR(c.model_config.telespeech_ctc); - - SHERPA_ONNX_DELETE_C_STR(c.lm_config.model); - - SHERPA_ONNX_DELETE_C_STR(c.decoding_method); - SHERPA_ONNX_DELETE_C_STR(c.hotwords_file); - SHERPA_ONNX_DELETE_C_STR(c.rule_fsts); - SHERPA_ONNX_DELETE_C_STR(c.rule_fars); - - if (!recognizer) { - Napi::TypeError::New(env, "Please check your config!") - .ThrowAsJavaScriptException(); - - return {}; - } - - return Napi::External::New( - env, const_cast(recognizer), - [](Napi::Env env, SherpaOnnxOfflineRecognizer *recognizer) { - SherpaOnnxDestroyOfflineRecognizer(recognizer); - }); -} - -static Napi::External CreateOfflineStreamWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New( - env, - "You should pass an offline recognizer pointer as the only argument") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxOfflineRecognizer *recognizer = - info[0].As>().Data(); - - const SherpaOnnxOfflineStream *stream = - SherpaOnnxCreateOfflineStream(recognizer); - - return Napi::External::New( - env, const_cast(stream), - [](Napi::Env env, SherpaOnnxOfflineStream *stream) { - SherpaOnnxDestroyOfflineStream(stream); - }); -} - -static void AcceptWaveformOfflineWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be an online stream pointer.") - .ThrowAsJavaScriptException(); - - return; - } - - SherpaOnnxOfflineStream *stream = - info[0].As>().Data(); - - if (!info[1].IsObject()) { - Napi::TypeError::New(env, "Argument 1 should be an object") - .ThrowAsJavaScriptException(); - - return; - } - - Napi::Object obj = info[1].As(); - - if (!obj.Has("samples")) { - Napi::TypeError::New(env, "The argument object should have a field samples") - .ThrowAsJavaScriptException(); - - return; - } - - if (!obj.Get("samples").IsTypedArray()) { - Napi::TypeError::New(env, "The object['samples'] should be a typed array") - .ThrowAsJavaScriptException(); - - return; - } - - if (!obj.Has("sampleRate")) { - Napi::TypeError::New(env, - "The argument object should have a field sampleRate") - .ThrowAsJavaScriptException(); - - return; - } - - if (!obj.Get("sampleRate").IsNumber()) { - Napi::TypeError::New(env, "The object['samples'] should be a number") - .ThrowAsJavaScriptException(); - - return; - } - - Napi::Float32Array samples = obj.Get("samples").As(); - int32_t sample_rate = obj.Get("sampleRate").As().Int32Value(); - - SherpaOnnxAcceptWaveformOffline(stream, sample_rate, samples.Data(), - samples.ElementLength()); -} - -static void DecodeOfflineStreamWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, - "Argument 0 should be an offline recognizer pointer.") - .ThrowAsJavaScriptException(); - - return; - } - - if (!info[1].IsExternal()) { - Napi::TypeError::New(env, "Argument 1 should be an offline stream pointer.") - .ThrowAsJavaScriptException(); - - return; - } - - SherpaOnnxOfflineRecognizer *recognizer = - info[0].As>().Data(); - - SherpaOnnxOfflineStream *stream = - info[1].As>().Data(); - - SherpaOnnxDecodeOfflineStream(recognizer, stream); -} - -static Napi::String GetOfflineStreamResultAsJsonWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be an online stream pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxOfflineStream *stream = - info[0].As>().Data(); - - const char *json = SherpaOnnxGetOfflineStreamResultAsJson(stream); - Napi::String s = Napi::String::New(env, json); - - SherpaOnnxDestroyOfflineStreamResultJson(json); - - return s; -} - -void InitNonStreamingAsr(Napi::Env env, Napi::Object exports) { - exports.Set(Napi::String::New(env, "createOfflineRecognizer"), - Napi::Function::New(env, CreateOfflineRecognizerWrapper)); - - exports.Set(Napi::String::New(env, "createOfflineStream"), - Napi::Function::New(env, CreateOfflineStreamWrapper)); - - exports.Set(Napi::String::New(env, "acceptWaveformOffline"), - Napi::Function::New(env, AcceptWaveformOfflineWrapper)); - - exports.Set(Napi::String::New(env, "decodeOfflineStream"), - Napi::Function::New(env, DecodeOfflineStreamWrapper)); - - exports.Set(Napi::String::New(env, "getOfflineStreamResultAsJson"), - Napi::Function::New(env, GetOfflineStreamResultAsJsonWrapper)); -} diff --git a/scripts/node-addon-api/src/non-streaming-asr.cc b/scripts/node-addon-api/src/non-streaming-asr.cc new file mode 120000 index 000000000..12f89f891 --- /dev/null +++ b/scripts/node-addon-api/src/non-streaming-asr.cc @@ -0,0 +1 @@ +../../../harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-asr.cc \ No newline at end of file diff --git a/scripts/node-addon-api/src/non-streaming-speaker-diarization.cc b/scripts/node-addon-api/src/non-streaming-speaker-diarization.cc deleted file mode 100644 index a35f7924a..000000000 --- a/scripts/node-addon-api/src/non-streaming-speaker-diarization.cc +++ /dev/null @@ -1,310 +0,0 @@ -// scripts/node-addon-api/src/non-streaming-speaker-diarization.cc -// -// Copyright (c) 2024 Xiaomi Corporation - -#include -#include - -#include "macros.h" // NOLINT -#include "napi.h" // NOLINT -#include "sherpa-onnx/c-api/c-api.h" - -static SherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig -GetOfflineSpeakerSegmentationPyannoteModelConfig(Napi::Object obj) { - SherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("pyannote") || !obj.Get("pyannote").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("pyannote").As(); - SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); - - return c; -} - -static SherpaOnnxOfflineSpeakerSegmentationModelConfig -GetOfflineSpeakerSegmentationModelConfig(Napi::Object obj) { - SherpaOnnxOfflineSpeakerSegmentationModelConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("segmentation") || !obj.Get("segmentation").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("segmentation").As(); - - c.pyannote = GetOfflineSpeakerSegmentationPyannoteModelConfig(o); - - SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); - - if (o.Has("debug") && - (o.Get("debug").IsNumber() || o.Get("debug").IsBoolean())) { - if (o.Get("debug").IsBoolean()) { - c.debug = o.Get("debug").As().Value(); - } else { - c.debug = o.Get("debug").As().Int32Value(); - } - } - - SHERPA_ONNX_ASSIGN_ATTR_STR(provider, provider); - - return c; -} - -static SherpaOnnxSpeakerEmbeddingExtractorConfig -GetSpeakerEmbeddingExtractorConfig(Napi::Object obj) { - SherpaOnnxSpeakerEmbeddingExtractorConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("embedding") || !obj.Get("embedding").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("embedding").As(); - - SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); - SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); - - if (o.Has("debug") && - (o.Get("debug").IsNumber() || o.Get("debug").IsBoolean())) { - if (o.Get("debug").IsBoolean()) { - c.debug = o.Get("debug").As().Value(); - } else { - c.debug = o.Get("debug").As().Int32Value(); - } - } - - SHERPA_ONNX_ASSIGN_ATTR_STR(provider, provider); - - return c; -} - -static SherpaOnnxFastClusteringConfig GetFastClusteringConfig( - Napi::Object obj) { - SherpaOnnxFastClusteringConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("clustering") || !obj.Get("clustering").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("clustering").As(); - - SHERPA_ONNX_ASSIGN_ATTR_INT32(num_clusters, numClusters); - SHERPA_ONNX_ASSIGN_ATTR_FLOAT(threshold, threshold); - - return c; -} - -static Napi::External -CreateOfflineSpeakerDiarizationWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsObject()) { - Napi::TypeError::New(env, "Expect an object as the argument") - .ThrowAsJavaScriptException(); - - return {}; - } - - Napi::Object o = info[0].As(); - - SherpaOnnxOfflineSpeakerDiarizationConfig c; - memset(&c, 0, sizeof(c)); - - c.segmentation = GetOfflineSpeakerSegmentationModelConfig(o); - c.embedding = GetSpeakerEmbeddingExtractorConfig(o); - c.clustering = GetFastClusteringConfig(o); - - SHERPA_ONNX_ASSIGN_ATTR_FLOAT(min_duration_on, minDurationOn); - SHERPA_ONNX_ASSIGN_ATTR_FLOAT(min_duration_off, minDurationOff); - - const SherpaOnnxOfflineSpeakerDiarization *sd = - SherpaOnnxCreateOfflineSpeakerDiarization(&c); - - if (c.segmentation.pyannote.model) { - delete[] c.segmentation.pyannote.model; - } - - if (c.segmentation.provider) { - delete[] c.segmentation.provider; - } - - if (c.embedding.model) { - delete[] c.embedding.model; - } - - if (c.embedding.provider) { - delete[] c.embedding.provider; - } - - if (!sd) { - Napi::TypeError::New(env, "Please check your config!") - .ThrowAsJavaScriptException(); - - return {}; - } - - return Napi::External::New( - env, const_cast(sd), - [](Napi::Env env, SherpaOnnxOfflineSpeakerDiarization *sd) { - SherpaOnnxDestroyOfflineSpeakerDiarization(sd); - }); -} - -static Napi::Number OfflineSpeakerDiarizationGetSampleRateWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New( - env, "Argument 0 should be an offline speaker diarization pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - const SherpaOnnxOfflineSpeakerDiarization *sd = - info[0].As>().Data(); - - int32_t sample_rate = SherpaOnnxOfflineSpeakerDiarizationGetSampleRate(sd); - - return Napi::Number::New(env, sample_rate); -} - -static Napi::Array OfflineSpeakerDiarizationProcessWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New( - env, "Argument 0 should be an offline speaker diarization pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - const SherpaOnnxOfflineSpeakerDiarization *sd = - info[0].As>().Data(); - - if (!info[1].IsTypedArray()) { - Napi::TypeError::New(env, "Argument 1 should be a typed array") - .ThrowAsJavaScriptException(); - - return {}; - } - - Napi::Float32Array samples = info[1].As(); - - const SherpaOnnxOfflineSpeakerDiarizationResult *r = - SherpaOnnxOfflineSpeakerDiarizationProcess(sd, samples.Data(), - samples.ElementLength()); - - int32_t num_segments = - SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments(r); - - const SherpaOnnxOfflineSpeakerDiarizationSegment *segments = - SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime(r); - - Napi::Array ans = Napi::Array::New(env, num_segments); - - for (int32_t i = 0; i != num_segments; ++i) { - Napi::Object obj = Napi::Object::New(env); - - obj.Set(Napi::String::New(env, "start"), segments[i].start); - obj.Set(Napi::String::New(env, "end"), segments[i].end); - obj.Set(Napi::String::New(env, "speaker"), segments[i].speaker); - - ans.Set(i, obj); - } - - SherpaOnnxOfflineSpeakerDiarizationDestroySegment(segments); - SherpaOnnxOfflineSpeakerDiarizationDestroyResult(r); - - return ans; -} - -static void OfflineSpeakerDiarizationSetConfigWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New( - env, "Argument 0 should be an offline speaker diarization pointer.") - .ThrowAsJavaScriptException(); - - return; - } - - const SherpaOnnxOfflineSpeakerDiarization *sd = - info[0].As>().Data(); - - if (!info[1].IsObject()) { - Napi::TypeError::New(env, "Expect an object as the argument") - .ThrowAsJavaScriptException(); - - return; - } - - Napi::Object o = info[0].As(); - - SherpaOnnxOfflineSpeakerDiarizationConfig c; - memset(&c, 0, sizeof(c)); - - c.clustering = GetFastClusteringConfig(o); - SherpaOnnxOfflineSpeakerDiarizationSetConfig(sd, &c); -} - -void InitNonStreamingSpeakerDiarization(Napi::Env env, Napi::Object exports) { - exports.Set(Napi::String::New(env, "createOfflineSpeakerDiarization"), - Napi::Function::New(env, CreateOfflineSpeakerDiarizationWrapper)); - - exports.Set( - Napi::String::New(env, "getOfflineSpeakerDiarizationSampleRate"), - Napi::Function::New(env, OfflineSpeakerDiarizationGetSampleRateWrapper)); - - exports.Set( - Napi::String::New(env, "offlineSpeakerDiarizationProcess"), - Napi::Function::New(env, OfflineSpeakerDiarizationProcessWrapper)); - - exports.Set( - Napi::String::New(env, "offlineSpeakerDiarizationSetConfig"), - Napi::Function::New(env, OfflineSpeakerDiarizationSetConfigWrapper)); -} diff --git a/scripts/node-addon-api/src/non-streaming-speaker-diarization.cc b/scripts/node-addon-api/src/non-streaming-speaker-diarization.cc new file mode 120000 index 000000000..5c325a885 --- /dev/null +++ b/scripts/node-addon-api/src/non-streaming-speaker-diarization.cc @@ -0,0 +1 @@ +../../../harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-speaker-diarization.cc \ No newline at end of file diff --git a/scripts/node-addon-api/src/non-streaming-tts.cc b/scripts/node-addon-api/src/non-streaming-tts.cc deleted file mode 100644 index 70d97cddb..000000000 --- a/scripts/node-addon-api/src/non-streaming-tts.cc +++ /dev/null @@ -1,329 +0,0 @@ -// scripts/node-addon-api/src/non-streaming-tts.cc -// -// Copyright (c) 2024 Xiaomi Corporation - -#include -#include - -#include "macros.h" // NOLINT -#include "napi.h" // NOLINT -#include "sherpa-onnx/c-api/c-api.h" - -static SherpaOnnxOfflineTtsVitsModelConfig GetOfflineTtsVitsModelConfig( - Napi::Object obj) { - SherpaOnnxOfflineTtsVitsModelConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("vits") || !obj.Get("vits").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("vits").As(); - SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); - SHERPA_ONNX_ASSIGN_ATTR_STR(lexicon, lexicon); - SHERPA_ONNX_ASSIGN_ATTR_STR(tokens, tokens); - SHERPA_ONNX_ASSIGN_ATTR_STR(data_dir, dataDir); - SHERPA_ONNX_ASSIGN_ATTR_FLOAT(noise_scale, noiseScale); - SHERPA_ONNX_ASSIGN_ATTR_FLOAT(noise_scale_w, noiseScaleW); - SHERPA_ONNX_ASSIGN_ATTR_FLOAT(length_scale, lengthScale); - SHERPA_ONNX_ASSIGN_ATTR_STR(dict_dir, dictDir); - - return c; -} - -static SherpaOnnxOfflineTtsModelConfig GetOfflineTtsModelConfig( - Napi::Object obj) { - SherpaOnnxOfflineTtsModelConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("model") || !obj.Get("model").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("model").As(); - - c.vits = GetOfflineTtsVitsModelConfig(o); - - SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); - - if (o.Has("debug") && - (o.Get("debug").IsNumber() || o.Get("debug").IsBoolean())) { - if (o.Get("debug").IsBoolean()) { - c.debug = o.Get("debug").As().Value(); - } else { - c.debug = o.Get("debug").As().Int32Value(); - } - } - - SHERPA_ONNX_ASSIGN_ATTR_STR(provider, provider); - - return c; -} - -static Napi::External CreateOfflineTtsWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsObject()) { - Napi::TypeError::New(env, "Expect an object as the argument") - .ThrowAsJavaScriptException(); - - return {}; - } - - Napi::Object o = info[0].As(); - - SherpaOnnxOfflineTtsConfig c; - memset(&c, 0, sizeof(c)); - - c.model = GetOfflineTtsModelConfig(o); - - SHERPA_ONNX_ASSIGN_ATTR_STR(rule_fsts, ruleFsts); - SHERPA_ONNX_ASSIGN_ATTR_INT32(max_num_sentences, maxNumSentences); - SHERPA_ONNX_ASSIGN_ATTR_STR(rule_fars, ruleFars); - - SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTts(&c); - - if (c.model.vits.model) { - delete[] c.model.vits.model; - } - - if (c.model.vits.lexicon) { - delete[] c.model.vits.lexicon; - } - - if (c.model.vits.tokens) { - delete[] c.model.vits.tokens; - } - - if (c.model.vits.data_dir) { - delete[] c.model.vits.data_dir; - } - - if (c.model.vits.dict_dir) { - delete[] c.model.vits.dict_dir; - } - - if (c.model.provider) { - delete[] c.model.provider; - } - - if (c.rule_fsts) { - delete[] c.rule_fsts; - } - - if (c.rule_fars) { - delete[] c.rule_fars; - } - - if (!tts) { - Napi::TypeError::New(env, "Please check your config!") - .ThrowAsJavaScriptException(); - - return {}; - } - - return Napi::External::New( - env, tts, [](Napi::Env env, SherpaOnnxOfflineTts *tts) { - SherpaOnnxDestroyOfflineTts(tts); - }); -} - -static Napi::Number OfflineTtsSampleRateWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be an offline tts pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxOfflineTts *tts = - info[0].As>().Data(); - - int32_t sample_rate = SherpaOnnxOfflineTtsSampleRate(tts); - - return Napi::Number::New(env, sample_rate); -} - -static Napi::Number OfflineTtsNumSpeakersWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be an offline tts pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxOfflineTts *tts = - info[0].As>().Data(); - - int32_t num_speakers = SherpaOnnxOfflineTtsNumSpeakers(tts); - - return Napi::Number::New(env, num_speakers); -} - -static Napi::Object OfflineTtsGenerateWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be an offline tts pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxOfflineTts *tts = - info[0].As>().Data(); - - if (!info[1].IsObject()) { - Napi::TypeError::New(env, "Argument 1 should be an object") - .ThrowAsJavaScriptException(); - - return {}; - } - - Napi::Object obj = info[1].As(); - - if (!obj.Has("text")) { - Napi::TypeError::New(env, "The argument object should have a field text") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Get("text").IsString()) { - Napi::TypeError::New(env, "The object['text'] should be a string") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Has("sid")) { - Napi::TypeError::New(env, "The argument object should have a field sid") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Get("sid").IsNumber()) { - Napi::TypeError::New(env, "The object['sid'] should be a number") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Has("speed")) { - Napi::TypeError::New(env, "The argument object should have a field speed") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Get("speed").IsNumber()) { - Napi::TypeError::New(env, "The object['speed'] should be a number") - .ThrowAsJavaScriptException(); - - return {}; - } - - bool enable_external_buffer = true; - if (obj.Has("enableExternalBuffer") && - obj.Get("enableExternalBuffer").IsBoolean()) { - enable_external_buffer = - obj.Get("enableExternalBuffer").As().Value(); - } - - Napi::String _text = obj.Get("text").As(); - std::string text = _text.Utf8Value(); - int32_t sid = obj.Get("sid").As().Int32Value(); - float speed = obj.Get("speed").As().FloatValue(); - - const SherpaOnnxGeneratedAudio *audio = - SherpaOnnxOfflineTtsGenerate(tts, text.c_str(), sid, speed); - - if (enable_external_buffer) { - Napi::ArrayBuffer arrayBuffer = Napi::ArrayBuffer::New( - env, const_cast(audio->samples), sizeof(float) * audio->n, - [](Napi::Env /*env*/, void * /*data*/, - const SherpaOnnxGeneratedAudio *hint) { - SherpaOnnxDestroyOfflineTtsGeneratedAudio(hint); - }, - audio); - Napi::Float32Array float32Array = - Napi::Float32Array::New(env, audio->n, arrayBuffer, 0); - - Napi::Object ans = Napi::Object::New(env); - ans.Set(Napi::String::New(env, "samples"), float32Array); - ans.Set(Napi::String::New(env, "sampleRate"), audio->sample_rate); - return ans; - } else { - // don't use external buffer - Napi::ArrayBuffer arrayBuffer = - Napi::ArrayBuffer::New(env, sizeof(float) * audio->n); - - Napi::Float32Array float32Array = - Napi::Float32Array::New(env, audio->n, arrayBuffer, 0); - - std::copy(audio->samples, audio->samples + audio->n, float32Array.Data()); - - Napi::Object ans = Napi::Object::New(env); - ans.Set(Napi::String::New(env, "samples"), float32Array); - ans.Set(Napi::String::New(env, "sampleRate"), audio->sample_rate); - SherpaOnnxDestroyOfflineTtsGeneratedAudio(audio); - return ans; - } -} - -void InitNonStreamingTts(Napi::Env env, Napi::Object exports) { - exports.Set(Napi::String::New(env, "createOfflineTts"), - Napi::Function::New(env, CreateOfflineTtsWrapper)); - - exports.Set(Napi::String::New(env, "getOfflineTtsSampleRate"), - Napi::Function::New(env, OfflineTtsSampleRateWrapper)); - - exports.Set(Napi::String::New(env, "getOfflineTtsNumSpeakers"), - Napi::Function::New(env, OfflineTtsNumSpeakersWrapper)); - - exports.Set(Napi::String::New(env, "offlineTtsGenerate"), - Napi::Function::New(env, OfflineTtsGenerateWrapper)); -} diff --git a/scripts/node-addon-api/src/non-streaming-tts.cc b/scripts/node-addon-api/src/non-streaming-tts.cc new file mode 120000 index 000000000..fd4eb93f2 --- /dev/null +++ b/scripts/node-addon-api/src/non-streaming-tts.cc @@ -0,0 +1 @@ +../../../harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc \ No newline at end of file diff --git a/scripts/node-addon-api/src/punctuation.cc b/scripts/node-addon-api/src/punctuation.cc deleted file mode 100644 index df079b96d..000000000 --- a/scripts/node-addon-api/src/punctuation.cc +++ /dev/null @@ -1,135 +0,0 @@ -// scripts/node-addon-api/src/punctuation.cc -// -// Copyright (c) 2024 Xiaomi Corporation -#include - -#include "macros.h" // NOLINT -#include "napi.h" // NOLINT -#include "sherpa-onnx/c-api/c-api.h" - -static SherpaOnnxOfflinePunctuationModelConfig GetOfflinePunctuationModelConfig( - Napi::Object obj) { - SherpaOnnxOfflinePunctuationModelConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("model") || !obj.Get("model").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("model").As(); - - SHERPA_ONNX_ASSIGN_ATTR_STR(ct_transformer, ctTransformer); - - SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); - - if (o.Has("debug") && - (o.Get("debug").IsNumber() || o.Get("debug").IsBoolean())) { - if (o.Get("debug").IsBoolean()) { - c.debug = o.Get("debug").As().Value(); - } else { - c.debug = o.Get("debug").As().Int32Value(); - } - } - SHERPA_ONNX_ASSIGN_ATTR_STR(provider, provider); - - return c; -} - -static Napi::External -CreateOfflinePunctuationWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsObject()) { - Napi::TypeError::New(env, "You should pass an object as the only argument.") - .ThrowAsJavaScriptException(); - - return {}; - } - - Napi::Object o = info[0].As(); - - SherpaOnnxOfflinePunctuationConfig c; - memset(&c, 0, sizeof(c)); - c.model = GetOfflinePunctuationModelConfig(o); - - const SherpaOnnxOfflinePunctuation *punct = - SherpaOnnxCreateOfflinePunctuation(&c); - - if (c.model.ct_transformer) { - delete[] c.model.ct_transformer; - } - - if (c.model.provider) { - delete[] c.model.provider; - } - - if (!punct) { - Napi::TypeError::New(env, "Please check your config!") - .ThrowAsJavaScriptException(); - - return {}; - } - - return Napi::External::New( - env, const_cast(punct), - [](Napi::Env env, SherpaOnnxOfflinePunctuation *punct) { - SherpaOnnxDestroyOfflinePunctuation(punct); - }); -} - -static Napi::String OfflinePunctuationAddPunctWraper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New( - env, - "You should pass an offline punctuation pointer as the first argument") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[1].IsString()) { - Napi::TypeError::New(env, "You should pass a string as the second argument") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxOfflinePunctuation *punct = - info[0].As>().Data(); - Napi::String js_text = info[1].As(); - std::string text = js_text.Utf8Value(); - - const char *punct_text = - SherpaOfflinePunctuationAddPunct(punct, text.c_str()); - - Napi::String ans = Napi::String::New(env, punct_text); - SherpaOfflinePunctuationFreeText(punct_text); - return ans; -} - -void InitPunctuation(Napi::Env env, Napi::Object exports) { - exports.Set(Napi::String::New(env, "createOfflinePunctuation"), - Napi::Function::New(env, CreateOfflinePunctuationWrapper)); - - exports.Set(Napi::String::New(env, "offlinePunctuationAddPunct"), - Napi::Function::New(env, OfflinePunctuationAddPunctWraper)); -} diff --git a/scripts/node-addon-api/src/punctuation.cc b/scripts/node-addon-api/src/punctuation.cc new file mode 120000 index 000000000..a0d6b08e1 --- /dev/null +++ b/scripts/node-addon-api/src/punctuation.cc @@ -0,0 +1 @@ +../../../harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/punctuation.cc \ No newline at end of file diff --git a/scripts/node-addon-api/src/sherpa-onnx-node-addon-api.cc b/scripts/node-addon-api/src/sherpa-onnx-node-addon-api.cc deleted file mode 100644 index 3f0affd79..000000000 --- a/scripts/node-addon-api/src/sherpa-onnx-node-addon-api.cc +++ /dev/null @@ -1,47 +0,0 @@ -// scripts/node-addon-api/src/sherpa-onnx-node-addon-api.cc -// -// Copyright (c) 2024 Xiaomi Corporation -#include "napi.h" // NOLINT - -void InitStreamingAsr(Napi::Env env, Napi::Object exports); - -void InitNonStreamingAsr(Napi::Env env, Napi::Object exports); - -void InitNonStreamingTts(Napi::Env env, Napi::Object exports); - -void InitVad(Napi::Env env, Napi::Object exports); - -void InitWaveReader(Napi::Env env, Napi::Object exports); - -void InitWaveWriter(Napi::Env env, Napi::Object exports); - -void InitSpokenLanguageID(Napi::Env env, Napi::Object exports); - -void InitSpeakerID(Napi::Env env, Napi::Object exports); - -void InitAudioTagging(Napi::Env env, Napi::Object exports); - -void InitPunctuation(Napi::Env env, Napi::Object exports); - -void InitKeywordSpotting(Napi::Env env, Napi::Object exports); - -void InitNonStreamingSpeakerDiarization(Napi::Env env, Napi::Object exports); - -Napi::Object Init(Napi::Env env, Napi::Object exports) { - InitStreamingAsr(env, exports); - InitNonStreamingAsr(env, exports); - InitNonStreamingTts(env, exports); - InitVad(env, exports); - InitWaveReader(env, exports); - InitWaveWriter(env, exports); - InitSpokenLanguageID(env, exports); - InitSpeakerID(env, exports); - InitAudioTagging(env, exports); - InitPunctuation(env, exports); - InitKeywordSpotting(env, exports); - InitNonStreamingSpeakerDiarization(env, exports); - - return exports; -} - -NODE_API_MODULE(addon, Init) diff --git a/scripts/node-addon-api/src/sherpa-onnx-node-addon-api.cc b/scripts/node-addon-api/src/sherpa-onnx-node-addon-api.cc new file mode 120000 index 000000000..9394c068a --- /dev/null +++ b/scripts/node-addon-api/src/sherpa-onnx-node-addon-api.cc @@ -0,0 +1 @@ +../../../harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/sherpa-onnx-node-addon-api.cc \ No newline at end of file diff --git a/scripts/node-addon-api/src/speaker-identification.cc b/scripts/node-addon-api/src/speaker-identification.cc deleted file mode 100644 index a08a6ed66..000000000 --- a/scripts/node-addon-api/src/speaker-identification.cc +++ /dev/null @@ -1,808 +0,0 @@ -// scripts/node-addon-api/src/speaker-identification.cc -// -// Copyright (c) 2024 Xiaomi Corporation -#include -#include - -#include "macros.h" // NOLINT -#include "napi.h" // NOLINT -#include "sherpa-onnx/c-api/c-api.h" - -static Napi::External -CreateSpeakerEmbeddingExtractorWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsObject()) { - Napi::TypeError::New(env, "You should pass an object as the only argument.") - .ThrowAsJavaScriptException(); - - return {}; - } - - Napi::Object o = info[0].As(); - - SherpaOnnxSpeakerEmbeddingExtractorConfig c; - memset(&c, 0, sizeof(c)); - - SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); - SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); - - if (o.Has("debug") && - (o.Get("debug").IsNumber() || o.Get("debug").IsBoolean())) { - if (o.Get("debug").IsBoolean()) { - c.debug = o.Get("debug").As().Value(); - } else { - c.debug = o.Get("debug").As().Int32Value(); - } - } - - SHERPA_ONNX_ASSIGN_ATTR_STR(provider, provider); - - const SherpaOnnxSpeakerEmbeddingExtractor *extractor = - SherpaOnnxCreateSpeakerEmbeddingExtractor(&c); - - if (c.model) { - delete[] c.model; - } - - if (c.provider) { - delete[] c.provider; - } - - if (!extractor) { - Napi::TypeError::New(env, "Please check your config!") - .ThrowAsJavaScriptException(); - - return {}; - } - - return Napi::External::New( - env, const_cast(extractor), - [](Napi::Env env, SherpaOnnxSpeakerEmbeddingExtractor *extractor) { - SherpaOnnxDestroySpeakerEmbeddingExtractor(extractor); - }); -} - -static Napi::Number SpeakerEmbeddingExtractorDimWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New( - env, "Argument 0 should be a speaker embedding extractor pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxSpeakerEmbeddingExtractor *extractor = - info[0].As>().Data(); - - int32_t dim = SherpaOnnxSpeakerEmbeddingExtractorDim(extractor); - - return Napi::Number::New(env, dim); -} - -static Napi::External -SpeakerEmbeddingExtractorCreateStreamWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, - "You should pass a speaker embedding extractor " - "pointer as the only argument") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxSpeakerEmbeddingExtractor *extractor = - info[0].As>().Data(); - - const SherpaOnnxOnlineStream *stream = - SherpaOnnxSpeakerEmbeddingExtractorCreateStream(extractor); - - return Napi::External::New( - env, const_cast(stream), - [](Napi::Env env, SherpaOnnxOnlineStream *stream) { - SherpaOnnxDestroyOnlineStream(stream); - }); -} - -static Napi::Boolean SpeakerEmbeddingExtractorIsReadyWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New( - env, "Argument 0 should be a speaker embedding extractor pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[1].IsExternal()) { - Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxSpeakerEmbeddingExtractor *extractor = - info[0].As>().Data(); - - SherpaOnnxOnlineStream *stream = - info[1].As>().Data(); - - int32_t is_ready = - SherpaOnnxSpeakerEmbeddingExtractorIsReady(extractor, stream); - - return Napi::Boolean::New(env, is_ready); -} - -static Napi::Float32Array SpeakerEmbeddingExtractorComputeEmbeddingWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 2 && info.Length() != 3) { - std::ostringstream os; - os << "Expect only 2 or 3 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New( - env, "Argument 0 should be a speaker embedding extractor pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[1].IsExternal()) { - Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - bool enable_external_buffer = true; - if (info.Length() == 3) { - if (info[2].IsBoolean()) { - enable_external_buffer = info[2].As().Value(); - } else { - Napi::TypeError::New(env, "Argument 2 should be a boolean.") - .ThrowAsJavaScriptException(); - } - } - - SherpaOnnxSpeakerEmbeddingExtractor *extractor = - info[0].As>().Data(); - - SherpaOnnxOnlineStream *stream = - info[1].As>().Data(); - - const float *v = - SherpaOnnxSpeakerEmbeddingExtractorComputeEmbedding(extractor, stream); - - int32_t dim = SherpaOnnxSpeakerEmbeddingExtractorDim(extractor); - - if (enable_external_buffer) { - Napi::ArrayBuffer arrayBuffer = Napi::ArrayBuffer::New( - env, const_cast(v), sizeof(float) * dim, - [](Napi::Env /*env*/, void *data) { - SherpaOnnxSpeakerEmbeddingExtractorDestroyEmbedding( - reinterpret_cast(data)); - }); - - return Napi::Float32Array::New(env, dim, arrayBuffer, 0); - } else { - // don't use external buffer - Napi::ArrayBuffer arrayBuffer = - Napi::ArrayBuffer::New(env, sizeof(float) * dim); - - Napi::Float32Array float32Array = - Napi::Float32Array::New(env, dim, arrayBuffer, 0); - - std::copy(v, v + dim, float32Array.Data()); - - SherpaOnnxSpeakerEmbeddingExtractorDestroyEmbedding(v); - - return float32Array; - } -} - -static Napi::External -CreateSpeakerEmbeddingManagerWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsNumber()) { - Napi::TypeError::New(env, - "You should pass an integer as the only argument.") - .ThrowAsJavaScriptException(); - - return {}; - } - - int32_t dim = info[0].As().Int32Value(); - - const SherpaOnnxSpeakerEmbeddingManager *manager = - SherpaOnnxCreateSpeakerEmbeddingManager(dim); - - if (!manager) { - Napi::TypeError::New(env, "Please check your input dim!") - .ThrowAsJavaScriptException(); - - return {}; - } - - return Napi::External::New( - env, const_cast(manager), - [](Napi::Env env, SherpaOnnxSpeakerEmbeddingManager *manager) { - SherpaOnnxDestroySpeakerEmbeddingManager(manager); - }); -} - -static Napi::Boolean SpeakerEmbeddingManagerAddWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, - "You should pass a speaker embedding manager pointer " - "as the first argument.") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[1].IsObject()) { - Napi::TypeError::New(env, "Argument 1 should be an object") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxSpeakerEmbeddingManager *manager = - info[0].As>().Data(); - - Napi::Object obj = info[1].As(); - - if (!obj.Has("v")) { - Napi::TypeError::New(env, "The argument object should have a field v") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Get("v").IsTypedArray()) { - Napi::TypeError::New(env, "The object['v'] should be a typed array") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Has("name")) { - Napi::TypeError::New(env, "The argument object should have a field name") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Get("name").IsString()) { - Napi::TypeError::New(env, "The object['name'] should be a string") - .ThrowAsJavaScriptException(); - - return {}; - } - - Napi::Float32Array v = obj.Get("v").As(); - Napi::String js_name = obj.Get("name").As(); - std::string name = js_name.Utf8Value(); - - int32_t ok = - SherpaOnnxSpeakerEmbeddingManagerAdd(manager, name.c_str(), v.Data()); - return Napi::Boolean::New(env, ok); -} - -static Napi::Boolean SpeakerEmbeddingManagerAddListFlattenedWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, - "You should pass a speaker embedding manager pointer " - "as the first argument.") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[1].IsObject()) { - Napi::TypeError::New(env, "Argument 1 should be an object") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxSpeakerEmbeddingManager *manager = - info[0].As>().Data(); - - Napi::Object obj = info[1].As(); - - if (!obj.Has("vv")) { - Napi::TypeError::New(env, "The argument object should have a field vv") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Get("vv").IsTypedArray()) { - Napi::TypeError::New(env, "The object['vv'] should be a typed array") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Has("name")) { - Napi::TypeError::New(env, "The argument object should have a field name") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Get("name").IsString()) { - Napi::TypeError::New(env, "The object['name'] should be a string") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Has("n")) { - Napi::TypeError::New(env, "The argument object should have a field n") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Get("n").IsNumber()) { - Napi::TypeError::New(env, "The object['n'] should be an integer") - .ThrowAsJavaScriptException(); - - return {}; - } - - Napi::Float32Array v = obj.Get("vv").As(); - Napi::String js_name = obj.Get("name").As(); - int32_t n = obj.Get("n").As().Int32Value(); - - std::string name = js_name.Utf8Value(); - - int32_t ok = SherpaOnnxSpeakerEmbeddingManagerAddListFlattened( - manager, name.c_str(), v.Data(), n); - - return Napi::Boolean::New(env, ok); -} - -static Napi::Boolean SpeakerEmbeddingManagerRemoveWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, - "You should pass a speaker embedding manager pointer " - "as the first argument.") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[1].IsString()) { - Napi::TypeError::New(env, "Argument 1 should be string") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxSpeakerEmbeddingManager *manager = - info[0].As>().Data(); - - Napi::String js_name = info[1].As(); - std::string name = js_name.Utf8Value(); - - int32_t ok = SherpaOnnxSpeakerEmbeddingManagerRemove(manager, name.c_str()); - - return Napi::Boolean::New(env, ok); -} - -static Napi::String SpeakerEmbeddingManagerSearchWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, - "You should pass a speaker embedding manager pointer " - "as the first argument.") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[1].IsObject()) { - Napi::TypeError::New(env, "Argument 1 should be an object") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxSpeakerEmbeddingManager *manager = - info[0].As>().Data(); - - Napi::Object obj = info[1].As(); - - if (!obj.Has("v")) { - Napi::TypeError::New(env, "The argument object should have a field v") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Get("v").IsTypedArray()) { - Napi::TypeError::New(env, "The object['v'] should be a typed array") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Has("threshold")) { - Napi::TypeError::New(env, - "The argument object should have a field threshold") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Get("threshold").IsNumber()) { - Napi::TypeError::New(env, "The object['threshold'] should be a float") - .ThrowAsJavaScriptException(); - - return {}; - } - - Napi::Float32Array v = obj.Get("v").As(); - float threshold = obj.Get("threshold").As().FloatValue(); - - const char *name = - SherpaOnnxSpeakerEmbeddingManagerSearch(manager, v.Data(), threshold); - const char *p = name; - if (!p) { - p = ""; - } - - Napi::String js_name = Napi::String::New(env, p); - SherpaOnnxSpeakerEmbeddingManagerFreeSearch(name); - - return js_name; -} - -static Napi::Boolean SpeakerEmbeddingManagerVerifyWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, - "You should pass a speaker embedding manager pointer " - "as the first argument.") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[1].IsObject()) { - Napi::TypeError::New(env, "Argument 1 should be an object") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxSpeakerEmbeddingManager *manager = - info[0].As>().Data(); - - Napi::Object obj = info[1].As(); - - if (!obj.Has("v")) { - Napi::TypeError::New(env, "The argument object should have a field v") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Get("v").IsTypedArray()) { - Napi::TypeError::New(env, "The object['v'] should be a typed array") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Has("threshold")) { - Napi::TypeError::New(env, - "The argument object should have a field threshold") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Get("threshold").IsNumber()) { - Napi::TypeError::New(env, "The object['threshold'] should be a float") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Has("name")) { - Napi::TypeError::New(env, "The argument object should have a field name") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Get("name").IsString()) { - Napi::TypeError::New(env, "The object['name'] should be a string") - .ThrowAsJavaScriptException(); - - return {}; - } - - Napi::Float32Array v = obj.Get("v").As(); - float threshold = obj.Get("threshold").As().FloatValue(); - - Napi::String js_name = obj.Get("name").As(); - std::string name = js_name.Utf8Value(); - - int32_t found = SherpaOnnxSpeakerEmbeddingManagerVerify(manager, name.c_str(), - v.Data(), threshold); - - return Napi::Boolean::New(env, found); -} - -static Napi::Boolean SpeakerEmbeddingManagerContainsWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, - "You should pass a speaker embedding manager pointer " - "as the first argument.") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[1].IsString()) { - Napi::TypeError::New(env, "Argument 1 should be a string") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxSpeakerEmbeddingManager *manager = - info[0].As>().Data(); - - Napi::String js_name = info[1].As(); - std::string name = js_name.Utf8Value(); - - int32_t exists = - SherpaOnnxSpeakerEmbeddingManagerContains(manager, name.c_str()); - - return Napi::Boolean::New(env, exists); -} - -static Napi::Number SpeakerEmbeddingManagerNumSpeakersWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, - "You should pass a speaker embedding manager pointer " - "as the first argument.") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxSpeakerEmbeddingManager *manager = - info[0].As>().Data(); - - int32_t num_speakers = SherpaOnnxSpeakerEmbeddingManagerNumSpeakers(manager); - - return Napi::Number::New(env, num_speakers); -} - -static Napi::Array SpeakerEmbeddingManagerGetAllSpeakersWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, - "You should pass a speaker embedding manager pointer " - "as the first argument.") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxSpeakerEmbeddingManager *manager = - info[0].As>().Data(); - - int32_t num_speakers = SherpaOnnxSpeakerEmbeddingManagerNumSpeakers(manager); - if (num_speakers == 0) { - return {}; - } - - const char *const *all_speaker_names = - SherpaOnnxSpeakerEmbeddingManagerGetAllSpeakers(manager); - - Napi::Array ans = Napi::Array::New(env, num_speakers); - for (uint32_t i = 0; i != num_speakers; ++i) { - ans[i] = Napi::String::New(env, all_speaker_names[i]); - } - SherpaOnnxSpeakerEmbeddingManagerFreeAllSpeakers(all_speaker_names); - return ans; -} - -void InitSpeakerID(Napi::Env env, Napi::Object exports) { - exports.Set(Napi::String::New(env, "createSpeakerEmbeddingExtractor"), - Napi::Function::New(env, CreateSpeakerEmbeddingExtractorWrapper)); - - exports.Set(Napi::String::New(env, "speakerEmbeddingExtractorDim"), - Napi::Function::New(env, SpeakerEmbeddingExtractorDimWrapper)); - - exports.Set( - Napi::String::New(env, "speakerEmbeddingExtractorCreateStream"), - Napi::Function::New(env, SpeakerEmbeddingExtractorCreateStreamWrapper)); - - exports.Set( - Napi::String::New(env, "speakerEmbeddingExtractorIsReady"), - Napi::Function::New(env, SpeakerEmbeddingExtractorIsReadyWrapper)); - - exports.Set( - Napi::String::New(env, "speakerEmbeddingExtractorComputeEmbedding"), - Napi::Function::New(env, - SpeakerEmbeddingExtractorComputeEmbeddingWrapper)); - - exports.Set(Napi::String::New(env, "createSpeakerEmbeddingManager"), - Napi::Function::New(env, CreateSpeakerEmbeddingManagerWrapper)); - - exports.Set(Napi::String::New(env, "speakerEmbeddingManagerAdd"), - Napi::Function::New(env, SpeakerEmbeddingManagerAddWrapper)); - - exports.Set( - Napi::String::New(env, "speakerEmbeddingManagerAddListFlattened"), - Napi::Function::New(env, SpeakerEmbeddingManagerAddListFlattenedWrapper)); - - exports.Set(Napi::String::New(env, "speakerEmbeddingManagerRemove"), - Napi::Function::New(env, SpeakerEmbeddingManagerRemoveWrapper)); - - exports.Set(Napi::String::New(env, "speakerEmbeddingManagerSearch"), - Napi::Function::New(env, SpeakerEmbeddingManagerSearchWrapper)); - - exports.Set(Napi::String::New(env, "speakerEmbeddingManagerVerify"), - Napi::Function::New(env, SpeakerEmbeddingManagerVerifyWrapper)); - - exports.Set(Napi::String::New(env, "speakerEmbeddingManagerContains"), - Napi::Function::New(env, SpeakerEmbeddingManagerContainsWrapper)); - - exports.Set( - Napi::String::New(env, "speakerEmbeddingManagerNumSpeakers"), - Napi::Function::New(env, SpeakerEmbeddingManagerNumSpeakersWrapper)); - - exports.Set( - Napi::String::New(env, "speakerEmbeddingManagerGetAllSpeakers"), - Napi::Function::New(env, SpeakerEmbeddingManagerGetAllSpeakersWrapper)); -} diff --git a/scripts/node-addon-api/src/speaker-identification.cc b/scripts/node-addon-api/src/speaker-identification.cc new file mode 120000 index 000000000..f83455f6b --- /dev/null +++ b/scripts/node-addon-api/src/speaker-identification.cc @@ -0,0 +1 @@ +../../../harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/speaker-identification.cc \ No newline at end of file diff --git a/scripts/node-addon-api/src/spoken-language-identification.cc b/scripts/node-addon-api/src/spoken-language-identification.cc deleted file mode 100644 index 35ade6541..000000000 --- a/scripts/node-addon-api/src/spoken-language-identification.cc +++ /dev/null @@ -1,188 +0,0 @@ -// scripts/node-addon-api/src/spoken-language-identification.cc -// -// Copyright (c) 2024 Xiaomi Corporation - -#include - -#include "macros.h" // NOLINT -#include "napi.h" // NOLINT -#include "sherpa-onnx/c-api/c-api.h" - -static SherpaOnnxSpokenLanguageIdentificationWhisperConfig -GetSpokenLanguageIdentificationWhisperConfig(Napi::Object obj) { - SherpaOnnxSpokenLanguageIdentificationWhisperConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("whisper") || !obj.Get("whisper").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("whisper").As(); - - SHERPA_ONNX_ASSIGN_ATTR_STR(encoder, encoder); - SHERPA_ONNX_ASSIGN_ATTR_STR(decoder, decoder); - SHERPA_ONNX_ASSIGN_ATTR_INT32(tail_paddings, tailPaddings); - - return c; -} - -static Napi::External -CreateSpokenLanguageIdentificationWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsObject()) { - Napi::TypeError::New(env, "You should pass an object as the only argument.") - .ThrowAsJavaScriptException(); - - return {}; - } - - Napi::Object o = info[0].As(); - - SherpaOnnxSpokenLanguageIdentificationConfig c; - memset(&c, 0, sizeof(c)); - c.whisper = GetSpokenLanguageIdentificationWhisperConfig(o); - - SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); - - if (o.Has("debug") && - (o.Get("debug").IsNumber() || o.Get("debug").IsBoolean())) { - if (o.Get("debug").IsBoolean()) { - c.debug = o.Get("debug").As().Value(); - } else { - c.debug = o.Get("debug").As().Int32Value(); - } - } - SHERPA_ONNX_ASSIGN_ATTR_STR(provider, provider); - - const SherpaOnnxSpokenLanguageIdentification *slid = - SherpaOnnxCreateSpokenLanguageIdentification(&c); - - if (c.whisper.encoder) { - delete[] c.whisper.encoder; - } - - if (c.whisper.decoder) { - delete[] c.whisper.decoder; - } - - if (c.provider) { - delete[] c.provider; - } - - if (!slid) { - Napi::TypeError::New(env, "Please check your config!") - .ThrowAsJavaScriptException(); - - return {}; - } - - return Napi::External::New( - env, const_cast(slid), - [](Napi::Env env, SherpaOnnxSpokenLanguageIdentification *slid) { - SherpaOnnxDestroySpokenLanguageIdentification(slid); - }); -} - -static Napi::External -SpokenLanguageIdentificationCreateOfflineStreamWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New( - env, - "You should pass an offline language ID pointer as the only argument") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxSpokenLanguageIdentification *slid = - info[0] - .As>() - .Data(); - - SherpaOnnxOfflineStream *stream = - SherpaOnnxSpokenLanguageIdentificationCreateOfflineStream(slid); - - return Napi::External::New( - env, stream, [](Napi::Env env, SherpaOnnxOfflineStream *stream) { - SherpaOnnxDestroyOfflineStream(stream); - }); -} - -static Napi::String SpokenLanguageIdentificationComputeWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New( - env, "Argument 0 should be an offline spoken language ID pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[1].IsExternal()) { - Napi::TypeError::New(env, "Argument 1 should be an offline stream pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxSpokenLanguageIdentification *slid = - info[0] - .As>() - .Data(); - - SherpaOnnxOfflineStream *stream = - info[1].As>().Data(); - - const SherpaOnnxSpokenLanguageIdentificationResult *r = - SherpaOnnxSpokenLanguageIdentificationCompute(slid, stream); - - std::string lang = r->lang; - SherpaOnnxDestroySpokenLanguageIdentificationResult(r); - - return Napi::String::New(env, lang); -} - -void InitSpokenLanguageID(Napi::Env env, Napi::Object exports) { - exports.Set( - Napi::String::New(env, "createSpokenLanguageIdentification"), - Napi::Function::New(env, CreateSpokenLanguageIdentificationWrapper)); - - exports.Set( - Napi::String::New(env, "createSpokenLanguageIdentificationOfflineStream"), - Napi::Function::New( - env, SpokenLanguageIdentificationCreateOfflineStreamWrapper)); - - exports.Set( - Napi::String::New(env, "spokenLanguageIdentificationCompute"), - Napi::Function::New(env, SpokenLanguageIdentificationComputeWrapper)); -} diff --git a/scripts/node-addon-api/src/spoken-language-identification.cc b/scripts/node-addon-api/src/spoken-language-identification.cc new file mode 120000 index 000000000..39c77df75 --- /dev/null +++ b/scripts/node-addon-api/src/spoken-language-identification.cc @@ -0,0 +1 @@ +../../../harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/spoken-language-identification.cc \ No newline at end of file diff --git a/scripts/node-addon-api/src/streaming-asr.cc b/scripts/node-addon-api/src/streaming-asr.cc deleted file mode 100644 index 4652be5f5..000000000 --- a/scripts/node-addon-api/src/streaming-asr.cc +++ /dev/null @@ -1,708 +0,0 @@ -// scripts/node-addon-api/src/streaming-asr.cc -// -// Copyright (c) 2024 Xiaomi Corporation -#include - -#include "macros.h" // NOLINT -#include "napi.h" // NOLINT -#include "sherpa-onnx/c-api/c-api.h" -/* -{ - 'featConfig': { - 'sampleRate': 16000, - 'featureDim': 80, - } -}; - */ -SherpaOnnxFeatureConfig GetFeatureConfig(Napi::Object obj) { - SherpaOnnxFeatureConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("featConfig") || !obj.Get("featConfig").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("featConfig").As(); - - SHERPA_ONNX_ASSIGN_ATTR_INT32(sample_rate, sampleRate); - SHERPA_ONNX_ASSIGN_ATTR_INT32(feature_dim, featureDim); - - return c; -} -/* -{ - 'transducer': { - 'encoder': './encoder.onnx', - 'decoder': './decoder.onnx', - 'joiner': './joiner.onnx', - } -} - */ - -static SherpaOnnxOnlineTransducerModelConfig GetOnlineTransducerModelConfig( - Napi::Object obj) { - SherpaOnnxOnlineTransducerModelConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("transducer") || !obj.Get("transducer").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("transducer").As(); - - SHERPA_ONNX_ASSIGN_ATTR_STR(encoder, encoder); - SHERPA_ONNX_ASSIGN_ATTR_STR(decoder, decoder); - SHERPA_ONNX_ASSIGN_ATTR_STR(joiner, joiner); - - return c; -} - -static SherpaOnnxOnlineZipformer2CtcModelConfig -GetOnlineZipformer2CtcModelConfig(Napi::Object obj) { - SherpaOnnxOnlineZipformer2CtcModelConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("zipformer2Ctc") || !obj.Get("zipformer2Ctc").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("zipformer2Ctc").As(); - - SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); - - return c; -} - -static SherpaOnnxOnlineParaformerModelConfig GetOnlineParaformerModelConfig( - Napi::Object obj) { - SherpaOnnxOnlineParaformerModelConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("paraformer") || !obj.Get("paraformer").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("paraformer").As(); - - SHERPA_ONNX_ASSIGN_ATTR_STR(encoder, encoder); - SHERPA_ONNX_ASSIGN_ATTR_STR(decoder, decoder); - - return c; -} - -SherpaOnnxOnlineModelConfig GetOnlineModelConfig(Napi::Object obj) { - SherpaOnnxOnlineModelConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("modelConfig") || !obj.Get("modelConfig").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("modelConfig").As(); - - c.transducer = GetOnlineTransducerModelConfig(o); - c.paraformer = GetOnlineParaformerModelConfig(o); - c.zipformer2_ctc = GetOnlineZipformer2CtcModelConfig(o); - - SHERPA_ONNX_ASSIGN_ATTR_STR(tokens, tokens); - SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); - SHERPA_ONNX_ASSIGN_ATTR_STR(provider, provider); - - if (o.Has("debug") && - (o.Get("debug").IsNumber() || o.Get("debug").IsBoolean())) { - if (o.Get("debug").IsBoolean()) { - c.debug = o.Get("debug").As().Value(); - } else { - c.debug = o.Get("debug").As().Int32Value(); - } - } - - SHERPA_ONNX_ASSIGN_ATTR_STR(model_type, modelType); - SHERPA_ONNX_ASSIGN_ATTR_STR(modeling_unit, modelingUnit); - SHERPA_ONNX_ASSIGN_ATTR_STR(bpe_vocab, bpeVocab); - SHERPA_ONNX_ASSIGN_ATTR_STR(tokens_buf, tokensBuf); - SHERPA_ONNX_ASSIGN_ATTR_INT32(tokens_buf_size, tokensBufSize); - - return c; -} - -static SherpaOnnxOnlineCtcFstDecoderConfig GetCtcFstDecoderConfig( - Napi::Object obj) { - SherpaOnnxOnlineCtcFstDecoderConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("ctcFstDecoderConfig") || - !obj.Get("ctcFstDecoderConfig").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("ctcFstDecoderConfig").As(); - - SHERPA_ONNX_ASSIGN_ATTR_STR(graph, graph); - SHERPA_ONNX_ASSIGN_ATTR_INT32(max_active, maxActive); - - return c; -} - -static Napi::External CreateOnlineRecognizerWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsObject()) { - Napi::TypeError::New(env, "Expect an object as the argument") - .ThrowAsJavaScriptException(); - - return {}; - } - - Napi::Object o = info[0].As(); - SherpaOnnxOnlineRecognizerConfig c; - memset(&c, 0, sizeof(c)); - c.feat_config = GetFeatureConfig(o); - c.model_config = GetOnlineModelConfig(o); - - SHERPA_ONNX_ASSIGN_ATTR_STR(decoding_method, decodingMethod); - SHERPA_ONNX_ASSIGN_ATTR_INT32(max_active_paths, maxActivePaths); - - // enableEndpoint can be either a boolean or an integer - if (o.Has("enableEndpoint") && (o.Get("enableEndpoint").IsNumber() || - o.Get("enableEndpoint").IsBoolean())) { - if (o.Get("enableEndpoint").IsNumber()) { - c.enable_endpoint = - o.Get("enableEndpoint").As().Int32Value(); - } else { - c.enable_endpoint = o.Get("enableEndpoint").As().Value(); - } - } - - SHERPA_ONNX_ASSIGN_ATTR_FLOAT(rule1_min_trailing_silence, - rule1MinTrailingSilence); - SHERPA_ONNX_ASSIGN_ATTR_FLOAT(rule2_min_trailing_silence, - rule2MinTrailingSilence); - SHERPA_ONNX_ASSIGN_ATTR_FLOAT(rule3_min_utterance_length, - rule3MinUtteranceLength); - SHERPA_ONNX_ASSIGN_ATTR_STR(hotwords_file, hotwordsFile); - SHERPA_ONNX_ASSIGN_ATTR_FLOAT(hotwords_score, hotwordsScore); - SHERPA_ONNX_ASSIGN_ATTR_STR(rule_fsts, ruleFsts); - SHERPA_ONNX_ASSIGN_ATTR_STR(rule_fars, ruleFars); - SHERPA_ONNX_ASSIGN_ATTR_FLOAT(blank_penalty, blankPenalty); - SHERPA_ONNX_ASSIGN_ATTR_STR(hotwords_buf, hotwordsBuf); - SHERPA_ONNX_ASSIGN_ATTR_INT32(hotwords_buf_size, hotwordsBufSize); - - c.ctc_fst_decoder_config = GetCtcFstDecoderConfig(o); - - const SherpaOnnxOnlineRecognizer *recognizer = - SherpaOnnxCreateOnlineRecognizer(&c); - - if (c.model_config.transducer.encoder) { - delete[] c.model_config.transducer.encoder; - } - - if (c.model_config.transducer.decoder) { - delete[] c.model_config.transducer.decoder; - } - - if (c.model_config.transducer.joiner) { - delete[] c.model_config.transducer.joiner; - } - - if (c.model_config.paraformer.encoder) { - delete[] c.model_config.paraformer.encoder; - } - - if (c.model_config.paraformer.decoder) { - delete[] c.model_config.paraformer.decoder; - } - - if (c.model_config.zipformer2_ctc.model) { - delete[] c.model_config.zipformer2_ctc.model; - } - - if (c.model_config.tokens) { - delete[] c.model_config.tokens; - } - - if (c.model_config.provider) { - delete[] c.model_config.provider; - } - - if (c.model_config.model_type) { - delete[] c.model_config.model_type; - } - - if (c.model_config.modeling_unit) { - delete[] c.model_config.modeling_unit; - } - - if (c.model_config.bpe_vocab) { - delete[] c.model_config.bpe_vocab; - } - - if (c.model_config.tokens_buf) { - delete[] c.model_config.tokens_buf; - } - - if (c.decoding_method) { - delete[] c.decoding_method; - } - - if (c.hotwords_file) { - delete[] c.hotwords_file; - } - - if (c.rule_fsts) { - delete[] c.rule_fsts; - } - - if (c.rule_fars) { - delete[] c.rule_fars; - } - - if (c.hotwords_buf) { - delete[] c.hotwords_buf; - } - - if (c.ctc_fst_decoder_config.graph) { - delete[] c.ctc_fst_decoder_config.graph; - } - - if (!recognizer) { - Napi::TypeError::New(env, "Please check your config!") - .ThrowAsJavaScriptException(); - - return {}; - } - - return Napi::External::New( - env, const_cast(recognizer), - [](Napi::Env env, SherpaOnnxOnlineRecognizer *recognizer) { - SherpaOnnxDestroyOnlineRecognizer(recognizer); - }); -} - -static Napi::External CreateOnlineStreamWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New( - env, - "You should pass an online recognizer pointer as the only argument") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxOnlineRecognizer *recognizer = - info[0].As>().Data(); - - const SherpaOnnxOnlineStream *stream = - SherpaOnnxCreateOnlineStream(recognizer); - - return Napi::External::New( - env, const_cast(stream), - [](Napi::Env env, SherpaOnnxOnlineStream *stream) { - SherpaOnnxDestroyOnlineStream(stream); - }); -} - -static void AcceptWaveformWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be an online stream pointer.") - .ThrowAsJavaScriptException(); - - return; - } - - SherpaOnnxOnlineStream *stream = - info[0].As>().Data(); - - if (!info[1].IsObject()) { - Napi::TypeError::New(env, "Argument 1 should be an object") - .ThrowAsJavaScriptException(); - - return; - } - - Napi::Object obj = info[1].As(); - - if (!obj.Has("samples")) { - Napi::TypeError::New(env, "The argument object should have a field samples") - .ThrowAsJavaScriptException(); - - return; - } - - if (!obj.Get("samples").IsTypedArray()) { - Napi::TypeError::New(env, "The object['samples'] should be a typed array") - .ThrowAsJavaScriptException(); - - return; - } - - if (!obj.Has("sampleRate")) { - Napi::TypeError::New(env, - "The argument object should have a field sampleRate") - .ThrowAsJavaScriptException(); - - return; - } - - if (!obj.Get("sampleRate").IsNumber()) { - Napi::TypeError::New(env, "The object['samples'] should be a number") - .ThrowAsJavaScriptException(); - - return; - } - - Napi::Float32Array samples = obj.Get("samples").As(); - int32_t sample_rate = obj.Get("sampleRate").As().Int32Value(); - - SherpaOnnxOnlineStreamAcceptWaveform(stream, sample_rate, samples.Data(), - samples.ElementLength()); -} - -static Napi::Boolean IsOnlineStreamReadyWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, - "Argument 0 should be an online recognizer pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[1].IsExternal()) { - Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxOnlineRecognizer *recognizer = - info[0].As>().Data(); - - SherpaOnnxOnlineStream *stream = - info[1].As>().Data(); - - int32_t is_ready = SherpaOnnxIsOnlineStreamReady(recognizer, stream); - - return Napi::Boolean::New(env, is_ready); -} - -static void DecodeOnlineStreamWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, - "Argument 0 should be an online recognizer pointer.") - .ThrowAsJavaScriptException(); - - return; - } - - if (!info[1].IsExternal()) { - Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.") - .ThrowAsJavaScriptException(); - - return; - } - - SherpaOnnxOnlineRecognizer *recognizer = - info[0].As>().Data(); - - SherpaOnnxOnlineStream *stream = - info[1].As>().Data(); - - SherpaOnnxDecodeOnlineStream(recognizer, stream); -} - -static Napi::String GetOnlineStreamResultAsJsonWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, - "Argument 0 should be an online recognizer pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[1].IsExternal()) { - Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxOnlineRecognizer *recognizer = - info[0].As>().Data(); - - SherpaOnnxOnlineStream *stream = - info[1].As>().Data(); - - const char *json = SherpaOnnxGetOnlineStreamResultAsJson(recognizer, stream); - Napi::String s = Napi::String::New(env, json); - - SherpaOnnxDestroyOnlineStreamResultJson(json); - - return s; -} - -static void InputFinishedWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be an online stream pointer.") - .ThrowAsJavaScriptException(); - - return; - } - - SherpaOnnxOnlineStream *stream = - info[0].As>().Data(); - - SherpaOnnxOnlineStreamInputFinished(stream); -} - -static void ResetOnlineStreamWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, - "Argument 0 should be an online recognizer pointer.") - .ThrowAsJavaScriptException(); - - return; - } - - if (!info[1].IsExternal()) { - Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.") - .ThrowAsJavaScriptException(); - - return; - } - - SherpaOnnxOnlineRecognizer *recognizer = - info[0].As>().Data(); - - SherpaOnnxOnlineStream *stream = - info[1].As>().Data(); - - SherpaOnnxOnlineStreamReset(recognizer, stream); -} - -static Napi::Boolean IsEndpointWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, - "Argument 0 should be an online recognizer pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[1].IsExternal()) { - Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxOnlineRecognizer *recognizer = - info[0].As>().Data(); - - SherpaOnnxOnlineStream *stream = - info[1].As>().Data(); - - int32_t is_endpoint = SherpaOnnxOnlineStreamIsEndpoint(recognizer, stream); - - return Napi::Boolean::New(env, is_endpoint); -} - -static Napi::External CreateDisplayWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsNumber()) { - Napi::TypeError::New(env, "Expect a number as the argument") - .ThrowAsJavaScriptException(); - - return {}; - } - int32_t max_word_per_line = info[0].As().Int32Value(); - - const SherpaOnnxDisplay *display = SherpaOnnxCreateDisplay(max_word_per_line); - - return Napi::External::New( - env, const_cast(display), - [](Napi::Env env, SherpaOnnxDisplay *display) { - SherpaOnnxDestroyDisplay(display); - }); -} - -static void PrintWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 3) { - std::ostringstream os; - os << "Expect only 3 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be an online stream pointer.") - .ThrowAsJavaScriptException(); - - return; - } - - if (!info[1].IsNumber()) { - Napi::TypeError::New(env, "Argument 1 should be a number.") - .ThrowAsJavaScriptException(); - - return; - } - - if (!info[2].IsString()) { - Napi::TypeError::New(env, "Argument 2 should be a string.") - .ThrowAsJavaScriptException(); - - return; - } - - SherpaOnnxDisplay *display = - info[0].As>().Data(); - - int32_t idx = info[1].As().Int32Value(); - - Napi::String text = info[2].As(); - std::string s = text.Utf8Value(); - SherpaOnnxPrint(display, idx, s.c_str()); -} - -void InitStreamingAsr(Napi::Env env, Napi::Object exports) { - exports.Set(Napi::String::New(env, "createOnlineRecognizer"), - Napi::Function::New(env, CreateOnlineRecognizerWrapper)); - - exports.Set(Napi::String::New(env, "createOnlineStream"), - Napi::Function::New(env, CreateOnlineStreamWrapper)); - - exports.Set(Napi::String::New(env, "acceptWaveformOnline"), - Napi::Function::New(env, AcceptWaveformWrapper)); - - exports.Set(Napi::String::New(env, "isOnlineStreamReady"), - Napi::Function::New(env, IsOnlineStreamReadyWrapper)); - - exports.Set(Napi::String::New(env, "decodeOnlineStream"), - Napi::Function::New(env, DecodeOnlineStreamWrapper)); - - exports.Set(Napi::String::New(env, "getOnlineStreamResultAsJson"), - Napi::Function::New(env, GetOnlineStreamResultAsJsonWrapper)); - - exports.Set(Napi::String::New(env, "inputFinished"), - Napi::Function::New(env, InputFinishedWrapper)); - - exports.Set(Napi::String::New(env, "reset"), - Napi::Function::New(env, ResetOnlineStreamWrapper)); - - exports.Set(Napi::String::New(env, "isEndpoint"), - Napi::Function::New(env, IsEndpointWrapper)); - - exports.Set(Napi::String::New(env, "createDisplay"), - Napi::Function::New(env, CreateDisplayWrapper)); - - exports.Set(Napi::String::New(env, "print"), - Napi::Function::New(env, PrintWrapper)); -} diff --git a/scripts/node-addon-api/src/streaming-asr.cc b/scripts/node-addon-api/src/streaming-asr.cc new file mode 120000 index 000000000..a3d3201fb --- /dev/null +++ b/scripts/node-addon-api/src/streaming-asr.cc @@ -0,0 +1 @@ +../../../harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/streaming-asr.cc \ No newline at end of file diff --git a/scripts/node-addon-api/src/vad.cc b/scripts/node-addon-api/src/vad.cc deleted file mode 100644 index eaed2aeea..000000000 --- a/scripts/node-addon-api/src/vad.cc +++ /dev/null @@ -1,668 +0,0 @@ -// scripts/node-addon-api/src/vad.cc -// -// Copyright (c) 2024 Xiaomi Corporation - -#include -#include - -#include "macros.h" // NOLINT -#include "napi.h" // NOLINT -#include "sherpa-onnx/c-api/c-api.h" - -static Napi::External CreateCircularBufferWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsNumber()) { - Napi::TypeError::New(env, "You should pass an integer as the argument.") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxCircularBuffer *buf = - SherpaOnnxCreateCircularBuffer(info[0].As().Int32Value()); - - return Napi::External::New( - env, buf, [](Napi::Env env, SherpaOnnxCircularBuffer *p) { - SherpaOnnxDestroyCircularBuffer(p); - }); -} - -static void CircularBufferPushWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be an CircularBuffer pointer.") - .ThrowAsJavaScriptException(); - - return; - } - - SherpaOnnxCircularBuffer *buf = - info[0].As>().Data(); - - if (!info[1].IsTypedArray()) { - Napi::TypeError::New(env, "Argument 1 should be a Float32Array.") - .ThrowAsJavaScriptException(); - - return; - } - - Napi::Float32Array data = info[1].As(); - SherpaOnnxCircularBufferPush(buf, data.Data(), data.ElementLength()); -} - -// see https://github.com/nodejs/node-addon-api/blob/main/doc/typed_array.md -// https://github.com/nodejs/node-addon-examples/blob/main/src/2-js-to-native-conversion/typed_array_to_native/node-addon-api/typed_array_to_native.cc -static Napi::Float32Array CircularBufferGetWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 3 && info.Length() != 4) { - std::ostringstream os; - os << "Expect only 3 or 4 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be an CircularBuffer pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxCircularBuffer *buf = - info[0].As>().Data(); - - if (!info[1].IsNumber()) { - Napi::TypeError::New(env, "Argument 1 should be an integer (startIndex).") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[2].IsNumber()) { - Napi::TypeError::New(env, "Argument 2 should be an integer (n).") - .ThrowAsJavaScriptException(); - - return {}; - } - - bool enable_external_buffer = true; - if (info.Length() == 4) { - if (info[3].IsBoolean()) { - enable_external_buffer = info[3].As().Value(); - } else { - Napi::TypeError::New(env, "Argument 3 should be a boolean.") - .ThrowAsJavaScriptException(); - } - } - - int32_t start_index = info[1].As().Int32Value(); - int32_t n = info[2].As().Int32Value(); - - const float *data = SherpaOnnxCircularBufferGet(buf, start_index, n); - - if (enable_external_buffer) { - Napi::ArrayBuffer arrayBuffer = Napi::ArrayBuffer::New( - env, const_cast(data), sizeof(float) * n, - [](Napi::Env /*env*/, void *p) { - SherpaOnnxCircularBufferFree(reinterpret_cast(p)); - }); - - Napi::Float32Array float32Array = - Napi::Float32Array::New(env, n, arrayBuffer, 0); - - return float32Array; - } else { - // don't use external buffer - Napi::ArrayBuffer arrayBuffer = - Napi::ArrayBuffer::New(env, sizeof(float) * n); - - Napi::Float32Array float32Array = - Napi::Float32Array::New(env, n, arrayBuffer, 0); - - std::copy(data, data + n, float32Array.Data()); - - SherpaOnnxCircularBufferFree(data); - - return float32Array; - } -} - -static void CircularBufferPopWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be an CircularBuffer pointer.") - .ThrowAsJavaScriptException(); - - return; - } - - SherpaOnnxCircularBuffer *buf = - info[0].As>().Data(); - - if (!info[1].IsNumber()) { - Napi::TypeError::New(env, "Argument 1 should be an integer (n).") - .ThrowAsJavaScriptException(); - - return; - } - - int32_t n = info[1].As().Int32Value(); - - SherpaOnnxCircularBufferPop(buf, n); -} - -static Napi::Number CircularBufferSizeWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be an CircularBuffer pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxCircularBuffer *buf = - info[0].As>().Data(); - - int32_t size = SherpaOnnxCircularBufferSize(buf); - - return Napi::Number::New(env, size); -} - -static Napi::Number CircularBufferHeadWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be an CircularBuffer pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxCircularBuffer *buf = - info[0].As>().Data(); - - int32_t size = SherpaOnnxCircularBufferHead(buf); - - return Napi::Number::New(env, size); -} - -static void CircularBufferResetWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be an CircularBuffer pointer.") - .ThrowAsJavaScriptException(); - - return; - } - - SherpaOnnxCircularBuffer *buf = - info[0].As>().Data(); - - SherpaOnnxCircularBufferReset(buf); -} - -static SherpaOnnxSileroVadModelConfig GetSileroVadConfig( - const Napi::Object &obj) { - SherpaOnnxSileroVadModelConfig c; - memset(&c, 0, sizeof(c)); - - if (!obj.Has("sileroVad") || !obj.Get("sileroVad").IsObject()) { - return c; - } - - Napi::Object o = obj.Get("sileroVad").As(); - SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); - SHERPA_ONNX_ASSIGN_ATTR_FLOAT(threshold, threshold); - SHERPA_ONNX_ASSIGN_ATTR_FLOAT(min_silence_duration, minSilenceDuration); - SHERPA_ONNX_ASSIGN_ATTR_FLOAT(min_speech_duration, minSpeechDuration); - SHERPA_ONNX_ASSIGN_ATTR_INT32(window_size, windowSize); - SHERPA_ONNX_ASSIGN_ATTR_FLOAT(max_speech_duration, maxSpeechDuration); - - return c; -} - -static Napi::External -CreateVoiceActivityDetectorWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsObject()) { - Napi::TypeError::New(env, - "You should pass an object as the first argument.") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[1].IsNumber()) { - Napi::TypeError::New(env, - "You should pass an integer as the second argument.") - .ThrowAsJavaScriptException(); - - return {}; - } - - Napi::Object o = info[0].As(); - - SherpaOnnxVadModelConfig c; - memset(&c, 0, sizeof(c)); - c.silero_vad = GetSileroVadConfig(o); - - SHERPA_ONNX_ASSIGN_ATTR_INT32(sample_rate, sampleRate); - SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); - SHERPA_ONNX_ASSIGN_ATTR_STR(provider, provider); - - if (o.Has("debug") && - (o.Get("debug").IsNumber() || o.Get("debug").IsBoolean())) { - if (o.Get("debug").IsBoolean()) { - c.debug = o.Get("debug").As().Value(); - } else { - c.debug = o.Get("debug").As().Int32Value(); - } - } - - float buffer_size_in_seconds = info[1].As().FloatValue(); - - SherpaOnnxVoiceActivityDetector *vad = - SherpaOnnxCreateVoiceActivityDetector(&c, buffer_size_in_seconds); - - if (c.silero_vad.model) { - delete[] c.silero_vad.model; - } - - if (c.provider) { - delete[] c.provider; - } - - return Napi::External::New( - env, vad, [](Napi::Env env, SherpaOnnxVoiceActivityDetector *p) { - SherpaOnnxDestroyVoiceActivityDetector(p); - }); -} - -static void VoiceActivityDetectorAcceptWaveformWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be a VAD pointer.") - .ThrowAsJavaScriptException(); - - return; - } - - SherpaOnnxVoiceActivityDetector *vad = - info[0].As>().Data(); - - if (!info[1].IsTypedArray()) { - Napi::TypeError::New( - env, "Argument 1 should be a Float32Array containing samples") - .ThrowAsJavaScriptException(); - - return; - } - - Napi::Float32Array samples = info[1].As(); - - SherpaOnnxVoiceActivityDetectorAcceptWaveform(vad, samples.Data(), - samples.ElementLength()); -} - -static Napi::Boolean VoiceActivityDetectorEmptyWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be a VAD pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxVoiceActivityDetector *vad = - info[0].As>().Data(); - - int32_t is_empty = SherpaOnnxVoiceActivityDetectorEmpty(vad); - - return Napi::Boolean::New(env, is_empty); -} - -static Napi::Boolean VoiceActivityDetectorDetectedWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be a VAD pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - SherpaOnnxVoiceActivityDetector *vad = - info[0].As>().Data(); - - int32_t is_detected = SherpaOnnxVoiceActivityDetectorDetected(vad); - - return Napi::Boolean::New(env, is_detected); -} - -static void VoiceActivityDetectorPopWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be a VAD pointer.") - .ThrowAsJavaScriptException(); - - return; - } - - SherpaOnnxVoiceActivityDetector *vad = - info[0].As>().Data(); - - SherpaOnnxVoiceActivityDetectorPop(vad); -} - -static void VoiceActivityDetectorClearWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be a VAD pointer.") - .ThrowAsJavaScriptException(); - - return; - } - - SherpaOnnxVoiceActivityDetector *vad = - info[0].As>().Data(); - - SherpaOnnxVoiceActivityDetectorClear(vad); -} - -static Napi::Object VoiceActivityDetectorFrontWrapper( - const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 1 && info.Length() != 2) { - std::ostringstream os; - os << "Expect only 1 or 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be a VAD pointer.") - .ThrowAsJavaScriptException(); - - return {}; - } - - bool enable_external_buffer = true; - if (info.Length() == 2) { - if (info[1].IsBoolean()) { - enable_external_buffer = info[1].As().Value(); - } else { - Napi::TypeError::New(env, "Argument 1 should be a boolean.") - .ThrowAsJavaScriptException(); - } - } - - SherpaOnnxVoiceActivityDetector *vad = - info[0].As>().Data(); - - const SherpaOnnxSpeechSegment *segment = - SherpaOnnxVoiceActivityDetectorFront(vad); - - if (enable_external_buffer) { - Napi::ArrayBuffer arrayBuffer = Napi::ArrayBuffer::New( - env, const_cast(segment->samples), sizeof(float) * segment->n, - [](Napi::Env /*env*/, void * /*data*/, - const SherpaOnnxSpeechSegment *hint) { - SherpaOnnxDestroySpeechSegment(hint); - }, - segment); - - Napi::Float32Array float32Array = - Napi::Float32Array::New(env, segment->n, arrayBuffer, 0); - - Napi::Object obj = Napi::Object::New(env); - obj.Set(Napi::String::New(env, "start"), segment->start); - obj.Set(Napi::String::New(env, "samples"), float32Array); - - return obj; - } else { - Napi::ArrayBuffer arrayBuffer = - Napi::ArrayBuffer::New(env, sizeof(float) * segment->n); - - Napi::Float32Array float32Array = - Napi::Float32Array::New(env, segment->n, arrayBuffer, 0); - - std::copy(segment->samples, segment->samples + segment->n, - float32Array.Data()); - - Napi::Object obj = Napi::Object::New(env); - obj.Set(Napi::String::New(env, "start"), segment->start); - obj.Set(Napi::String::New(env, "samples"), float32Array); - - SherpaOnnxDestroySpeechSegment(segment); - - return obj; - } -} - -static void VoiceActivityDetectorResetWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be a VAD pointer.") - .ThrowAsJavaScriptException(); - - return; - } - - SherpaOnnxVoiceActivityDetector *vad = - info[0].As>().Data(); - - SherpaOnnxVoiceActivityDetectorReset(vad); -} - -static void VoiceActivityDetectorFlushWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 1) { - std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return; - } - - if (!info[0].IsExternal()) { - Napi::TypeError::New(env, "Argument 0 should be a VAD pointer.") - .ThrowAsJavaScriptException(); - - return; - } - - SherpaOnnxVoiceActivityDetector *vad = - info[0].As>().Data(); - - SherpaOnnxVoiceActivityDetectorFlush(vad); -} - -void InitVad(Napi::Env env, Napi::Object exports) { - exports.Set(Napi::String::New(env, "createCircularBuffer"), - Napi::Function::New(env, CreateCircularBufferWrapper)); - - exports.Set(Napi::String::New(env, "circularBufferPush"), - Napi::Function::New(env, CircularBufferPushWrapper)); - - exports.Set(Napi::String::New(env, "circularBufferGet"), - Napi::Function::New(env, CircularBufferGetWrapper)); - - exports.Set(Napi::String::New(env, "circularBufferPop"), - Napi::Function::New(env, CircularBufferPopWrapper)); - - exports.Set(Napi::String::New(env, "circularBufferSize"), - Napi::Function::New(env, CircularBufferSizeWrapper)); - - exports.Set(Napi::String::New(env, "circularBufferHead"), - Napi::Function::New(env, CircularBufferHeadWrapper)); - - exports.Set(Napi::String::New(env, "circularBufferReset"), - Napi::Function::New(env, CircularBufferResetWrapper)); - - exports.Set(Napi::String::New(env, "createVoiceActivityDetector"), - Napi::Function::New(env, CreateVoiceActivityDetectorWrapper)); - - exports.Set( - Napi::String::New(env, "voiceActivityDetectorAcceptWaveform"), - Napi::Function::New(env, VoiceActivityDetectorAcceptWaveformWrapper)); - - exports.Set(Napi::String::New(env, "voiceActivityDetectorIsEmpty"), - Napi::Function::New(env, VoiceActivityDetectorEmptyWrapper)); - - exports.Set(Napi::String::New(env, "voiceActivityDetectorIsDetected"), - Napi::Function::New(env, VoiceActivityDetectorDetectedWrapper)); - - exports.Set(Napi::String::New(env, "voiceActivityDetectorPop"), - Napi::Function::New(env, VoiceActivityDetectorPopWrapper)); - - exports.Set(Napi::String::New(env, "voiceActivityDetectorClear"), - Napi::Function::New(env, VoiceActivityDetectorClearWrapper)); - - exports.Set(Napi::String::New(env, "voiceActivityDetectorFront"), - Napi::Function::New(env, VoiceActivityDetectorFrontWrapper)); - - exports.Set(Napi::String::New(env, "voiceActivityDetectorReset"), - Napi::Function::New(env, VoiceActivityDetectorResetWrapper)); - - exports.Set(Napi::String::New(env, "voiceActivityDetectorFlush"), - Napi::Function::New(env, VoiceActivityDetectorFlushWrapper)); -} diff --git a/scripts/node-addon-api/src/vad.cc b/scripts/node-addon-api/src/vad.cc new file mode 120000 index 000000000..5650274b8 --- /dev/null +++ b/scripts/node-addon-api/src/vad.cc @@ -0,0 +1 @@ +../../../harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/vad.cc \ No newline at end of file diff --git a/scripts/node-addon-api/src/wave-reader.cc b/scripts/node-addon-api/src/wave-reader.cc deleted file mode 100644 index 874f61bab..000000000 --- a/scripts/node-addon-api/src/wave-reader.cc +++ /dev/null @@ -1,91 +0,0 @@ -// scripts/node-addon-api/src/wave-reader.cc -// -// Copyright (c) 2024 Xiaomi Corporation - -#include -#include - -#include "napi.h" // NOLINT -#include "sherpa-onnx/c-api/c-api.h" - -static Napi::Object ReadWaveWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() > 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsString()) { - Napi::TypeError::New(env, "Argument 0 should be a string") - .ThrowAsJavaScriptException(); - - return {}; - } - - std::string filename = info[0].As().Utf8Value(); - - bool enable_external_buffer = true; - if (info.Length() == 2) { - if (info[1].IsBoolean()) { - enable_external_buffer = info[1].As().Value(); - } else { - Napi::TypeError::New(env, "Argument 1 should be a boolean") - .ThrowAsJavaScriptException(); - - return {}; - } - } - - const SherpaOnnxWave *wave = SherpaOnnxReadWave(filename.c_str()); - if (!wave) { - std::ostringstream os; - os << "Failed to read '" << filename << "'"; - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (enable_external_buffer) { - Napi::ArrayBuffer arrayBuffer = Napi::ArrayBuffer::New( - env, const_cast(wave->samples), - sizeof(float) * wave->num_samples, - [](Napi::Env /*env*/, void * /*data*/, const SherpaOnnxWave *hint) { - SherpaOnnxFreeWave(hint); - }, - wave); - Napi::Float32Array float32Array = - Napi::Float32Array::New(env, wave->num_samples, arrayBuffer, 0); - - Napi::Object obj = Napi::Object::New(env); - obj.Set(Napi::String::New(env, "samples"), float32Array); - obj.Set(Napi::String::New(env, "sampleRate"), wave->sample_rate); - return obj; - } else { - // don't use external buffer - Napi::ArrayBuffer arrayBuffer = - Napi::ArrayBuffer::New(env, sizeof(float) * wave->num_samples); - - Napi::Float32Array float32Array = - Napi::Float32Array::New(env, wave->num_samples, arrayBuffer, 0); - - std::copy(wave->samples, wave->samples + wave->num_samples, - float32Array.Data()); - - Napi::Object obj = Napi::Object::New(env); - obj.Set(Napi::String::New(env, "samples"), float32Array); - obj.Set(Napi::String::New(env, "sampleRate"), wave->sample_rate); - - SherpaOnnxFreeWave(wave); - - return obj; - } -} - -void InitWaveReader(Napi::Env env, Napi::Object exports) { - exports.Set(Napi::String::New(env, "readWave"), - Napi::Function::New(env, ReadWaveWrapper)); -} diff --git a/scripts/node-addon-api/src/wave-reader.cc b/scripts/node-addon-api/src/wave-reader.cc new file mode 120000 index 000000000..839b562d0 --- /dev/null +++ b/scripts/node-addon-api/src/wave-reader.cc @@ -0,0 +1 @@ +../../../harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-reader.cc \ No newline at end of file diff --git a/scripts/node-addon-api/src/wave-writer.cc b/scripts/node-addon-api/src/wave-writer.cc deleted file mode 100644 index 3ade695a0..000000000 --- a/scripts/node-addon-api/src/wave-writer.cc +++ /dev/null @@ -1,81 +0,0 @@ -// scripts/node-addon-api/src/wave-writer.cc -// -// Copyright (c) 2024 Xiaomi Corporation - -#include - -#include "napi.h" // NOLINT -#include "sherpa-onnx/c-api/c-api.h" - -// (filename, {samples: samples, sampleRate: sampleRate} -static Napi::Boolean WriteWaveWrapper(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - - if (info.Length() != 2) { - std::ostringstream os; - os << "Expect only 2 arguments. Given: " << info.Length(); - - Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[0].IsString()) { - Napi::TypeError::New(env, "Argument 0 should be a string") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!info[1].IsObject()) { - Napi::TypeError::New(env, "Argument 1 should be an object") - .ThrowAsJavaScriptException(); - - return {}; - } - - Napi::Object obj = info[1].As(); - - if (!obj.Has("samples")) { - Napi::TypeError::New(env, "The argument object should have a field samples") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Get("samples").IsTypedArray()) { - Napi::TypeError::New(env, "The object['samples'] should be a typed array") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Has("sampleRate")) { - Napi::TypeError::New(env, - "The argument object should have a field sampleRate") - .ThrowAsJavaScriptException(); - - return {}; - } - - if (!obj.Get("sampleRate").IsNumber()) { - Napi::TypeError::New(env, "The object['samples'] should be a number") - .ThrowAsJavaScriptException(); - - return {}; - } - - Napi::Float32Array samples = obj.Get("samples").As(); - int32_t sample_rate = obj.Get("sampleRate").As().Int32Value(); - - int32_t ok = - SherpaOnnxWriteWave(samples.Data(), samples.ElementLength(), sample_rate, - info[0].As().Utf8Value().c_str()); - - return Napi::Boolean::New(env, ok); -} - -void InitWaveWriter(Napi::Env env, Napi::Object exports) { - exports.Set(Napi::String::New(env, "writeWave"), - Napi::Function::New(env, WriteWaveWrapper)); -} diff --git a/scripts/node-addon-api/src/wave-writer.cc b/scripts/node-addon-api/src/wave-writer.cc new file mode 120000 index 000000000..03c3818c3 --- /dev/null +++ b/scripts/node-addon-api/src/wave-writer.cc @@ -0,0 +1 @@ +../../../harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-writer.cc \ No newline at end of file diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index fbc3a010a..a6ad0772e 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -1201,6 +1202,28 @@ const SherpaOnnxWave *SherpaOnnxReadWave(const char *filename) { return wave; } +const SherpaOnnxWave *SherpaOnnxReadWaveFromBinaryData(const char *data, + int32_t n) { + int32_t sample_rate = -1; + bool is_ok = false; + + std::istrstream is(data, n); + + std::vector samples = sherpa_onnx::ReadWave(is, &sample_rate, &is_ok); + if (!is_ok) { + return nullptr; + } + + float *c_samples = new float[samples.size()]; + std::copy(samples.begin(), samples.end(), c_samples); + + SherpaOnnxWave *wave = new SherpaOnnxWave; + wave->samples = c_samples; + wave->sample_rate = sample_rate; + wave->num_samples = samples.size(); + return wave; +} + void SherpaOnnxFreeWave(const SherpaOnnxWave *wave) { if (wave) { delete[] wave->samples; diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index 1feaab306..5251413a8 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -1001,6 +1001,14 @@ SHERPA_ONNX_API typedef struct SherpaOnnxWave { // SherpaOnnxFreeWave() to free the returned pointer to avoid memory leak. SHERPA_ONNX_API const SherpaOnnxWave *SherpaOnnxReadWave(const char *filename); +// Similar to SherpaOnnxReadWave(), it has read the content of `filename` +// into the array `data`. +// +// If the returned pointer is not NULL, the user has to invoke +// SherpaOnnxFreeWave() to free the returned pointer to avoid memory leak. +SHERPA_ONNX_API const SherpaOnnxWave *SherpaOnnxReadWaveFromBinaryData( + const char *data, int32_t n); + SHERPA_ONNX_API void SherpaOnnxFreeWave(const SherpaOnnxWave *wave); // ============================================================ From f3f896146268bf748cf58ad717e82d89d1a913d1 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 28 Nov 2024 22:59:56 +0800 Subject: [PATCH 083/183] Add VAD+ASR demo for HarmonyOS (#1573) --- harmony-os/README.md | 9 + harmony-os/SherpaOnnxVadAsr/.gitignore | 12 + .../SherpaOnnxVadAsr/AppScope/app.json5 | 10 + .../resources/base/element/string.json | 8 + .../resources/base/media/app_icon.png | Bin 0 -> 2777 bytes .../SherpaOnnxVadAsr/build-profile.json5 | 40 +++ harmony-os/SherpaOnnxVadAsr/code-linter.json5 | 20 ++ harmony-os/SherpaOnnxVadAsr/entry/.gitignore | 7 + harmony-os/SherpaOnnxVadAsr/entry/README.md | 7 + .../entry/build-profile.json5 | 33 +++ .../SherpaOnnxVadAsr/entry/hvigorfile.ts | 6 + .../entry/obfuscation-rules.txt | 23 ++ .../SherpaOnnxVadAsr/entry/oh-package.json5 | 10 + .../main/ets/entryability/EntryAbility.ets | 43 ++++ .../entrybackupability/EntryBackupAbility.ets | 12 + .../entry/src/main/ets/pages/Index.ets | 173 +++++++++++++ .../main/ets/pages/NonStreamingAsrModels.ets | 237 ++++++++++++++++++ .../workers/NonStreamingAsrWithVadWorker.ets | 177 +++++++++++++ .../entry/src/main/module.json5 | 52 ++++ .../main/resources/base/element/color.json | 8 + .../main/resources/base/element/string.json | 16 ++ .../main/resources/base/media/background.png | Bin 0 -> 57364 bytes .../main/resources/base/media/foreground.png | Bin 0 -> 12430 bytes .../base/media/ic_public_input_voice.png | Bin 0 -> 474 bytes .../media/ic_public_input_voice_default.png | Bin 0 -> 438 bytes .../main/resources/base/media/icon_doc.png | Bin 0 -> 295 bytes .../resources/base/media/icon_doc_default.png | Bin 0 -> 295 bytes .../main/resources/base/media/info_circle.png | Bin 0 -> 636 bytes .../base/media/info_circle_default.png | Bin 0 -> 636 bytes .../resources/base/media/layered_image.json | 7 + .../main/resources/base/media/startIcon.png | Bin 0 -> 20093 bytes .../resources/base/profile/backup_config.json | 3 + .../resources/base/profile/main_pages.json | 5 + .../main/resources/en_US/element/string.json | 16 ++ .../main/resources/zh_CN/element/string.json | 16 ++ .../src/ohosTest/ets/test/Ability.test.ets | 35 +++ .../entry/src/ohosTest/ets/test/List.test.ets | 5 + .../entry/src/ohosTest/module.json5 | 13 + .../entry/src/test/List.test.ets | 5 + .../entry/src/test/LocalUnit.test.ets | 33 +++ .../hvigor/hvigor-config.json5 | 22 ++ harmony-os/SherpaOnnxVadAsr/hvigorfile.ts | 6 + .../SherpaOnnxVadAsr/oh-package-lock.json5 | 19 ++ harmony-os/SherpaOnnxVadAsr/oh-package.json5 | 14 ++ 44 files changed, 1102 insertions(+) create mode 100644 harmony-os/README.md create mode 100644 harmony-os/SherpaOnnxVadAsr/.gitignore create mode 100644 harmony-os/SherpaOnnxVadAsr/AppScope/app.json5 create mode 100644 harmony-os/SherpaOnnxVadAsr/AppScope/resources/base/element/string.json create mode 100644 harmony-os/SherpaOnnxVadAsr/AppScope/resources/base/media/app_icon.png create mode 100644 harmony-os/SherpaOnnxVadAsr/build-profile.json5 create mode 100644 harmony-os/SherpaOnnxVadAsr/code-linter.json5 create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/.gitignore create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/README.md create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/build-profile.json5 create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/hvigorfile.ts create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/obfuscation-rules.txt create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/entryability/EntryAbility.ets create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/NonStreamingAsrModels.ets create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/workers/NonStreamingAsrWithVadWorker.ets create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/module.json5 create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/element/color.json create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/element/string.json create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/background.png create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/foreground.png create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/ic_public_input_voice.png create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/ic_public_input_voice_default.png create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_doc.png create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_doc_default.png create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/info_circle.png create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/info_circle_default.png create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/layered_image.json create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/startIcon.png create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/profile/backup_config.json create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/profile/main_pages.json create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/en_US/element/string.json create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/zh_CN/element/string.json create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/ohosTest/ets/test/Ability.test.ets create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/ohosTest/ets/test/List.test.ets create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/ohosTest/module.json5 create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/test/List.test.ets create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/test/LocalUnit.test.ets create mode 100644 harmony-os/SherpaOnnxVadAsr/hvigor/hvigor-config.json5 create mode 100644 harmony-os/SherpaOnnxVadAsr/hvigorfile.ts create mode 100644 harmony-os/SherpaOnnxVadAsr/oh-package-lock.json5 create mode 100644 harmony-os/SherpaOnnxVadAsr/oh-package.json5 diff --git a/harmony-os/README.md b/harmony-os/README.md new file mode 100644 index 000000000..280258bd8 --- /dev/null +++ b/harmony-os/README.md @@ -0,0 +1,9 @@ +# Introduction + +- [./SherpaOnnxHar](./SherpaOnnxHar) It is for building `sherpa_onnx.har`. + If you don't need to change the C++ or Typescript code of sherpa-onnx, then + you can download pre-built `sherpa_onnx.har` from us. Please refer to + our [doc](https://k2-fsa.github.io/sherpa/onnx) for how to download it. + +- [./SherpaOnnxVadAsr](./SherpaOnnxVadAsr) It shows how to use + VAD + Non-streaming ASR for speech recognition. diff --git a/harmony-os/SherpaOnnxVadAsr/.gitignore b/harmony-os/SherpaOnnxVadAsr/.gitignore new file mode 100644 index 000000000..d2ff20141 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/.gitignore @@ -0,0 +1,12 @@ +/node_modules +/oh_modules +/local.properties +/.idea +**/build +/.hvigor +.cxx +/.clangd +/.clang-format +/.clang-tidy +**/.test +/.appanalyzer \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/AppScope/app.json5 b/harmony-os/SherpaOnnxVadAsr/AppScope/app.json5 new file mode 100644 index 000000000..3a141cf2e --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/AppScope/app.json5 @@ -0,0 +1,10 @@ +{ + "app": { + "bundleName": "com.k2fsa.sherpa.onnx.vad.asr", + "vendor": "example", + "versionCode": 1000000, + "versionName": "1.0.0", + "icon": "$media:app_icon", + "label": "$string:app_name" + } +} diff --git a/harmony-os/SherpaOnnxVadAsr/AppScope/resources/base/element/string.json b/harmony-os/SherpaOnnxVadAsr/AppScope/resources/base/element/string.json new file mode 100644 index 000000000..9e27cd604 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/AppScope/resources/base/element/string.json @@ -0,0 +1,8 @@ +{ + "string": [ + { + "name": "app_name", + "value": "SherpaOnnxVadAsr" + } + ] +} diff --git a/harmony-os/SherpaOnnxVadAsr/AppScope/resources/base/media/app_icon.png b/harmony-os/SherpaOnnxVadAsr/AppScope/resources/base/media/app_icon.png new file mode 100644 index 0000000000000000000000000000000000000000..a39445dc87828b76fed6d2ec470dd455c45319e3 GIT binary patch literal 2777 zcmV;~3MTc5P)9*YHQQH znh@I(s7WDIN`nJ+5@|<)iZcg=qN74U#DNnD1Se7u4fs(|1ivr?9ayP|B3iYCD$mfQ zCQ{S1n2)}^yxe#1J=_0pt-a1UPwQ^Z*?X_`Uu*sM+8<}X+baE^a`3seUF}?bEaiMO zrD`Qrd5@qw^epHZ>Df|p-qKBUEB%*?!m0{PHC6j|RplEgR~PkM5a^}N)Sfwi>W;Uz zdhwo_4HXBU%kRl^w@&7iKPx$e-n9%#IU!&oMI~iNsw0n19qSX;dS>I`G_G=WdcN9r z;_Rtv9XC<7kbL+HHxJ782T~pg05t)tf^>2vNJqfYt{YmqQDoBxkv+ra*BxxhcuK2v zm5%@Y)biQz)R8O%e=o%n${;ojY;EUP>`Qj6Cq)7GHm)C%2%^+hI;Z4T#a|oKIvshv z5H%!I+|I4PEXaXj04%ybsVolr%vhKnW7AEhC?eP!o1{y;8m2R#;}{6VZPc!+)ou0C zVWz$|1#2(|L5z%EYRxOzP+uLB>qYGuajX-<#^u;Kw&2uh&93)h>nHaFA%{&2PW=Nn zr?*a;gk3xvRhQIRa1de-!r(ss&?tRmZ=L2FMkhxI3lK6Jn<>5c*ID|@KU#^MCIo6> zpFA{|R(4fsBwHIW z9v!7G|7enadv4}~*8q_h%tD^j$7=PCnn0=dR0GKA(fgb9`2IRg6ksBIo+Gdw#|-3eSe=3tmDe zIqVN)tScM`0W#Z>2wc>~2Uv=3L)~D4gXqZtPQ8rifbYJqwkG>bv}95G7+};9Br?hF zWSa3b)X}z#79W9kukM%6-b_54WDJm~Ub=gsrJ0lz-8&lrQ7zfK1qzuZQkZvcE3|~S zZWmk0ETaNIHnMALn>akuvHLf5c4`y%!f+u>ZGp%@q_;T!`76_snc_?K;Wx%YpF;5K zw^F+BCYUPy`fpRif@5O@Im5cf?evD$>KlAgX;D0*HiO0`Yg3j;R4jT(9h(L_TsY6yxk*@ZBe%+dMqY=cB5oGs{D$QwOFbH)G$iVf<3Olcd7^#fr- zM{!ILWt#coT)s9ySkwDCPHv0oww8g8K%Yr{aR}msELVX(}JQr%F4Q8=KKn*OjSO*uSp;JK%GwhRF_K??vGC$ZqmJX z@+}8sQ)9Z}3*DiWl+L_7OXn_^{SW~2&C*b^;%IP!j$lkre7H&bMR1}7aTT*G8P}|G zHM1)hZDe{r_E3{{Y=d}}_PxJO_w4MaE4)$<<3JwzPdwPzfNemK(-X;{UCzmVr0zu5 zEnT}fzx)oVd!*W77`1Ig`DFcZ6TkPaI$hO1+`cGb$({ukz&{p4Ic-Xnwrg-KEkDqW zW3l$7Q`V$!1T(=QL1jgjIachdr75>-8>1A^h+;rTrD^nnwf?bw(Rang!*16Odj$Pn z@)JN5&5w~}ae6d};oa|&G>sT!)ixE#5;QW(u(=bqYHXcOflE%@t4A?n5fTUm0F~8_ zwpoz9rrU`@G=vsNjDRY(CrF(jIjqg8bd|CP02>eFag7T?u;C^ir+Z7YKmBYw;%%XdT2T}a$X4yR7EI;zaof3a)5Z;`OwVi%D?gbkBj!{;z2tOBSFk&E1DeiZXD**uvNqL}+|pO{ ztO$}2NMRit2ddU?)7Prq&*&H3X>&=E{-+j4iUz zrvL;?0$^@lyl=LHz9G^$SJV6ID__@7z->Bh>Vm=6AK&5bP%@heveHja5F@agGgUsY z@L@W2+^*NVoId0!kS~4XkWb%y;f}XBf>S+NIw9aHK;vN+4mJ|em)_QjIVfb2$;bwv zDKmoq6AThgKydS6Hs+UpKPWq|UA}s=UOEBZNM3oNT5qTAabY)X>L6jxfGDuu7&GD_ z=@@m?sJ-o2GS}&hNRW}-zHkr>o4&138@a8IC-FjSBxzjx?(*3@YmdmWGAd%0QvXzS zJ53JpX%Fp!=>v&`Hd7F@+Atw2vx9%^2M-APg0Jd|ePsRn3*B$#9Z5hCou4fo7W#SN z#}-@-N=##yQDh26pNzr9f*Q88krhI5@DHcf{dU-~PLSs}MvI4s1i|<=qxD~9`7>*~ znlw5lr$_6mTG4XbBNF_79BzvZ!TeIP)exdk3)kSHjYdW1P10ZJ_NCJSlrCuIU#gqw f88(SSw!Z%ZUzhC#9QlKF00000NkvXXu0mjfG$}gK literal 0 HcmV?d00001 diff --git a/harmony-os/SherpaOnnxVadAsr/build-profile.json5 b/harmony-os/SherpaOnnxVadAsr/build-profile.json5 new file mode 100644 index 000000000..8e63d9768 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/build-profile.json5 @@ -0,0 +1,40 @@ +{ + "app": { + "signingConfigs": [], + "products": [ + { + "name": "default", + "signingConfig": "default", + "compatibleSdkVersion": "4.0.0(10)", + "runtimeOS": "HarmonyOS", + "buildOption": { + "strictMode": { + "caseSensitiveCheck": true, + } + } + } + ], + "buildModeSet": [ + { + "name": "debug", + }, + { + "name": "release" + } + ] + }, + "modules": [ + { + "name": "entry", + "srcPath": "./entry", + "targets": [ + { + "name": "default", + "applyToProducts": [ + "default" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/code-linter.json5 b/harmony-os/SherpaOnnxVadAsr/code-linter.json5 new file mode 100644 index 000000000..77b31b517 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/code-linter.json5 @@ -0,0 +1,20 @@ +{ + "files": [ + "**/*.ets" + ], + "ignore": [ + "**/src/ohosTest/**/*", + "**/src/test/**/*", + "**/src/mock/**/*", + "**/node_modules/**/*", + "**/oh_modules/**/*", + "**/build/**/*", + "**/.preview/**/*" + ], + "ruleSet": [ + "plugin:@performance/recommended", + "plugin:@typescript-eslint/recommended" + ], + "rules": { + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/.gitignore b/harmony-os/SherpaOnnxVadAsr/entry/.gitignore new file mode 100644 index 000000000..53823fed8 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/.gitignore @@ -0,0 +1,7 @@ +/node_modules +/oh_modules +/.preview +/build +/.cxx +/.test +*.har diff --git a/harmony-os/SherpaOnnxVadAsr/entry/README.md b/harmony-os/SherpaOnnxVadAsr/entry/README.md new file mode 100644 index 000000000..fcdfe710b --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/README.md @@ -0,0 +1,7 @@ +# Introduction + +Please download ./sherpa_onnx-v1.10.32.har +from + +Hint: For users who have no access to huggingface, please use + diff --git a/harmony-os/SherpaOnnxVadAsr/entry/build-profile.json5 b/harmony-os/SherpaOnnxVadAsr/entry/build-profile.json5 new file mode 100644 index 000000000..fa8ffe231 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/build-profile.json5 @@ -0,0 +1,33 @@ +{ + "apiType": "stageMode", + "buildOption": { + "sourceOption": { + "workers": [ + './src/main/ets/workers/NonStreamingAsrWithVadWorker.ets' + ] + } + }, + "buildOptionSet": [ + { + "name": "release", + "arkOptions": { + "obfuscation": { + "ruleOptions": { + "enable": false, + "files": [ + "./obfuscation-rules.txt" + ] + } + } + } + }, + ], + "targets": [ + { + "name": "default" + }, + { + "name": "ohosTest", + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/hvigorfile.ts b/harmony-os/SherpaOnnxVadAsr/entry/hvigorfile.ts new file mode 100644 index 000000000..c6edcd904 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/hvigorfile.ts @@ -0,0 +1,6 @@ +import { hapTasks } from '@ohos/hvigor-ohos-plugin'; + +export default { + system: hapTasks, /* Built-in plugin of Hvigor. It cannot be modified. */ + plugins:[] /* Custom plugin to extend the functionality of Hvigor. */ +} diff --git a/harmony-os/SherpaOnnxVadAsr/entry/obfuscation-rules.txt b/harmony-os/SherpaOnnxVadAsr/entry/obfuscation-rules.txt new file mode 100644 index 000000000..272efb6ca --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/obfuscation-rules.txt @@ -0,0 +1,23 @@ +# Define project specific obfuscation rules here. +# You can include the obfuscation configuration files in the current module's build-profile.json5. +# +# For more details, see +# https://developer.huawei.com/consumer/cn/doc/harmonyos-guides-V5/source-obfuscation-V5 + +# Obfuscation options: +# -disable-obfuscation: disable all obfuscations +# -enable-property-obfuscation: obfuscate the property names +# -enable-toplevel-obfuscation: obfuscate the names in the global scope +# -compact: remove unnecessary blank spaces and all line feeds +# -remove-log: remove all console.* statements +# -print-namecache: print the name cache that contains the mapping from the old names to new names +# -apply-namecache: reuse the given cache file + +# Keep options: +# -keep-property-name: specifies property names that you want to keep +# -keep-global-name: specifies names that you want to keep in the global scope + +-enable-property-obfuscation +-enable-toplevel-obfuscation +-enable-filename-obfuscation +-enable-export-obfuscation \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 b/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 new file mode 100644 index 000000000..248c3b754 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 @@ -0,0 +1,10 @@ +{ + "name": "entry", + "version": "1.0.0", + "description": "Please describe the basic information.", + "main": "", + "author": "", + "license": "", + "dependencies": {} +} + diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/entryability/EntryAbility.ets b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/entryability/EntryAbility.ets new file mode 100644 index 000000000..679d91453 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/entryability/EntryAbility.ets @@ -0,0 +1,43 @@ +import AbilityConstant from '@ohos.app.ability.AbilityConstant'; +import hilog from '@ohos.hilog'; +import UIAbility from '@ohos.app.ability.UIAbility'; +import Want from '@ohos.app.ability.Want'; +import window from '@ohos.window'; + +export default class EntryAbility extends UIAbility { + onCreate(want: Want, launchParam: AbilityConstant.LaunchParam): void { + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onCreate'); + } + + onDestroy(): void { + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onDestroy'); + } + + onWindowStageCreate(windowStage: window.WindowStage): void { + // Main window is created, set main page for this ability + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onWindowStageCreate'); + + windowStage.loadContent('pages/Index', (err) => { + if (err.code) { + hilog.error(0x0000, 'testTag', 'Failed to load the content. Cause: %{public}s', JSON.stringify(err) ?? ''); + return; + } + hilog.info(0x0000, 'testTag', 'Succeeded in loading the content.'); + }); + } + + onWindowStageDestroy(): void { + // Main window is destroyed, release UI related resources + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onWindowStageDestroy'); + } + + onForeground(): void { + // Ability has brought to foreground + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onForeground'); + } + + onBackground(): void { + // Ability has back to background + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onBackground'); + } +} diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets new file mode 100644 index 000000000..d2c48b421 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets @@ -0,0 +1,12 @@ +import hilog from '@ohos.hilog'; +import BackupExtensionAbility, { BundleVersion } from '@ohos.application.BackupExtensionAbility'; + +export default class EntryBackupAbility extends BackupExtensionAbility { + async onBackup() { + hilog.info(0x0000, 'testTag', 'onBackup ok'); + } + + async onRestore(bundleVersion: BundleVersion) { + hilog.info(0x0000, 'testTag', 'onRestore ok %{public}s', JSON.stringify(bundleVersion)); + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets new file mode 100644 index 000000000..e32b4eb76 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets @@ -0,0 +1,173 @@ +import { LengthUnit } from '@kit.ArkUI'; +import worker, { MessageEvents } from '@ohos.worker'; +import { BusinessError } from '@kit.BasicServicesKit'; +import { picker } from '@kit.CoreFileKit'; + + +@Entry +@Component +struct Index { + @State currentIndex: number = 0; + @State resultFromFile: string = ''; + @State progressForFile: number = 0; + @State selectFileBtnEnabled: boolean = false; + @State message: string = 'To be implemented'; + @State lang: string = 'English'; + private controller: TabsController = new TabsController(); + private workerInstance?: worker.ThreadWorker + private readonly scriptURL: string = 'entry/ets/workers/NonStreamingAsrWithVadWorker.ets' + + aboutToAppear(): void { + this.workerInstance = new worker.ThreadWorker(this.scriptURL, { + name: 'NonStreaming ASR worker' + }); + + this.workerInstance.onmessage = (e: MessageEvents) => { + const msgType = e.data['msgType'] as string; + console.log(`received data ${msgType}`); + + if (msgType == 'init-non-streaming-asr-done') { + this.selectFileBtnEnabled = true; + } + + if (msgType == 'non-streaming-asr-vad-decode-done') { + this.resultFromFile = e.data['text'] as string + '\n'; + } + + if (msgType == 'non-streaming-asr-vad-decode-partial') { + if (this.resultFromFile == '') { + this.resultFromFile = e.data['text'] as string; + } else { + this.resultFromFile += '\n\n' + e.data['text'] as string; + } + } + + if (msgType == 'non-streaming-asr-vad-decode-error') { + this.resultFromFile = e.data['text'] as string; + } + + if (msgType == 'non-streaming-asr-vad-decode-progress') { + this.progressForFile = e.data['progress'] as number; + + this.selectFileBtnEnabled = this.progressForFile >= 100; + } + } + + const context = getContext(); + this.workerInstance.postMessage({ msgType: 'init-vad', context }); + this.workerInstance.postMessage({ msgType: 'init-non-streaming-asr', context }); + } + + @Builder + TabBuilder(title: string, targetIndex: number, selectedImg: Resource, normalImg: Resource) { + Column() { + Image(this.currentIndex == targetIndex ? selectedImg : normalImg) + .size({ width: 25, height: 25 }) + Text(title) + .fontColor(this.currentIndex == targetIndex ? '#28bff1' : '#8a8a8a') + } + .width('100%') + .height(50) + .justifyContent(FlexAlign.Center) + .onClick(() => { + this.currentIndex = targetIndex; + this.controller.changeIndex(this.currentIndex); + }) + } + + build() { + Column() { + Tabs({ barPosition: BarPosition.End, controller: this.controller }) { + TabContent() { + Column({ space: 10 }) { + Text('Next-gen Kaldi: VAD + ASR') + .fontColor('#182431') + .fontSize(25) + .lineHeight(41) + .fontWeight(500) + + Button('Select .wav file ') + .enabled(this.selectFileBtnEnabled) + .fontSize(13) + .width(296) + .height(60) + .onClick(() => { + this.resultFromFile = ''; + this.progressForFile = 0; + + const documentSelectOptions = new picker.DocumentSelectOptions(); + documentSelectOptions.maxSelectNumber = 1; + documentSelectOptions.fileSuffixFilters = ['.wav']; + const documentViewPicker = new picker.DocumentViewPicker(); + documentViewPicker.select(documentSelectOptions).then((result: Array) => { + console.log(`Result: ${result}`); + + if (!result[0]) { + this.resultFromFile = 'Please select a file to decode'; + this.selectFileBtnEnabled = true; + return; + } + + if (this.workerInstance) { + this.workerInstance.postMessage({ + msgType: 'non-streaming-asr-vad-decode', + filename: result[0], + }); + } else { + console.log(`this worker instance is undefined ${this.workerInstance}`); + } + }).catch((err: BusinessError) => { + console.error(`Failed to select file, code is ${err.code}, message is ${err.message}`); + }) + + }) + + Text(`Supported languages: ${this.lang}`) + + if (this.progressForFile > 0) { + Row() { + Progress({ value: 0, total: 100, type: ProgressType.Capsule }) + .width('80%') + .height(20) + .value(this.progressForFile); + + Text(`${this.progressForFile.toFixed(2)}%`).width('15%') + }.width('100%').justifyContent(FlexAlign.Center) + } + + TextArea({ text: this.resultFromFile }).width('100%').lineSpacing({ value: 10, unit: LengthUnit.VP }); + + } + .alignItems(HorizontalAlign.Center) + .justifyContent(FlexAlign.Start) + }.tabBar(this.TabBuilder('From file', 0, $r('app.media.icon_doc'), $r('app.media.icon_doc_default'))) + + TabContent() { + Column() { + Text(this.message) + .fontSize(50) + .fontWeight(FontWeight.Bold); + } + } + .tabBar(this.TabBuilder('From mic', 1, $r('app.media.ic_public_input_voice'), + $r('app.media.ic_public_input_voice_default'))) + + TabContent() { + Column() { + Text("Everything is open-sourced"); + Divider(); + Text("It runs locally, without accessing the network"); + Divider(); + Text("See also https://github.com/k2-fsa/sherpa-onnx"); + Divider(); + Text("and https://k2-fsa.github.io/sherpa/social-groups.html"); + }.justifyContent(FlexAlign.Start) + }.tabBar(this.TabBuilder('Help', 2, $r('app.media.info_circle'), + $r('app.media.info_circle_default'))) + + }.scrollable(false) + } + .width('100%') + .justifyContent(FlexAlign.Start) + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/NonStreamingAsrModels.ets b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/NonStreamingAsrModels.ets new file mode 100644 index 000000000..3dc945025 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/NonStreamingAsrModels.ets @@ -0,0 +1,237 @@ +// Please keep in sync with +// https://github.com/k2-fsa/sherpa-onnx/blob/master/sherpa-onnx/kotlin-api/OfflineRecognizer.kt#L184 + +import { OfflineModelConfig } from 'sherpa_onnx'; + +export function getOfflineModelConfig(type: number): OfflineModelConfig { + const c = new OfflineModelConfig(); + switch (type) { + case 0: { + const modelDir = 'sherpa-onnx-paraformer-zh-2023-09-14' + c.paraformer.model = `${modelDir}/model.int8.onnx`; + c.tokens = `${modelDir}/tokens.txt`; + c.modelType = "paraformer"; + + break; + } + + case 1: { + const modelDir = 'icefall-asr-multidataset-pruned_transducer_stateless7-2023-05-04' + c.transducer.encoder = `$modelDir}/encoder-epoch-30-avg-4.int8.onnx`; + c.transducer.decoder = `${modelDir}/decoder-epoch-30-avg-4.onnx`; + c.transducer.encoder = `${modelDir}/joiner-epoch-30-avg-4.onnx`; + c.tokens = `${modelDir}/tokens.txt`; + c.modelType = "transducer"; + + break; + } + + case 2: { + const modelDir = 'sherpa-onnx-whisper-tiny.en'; + c.whisper.encoder = `${modelDir}/tiny.en-encoder.int8.onnx`; + c.whisper.decoder = `${modelDir}/tiny.en-decoder.int8.onnx`; + c.tokens = `${modelDir}/tiny.en-tokens.txt`; + c.modelType = "whisper"; + + break; + } + + case 3: { + const modelDir = 'sherpa-onnx-whisper-base.en'; + c.whisper.encoder = `${modelDir}/base.en-encoder.int8.onnx`; + c.whisper.decoder = `${modelDir}/base.en-decoder.int8.onnx`; + c.tokens = `${modelDir}/base.en-tokens.txt`; + c.modelType = "whisper"; + + break; + } + + case 4: { + const modelDir = "icefall-asr-zipformer-wenetspeech-20230615"; + c.transducer.encoder = `${modelDir}/encoder-epoch-12-avg-4.int8.onnx`; + c.transducer.decoder = `${modelDir}/decoder-epoch-12-avg-4.onnx`; + c.transducer.joiner = `${modelDir}/joiner-epoch-12-avg-4.int8.onnx`; + c.tokens = `${modelDir}/tokens.txt`; + c.modelType = "transducer"; + + break; + } + + case 5: { + const modelDir = "sherpa-onnx-zipformer-multi-zh-hans-2023-9-2"; + c.transducer.encoder = `${modelDir}/encoder-epoch-20-avg-1.int8.onnx`; + c.transducer.decoder = `${modelDir}/decoder-epoch-20-avg-1.onnx`; + c.transducer.joiner = `${modelDir}/joiner-epoch-20-avg-1.int8.onnx`; + c.tokens = `${modelDir}/tokens.txt`; + c.modelType = "transducer"; + + break; + } + + case 6: { + const modelDir = "sherpa-onnx-nemo-ctc-en-citrinet-512"; + c.nemoCtc.model = `${modelDir}/model.int8.onnx`; + c.tokens = `${modelDir}/tokens.txt`; + + break; + } + + case 7: { + const modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-be-de-en-es-fr-hr-it-pl-ru-uk-20k" + c.nemoCtc.model = `${modelDir}/model.onnx`; + c.tokens = `${modelDir}/tokens.txt`; + + break; + } + + case 8: { + const modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-en-24500" + c.nemoCtc.model = `${modelDir}/model.onnx`; + c.tokens = `${modelDir}/tokens.txt`; + + break; + } + + case 9: { + const modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-en-de-es-fr-14288" + c.nemoCtc.model = `${modelDir}/model.onnx`; + c.tokens = `${modelDir}/tokens.txt`; + + break; + } + + case 10: { + const modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-es-1424" + c.nemoCtc.model = `${modelDir}/model.onnx`; + c.tokens = `${modelDir}/tokens.txt`; + + break; + } + + case 11: { + const modelDir = "sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04" + c.telespeechCtc = `${modelDir}/model.int8.onnx`; + c.tokens = `${modelDir}/tokens.txt`; + c.modelType = "telespeech_ctc"; + + break; + } + + case 12: { + const modelDir = "sherpa-onnx-zipformer-thai-2024-06-20" + c.transducer.encoder = `${modelDir}/encoder-epoch-12-avg-5.int8.onnx`; + c.transducer.decoder = `${modelDir}/decoder-epoch-12-avg-5.onnx`; + c.transducer.joiner = `${modelDir}/joiner-epoch-12-avg-5.int8.onnx`; + c.tokens = `${modelDir}/tokens.txt`; + c.modelType = "transducer"; + + break; + } + + case 13: { + const modelDir = "sherpa-onnx-zipformer-korean-2024-06-24"; + c.transducer.encoder = `${modelDir}/encoder-epoch-99-avg-1.int8.onnx`; + c.transducer.decoder = `${modelDir}/decoder-epoch-99-avg-1.onnx`; + c.transducer.joiner = `${modelDir}/joiner-epoch-99-avg-1.int8.onnx`; + c.tokens = `${modelDir}/tokens.txt`; + c.modelType = "transducer"; + + break; + } + + case 14: { + const modelDir = "sherpa-onnx-paraformer-zh-small-2024-03-09"; + c.paraformer.model = `${modelDir}/model.int8.onnx`; + c.tokens = `${modelDir}/tokens.txt`; + c.modelType = "paraformer"; + + break; + } + + case 15: { + const modelDir = "sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17"; + c.senseVoice.model = `${modelDir}/model.int8.onnx`; + c.tokens = `${modelDir}/tokens.txt`; + + break; + } + + case 16: { + const modelDir = "sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01"; + c.transducer.encoder = `${modelDir}/encoder-epoch-99-avg-1.int8.onnx`; + c.transducer.decoder = `${modelDir}/decoder-epoch-99-avg-1.onnx`; + c.transducer.joiner = `${modelDir}/joiner-epoch-99-avg-1.int8.onnx`; + c.tokens = `${modelDir}/tokens.txt`; + c.modelType = "transducer"; + + break; + } + + case 17: { + const modelDir = "sherpa-onnx-zipformer-ru-2024-09-18"; + c.transducer.encoder = `${modelDir}/encoder.int8.onnx`; + c.transducer.decoder = `${modelDir}/decoder.onnx`; + c.transducer.joiner = `${modelDir}/joiner.int8.onnx`; + c.tokens = `${modelDir}/tokens.txt`; + c.modelType = "transducer"; + + break; + } + + case 18: { + const modelDir = "sherpa-onnx-small-zipformer-ru-2024-09-18"; + c.transducer.encoder = `${modelDir}/encoder.int8.onnx`; + c.transducer.decoder = `${modelDir}/decoder.onnx`; + c.transducer.joiner = `${modelDir}/joiner.int8.onnx`; + c.tokens = `${modelDir}/tokens.txt`; + c.modelType = "transducer"; + + break; + } + + case 19: { + const modelDir = "sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24"; + c.nemoCtc.model = `${modelDir}/model.int8.onnx`; + c.tokens = `${modelDir}/tokens.txt`; + + break; + } + + case 20: { + const modelDir = "sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24"; + c.transducer.encoder = `${modelDir}/encoder.int8.onnx`; + c.transducer.decoder = `${modelDir}/decoder.onnx`; + c.transducer.joiner = `${modelDir}/joiner.onnx`; + c.tokens = `${modelDir}/tokens.txt`; + c.modelType = "nemo_transducer"; + + break; + } + + case 21: { + const modelDir = "sherpa-onnx-moonshine-tiny-en-int8"; + c.moonshine.preprocessor = `${modelDir}/preprocess.onnx`; + c.moonshine.encoder = `${modelDir}/encode.int8.onnx`; + c.moonshine.uncachedDecoder = `${modelDir}/uncached_decode.int8.onnx`; + c.moonshine.cachedDecoder = `${modelDir}/cached_decode.int8.onnx`; + c.tokens = `${modelDir}/tokens.txt`; + + break; + } + + case 22: { + const modelDir = "sherpa-onnx-moonshine-base-en-int8"; + c.moonshine.preprocessor = `${modelDir}/preprocess.onnx`; + c.moonshine.encoder = `${modelDir}/encode.int8.onnx`; + c.moonshine.uncachedDecoder = `${modelDir}/uncached_decode.int8.onnx`; + c.moonshine.cachedDecoder = `${modelDir}/cached_decode.int8.onnx`; + c.tokens = `${modelDir}/tokens.txt`; + + break; + } + } + + console.log(`Please specify a supported type. Given type ${type}`); + + return c; +} diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/workers/NonStreamingAsrWithVadWorker.ets b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/workers/NonStreamingAsrWithVadWorker.ets new file mode 100644 index 000000000..3ffe18671 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/workers/NonStreamingAsrWithVadWorker.ets @@ -0,0 +1,177 @@ +import { ErrorEvent, MessageEvents, ThreadWorkerGlobalScope, worker } from '@kit.ArkTS'; +import { + OfflineRecognizer, + OfflineRecognizerConfig, + readWaveFromBinary, + SileroVadConfig, + Vad, + VadConfig, +} from 'sherpa_onnx'; +import { Context } from '@kit.AbilityKit'; +import { fileIo } from '@kit.CoreFileKit'; +import { getOfflineModelConfig } from '../pages/NonStreamingAsrModels'; + +const workerPort: ThreadWorkerGlobalScope = worker.workerPort; + +let recognizer: OfflineRecognizer; +let vad: Vad; // vad for decoding files + +function initVad(context: Context): Vad { + let mgr = context.resourceManager; + const config = new VadConfig( + new SileroVadConfig( + 'silero_vad.onnx', + 0.5, + 0.25, + 0.5, + 512, + ), + 16000, + true, + 1, + ); + + const bufferSizeInSeconds = 60; + return new Vad(config, bufferSizeInSeconds, mgr); +} + +function initNonStreamingAsr(context: Context): OfflineRecognizer { + let mgr = context.resourceManager; + const config = new OfflineRecognizerConfig(); + + // Note that you can switch to a new model by changing type + // + // If you use type = 2, which means you will use + // sherpa-onnx-whisper-tiny.en + // we assume you have the following folder structure in you resources/rawfile + /* + (py38) fangjuns-MacBook-Pro:main fangjun$ pwd + /Users/fangjun/open-source/sherpa-onnx/harmony-os/SherpaOnnxVadAsr/entry/src/main + (py38) fangjuns-MacBook-Pro:main fangjun$ tree resources/rawfile/ + resources/rawfile/ + ├── sherpa-onnx-whisper-tiny.en + │ ├── README.md + │ ├── tiny.en-decoder.int8.onnx + │ ├── tiny.en-encoder.int8.onnx + │ └── tiny.en-tokens.txt + └── silero_vad.onnx + + 1 directory, 5 files + */ + const type = 2; + config.modelConfig = getOfflineModelConfig(type); + config.modelConfig.debug = true; + return new OfflineRecognizer(config, mgr) +} + +function decode(filename: string): string { + vad.reset(); + + const fp = fileIo.openSync(filename); + const stat = fileIo.statSync(fp.fd); + const arrayBuffer = new ArrayBuffer(stat.size); + fileIo.readSync(fp.fd, arrayBuffer); + const data = new Uint8Array(arrayBuffer); + + const wave = readWaveFromBinary(data); + + console.log(`sample rate ${wave.sampleRate}`); + console.log(`samples length ${wave.samples.length}`); + const resultList: string[] = []; + + const windowSize = vad.config.sileroVad.windowSize; + for (let i = 0; i < wave.samples.length; i += windowSize) { + const thisWindow = wave.samples.subarray(i, i + windowSize) + vad.acceptWaveform(thisWindow); + if (i + windowSize >= wave.samples.length) { + vad.flush(); + } + while (!vad.isEmpty()) { + const segment = vad.front(); + const _startTime = (segment.start / wave.sampleRate); + const _endTime = _startTime + segment.samples.length / wave.sampleRate; + + if (_endTime - _startTime < 0.2) { + vad.pop(); + continue; + } + + const startTime = _startTime.toFixed(2); + const endTime = _endTime.toFixed(2); + + const progress = (segment.start + segment.samples.length) / wave.samples.length * 100; + + workerPort.postMessage({ 'msgType': 'non-streaming-asr-vad-decode-progress', progress }); + + const stream = recognizer.createStream(); + stream.acceptWaveform({ samples: segment.samples, sampleRate: wave.sampleRate }); + recognizer.decode(stream); + const result = recognizer.getResult(stream); + + const text = `${startTime} -- ${endTime} ${result.text}` + resultList.push(text); + console.log(`partial result ${text}`); + + workerPort.postMessage({ 'msgType': 'non-streaming-asr-vad-decode-partial', text }); + + vad.pop(); + } + } + + return resultList.join('\n\n'); +} + +/** + * Defines the event handler to be called when the worker thread receives a message sent by the host thread. + * The event handler is executed in the worker thread. + * + * @param e message data + */ +workerPort.onmessage = (e: MessageEvents) => { + const msgType = e.data['msgType'] as string; + console.log(`msg-type: ${msgType}`) + if (msgType == 'init-vad' && !vad) { + const context = e.data['context'] as Context; + vad = initVad(context); + console.log('init vad done'); + workerPort.postMessage({ 'msgType': 'init-vad-done' }); + } + + if (msgType == 'init-non-streaming-asr' && !recognizer) { + const context = e.data['context'] as Context; + recognizer = initNonStreamingAsr(context); + console.log('init non streaming ASR done'); + workerPort.postMessage({ 'msgType': 'init-non-streaming-asr-done' }); + } + + if (msgType == 'non-streaming-asr-vad-decode') { + const filename = e.data['filename'] as string; + console.log(`decoding ${filename}`); + try { + const text = decode(filename); + workerPort.postMessage({ msgType: 'non-streaming-asr-vad-decode-done', text }); + } catch (e) { + workerPort.postMessage({ msgType: 'non-streaming-asr-vad-decode-error', text: `Failed to decode ${filename}` }); + } + + workerPort.postMessage({ 'msgType': 'non-streaming-asr-vad-decode-progress', progress: 100 }); + } +} + +/** + * Defines the event handler to be called when the worker receives a message that cannot be deserialized. + * The event handler is executed in the worker thread. + * + * @param e message data + */ +workerPort.onmessageerror = (e: MessageEvents) => { +} + +/** + * Defines the event handler to be called when an exception occurs during worker execution. + * The event handler is executed in the worker thread. + * + * @param e error message + */ +workerPort.onerror = (e: ErrorEvent) => { +} diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/module.json5 b/harmony-os/SherpaOnnxVadAsr/entry/src/main/module.json5 new file mode 100644 index 000000000..a1cea8b6a --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/module.json5 @@ -0,0 +1,52 @@ +{ + "module": { + "name": "entry", + "type": "entry", + "description": "$string:module_desc", + "mainElement": "EntryAbility", + "deviceTypes": [ + "phone", + "tablet", + "2in1" + ], + "deliveryWithInstall": true, + "installationFree": false, + "pages": "$profile:main_pages", + "abilities": [ + { + "name": "EntryAbility", + "srcEntry": "./ets/entryability/EntryAbility.ets", + "description": "$string:EntryAbility_desc", + "icon": "$media:layered_image", + "label": "$string:EntryAbility_label", + "startWindowIcon": "$media:startIcon", + "startWindowBackground": "$color:start_window_background", + "exported": true, + "skills": [ + { + "entities": [ + "entity.system.home" + ], + "actions": [ + "action.system.home" + ] + } + ] + } + ], + "extensionAbilities": [ + { + "name": "EntryBackupAbility", + "srcEntry": "./ets/entrybackupability/EntryBackupAbility.ets", + "type": "backup", + "exported": false, + "metadata": [ + { + "name": "ohos.extension.backup", + "resource": "$profile:backup_config" + } + ], + } + ] + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/element/color.json b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/element/color.json new file mode 100644 index 000000000..3c712962d --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/element/color.json @@ -0,0 +1,8 @@ +{ + "color": [ + { + "name": "start_window_background", + "value": "#FFFFFF" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/element/string.json b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/element/string.json new file mode 100644 index 000000000..3b3015be7 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/element/string.json @@ -0,0 +1,16 @@ +{ + "string": [ + { + "name": "module_desc", + "value": "VAD+ASR with Next-gen Kaldi" + }, + { + "name": "EntryAbility_desc", + "value": "VAD+ASR" + }, + { + "name": "EntryAbility_label", + "value": "VAD_ASR" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/background.png b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/background.png new file mode 100644 index 0000000000000000000000000000000000000000..f939c9fa8cc8914832e602198745f592a0dfa34d GIT binary patch literal 57364 zcmYIuc|6qL_rIk#Su&MMQlYU)cz{|$Qc0x~A^BEf( z`{n=HaSk>%wsfNM*uUkN^8dI{qxxW z*@b_`#>VlLWSG9 z0>QdPQ-&i_RCVdp2s$-u%S362^SHV0`EO6;@n(xK));G>#qwhPWrDXGk@OBMV}H!J za!48&`xhWJKj{_+f3ir<>Jg6Ax<&Xgn;)U7UJyAw{(u?zlf{oLsJTS-_o1?+lSg-j z8fcZj1*Ad(!X>WuuxM!H5t@V3*8vLL6`QnC!q!BwQjI{yk*;~@|3;B)`p$WYcDmnZ zt`R zr=oS6o-D$WZsYKh1PiOdhhX&YWGOzpc<6ITKzr^zi-#>z){t;yz3tu_a!>)(tTU9d zd}COuy~Tb}UIRNX@aVGJqEKUa)1#E-u}pl!sY)z4cu+Hu9==`6=0Ob#x-%q}t@jBp zmoiZDcfF1WL{PB0ZO**8yZ+%;LF6K*JDUoHrJkl0Wzak+Y%E( znUmuA^p@Jv6{%Y;MsiZ4O?#ID2b2ssEq6_KGL z8T%zdA3YhMnkBu19bNsa_$$_1^16jadx`0ZzPx`M%T>qZpYyNYOeDdmqLTNWpR5T% zOlRrW_xNCD+*3_WSxvt4P-@qQ9g_$aedDk-hcV~t>Oxw;UaAk1V?9m5<2k4%VrM$- z?{KH{)m_U~yJcBbX+vqVfq&4)Vf+FvAHd|s{V34=f#uJM!Tp?b32THmfzNn1unwY& zPNtaE{ZZ=OkZFh*xW2FT&fDF?64Q%l>dwdZ#Bg;^v;dAbU*QLEQG@_|ucNXFyx~H( z#h?kJKeI3jD^U~`e`*^zcm?PlIWj|tL_a8NC?HVl*gX%;5PW5Y%ZZ*G=jPn5#o+Sh zhnE>D@Wb!f*O>cZ0}ZT=HlEdoWVWk}5H1S;$vxe#Rv~;l5rJ=w--wPl621jCW}B|gxECKzT9 z3FKlD{=OfN5$J3?Ag0g4F5t8_D(RvO8W!*~?#q{Dhx(Sj=)^9ZlE|LyI?p1NXMWr| zGGbzFN^3)5?P^vfnD7XZo*8yf&O&>7XULUUvhJT@rHcF>PmjodH~u4RSmX4TH?v`IKg2cy7 z(T@e4&pPRHRczikEvwvO?jbblSVp z2qpyT+LHUFhHwcunP(^h)G#uA95vF`Gd&1k%F@wuCk3DnjNjw;b}*;dY{F5{7tNsg zLf4y|)RTV`PjQ^!NoWB3YA@S@Cw zUAr?mUcn7g)n!3J`D7*H`y{%TuT$wNY;))rP(y@kdFdPH#h|rjcW2#oRybxTchXlQ zwMW{bVcqRRc_2r^tI)Zav_+qLwdd|Bw=*pM!|pflbT%K&Eof^{6+|k{2_;HcrWd3? z#z;>@Y3dp#B^R5c9RhH8lT5MRr*;>xd<%C3sV2Y}>{On@a*oump`g#H<6V&DKeZ-?Zic$S$>ulEiZvJG8kHMeSzVE(R|E-<}cEG^n2E*Cp z-25-DQv_Mf+&WhT3r?23Phid$q`Z3HE($RgC{EJA0Yc1SP6(a(oZ4RU2L1~H6k0Q< zHY1Mj{)b(ll3Wr=HakbiEk13zYKN&f#9*}tMZiQ7h@Us+N(Jk`aWQHf)r!ObZAT>_STJuzjuO{qHMlTjN9^hPZ8sZBMl zl&MX}xk{d5VUEInRK9r^Tnx#HE2;hFoa7?NDufAxZV6Mj9B^NaAt4;oStAtWfVg8< zjQAfLPj#u>Xp*sALAi;M(f1>la|_-k(E*-1Sa_Vdt$KsCNAwAbm8CmvpDbwL$`Cx8 zkBC0&3#@q@7E3LVtGQcrGS=s-uh6FHuC)WTtU_@t5c_T~`Wv+F0Jd$a9s(?ucd&l{ zWThjQ*u4YqU6Wq{+^0sC%S;vXx~qO|+s%Am1m-N}zkd84>GT;5u}a1*p9&!g%3wk2 zl=rj+H9g>!z4_zdU1iItL}Zox?lwK^ykQ+_#Ym~p>s8CgcLQYV4wezL^K-_HzM$r! z1m$U&G13HqDckgHschNcoe73o=)$P$j46Y)SnaZK(U|F7d#{AGb%>@b+jX#5*Rf5x zq}@ejPTyyn&&@n|dDGl-o-=XF%6dndW+}@7JDd?6b}Mt-SX_GV^3{!3Yz5a~X@$Fw zyDIkaWq*rtn{8knumG6=yF(6lzQnq)&M@%8RzdC%{%-0Ey{v&0pp-aIPP$bTrF|=~!MvLftx2pd=0-86i#@A;;(b^r-TzBJn~W4d42|-V*)} zt}h95!TwDQ%xWD9TFS{BwGO@d9P>kia=+LQ@r>0>5VvEV8(&tEuw%+YP*Qm6KzZs9 z#qL6SPwl9DtPZ{0`)Vph`^ryNV|=I7r2Vf@LrX3<=$f6zv1^z*!<6j{f$|6Jw=%s2 zb)|d{?()1m_Xoab$B5r9#&taTI^E@0yTQ$UB1_f0nc<oQhFOi;b@!o=y6w&Tsrw|K5XXEJEA>@Eb?8hi( zlT-*bXZd6g*C+W9V6B5iF$2f(_-ek(ko^JW%$@}`#GJVV0S8A~FwzM(JdY)c1B&ls(qJ=bvy&S10cqD8@1Clbooq|3kmbD_she z@O#tu^ibgYfM#HD%WIF%%uf7+)sc&Dejs@WRQE+Q1jXlN2z>9dB;X9e>Y3a-&-A;T z>||D+o$j^$E>F`4y02DTELRMYH*biv(5+ed(cQq&82Gu z2~UNnOcNc&MwT3lD@S}nPJMsvOT%0L{`dN}DU&^Z#6?2^aE!5ulUV_Zct}2~K6R!_ z4ReuaX$@AP?M!XMpi&ZJwsY2up5F-xe0{ym`9#@pr%63v->d&@UoFthcC1`k$L=ze zYX1{xl49Q=z953h>NzyMc3UuH96t7)-k|lRw-P=T%Q`;dC7@r`uCOq8Eqi7gKC)~7 zb(*Q>H|T2(e>5DVf9nswM~C%V2G2 z#B|VOitZm{FlV>EydvsFF|Ue~ium0%0KOaFiMOLk(X}jHq@dI@*AM2G6XzCU zSpFR?#U4MPz~VZR>RA@a!CZu45#f<)^f#kJ+ULtRLJKzSj=cf+NxQ}Kw)Yme6wJz; zu3W=Jz<}rEm$g7sNy>yr-Z|OiI>qQ4m37~);`_~Xgr~N4wOAssk(HTh5er1XtFm+! zb`5FT&FoKA{ADaUP!Y#o^sGPb?mT2wBY9ZfQ}ujLk`C_dyTvT&)34sj!RXJcZ%lCzF?kE~i-xCSGh{ zy%iUR0+S_JP(#%W9!Npk=RL(8tFB7(up1ms-Q#8 z$-{dva97!EQB<5#@0KgW&2S|ddKN*<(?}37-=O@1bF668sG)3(D61=Ech&sJ;j|An zqu1a;`}bcMj;#tF3l~&Ue9ES7GRw~kIPKK&q&^No_3M#yjp?ygI;To&wcXbe%ju*z zpMI!gbi8@{AJVkgXR+py{dMSfko}H`^q^elZQ-5<2bG-K8tYq8Q@*4t)`Blvz!#v> zE;3kk_e^|Kew4?}eU;3n)q48yWgAm)d+F(;W(>jPB_glQLiH|IE=EDVFI*j_FBebS0vXyh5@x9LS?RNi7vXf?RckfXjvy^Pifki$9G zzwp&k7S+aNOI8%DUON~#xxv+a5rJDE+^6;@RcjnwKZ|%#%Ukq~@&vL#Pts;`f?jwYL)Y zDOROB^T8hlFfA@(=$bFYKWy{F^5$#{h*A1FG5GZZ1?>Y+!}UULap(oEekfHZCJkpC zppRS@+Uvrs>_Df!YT#HWpuaEwRq)V49)TgZ7Jf{A6@tpv&>tG)c9F&eZWo)(tDPDB z4Fkl6@ov*S4!gboeokhZ>My7@q%!Z93-zy>Y(_9axnH2W2Ie&#X2Z->o1A6ZoV(OgY z@PpdL`E%U}QN-vzdLCdkVX)Vp-z|CGg)^e06LvMfbj%1)ZdXNB>r>{Jk&ApwTkkLr z-2C5e31{3c{*xsm?)EItQ%pSW(%723B}AHgke#M{7KJW6TT*>9^+`FIe4;VHRwSF$ z9rBta7_>vwCuV;vFY=|NZ2KlX$A`EUk*phH=Pd~I8Kkr|v!j3sBAD^fPD!FoPpnHf zqP&jc&^s{jm0M&oBNXjUol2${7|G^u7UtOd2kxA0b?japS#xlwo_TaY+jh-`+$sfO zFLgfqb~kaemX{ErUn7}?_tb>g?G@UyT99HoY^;BG(5|gh>F3J!9J* zvrz6TP+;XdE$<41%Vony^Y}i*aCz@+4v^38p)5?Nhw`m%Cbg5Lpz%VOxaBnlA9P;N z9D=#{(>`$N_!?&CKf9eJGzIc>dhWes8XtpX`{gOhP;HMklZ8~@Yu~YT1bZZ{VwrAffDNiZ6Mh5vEzpq z=5A;0ff@>1MG@vbwRU!?7ZFD-SYng>JN(=>uwrkrl@4u6M^n6jl1shsk;DM`t#|F? z(H9W(@&~b(mmUR)30H=vAZdIrX%9iR7rMruZ_I4$Eq7YnBI4Z8T zj5;RTUu8?(ZsW>30%Hk#$^zfAtgZ&y!|p@5%e_4oe7)3{Y6c^x>zv=o_XPiF*wI1y zNe5L3p=L;8_D7-+5I+LfNgDYrOIUD_Iu_VJQD^=4v=Gd z_u%h$8{Lobyu6%VkeZI%T_vssgc#J4yD+&6pVkdLYl@3@NdcQbwl!J%4{RC80oF1z z`ksIXyrZT=Apq3kOR#m795+y}-8NizKBNESZCmBS#jqG`n4kCydp-4DZ^BF-zWD2# z1@F?p*^9m)EPrkd^E&cimk<1mN+iwSCVTHpqz^#`_Dj;-5xURqxK*!kp5asE##*=< zc{bFC-`m;q4VL3=| zKN6@)%XIu=yS*-K-9Bw`jN+-lWBttd77x>|g)~$UgPB_qH0h&bm}j3#sdLfV&xcR^ zQFk=d3;U8~YLqm@^61C zmaLbHw=dJ0oLP?>eyJ&=wgtZm!2mS9V!i~62x+n`%jyesf0bKruxRDH-)c2uF;&qT z4Z0drBbHg-G#ueH1vVaEJFTw$U))8mlUjFz?!PDqNpcIqZ%B6$Ju$CzrK@_na@?na5LpJODS}`)`8j7i#>C z0RNEb>nnQ8v$qXrgh)-(=VVRFwj4 zZKH}5T4rlZ$PiI2z3_*{`av5A0jPJY!Y*RQ?XbKPZmNdwp6ufAH4m~1%r{gYeOJBR zai+gl7I{Z35P0Q7EoGmkkLGHe5rR^{bdxWyMiC1~&kI@I-bYJrdGv{=O7!p&kKxN3 ztOoyzWj_asX!zA>`fa~&>#$n@3{c@VVcl3(1m5=dCI-~1uR+4s;@87ozKCU|Z(EhE z7~Csbr}e|&-zPK~*W}WcKqB+rv-rNRzvqfY299AvP zA5u^Rs->xN6b@MzP_f(M+}|~RxUHs#zO%D772V@B$F;5<%Jx|0#Oh_?#%yrHfV>}I z!Lfe59_VCjJ!pEQOWyUr;CdyL z-RzERMQjU_j%}N!Av?++44uVMc#r_KCTZxxSZL>4`xbm)#)*?4I#nFDOZLv10s^{6 zAyo6zfA)w8n^jk|KBb4J;|Gbx9)grFflY-Nyl_v8_@}gizDNn(Y2l6TqM&aN(+9Qg zTBo#J4N$h%f!;K&2NqBlT~J6aqHGy6HI`Xn*)UV$w2>iLk~P=l)VTdah9Ab`z%}dg zxIvG$xPG=H0NRw|6_-~Bzh+BPv9&C;z)58?`7t~$HupdHcF!F5dirrGrn3d}wAHr! z^@&!aoW@3sENjl#i@LzRYOZ4b#v|Jk_Mo$-VYlgbE3LQVKniS1mH)uO`90X{bc~{1 z*%Wm4$E_2-W__`4`mDu;Ld(wv8e147=mMu!AKSC=mw*4n^8S>~fm9mJgf4~8t(bb> z^_3WSK>aAZ6lK3OZ#_7g@)?z1#pZ zoR2>rm%_enbG!+Y34#Jmal)V9@-s8li+_Le^~z8cxHeF5vR%p~{93TJv%YmeTB|@^ zc=}q4Gofbju_Z#%Iv9|44|pawNvh^mFGBA_KZ5C^rx-l~Ytqf4;%SxezE8%O)aJh& z>2it7b`epB=P&=s^y`mJMjMq&9Jvpdhn}6sFHk)q%d zE_RV6%-}?H)w7yAW9TA)&7XbMyu=N}tRA-JTl2iG6u8;@?;!BW;ykyof{i+alo zJu1v~ITow6y^)5crWdi)&;yNs0d)3*vN+aSszJ%`1`(%9X-Hi}3gH#iRg@{Svm?cP zM}T*)U{A8FTQ7b@oc$7vr_EeTIj6N%Cr}VI5VcfZk+@1UFc>zpJkm3S%cb<~=~`BV ztbyjzOPJuDkTJJ!hL^nLk}*=2EXd?->%+3NWrq&5a$%1G{r2~cLQT2W>8!pd$9G;K ziQIDUErsVk$XQPRm)pDFYVuLFlx&eiFlnoixT|jvAoB)ryM_}euaYFXrdKLqi|4AL zG`rnvWi4Qa>Wvo=;Y+t@ecMjl{#37K;?VkYdoSbT(2m}8!k~RT{yv0l8cPp{jtiXr z$7KAJAvM_g4ak}0Yo*q!sO%PN_CK)Pv>lC7xoB~vG1hs?Wv>^kpOBU0WV@$|oL!cE z1FV3%^4Pjr5Fqc)|Sv+upxx8BCM z9*cYQYi3jY(^pUL8`I|3rHf+5>sq98e!hkPsfNMQ1@y7Tnf4{F2p zx9AO&@zYO;WpCQUf4G@!d<{t43@&RHh2Ukg^D-8_;De`dc{hz?yPS_7BzU!x^P-tj zBWt_uk{g94M1uo_&0l?m1qh!Q>=dKy5cx zRa7mv(}`xYKJOm)h3s8goQ*XK1OT<#&Ozf35uTB^VD8m)Z6Bnlal5r-bkso}J^TcM zo)ZSc#2@`h0Si}lrnCFt67JFa*e&}2avKCL|IIk<$R2*5sILkv4P( zesTX_tP#NqXN#>Q{4oe!N=G{SZ_I#~%^kq5ilGc=Q63_5uRt!D^j$k=&$`Ha&bGlAjZ2&hWa=M};Cw|5onME2e;8le z)-hK+mgNbGw-4puLN6g_q5p6T?0XM^dMo810rSBSw7Rrl(jt2JNVBwhB0o3``lZ1y zBr`Dy8LdVilxv`X5b0N8#{#(y<2vQrLj;qv`XA#RZ+@Q~*aYa^UY~;#F>6BL>75+E zeH2(L#HhLeI=Mz1#%^96zY$Se;@N)biYOvM6H1p6-4LcvA=&GP()#?u=_WXgAoZl* z+bR{6BA52?12Rex)v?(LMRsKvf9{KzP<^4&NISV{2!a;wEhr&E)EloHqSR9%ezb)? zl9X;qQSTg@es%UevGs9-KQk6RqJ;Ui(v@S0=JpkXQVYgXlRKQcfFLT2A%*#c?7(b} zjki==Q^Y#Qf}ZVpFtF6<4SbGKkkU>I6wY*Ps*EAzemS5Z0r!-oD>~r!<<+c~fHK+{ z`u4nWcW&4!()0%2>r>@zr$F6$;5*IAuq5bc>cn-IEZ+B|hkO&NPeBi&47YiU-<$w0 zq-j9aGH~K;Y%0{D&e90RZ(J_@o*`(e0TgqWM zz>V1_2|7MMg_6zbeK`A2oW6>`dUuDIll*?4hKaK{^>2t!B*N9o7_!iC51?A=hss#S zTOD48mGM}}JkMLeB>f0zNw|zPj8Efyx1Qh?QyT7Bp*PsC1%+$kgboSqDR=rTEs%8X z-t2|68n3XC`A-sBYO9tXuQqE7{}pE3mRASQTvScN7(%JH0{M|k4t%rE7xh`qUf4A- zgEE3f#zcuMyMYyiu;w=#PFC-_W0rb;u#{l@E}K0uMy~Ec1MBz-KglT}I_AG%m9nb!XAkpoW-`_85Umy)5g0j(3(>`;o1;w;CKp zLKdGc@@LrE*Y6B#H>jMeTcD6nZx;FZw zZ?8nd;T;sv#~t>9Stu`V2=$pLBHrDq3VNw9{KZU-50LlNLK@?o*hLF?1Kjl3op`;u z=nFLXc(CuUKp%gcxwwBm08`iDki>51cyobB9Eypc5@0Uv%$x+m$P}vtzJ@yXv2Y(6 z%G|Dfw#*GyPhoZ)9Obc;u$h*k0~W zv)EW8ChYvHNP~Ws5(MQk4JSGnG!l*4I-odrw$8E;u9uTN)1sDTSK-9%H|jqRi1XpO z_RLbdR5?V7FZiM9a@_RLzrIa?o8u(&ct}&dJFEmRO#py=1J(LW)$S@B$xLi6T)SOw|;fa7Myzv z?MOZ*b$o!rCg?J9&v6SsP#m&goHWvlC%0`IUKT~X&=s1cU$O`0Ea`_f|aU@(<=bXW{`6+7W#cu@H9t zagx-Usc&&vez&!Mjqpdk+Ol(}Uo_B;A&JhUaOe-iG9|*Z<)SYRZ;!m{$5X=V;9Cl+ zs(#H}WR`823f+9`wmRKF;(;wyt*?b3@Y`H^;&@1GipUF_{Gb_RzIV!3$qMq++{iyr8Th+msVi*eA69cY1K|TmaXNA-rCXT%k z%$21aDiQY_-+BI`52BI$rv}FI)tg7-CaaD7_O`l9ngVYH9#Xu44ly2flHy-xuzEyCWC^6c-^K*QrZW zNG1PL`B#xfh_CD57q**Q+=Ty9EEolHUwT`)Z`SWJPvsxa-f8_iHO;AQOj^^?v$Pd6 zy~3pjahT&?UwB@2zW1)s8+UfK$SFAL~tHHx3whuvPyW4mh3w z`_Q5~nHOsoDT0sx@+N~J<-Y&TvqV4MCkgXgo^ntecjdoSopR%@?wkEfAuHDOIVHQe z|K0}d$IAWT3jC{=QJCD$*L3=%k#f)T)tT7R=nTHqn)i5$Q)sm)53ZV1w&{swK_X#n zpD3;2Eb$r)$CDg__L8tv=0-5U5hB))B~SI2(6`QM95phAkktAVs0hU305vOGT{|^t zH`?>)3!24y5TBnjRfAJG|J9jjj_JYwB?gujfD3QwPf@~K(A2Z4KynC|m! zMt!}`yx4=~u?@-#ab5-T?In;dGAUlGajcN(yFF%ypy(av6(B6-=d(A}}k7wcgUJ%c_TA&p~<@ZA~EU-mvx5S_ykM?O8{R|mH|RE75BR5QQ#CTpy{;f{(N zFpFjUOJ}EEwov(%eX6wm&~H5dD|PO&*VQvG&6Br6eo1I>i7L)sk`T?{8}`lQfCB2R z@nDF(51Rl?^;uv9K%Wz-qpmyIoZjoO+tGhY)P>lU7U1Rpv;b{^)mu_I7=1e%POI7M zneWYe`!E(sG!D4Pm@9XD2!jhItDw15w=Vl)ioN}tjFK(3~fxy=!h!`6@!cQuCP6#aH;{{dyV2@O1#ZX{Zl4pLmD z7*{Ip)`V*gV-QVaE+>|4R`><5Z1*;n%pfkb3AiZ1s39)5f5khONJ{XZ5dEj{AwE^i zj6G1{WVlyMNlC3!_Nyk^Z0DjKo$ha)xbx}7UO*rnNj8he_fyO?v!so#$d4^uhxAXf zZNG(a)^5wM7^{-xB|`JITdre*!q^0$>^GMLKm@oauH?5G^;l>0Hp)xxzomAmYTE02 z+c%CPd*0$Be%v~(u%mMywt>EgIlKPOZH{Q%Y5c6=;F0usNLUPph9Xez1H1>s1YOPG zz|s4D9}W5qUuupaM_2#&;|@Jl=mK~Bc0i~OYb643=Gzqz>i%czm6IJ}e-nj~`8ZFe zGWf#c?5=VP0hlqMCIlRJj0p>6ob8O5e(*AYuP~QI>C$d^Yi`)_a|r1LwH(~NZ9P?Y ze?ts^N2upq=Br??YX8%HZ%xopU$9Z$(sjX zPFNIynnhW{IRi^L#G9#+Ew!gHJ%T1dibisJk2~6dM4r$&WR1@Yh3+PZbrp7G519Z>UKXw(mZMT+M-ozzkggshV_x`b zthj%~?f*E&m2-P{17aTUsk&fyuduoa3w}G`Ii-fByRE*XlORaY&Ax;2q^Y}9DeUiq zyMK?>G}eX;GkTjbS%GZr z5T&~;Y#yW|>Ep#W|B^P_r=X1$4uFNPGyw?zjr2lT?F6>ZQaaY;=%~?w4R^35Z=yWu z?(pW}`Hbg{7^L5u3abb48R>Wz-8&e~ld& zG34mkg*Nsz8LkANRe$e1~y0OAYcFkLVXfFw#0X3 z=EB)RkCjS-zhk?;_Eww$ZWCeYf2AIt@_v0+O&5H%+nUcKQQZ*-D#Mj9~nh zx&c!=`cApy)!}O~mTV6{@dbum`*7{`e8wKXQ$qf(L_&%pEl%&9Hz4Ua`%w=5(|{Fe zG=KtAxRHvVR%isJiC+qS)RMDX`xiqORyFg!x&NkABWs5}rYfi3W6r|&5P*I>{#$0n zSspPdl-FAPCWDVqU+`hp5SJ)}U4;QxQ*A|gM$`7-D_HnBBw1Px+%y8Fr*ZBkK&P(5 zLO)g}sM)3#vqJr|zOLiUYMzC)Ip0^+BMHE(YMU_d9|WolPeKCgmx*JYTE6;S>Wa~2 z4x7~9yMFQiL85QHvJtCUi;sWX->6#j?bP;4-B$$B=t*-7v~dwa7d_l5=?cxUgm6Cd zaZr_|B^X5;{k6{%BEZY5G{tgIXaw~PMvhi$_PDnHbyno3v;_gj5-=Qm12)lz+O@kglm5{q;M_RZxMCq-* znMrLfk)rYkS^lo@-6`Sd+^FUeRw9NYH^+}naYE(H+Zh38KI`SA9vUIYM`w7n(({Fc z<0<5oW06nE*}@UB$5AV7a^dI2srSJRcWrClmn7EQdBmJ6?_NrBl@wo_%pe-;K3ph= zN1j@y%^Bw-|7I#-OsQL<1zRV2i1N8h%Jz zJwR0GxN$z5cL7T2`h@=Nn-d!(GsG9!?+6zh=pQ$E{l5S3TiBHQ1&Bvy(*8{} z3j>EOJw+p*2|#VfcRh@u)N+@NMx-@QrQhRg>Tr5cY}aHl3CA*moGLkK0}rdRVR=E^ z{#;gyR7l*RccCrEo*H}%3X|@5YPQ+FM>u|=k#sp-M{J+EGRGl7LH4Z8UIUZqJ%O1S$-a-TXZC__K^ zV}HQ($I)a#fHDGwtEVN4+}*Rszq5|ewZGZEuA5Iw2OpA6%g^thr!`g2lSe?v{V!Zs zZR|Oezz_e)(WIs7nejBn3Q;m~{el(T15QaA3slu+pDiHa->pWfN1C6rVtf%}cuYmO zgKLKj2iNqdxC5nzUkN5bWkY7QyW{~Jm`(yqq=456x~COUo&to>DhmwrE0T1u8eLBX zmGKaO;crc6pm6&VjM@?bZCAXTbba*pRUvkbglVZYwEkF8YfO`T(Y8Hj5McaI z|C{H>yx3qKlRMuy-lc?Sc1!2)CVr8jr{HCfqbxH-_?m>w0h)fl`U3oh{a{=<4u=GG zzB1dSG{rJNtgG}nPU<2q1UPrW{mUkc8)_`L7OAnol7dZB_a(SX@-|yK8Wwm(0F1NEm_aN1wVsURw>% zPcJ-K`1h9E5@B%#7Tn`q0}2)m8v1N;72R}2#~JeoV=z!u6nMx5Hh%7WcQf@>B}s}j zpX2a$CtQcsC3W?=6QyG8m#bS^7MwKolNJR0blaxwZnvS?S;Zd`$Td4sdlY4B=DpVj z;GB--4WcwwL>bZgwia+-FoH)nTd?9m$)`kWfURntsPevI9OkDUq}At_Fhr2*m>J<7 z|K^#22*1UDq{{(|XIx*ulqtAAdQ3OrRygED^IBKe*@i}bZ9_@AZve0qu;T?J2LZ}j zw%cP}y=TD%H^Z>GUW2*063o&E!US9==;FnvZpXFNHRbelmmD_~T)}7{w z&e;xBEsak%$=pypJ3t9=dtnbS!6w40@X`hEdjEiR%*$gfB`8X5t54B?{Y@k+{O-C( zyWn|kD&H^1e6{Z}+mjH!-{_d1n-62-&sj0eAIe`j`?O4m+Khn*F7;(ko`grc}wJs-Gcu{X=-q9>JtlE}duQ+wL-kpryH@ zy?9QcUQwlU%a{$3@vO{6uEg-;vQ6$i3UQK;nO(8qR*T1<;wvvr-5aev6Kzq@WY?yI z8CkJ-_v2o5#Cy<>1tkp7W+umyd18ce*OX=Fs(i}ooB^lb_(Z+B(#0c+peWSQ7vamb z`z_V8WZ6ITb0VsHVCX3uI!$aMYq+2H_VJv|H+xOae}8%g0Ho5T!|3N(fPIQlqqpY} zehINqo%!U~bwZHmWWWQHbG6yOu;gWGMqLHRHz7_bwPG8clq4AvuJY+yO|fZb!!O?8 zu}-gsTJ7>_YGOwb9ZP{7Y~E_-54t0uZ3t;;kkys%#n||9@a5P2V=teS?-R*n9l4LU zX`b4bjK#bVZd&U8y01tpmu%od$DMxAMMv9l&MoL=#mqz@UrVGR_l0_DR1(?*60e1Gde-2*c+IsqkdsUBQplCu zbAh}kVEU~E+wWc#ljwacB1;-}=6;qO#+T9U6+R*7gTqwax52TW8BT?9baXZbe&3!{KI_6)y4?e%W{LkWI2jCl?{Trz8L**uH#O^Q>E0F; zvZVDQPmj+y3P_#pP5&8F;btP7L{R3-N@^b&z}P6C*IselB-bHG;@9&O))tmx7<0R@ zq~8V%kqZ)eZcoE~O~sQ8B8+i&1Ue*r4H|9dY8S&zqWooS;5LT2)V0emG9SEr9t7AM z08Kh_ER&MkZz||l>!~yU@mi`?QQ4AitwkZp6F1DCU$U*G8x922-bf6%3pYrD#i2*< zwpz(G$kV;(&?c|bI?kVkWtK(xu`&B#;UTMoJn+{-FXYMJH&~sfC%3D^A2%%pYB~Fx zYFb@KR!L)a;xpqnrzd^@O_;-5c!|es9)R%NkQ;Y{;h&+Ck8^jTn&jZ}P=M)n>!7A9 zbI=`ms%#Cn4 zcD|SP<@REH*!8~greM*drUAx|97aK~i?nk84xe+fW zZ{VZUt^WcR{^_IyCA?BgZ6gdxVu5?G1|-aEz1&EUsaWP+cJ~=7?fk17Km5W&X3{&= zr6*juZl+Xa>izM!qk7^<2X1*30KepqIdjyV2i+e+zNXSxbK7Tpa}Fm~tK0+5Cmz|g zd=qVePKdNVx^>DVw^plZ?2M6Lxb`!8Ti#RkyDG;^w5l=4mTJ7GuF?>G>j?|lQi82< zNSi&Ar21!4wJGm%haIm3(&qHRaalgKQ+Zo*VUmdvO3d*r$tQiZdevGg?sUI{@hBMB z#c4dG%$ziRt^bWNf~3wy9fsIN_Xz#^hwnqZ)3n%{%nU9mIShVGJbF@_aV%R@{2`Bd zRRV1z;iLf8vnhQhV!*)}h_XFiU+=HG5zruPk-I(^EEW2+SP43iUg88Ktt+fn{a3`C zxH5^rzt^)}NibifBptLnWW>O$q<;o81Ytp^|JHO2c^)R9nQizz@=pua-L?WcDwzsk zqLYg1NS}l0EoS1SEwfU_n>3wtIkq4r(>>1vzP9Z)u* z7!cFZk(y94Ta9;@KGI}VuVTz%OclFRP84+NBUYBAN9)j18h-Dk(N_YxRc+#$@;E!G zk3>;{dx`$+A4-y+OCDz=U?O~&oq10pF2=@SEP`h*hn*uC*BdqRBV;NUWL%7GQwvf+ zy^@Jg8oV=aF&&>FIZfBUhPx!`mVdKBuW_kcOjuX6o{4h~GUS(Oc#=*IhjnUUK6V>q z3|r^NJ1i%lyLPs-RMaW{5i$=F!>FC4M0Pj0<<@G%muXC?eGi&&ai*KS|^#9Ba>V z1r&49PJmi&clkkAhrws5!q)&@Ng2>63rG~VPQPfM6P3_7JQhw!k2;x7`97!rb;o&f zj*N+5e^fk>D^vzYxcBT!!vc`_!+5f!_>XV3z@oz}r2l;7v?ybOOoLg1yQEm1p==et z8!M{V&DaVz@Xg1^2sOzN<|B~4p!Qqom;IvMJuhY^iq(pcg1vcJBD)9j$F|MVwyRM%Pf=l_jD+NyPHL%YE6 z$(-O5y>IX=Oj2(?JA*YBgFzC#Ok z9`8k0Tqim&9(eUu$uOl3X@wSOFmmcm0q`1mIA64Ve_<%3$nNID@10j(FXICMN0-)z_1h!Y(wFt@%rzn&KWkzAN|(aV{DA=J;-G z#?ZdfVo{uhmv0)tmnXPt7NlYVPN%)+Ps(HATs zB#a;EeCAVi=f9W$o`(OvXpJzf;CLh}-04ibR;6BeF3%HSpb7P|@BS;Ns&;?bSOo4F z4DlH!B~h1(AX80$!u6fC-}OET`Dlw`(|?}OMDd~ z>qFr8tnPYIjcmoZtVUn^-ei%&OQA5Tc=Z`Iz9m6b8v)SNDYgGI z&ufpuaQSeQ_2BtH5K)eKXd4pr>O-P(?zf3-LUZVNwLsusL-~7SqM_*WS%%V#M4_TG z{P&M5x)q1sQS4zgx}C=u@Q?t@YU*P&n!}ih@#Hx{2kRN*I*QhP*keYtJ=k?c?y9!B$5bcgrQql3d(MDOE& z$&4)a62X+@f)63w)4wmU=x5`h3F6ai?c0HhJ~iZLYXK!aa#)hyA>2 z|mZaulq=2%a+*w}~-#`f<0;rmBC$8kUReVyk83I8Vz z9h*!SORnHE+X=(t1767g6#NDfz8iGC>whkQKj)G}l@4r;Kv22N_b&h+TX2u|j7#Oj z(K3uiNL1XY*yk@SMq0V^nF^C4tY7F%Xkl1!XVbIhi9k&fR@zT?lM-aSH@RdqE*fzT z0x=nU5YhN`oe2_Me7X&Slwrh-emZTam}o^KV=~utowP0%qBQVdeF^BBD(JrsnqT=g z0Kw~8J^_6p*PaLgV@w0$mjgf4%j*&bCxW;?u04g`wLQC{3<iiFFlUUNQ@-0`3U0PTr^* zMu`6+{ji*^jscj}HzT-Ix^mFBSE+}Zet434IpXr-z;GbHM|<6Z$ud>QLOHm$q>Yj? zi=X^?XVKh5dmh63E6q?c-(MkM>f(9y>kJ)X*W=($$*zh%V_IowxHcM_Px=q^tBS~D z^CNokYN*qIzqTFLw@*J|W1E6Y93dEjFM7bVH;omm!&C=Z%kF zDZ!5^rmEV)HlD6O6Tr*st_e4;^fb1cMxb2+e*K7{dMXd+lY~LT*&%qoG(^LQ;xu2U zlX&3i8OG86!Vntf_USh9iF4*U|J`}Z=mVM)PeAs{D4WZ*4$7P zB%t)P&$2Kr04o8Xy;J`g@KPzWe`1T}m6IZ9MOy`GPfato?=$ik(>JsouPv<{^B1k$GpotiH# zAFc}^jX-(p!24l8(M&7@pUe|Pfm=;J8d^`&7M`y}lC2ikiklLO3&7s(v`TZM_wLvp z)BGvu*V#(5myOg0-#f?hZM~gOm)pbI4r6l2`c;x+BoKN zlf8pTUa5LIE_z>y*IP(5Wwu|3hR`D}LJe2Z{OO%LwF75itx_bm2;*V*L_d!<^U`113iZ?AUR2fo{~@G!O7S z8ry*a+L@ya1s~1tUwKIw=9Y$~W4(^vWXYd@p8Pzd41rg5Et!ZFn)0i|BZzsFQS{Ma z45FpX$A2OpdxJDya+vhWuRX!EMr)~=G60EB#(9=Cm{yUH#1~9tH^>Jf<0R6m#c}G< zi(K*ezx7%l*|KrLE}7Nbi?ghND_o~9`pZ1q-*}Q*Q`{_{6rWZ;i3So3-$FI8e&&NC zWaY{pZS>)b>-mE2`c_1^jB#|!C|63e+q*hQFKyk1RQ#UTkJI!M6}>*G=VmpY(8bq{tn;^1f`?7^Zc-BLmxn4n zI7ms3JW&2@wCq%Iun#b{=0FF4fUU|6)~D`fAdrMDf-%qb7}(_}O-Q%nk`;V~i0&E` znTDr*@a5IOZ9_&vz`~lLmNpX8``JG1kxEJD;}0!4K|3<0TVqBa%r23*zlrBZWH4U0 z5PQ(DoTHN$fb7YEFYgjdU<)3`W~2TCFZR=#A)q&Z+nJ$iP35--s`>pS@B(Z1_+$t{8(iqnGXFSA(Eez$U z(rAcMIv(%#M&j7W?q4q*k#Rn$E zuip+NtT*wwH#{;4u5GD8u}hZ<6@&20Q`j4GxWAW}!MyTY;KIYKaj~9lLj|ADb-{w> zXQV5^!qH%Z=(nxMKm85L9tLs3cFQNel6fR6KmK|2x@yy>gzqqyx%l2?3(eDsLCocG zdslQ2dcLqbO%Nc`$|v^)KCTKql8YQ&?l90WQGtlNjj$*dWc`kau){M=;cMhq|fFjQ_6$TE)+((=L zN}9jU#9gO~MwryIRsj`Atd^e}?`()lD^;B%s>2xr9u$3Ux0maqBQ-M>|74?_%Xg7K z!Rj9hvpde``3walaYgh+!5Q07qw5!{qQ@py4<7ToKiaHbesEVf#mwc)!Ha{sUwaYR zYil{4w$X?jszTm52%aZddax+>6ZVji-I*L2fukc8YS$2F;Fp7qW|#QMx9#UKh&WC@ z@b|j|WKkGzxI%6W_|)$N(vBy^<2S&=M}T&+nZ~}8nxXRO<)lH7nb=UnCA)@o7GYXG zo3mta!~WY5Dh@By(QrLSG!7x6di% zS9=>}2G(da?F-j0X5}QM<)9<2P^&l*D$0iYCMgnRBFhgP;FHiQ{{xc#7njIn&F46G z?iOCDCSZ+j2-Bt2p^J`aBdnQ2?1U{L4m?WeF)8Z<2czjUtR`T$m;{Z_29g z>0R-hEnP?RcHD}C;UCvlJW`!Q#=eH%5m;&(#~y)~Xxx)!XmTP*e;VXL8x+aO(;`p| z^Y7W=lRA)%A&Qg4Ci82P=5l54I9(e#7KD~f&prgcc-_0=Y$*(6kGR#%a+Hj=nMsHH z{nStbI?Mq~mcO0m3g4GMOW%!sg=~(F zHo*;$bSAPDVg*dJd-V~f&<4;QrUGPQ6G10(WzW(3hbT`A_0#Y>R2$q%MZMcYywII% z>aI2%Lsu?S5d6~Z&+thwjJ}cHCua1T#4KIVsE)J)J~nf3t4Di|CU2=n)FGexBvJ*U zcqjy-l@EC24Xf1KX1_uW^(#D5hrp2oIs)xY*_=Xl}7sic0DaxuVQ;Vj(H8jl6{ ztl@;=7&sO8d1Gy79NJS|g5yuZoY}H4{hxfL0oDiPGb?VB&s?rXwe~sbb+Sdvx96Mi zf7XvCdY<~>#8qEs6=adRIh)T#cly&iVqloGZYgq2DE$sBY(0R;w#HyO5m{Xi|j`ryzeJhFvObXi}zQ$^dkUa z8-=*j7t{_XJ~$Hv+WXY=obm2O&HfejylNDi~KEqaO>WLW#z~4D&S_4?L?|I7O zd9bOA>y97h8sWz}k$zJxC8agx00PU z=&q>}m9ckFl0H+8hHU7@QXQTDL?Q5QW~dH6U!?M-P2yvDhHyR=*S$jlFb&0tEg}In&YcQjdt18>ST2pa1*s+G_eQ z$i_(cvP~<#>q^Bp?-6%4Xz=QHw?E&1dQfBsGqE1{N7)PW@SLg91&af=IdJ<2o23%I z=B3MHDwg?zEY+b7?2pWuog5RCD;Ts$p6L=wk|sWaAE$aA+6Z*uB?%5v$opCbw9)s| zLe|cu36WL79#gea+kAOY86xuP@wbA8`P>mQkI<_463)vU;mhz}ev%wYe9GJV8DG zsI*WsdD7gNyjS4W75N&vocg7{z5xOXo$IkwyV2@+8uJ0z_5FJ|yr3t0HolQ8DNX*! z@UtBrYSwpRoJm))>Ui-&I|GfHtg}9}+AglmSHBzP+5p0(>?gKNG`pAQ!o9wA#@CUV?kk=n|xk;NAC7^On%cCA6GUg(8h74Mx zmW0D{fTc@BUs1k3M=8z#svN%Ei)~)D$!SRh)g|_VkdkQiW;lkt?N}oDiND=P-Idjx zkXC>GUNXXJwB{;*6!`ng08u+T37|1I=G#2R0wvra0A!Sc!<9r=?}l{$d_EW{5PB5< zwUrHoXWjP(om^Xc&*V*LNj~HwO;dHpPQq`eu13BY+nHVMI=pjOlsk;VH~8AK#p3E# z1Ayw~&8+%!P<)FVQz)NqdGfTyNTcPU!_)~5lQhDRYkp zC_%1KG3Srg*YlBCiN@6Rz58(IAeQR&A_FooBDOZM83P*b{nB%0neKaT#g$Y7rGmbH zHMCz_Yq+w?u72_rRDz6F4}2GfvaFfx80_zu;fIdvk1$FYLSXCbPQ#V%gzb)_Nq(}y zU3ZOC)Aq>!)bT44i|W`IwFgrG;@_%k*I%D4G6?l|eYRk%UGdM|8h^+cnFz~LymyV5 z5h^5j|4ieG`CvT0^v)hdx>x$4e6v^czfVQlAfgj#Fy_(pxneG?yXsOU8$@^>PX-We zw`wab$am3g+C&Uz4)|>7a*fvwKsEZ&?Ybqt9)qDXf}-cC5E22Loax}F)rj@7O7$(2 z?!By3nfztcBnGSUa1VZ)041(8iYs;m!`C^1Tiyg?|0l^IwgFc*BSY;i+Ru*Uh}%B( zpGlO&;XTgsH^=xdf>7^jmsz*4(_pfM?Wj~cXnBx z$yXh{O^XBq{@qVmy!3{Fe;!W@={=aK2j2UzP5%pMBJj0CeFX*AMz0*|e5> z0wrQ0n97T;j_W9N+s3LX;fTC8`{qy)IZ0K9riL!D!5uE5b9WPVf&!-Q=RVOjTSwBi z;k8~2s=sRnuy~C3mJ|d`StNjPSpD|gN1T; zzn|xTg~NK#smNy7NR@gBtcTMt3~%0kdbzV9%NPq6P)tbZzz0`C{C#mdv%>;Ao>|XF z9T!uW%f{;V^q70#wi`Y&^GyCG4UkW@$`FG>2r$|+R>cng%Ay@aip@1NWmZ1+gcN$V zGh=iq+^Iy7a|>y}@#KfqSDsgM>yr($WF&@~n1*KGhMF{vmm|Fakd5mo!~zM$Gew zn{T}s^aD5dq_;fJQ%))f`$5s3r1`G7tNu9Cv_YzL=G)n86=SkQN(esj_>Q{^f$Q0l zj$sILcM@Rv$kp*t$s4ktEp{iiV&b;eWR+O7^3?$9y^dc_N(V^%wbpl*ZmZW}s~61t zC)3`KlBcpmunVa)|J8NwWr3e`izfB^AQkzeKpWXQY){k@)2p5_!R@8GcPFT#3p_sS zU2P7<-pWbsgYLk%M&LUO#ycYKV59bKe8nkHyyH-9+I^Gtsekp|x9$Vh6x$K2JW4MH z?B97keW}HJL>CBgaJvcIuqZwH&v0t{zp6rmOjcJdt=5#U0gz%O;r5BPbli`~bn-B~x)jPcuX;Qa4p=fVKCY!AcXB)_9R@svcMQ3a+3Qf#anpAW6c zy`hp8b*Np5O#tA*6rhnIK0?8wYULw21)NewAS@DQyw=aryfmQb0zC~6F(8jHAmH%yD&YeYF3g2R$mBpYO8RPkdMs{f+{XJILUCPEi(lE9^uM}al?6z}`_pj_)mbUDDEc^i26 z^#|94ClCxrF#PNB6U=hBSP%DQzhg!rc^sg`bNY4$x@IgCJ_Sk>1Ce0sp47kZzXIY9 z|7!cT`@e6#M>bl%n(^E0X@sPdj`Wk)&2m9A|eG&Uv*S&;NUT2*W&tD|}H=7Wpy5$Op4C z;lrxxFPj050yU58a@~5snJrO;gF|XTcxBFwrycmk?zoNvu6Cu}Gr@DrqBwXLlharC zl1vBO)RIe=mBUAV+QtI_*stF9v3zwjExdyrp!b|Em z^Qi{xZ+SxKi*%CxJR`=belBN2@N*NRaj@ydsNK{UIK2gkP!gwG=z;sfD^oQzTA#La zO5vBp_e3}q=cE4-Kbqa{n-PV-zF=n@csZ2&dJ< zfPr0T)65}Y8PR7?#2yb`jv;P)6TsvSoOqenNdzgKy#1i7h!>dojt|V;PIc}Z;55sXdP=l9(^p|759HpLCBthH#}Aa`oZ`9GAO=*n{lX#bRAm^gh`ld{8~~gycM6iYEUB7zn&$9I}i%`)4W;V0V(Jht>^f zV!k8yO{{Cv1jw`yBk8d85UqHM5mK#FpJ3fnn2WQtrDy9`CEQO68Kxw??(_}4`m&iQ zn>(Hh5S=F6y#FT24V9j|Trq(4`!-UVkr>`Hu!LD=3vz0ks3PQsHSoStgeYXiK=vGzZpKaR8a6rQN!4etGo|kBLTOdJzt8YADqF*68=L zY+4i#i9+9$xs`EF*s$V5G6!#;J-EZDvfDh2F4xfkUa^ny{IpzpCqRC?vPY5~C+HEo zw2A<6CfR4qiAr<&J`>#S`=sNLi@g%rg=i@z|;p+JN}{J+d~3!bwR|1_p_WZ*zFg8JdY2H&$(=>qm|h~`0d88 zWfyZh%%J_j4Dq6hl=rxTCAnU4frH$_ytGsCU*D1mn`Z+sw9>F*#!002LkOF@J|RgG z&VYXmonzYG{uD{CvS4 z2zvgHZG^kGrEZme_YMX^>Jp5Ekly?SG)UqM2$JF;2kQZuO3HlZJBAWt5XB?QAtk6p z;PZBUYmLv}O4#vA`t8Ta9W!j|LYfuO*R{kX~Gkj&k=x{OR zgyuxc7eyW4QKwM~Y;XaJ4k9|Rj;;=@E%@FF)P+@9Wx#6|HcbPs9Er>v%et4vJrx)Y z3O+mlAgaHtAg>Nf|0Z2za?+B6+hfpony5lDAE$d(o?L1}N0%V|tJR#e1J<;%&1W}W z4sdoDCj#!=VGrjHHMfK~!Aastb2s_g)o|qjTPwpxh%bS!912Ze_R1@tsT?0hUX>l= z0g~f3qq>IyyT|fEsc3UU%%e9f@6tYuSbu!PUgly3^o}%#>ptxjwWfP1pM1AwR0`_Q z%ul*q5UsD$nLPe0@(4Nfp56?GD!KCH8Cq7Ut-*bUr}KB^_liJCg=aP&2w@$IA|4wz z09gyWU?8N!5TMlMU;(rK)zk;6jObF@{cH>4aH;$*7AvDf@#!;Um?R*(8&!b z5TAj!VC4&7_>dCm<;$(+T{TeoPk0>2{Bi?uVfbTXN!yb(S#~8f2){1p713Ty*{jc_ zRf2HseOZT8+!fPXa&@%N3i994vCh!EtP(;}!4)kKE%-$Ir&(6wqjxugE|6~v?;rNi z^h=ZRn^;Nzm0U~}M7eO*=BYA-tWFv8ZnP1qe?Ete!mwVw)ZOGc|2qNyR1{vBFqdt9 zt8xG7xKiWPD||`~g42zB1A?)^}Kb zHZN&k&5<=QopZ~J#!ma`OZ1?J|EfUB-SQyjl4>N4fd(x7L!Tv?k{Xl|Zi zj!2NPdK#Lr$aN7wpAeRyx5Er=tJ$^W!M|(Z|tTlIzdC>lf3BIlUt5Nq<^Tm~-|%FF_W;5qeHfl!yrS z9V6$z>|&Do^kuvZw?FH)k}b0zXk(QJeS<=)fX#LP&{-( zR1mXZ<8?!2fYl{@0Ezi8RS2-g=bTa3d*Q&5p}B_RA`OEM>K{D%u@0Na==gQGyV{eE z-kFU(OR^Kv7pt2ORs?Lq@qv7IXi2vKqKf33 zR~4e`{tcY0mG_o&UQI&*yPiUi5dRcXr0|&)XZQi&;?5gVlgjsGONiCF!slVgk!>pJ ztZJM|yhmK~(d5AOK36q1cB9m~^hW}b?T;y(@{Wy2Pli96zt0DS-1xLeo%g87+w+(p z>nEs|=n}0MPb;Eh_?gkGvf)rv3^I(x!*_Q~yK^$LoJi7p0jnH_?F3AMe?u6qKfACz zxBXJe>2EQe*q$tu`?_BD9)1(HV@WigmKpH)8qa8vN?apP0c^wh78>C_RjVEiq^C_M ziLc~F=qyRnDrNWFk00VNCHidqC;&lO-YJo^ilZH&&-2-nnG7s%+mw0h_s~!K*O8R3 zdXceMp|+2$u<*a4dybOy{rsWgc1HcLhxIs2qQ3&MoFc#~p7=ka}> zSXC^xPkO?8?qUqhJM_C!S!&(m8G3Jwc`Rc0Lv(=16$e0NUMq zg&0AcMq)4ca){?MH15c7r++038WzbRm^di@BInT7Q-|RVTyl#F$ zN#cH-@iNC$)^ouQ!q6}$)J3U?09q+e;jv%7R-)S-Tg~Fv-s)g$Za{wkkBTK+0U;hs zJXGJte6PM&iTX!8$oZr`sB{db{2cefDoJ1AZ*D#m-oYZdmG{q?_rL4IK4v0^_kBK= z-j#xDpZt3e8`$7C&CK}3T!m8lU>~eN6kQ*41SgS%V5hKZw=j)Y0#FP)dY2(Th|uUH z*sKv>v8vZVEx?Sto1+TzzFaFnv5g#17WrL9fQ9+6OXt`vpdPYF5qWs`#godJitEns zqdqueW_c6LUNyQ!6e)bV(zIh${I@c-qB98Qqq!2VR${EvJCyR!=6RF<@y{hl_Qyl2 zRdh>gWyr&rj-TmBVa~l0g-EWuk#WqPgx0ure2V|klh;4=KQV%yBZ<&=`Hd`3vbOwb zM`EK7C~{MW#PqMwf&TJ@9#J1^mA=^L?)=LLp?z4} zz^fRs$dnB19)LxSBwkz09b)2&L~W|Jf5_!{@4+(syl>;jtxMRO)@!;>_C* zf|Li*srkh>E${4jGP6<;xw<_rokHRO<7G2pVd?P#keF5p9sPK4xZ#+U7-rMwnLkG= zQp}}lGrZ!*cZq-z186@_t{%;RgXMksAD(?aQ)6-CqZ=`L_M!Oh1Io|y@hP=8=Z;nE6WMYM!8hA-?f{1$b8cd%+$!rUIY(C?#tyd?@}8%cbPu%fuV zHmJ?qK(RGCn^1^sz0*lppm$UUzNT_2bypgib!{*TbgoE-8kMliGrE|*OR;L`nD~#8B-YU(wWNs_(+5Un**Ep zff5*To$NlVS%x59R8Luue(S12jXGt_L*fDL?dgaseG8>+IdO-~L@F|zkWY>U^Dh1x z0rk7Qi)kd!8?2c~1Fy)kWslqI^)fQSdt)j@1z`Z2M)M41OCzTRx}ZKg!ot(XDZH5;arI>LD3nB^1q++cv|OT~`i z8ZoAX%GydeBvt!>ee56IT-VRx%(otrPQUJ(00XuH?IE}$Y?tClldCSub+=SuqEB+D zkt!~vrgb*u#_nbS1i$a3D{OkQhQ9C*_ovEATl&}ISmP<2KAlQ_-Grxw;okhm`w5qK z$_!LEkAFQ2I`dNsF(z*}iya2}T2Gyy!JHg6a?(VNYQ-;G6|4Wf_7F}vyw!Qmqj_bZ z4>QdG;vN z=^|&NU-I7b*sajdJc@(!q=!6FXSTadlX49Q)nc-2%~l9^p=1bvHRosomH4qXkdb@k zwK%z;z?zgB&4?-P8#|sLzsT z%{Y;tU%0KwHCb3~$ktLakPPO$8i3d~dkjW@-}c&{roA_Xy008E#BLYgH~|6E5d|T5 z1-=~Mav%F2rjId+NmKW#&3}4tNTnvK&2WU!&Nh^Zcj&P(k)yJceJO~@ zoS%KO6uItbmOcCzhD!{lYhWV4@#fZO*oy7o-8*q#kz1lxvw;y#OF@^7UpH9N5Gr9D zYX;BMkr2>|+2vZuzwSUhgC&IIbE^sZG9UEj@$y~S&z<4_c`&!!@pbI=$YmMMAVTzP z!hhUsnCf~c_FROUC;_J{ehp==1oXfm^pPqb?6%TBxJWN{YB}-$xNgnc47!yy?)4~9 zW6^M%8DbP(-}y*_8Fcpo(^}Ga9~-mB)pA8)~?JOV4olI{h0(@B+Q$xC5d~le-8b& zY#`>{j%RNi=Y+3Q8JeK8lqc~AWDpn6ABE0bo)xBW^l5+iByDp*_AG z{a+ch7yxnh2-*Dy0ou!wH}(i)Tdy_C+LlrjNC}H6oR&W~t|{>)!iqZ@y6F z{Z9uEMXfon-58Px??G!D5oo{xn_qE58U8r<{UL@3iFJ7md=6aaM45`lyZE<6eG8P0 zM+Mung>esC$yKLmsfO4+x7~jV3cjMTb@*iwBQd_KiT~bVMD7G_Fp-i#3Ag3VvwvgJ zeDa^SDwA}O33bLZdDOqk{PT2>}^ZuiwC z;D=h{g{AxG60UoTEx_=y8X}RY`67bD=rAHwZ~`vs`Cl9+)W^D#c=^|MK^l0IzPS41 z>RH|V-K#!>g^OjYfWDh6G?-KFP~=n8*#jfad4nU}&x-_VP)ifu|NZ2NXLv%`xe)Rm zaN2*^Is&#*_a^vh`05^UOnY*g&NH5O**!7oW}4H9xfyUZnHgZ~0K+~v_b!(td%2#s zA|rICEg_#ru(Op_*H7m-p+vt=$fN zl0Qxne}1|j#4)x@(su-^ZXsUZ&0`U>#&wsB4sdxCkP>pfg9q8I)PzY^z-%`J?NJ5B#wAUF*E2Sh8%o4VuZNg zhn+rNdZLtMTj=$|uiVd*tJpT=#8*~vliD`09q3=`vI~SPiE2whwhMl##D7H+MK?>c z9qx91xPZQD#cTSpLwZk5pbp&Wau1%yZ&}IM+_TuhJ}t1BDZ>aUr;y5D*_dLM_>Nhu zW{83uG!i$muzqsesr7=fVVV|SlyYf&jCFxqiSH+5-I=A@KglOh93TnIQ06WWwkHLi z`0(;_E#OI;>y-BS` zRm|I);;aH=hTh%rn;-wey*2XFe+YF-UJX&cX5d(H!3o{=vw*t1xcbYe_}x`48RXm( z2qznisI9=Rd#nlMm0S%6sVZoNE5d{J7WmoU2tT+%aICh?!;F{08 zghazF>D0pG24#JQ)Ma6K)cNP>Qr8}e3zM4XO&dkAwC6^+Tqz0GK((Yks9PR52Y)ee zaK?{9Fh z1OzF{6Z6zi=_B4F_4tM&(p6ufcX59*0K|pS-EFRos`0#BxB7L5LxZ5_UPTdAX^u+4 zk$9hZ+`{9j{Wzi@62z>L9lE~Nu3YmmKinE@mFXWlux76q1Ml#$2J zy~IT%@vm!(DmvUe<1z?0uks9UEt46=ExfsnMMi5nUL=8;h@pbhLh_fZRqa!_-VAAd zZ4kcH@p+K$r|y5suWeCLiF|VN$gz@cGdn9NDaOHVBs;=*wIW}drsdk;6KY3lo`2{AI5+U$BDWJUFm)aqj6;(x(Lbi7|Yf6yphgBoS@~ z@&3jP+jYo3-s7Jh6Ll86nw__T=~6!L{6`!G;#on#%J<>gaa>pc!8nirBEEOvD83b2DkFGe}n&vL_Vt7~BYWb7J?oTY5-bIK) zp$Wj)JV^Tv$30cGG-B}zio@Xc`g9iODv@tv5F<*T9f*EXNsILj(&5p#`)vj&LmKE@ zJYK=(vAM@6xoIfSeNoq*%i(xKmjsrk_OgAueO~k`*L~Z7e zG3nQs*XWS(`E4m7!$u$_u$@tYTjlC(IjL@S==w_alVmiyuJ(^(Bk{5D*_u!pd?>(} z^uz1f=n5YEtRF!919q7GvVTZ946bY&zn`pou#&sWCoFn+UqEnf?{`r&uIVIm^~=t0jOnZog6W`^$>?)m1L z2WWq_QHkKRuh>q}4<3bzfY;F?HpDLG%OYwa7>9-nN+Ul$mb z)}d>ObXR{(Il?cG)(n0iFAyZ)9h^xvS4GnJ9BiMuw#9}|PnZ4``H#`sEItn+NY_H$ zMv-g$J)?uqt%56~B=5pwGp^d|uO2)V^?gePPWIHo$*p{ z6+>TaHo3+CrpMqvE_U%n%+Vyhm-mR_ATK2a?1MwQ%*mg=@YteVRT%l&W=yGK4z;hMYLiI-d7jH45`uo~Q7q7}y zfK7gF5dWbfX3pw)gOG;zXTO37mt-de`NkO^)!O{6<{4L)>i%1|53+~T9A(i`akJ^c zVFDALp43U8v>D_o9SpxwQi_`DP?%B&Ku-1){GRrlX=HAikQD)Me2ovR&?D%ca(EBy zc=&6#_LtuIsY!%%sA6fY@p~ziWhoQ=OCt;>AmG}gWuKyRHw+T%Zbbhx{2bgE2x;5! zB)Z951iOh|T-)vNQ3|j7e*I<$-p-u(XT(}{B8#*cX%1cNXeg+HS=?>T`tI0~hTw>N zhzHIt z-wJuuWFu!DV+jd3l5|wjKaQ|98RQ;JOz;H4ncj#z+^U` zrh{^b3RJ;17r6k%*gQr2UScJ8CD{Z1z(^5DtkdW}FR`S0=iBIWdp-)hfq8OYqaLfU z1j)d>Q8r|9uSww}e2xa&1zfFBm|-k`-&=jWhFe5At#mxI%{ zxjnzZQw#Kz8CyxCor{W>(GN?%*p)0Xv_PMTs$O2ZtL9|Ug4sOdsva*IZz%yyz6G$* z;-;YwJo=@9yjDSv?qfC`PdR~rF{7Wd);QPDwHYZ!7!Y7Gm~U! zPTv^s34I*{I?#&xv?sFNk?XNy@n%dg#LZ~za)Xn18G{%qTRd_Op)?D{3rivId@I6w zWO>o~SO{H*=eR5;{Z(3$xo3UK!SZcP9P99=JicQ3&^^Dw^?L%;Fj+G>Xe>|_dx)<~~ZxS{*H1P97@Za9mlfgC*wjU)~yV?`)M#>TrI1Q(tWCw*OwNV6^i5qdA5vX?j-LrqYfo7yX$8s?i zB&WcgzHzMi`pM*atDU{M*6tg4=^GUi0(f9>GJ;sxPN-fqYe^WAM3x@MzT=A*ViVp~YzR!-_9svJmMlBU;YuI& zB7T*I{Ix8mee5wL*+JO8dUtdMBbwX!t(~x2fO~qFx(8f*9Neeg4#bHB=YUKSmdzEziS6~iVSC^u(*farDs5R(tY^Xw6_y%; z^E>>!^z6x7;=2R?S(xHg#>*bjZ>y12AMNW>=vUWb> z{bfD^cEU>vj`kl$t;6MidWc4%E?U$wc+7wgbwC7g>^gFH1o2o@d(9PE>al6T6J;pAt)TKLm zG5w}$NZ@v)%JyIY?_6iiObOg2t$}0#g|R3~p0~x^h4LjU-918XT5Vz;XmRa@&Ycu3 z)(0M;zK)$F*|@oUcs1eSgQp#Fq&9Ykc^C_x)1XTA82F*U+S-Oo?Gl)RDsMpc70trd zg3{VgqdG=0Xlem!%O1q5_Fj|y<8stHbqkYdB(dUj%{tB8qLLJj^v^mPDp^~H?Yw_~ zkM}I-*RTA&g+nbnt+uww4yo;%)&wz0L)F6@1q$e>4xDKg-+Bjx9RRI7H`SOGIGhxG zD$V_3JanT!yi%WTyM-NfD8m|uru{+MME}-aT@wny`_(~~bd+yN1DR4@833DS?Yqm-|<5+gF7u)C>4f?f}&Xc{@vbRpcB?YG2!*^m1M)UieMh zw~N)&APr53HF6MxBukt?E$KQC zB6A}^=jseIY#R|bC#fB9q)U-tfj;U+X^&&GiiY3hT${ym`!k$>pSFA(8+*`kFHK2q zAzFTtdV4^C+7<0JROnyM>u0C_Dqx*`=y-KKDM-PGzwiTFX!XdJu=tEBfkT!=(Tl@2 zz!_e0q8m8?nYo!t_k9D{N*svv7bn9Y-9Y^K|9x=S6m#G$rc(wM0aXw+(%A(J6C`6S z+jY@&Q3v8v$9>(}aL&d)Mz+jc8?^qi8FJ|+3TS_^d-=vx zKFR8FKAp!#ex_PL&W?_3Fw~_S;9jSiqaVR=65uVF2ImC3+dre!&uGe7NGn>-_jI%g zj1)1_#*OVA*!_CK(Ido zaR)cL>XJ5VK%w3MpW!cuVY9{^!l)JzJDwr6Wt#I@(nF-1rw-P0a_b2_`=<8rYuS%R zn@fUwb*pJhgylPNKPBuoI=lT3=wNYD@S8PXU>Ng(7z5dny=~6v-k$-tPIftYNyJ>U z?xgCCsQddaz=^zurlg+=_-(qqp4(*B$J19*IALzYuZaQ`@11i_r(kQ$$XLPN?V5ul ztIh)9K-#Qb2YiJJQQ=e?GR;ixB86K%-GlKjt=0`kRqn(XMeM=VLhc}^&#Nrh!uS!Z z%=x8p;9w~NqLaz$`v-5wrJWwMoZfd%!M#ExN&m;a5sYxy|6BkR&5lBpR{mTh@@O&V_ar;XKeAZ*~?F4PEGzjal z(F_R1QT?90Le7%LUCR^%S*B;lk?&Xf}{r(5{mwO-Y zdtT=}pA~+SSKH!J@e;dPI{T-7&!;Mo) zhWCtZ*wr{k8#RuE|LSgxnf`TL;vhKSL}Fe|-fQT_#Hv^@r}wor1OAm;t{17?V|QkK!+JqCehFni7@_sOh_S3HiwgNHRV6>J%EwIQdXB>rIBo^_yCT zUx(?^>NTtUQtkCi*6#=vlTx4KDH0{p%lDMb9ehT3K$6PS-39q>{<>NR zm;Q?W6vAX|ck2|BQDgYMp<*klK(QoAYGrbq4=m$~a^5f-DqP;d0LZwv)>vdBEqUwF z?B35U0^_!80O1I<#q$a!MkU*&>y`J=Xe70qdF45 zLGzB#Blk3N57~M-L{F*;N60obdO(5`~06DL?qHL$^kx= zZ&>@B(*8Qimsl>B)(;P+#*q84%;u=Ek}`aI!aucI3mFLhzspI#YoT0@i0}~-nO3_E zDiu&ZT^j5Nw_7~R0Uc8X{;+!2{NSTvIC|ETwaxem?A9u;`||VXmc*7E#)F&*ATbHv zj?(kR-LL>|!!}D=?QFPEMFY&xYl<>o-kl9bfhoN-f55_9j3*M>KMa%&U+A6Q==?T8*J;%dbIRf-;pYA&M@X;-D*1i z7wouNogBnKFJa&IvY1vA|Np5K0%Y}@FW<8GM&%{p(haA776W?f?_Mv${1}+&Q zwqiY{_>6{XZd(sSnX*69BnIb?zu+cD?|-WnbeUiUiP=Cb7RpQ7%e7+5?s6eMIPGjU zMc(O&B1N##BW-b~)1~Ec+1X2sfFAAk)10mHJw|})SYZD6SK$eyt{$9OJ5RosaMzLJ z@qN0pgrW5!b4zH;U{o#0Oxkph2JD)ao%=C$+BD)s}q-aJI zRv_?_7i8^a!G8}&9D*%hrhKzbbt~5$gZ}tty!?XPp?@Ohg+sdgud6Z$evIBSgEkXT zFr1qTb2_M+kCX*=cE4qSxQO0Am%3QRI=FZmSq1WSmxnWwXg9UZ0pewPh_EQq!vT$B zr>S6+p;SF961n^rFJk%>Kj-21{K4c)iIG$o^~lR*fyyIkfmj4G*VJ3y?UlA;T)-*a zp=(PXBLDCBos+S9)o-U49|Q;`3cK>Etz7xJ!nSU!y1itzR) zcpaG+%B%9lU;Vz;WQ^FyHr(GW*FsyJg463D9G~_TC+so+tAqkWkS-!KHj40C#{`l* z@5g&wi85gFTWcxhtDn3UdjRJ}c5X`dE&Yc1j-vS8=yex>-1SUo&?YGzuD55o#H zqu;vsdRpMw`G`-_89A+FfdAZcJ#8dhXy?z`q?WOEW2f^zGR>T^p?i$2tA|TIzp;O|ZwINSoEoHpO z^E$(+rz@ycjUiyXPQaOd?C_wNPj;M@oP$EzWCn~|6`|sxu74>Hp}A~W7KefshCT8b zZY3YJ-}z8ieFhH&N5sk1=sqV?ZB@rFo&V9j>vNdAyGs^Q74Y-L^v3&7USa)(Vqo1c z*5zUw$Za=yStsg^)izn$fK4x%YT71W=E>mxKY;sf4vwrkY(SY|Fjp_e{IVOMcoOc4 zBYBhHpj_^?LjFoa*>utBiIsMyQ@V}ACt~Wz&p*Z=u2;$4=%K9uhU=K}T6fqD3qnt6 z_Ex4S8z@F5T&vv?+}y$Pn2+97bMc2P!)8rU9w8Cxm-=O^ca2HiO^SPZ^kHQ^N3RZ3 zn+W1i7W+E(TVr>>r?uQoQ+&+)4>A`&%0+8##oi0TZ_aEC^L|Y{j6LF*@&GQ_?5jab zrX%chQIWK&3O!ckoBz6*12;xW2*!MMe)utN14?lyz_flV^mn2PeyuvTZ{Pz~mkkIT zr1h;iH3P;wql4n|Ul-NJdh5LF(CquRW$szN&1zH7&!q73bRHo4>4p z_O*+feaIKIZv$l?2Gf&nBNkyB^&~l@1^Q3dG@yj|SgBE~sQi*olYapT+1;qP(E>bwc?=sSAhQrrN8%ey; zNyxa1bNH2;zzrQCM0=>y?ZDv?KUsMKm%@$IezQbo_@!-LrzN8t3G=a3T@0a zB$-^g`m+gnEBCoI_3mL7Ge;chmf}$BJqKzRDc}&e3`-1tvp#zpbex7`E>-kQ&?V5D zkWlr)w}l|sG0r8O`?1v#OT6>NiuRwlNoE}v9m?EtsD539S1<-JyAHOvGW(MOqtivR zUB4Q;sFYMLIFAKT=UC1#c(OsEMdN4}N(^Zq&Z8jZFUuikG9>Ico@N`*let@10Tl(Y zbC$~O7v0(M5vm4Z+oCkt{#_J(M)qFM`u(zL!U213*Zz$$hVRCbb0cVg#W#mI6)wKqz$W>3pn>%45liDw^ETFqD7 z546xl)PqV8>K3nyXIzRANr|LDRv#!*t^i_!J?iea6g7O!@%edv&-;)sX=PAuebbj` zqEpWYQty;ciJrz*|Kr#seFjl)C~TS#4Ih^8k$!_A#CeVY@@!>jZ)W&*(%Tsr zj}x5JkSy%X3G|Zv3HdEXj6+p>{_qyd{MmjZ&}@cJp*ncyy`D~b>q7W5c~WvGCw9fM zNaFDRu#5~pGjbzF*2{1>A|n}^zn6s)%u+y$fIS8t{yUziuPEmB=+Wsbg3aB z7EG(0D^^&jBrb;}6|ftWg^pzVYVDc%nzm8BlQE}zQ|mCG>KU!47Otu}X*KH-1R`I= z)4z;tRejDuKHRN1*B1fL1VwgZ1>nmmpSO?Uj~`49|M#bIj)$#W9C*c>`Gehk?07k3 z(78ie-MDA#y(o2*M|;+BX}7$By<(i*_Xa##+seuG+HG=eH~@&fcYSN5-FIlu17Y*E z2_$t8*(BR_X4rhuvp+MTs9+YP{dyvo@iNGa-Mj0JtCoB-U%~-nIqt-xB?*}=> z!Q#P-xyS<}D9beLe4L>Zi=$P4<WAFo; z1Ik5R)Fjxf^$CpT&ueiU_YIUm`pf}vDZx(8A?rVxK4=Z%cKEL`0Jb!>PqtJYjIaDU zKhpWjZNCpjXWg}=86)5t8vLDqA>N$7%Sv93V{7^s47ba;MVFoI!dtYzOY4lLLHraP z{Y=_C2O5OG>}6~fQ);n(y!*!8gOq}HM&!ixtpb$Ui+17W2$zX+P@)YbqD7#Z7Uli@ zrBaXv_3QPT8-_iLxvgY&SSEYQfAa%5S=n{6$~%?4+)tzrzwZw zT9oli5B}_tx8nw}EAYME$%7l6^~*guhP7_*+|&J@9zd?Oovw*1$7qxG=RtGV6y%}b6qBb!V$-MA|P^@|a`8a$7bdCBCyi!vY_bmgYLMRl- zC%-38_HuR~B;;GTrED8rcYHy6*lTVa5=s}rBqW=k4$G%54}G`g`D$(!UGVeLts>`b zX&YhX&u!-8X@r_$1o}hKG^WKrW+{s6UTu_zk{_)}+9&ZZBNJcpnF>HJ+NF+zPVTLe zC`gtFHJvxE2sR`!ej2t$xyiSg@JRH|BE{jX_t8Q(xkFmFyo|;i9QMH#1m1AM)~i*d zTIk_OMO#hM`sjLjqTltyON}R#ZZvArA>`cua+RDPrn%e+5=P(<;Ah-3Vz4Lp4N&LH zxFthC3Pd#R>3@5}O64(uVZdIEBcGWk?Am*;&Z*F>usHRkvBd0*jQpX1?*)E^vjYY= zYkft|Zv{4_FmNj5&HkCEYsu$5J_r{A>k~PO_(1dJ=7$%DC%FOgM1$sU>8Zo<+Fu~p z*Q=UeemyYo&W}*W8z@1xM?C8KxauaW<-h`Pe60YT8g1atirF9wY4CVa97`{%{wv=; z+1u@n&6OWdOYmOgoto`9nd0RuKd&>1RD4LX^hNVT`OKcfM`ZyXMh-4fLu=X}QIxi>8fhws)z>zwT2V&}Dp=ov zjwy#+!j2DK(OvKeb9YW=MOyD` zHn>&8`!8^(u#|n@{FCd6DQuAQf@-&t->L#BaUzQUxV@5`cr*+w1yMhf)*=x zoV}dHfw3C!V@7Bp$F7vZWsJ)HjZfH!C*S(Kb*aS}>Lp!YXOK!kJ0i_y`faDq(0{xD z2nKPgCy!f>tS;~fHvM>m#5OGT3{UYbx{Fk>IQ7+)$Du0qsu}JQUG(tfXy{piOu5-Z zkz?7d-zLm-Kx4tYk?-DXIZ15C5PGD`+vJw90ZrWZxLXgDeIEVWy`@oi_L45W?ta$< zBh=UUHB$jU0?W}v{okg+(3ZlKg*x%X zHC`?fE9u5v?B)a`JCmh5_IysX;t>_gig{wKP81wYO9{SBx$nUv9T}2xaDa9k!ka?4 z&DbUi4gv@;bRiJWVL>8jdxUYU;8Pfn1~cVN`R_?Xi*sJGfqsoCbiK(uHypUK1>z!A zzcac|az+3kG3G|YIh~iHUwuMQs#il7Q@XDR(`(c~9Ou#QwU7A)c>#D{mj$BI^UsQB z7xL;e-g|u2fw^<$3=5!k}S?Xg7AhdpF^JUM^F zOR=@eQ?P3G^fD@hAATp$c>}y|;(kFo=|N_TZQM!K*wUvt|5;ABU))UOa{#8T8=p!D_~U8%ME>V2Irm^m$HnxvYMmNC$e1*MOmbXBYvJt*bW`1 zZl%R~Z_QFf%3Y7re)wrsQgiulGeY6N<00;VjPvB;e+PpC|KLiUb1}b z`5L?bC0VV^IW?ALoblV0#V?F57jW(KJ=;y%-;bb&k6> z!0N^Gqu>83e#7WZ`$k6l-^*%8ft&a@uz!c;G_D;OsdUPuZW_44LXBQ__Q(5^QL|z` zWp=nMwRRArI5a*G1PRzqnKU?jGy=MOA_knp2fEImd2qC8-M1(B+qU9O?5FO@g~`q@ ziUEPRl!rvLu5hd`=J|ojU?xJ=48cAEcC|Hf09TKV^Gf?R((Vw{{i)&#Swe1@dF_ z8bF7y|FPH!Ep$bKrghtD#m02`dBkvBzdsx(W*XooPL!RJ!_^jDZTs&a*I7Gb9M)hs z+C!(PgGdydXSb=V;dd#1YTSeYb~XavtesuF`G()j_UAli_Q-qbh5glUxc|&{6hQ3r ziu39m5)Z6t@7`?stYxs<7WY~pqtLi#@IPZcv(q0}=kfO9b4hyKeyJRERpi3jWuj3Nkcbl$TzOQTl|+a_wH&*%phVtk^V1ad--#iLN77V8e-0e?YT^! zf-HP+q75i=@h@uR7aS)VE_}KBaxahk+X!O%uYwB^P94otejug)@7Z3Smk0BMn*B6v zpMV354hSh?c~e8_r?@Ejo{6}9f-5|!J>mlv-R*u)`J4n;0UmEd++l+HQ;B>mZ~mNFY%`>JuCWKvbnPFLrOAxRE)+Xt}yt4YA&DG`lK z`7y57u`AO?yx_);#vn&)v1!MO&1;9o=l0aOqYy5ZZ z1?$>YqV;%#ds``o!_hVxyXpE4JEWHC@kz#hhZ=;tt3%0+z@_d?|A=NJD&79wGWo%P z(%wYTgS3r(0p#bZS{*x`8XR_0`thirMoGNqs4H`L`5)xT!q;>7s9dL4xF;iAC0TT1 zfP|s#-gv}OAEIj?N;S^BZe_oQ_h$_6gddG{ndaFJ z{3p4o5Z?DIu-fPK8|mU4dE{&pq&$9x}{~okfwzMlJ+Tjnua5nC<(Ge85&_ z`64SI==z}c8cueu@#f|oSyG^N3$Z*1>-~;V3o7|LKNe0MKe6>STsPbFOuZRb!R}zz zcFz@_i*lB(^B|J6rrT@Ya8V-vq)2Z8opKVK%SxV@4qOB$aU7e~1|>Mrq)Wa2dn^4Y zm8tFab)!=tG_x3jYhEmbe+(G`QT}dF#Ib_W=%M`wM5y2}$XWzOR+r=3xSscSDy1VS zDMimsiD~n%qigf;X+yE6@gt_V4=(f55_A4Rmnnmf8;gu<3acYF1ky+6-Zngk4|cA2 zgyChD{@&=f@4)6atG(O8+w0Nk_yQW>Y0+t2cJu`UT%6RxzSLN`UK+No{D8}$MLe%5Z7xd$z7+H zq_va|EGiLjYcUH9xi5511H5|1&kfa(>s0t#1^eMm5GKyaD+bCw4xax^0m9a%1R|Dx zEd1+sv_CkVrIy+^Txtd5L(1wNn=$)c>tu4w8r|#J3dQK0&F{aK#t1+sat2(mH(;1Q z=zOg*e?=Bf-e6@4YPMFKD-$^Q3b89UL9_R&L9YmcuLzdv53gQJm9)qglViHSw&l#z+UO)(6kwwhneyUv$=c z4&H zwY{VMxu?@_;7*V#@Hh=vZCQaooPCl(v||t{?w>40S2k&S{SArw1YqczbymV#lKXp8 zO;TC^Am-wvjQs0`V5sUl1pWa6(N9_h5cXaCl0X|bH7VOGLpBu|aOXcb^mQZ7+-+O+ zWwZi4gZ&cX_w_olH|F?d*Hb|E#Gy?T0);5%b}ajZwBJS>ncnpO_Q~0L=a0qLSy%}6 zKkc>Y?byWMqTL(ATr`x@r>T2un1M1cX%EEnEFjYmBdkmmS(^Cx>j7!31XiitqVsOB znK0ILnxm(VD?VS(^6KJ7L{&UuPOlF8B2Xc6>l@8>FfMw~Uvb2lCe{AqC!Ooh5t5rw z?6#CBZdJhUx)B7p}ImJCvuH2<%YgQ3N zo3;Os4HJxYYtnS|nqq`9$%vK@+m|f!u`nE@_!nRDk6{iE<4Lln_nH_&dUJLNe^ zL;DS3P(xnN@w+W))Rb{=^V2_Wgn*P`Oc{ynf1NPseSdg(lk&Cq$u16Z{C6B}4U>3=a)uaH0tg_D4~#r!ql5;4_VtN_)sb_o6B0(t)Ip)X7Ov6~Dq6e|Fw zpYm&PP(C)k9UHm7pwz`QsMse}gOYyTPDS!=-)-zNft-h!2S@euiZm86!15SCeRqgi zAkLdX*>8Wb!fFq$uU!IE!FYLRwmBJy)UGoQI=ueX`R!K!#1H?To*UY^Ik_oELCR`bWUXv9zn_v)e@D^=;u0Ms9Y|P7MD&>*TsBrGq4f5OL)4i# za<~Qos`b*53M0X?HI$NQ_)#qByNegESw(?*Z%Redvh~ZU7g0#cDI!|kO^U&R=LX*= zTG+}T_B%aW@NOrL+x2`Bh@`rX5OjKM>X*evOD7%q`z6eZQ`95xMZO+mvc%^?7s2=+ z!->Ust<%q(IyNmoj7YCjk~I&ry+cA|ZVL@7r9>(`^UeL`qbxT7^y2LSD}RQfMNO`c z#C=y1FC}eK%I}%m?JBhm3KObP#m0}uF*F}I1WFWN=XPH!e-FF!W+ep-7Dv!#0PjVC zT><#uJsSup`*_0S$2BCogeM{au9gl!9Zx)o1ml%hpa0lQN{4Ix+Vz0K0`Mz6?3avC z>ly^H6DRA1-NqUA$~IB@9Y~D1zN!^nS|QBkxz*K$P5IuM>yqotF(dxh8LY3k$P~GC zJNQa~_+Jv;ALsBCMv{41_o~bJr1kzKu<+UsY#7$3PuDaIX$ljg1TP?&c8dun`b6f+fPmOfc3*voorAuD8!)ALz z9zmE=$M(#ucTl0&f)2S$r7i%;8K-AK7e{pAhX6C}_7JKR!Q>=*E zI>zmtr1{dOf&z64lKZJ(FOABJ;)6a+3FP~I1>%;DVV~|x*b@YHBXHT8xY8#0=_2|4#`FMq=gy>8??~k+8Sri<=(^<)lp~ z(x7CwP&6=LW~EkW(uA;#Ip)W4GFVCdNL+Q3??o6xP~>Ize#cgUbMRg&d~VEgZ>@8D zV(L#8Bhc`&8jhMSpM1rQNcvVm<^fNn(c$ZFC-Z^v6>d@A48ne63-!K&@ezQI0NjcM zIm4fR4GVL52{XdHDj*+Mi0hq&PoJWMUGxj7HFZVAh2mzd*24onvm)(=CwVs;vtHb! z8(Nivy(f5J`3QNSY_l+kQvB7(G}iQ}XWJw{Rh!dbV;UeCP(eyS67`9(AOJmjvm&>$ zlAFXdqog{#Zg&OlxK}*-bZC9|lgrsqFXM(dbfl$&EaITOcg2A1wRA9|>s;nH7B-A;3h7$0;GOCM$ke znTned0rm$g0EK;N zDLIeIf4j~~dU|lsmuP;r(3G|gn)sT}*`Ie{1`H*kkBYZo{Da0SjiJl}@#nQ4HCTB1 z*ev>vS@?e*4;J6$pUL4-F`U>sXSMh%;F!^83$qK*nu*H!Spn#m2K?M`f4VidAc z964PLdw}u+G{J)IihQ#->zC5Cz&0Sm4}6}{*YPi3uh?S!^rTi>QJdLk4=~-7{QmA} z4usypjbj8c)}WgdJTLz({aR44rW)!b=(}?l55%NpA?+XY-4xE%MgFjYyi~y_UIw_H z5f;U*%QgQZ#-w8p;=|WtO{BNd)`}++rUNwaSKbG&Uq?iAq6rm37QfK3Hf8u1>9F_H zlYwaAtw6VV1n%)D_54O9xasz%W13G#^IPnDh4W)$^XK&(Ev6=yoqx86hIr{(YcPjqnS0dIglTK*jWdpr!eLkr;J&p5gns&Hb zc`F#s{4_L?{o>36d(v#65)*xDXY-LoHT7<3=vBza)TTL!wa1d^=By(Cz%w;b;g1@kCc95U9Rn zzI~K%GFGB(eMqj~a2Qcv3U@wx$6heU2BCF-EJyNxnruGA;cvtJbL!tlfVM=#lN{#) z4NK}~@~oVa?IvH+2w=%!tB7+bc0Ee*R-HnwFCL5!!f)jKj##!_aB*J>ygA}LGXF%f zm=XTk={<~2?$JeLLi3HD@^Wr|%hso?!~gVcGA7=`l1|sItgZ>L3yXP8Nc+#4J6iXJ zsWA!cj3s*FHLRd{5VSdvK@CW8t@5YDi$txkKc5|{c6a>2`X01E~3MgRA3_ws31vt+DENJiEr8BW+} zv%`C)s0`sD&%b}}b6{5l48Ko^Zh%fS(lKeqLBrgy2^mt-T+2y*@(<3}+>2{?xG5DM zl;?E3zf_IlZYqD41VTr(;C)6-CQ6#s=#KRpn;D{z{zg3BuOx4NyF|>LU?^S$VXN>- zdX?KJMwNO6QJuj&m!|{tYVcod>XJWAmk%Qd<1UH3e z3yX0ru`B%}3b)_}wFbrGL}5hZ($ThKeV%>Ausf!PTlF-bto&kBN>u&Fn+@jK8Q`Bi zh>v(+Z<>M%m*Z3Mea=a?vKn_$s@RqKUf<~$?;eKRnQ9HnZ0sFa!>-JBuk4G?m90Ps zmS#h0s9c7=;?ab+m&LOS*PfgHK)>ZZrKfM|tgJ*70C&1t$SWOFxaPeaQZiW4^Ka8M zTEJtc2DL{C(F|^j5%Iss5ZM?>WSS1XfMRl7_RwT)BF8rWuaxl8t_;SO<7o*N-Q3X} zfEytr(d6EQpers`Lna?0+fgJ!GyPDmUu?q7{{@3EzvX(I)H{W9kwO+fW++hAtP7$`Y@-OyKm|JCJij8#Te4JE&w3oa+S1`XXN4^!2|7Wsq?~-;?vr=a7N|`_E-FE zEPE&={pK8g?mQ4v2GXJ{W&?+FOUA$Vj_rBh=H_%mg{v8p6!%D*2z3>!G*rJqni7A8z;wiCOhVZt;3!|9xfM-^RWFyi{)#7W_zr{q67dT1+DxI{BvNk%ok zo@Dd!DU`@dQZ}=Lr0kY3d;f{0EX&*+^g&uWFP%PCZJ1PlQ@G**JQmp`#Wh3Tu>ZwN zsXigqr9eOo7g?vBcP8B|Z22-m{hIlvsc-6xW4$@6{Fs z=eX>H3uwH*eUQjtLAm1cgY83?^BG#+@(*~RibD}UXfAp4(F4PvNukrBruIW22l-~v zd>6Bg56qE?YpbrcT%KPP%7Xz%WWjA;2O_ zzy0!a)Wkby1BaVnMdzVNz(TRWN9GO2E%WjB_8W|TxL|G(fjY<^1qm;4#Ci9(1a7}F z$qz(1QUUpOICJ_7R52-pMh6<93VAyj89U9(pc}4&nT?H~c#cy@ECDB_5||$G_#1L` z`{>zqRgXjx2+a!sQehS<8!*+oyt-=ESJU)=Xv_l{H-662Zj_NQfAV`Kmg?J*xPjXB z6ga{9RaE#UMt=Upy$J%3zq4<&r))&V=vd268jsvXDONCeRcq6{4k%0v>&7}vVvY8G zrvWEdqe^V9rEqzoiG%Z|1Rx}OsCtJL^u5-b8f}V4!P8EjDSpd-3-D_i`C4;P4pR7p zt4KrKxV^f#xB5dO!e>_%~x1xshps8f^f6`A1 zTP$J76FV&k@?A=>+lptg7~$S$;Mrzq?RJ+=nzCZ3rZwAtv>S7GQWA2m?tIcvk>WT_{TrDw+JD;PtZ$m!g7EYLiyx-oe z=3)h5oijW@*_^?OEaK!N=h~;WDdL9rviT=0aeU0oy-&fDO_Ol-!vOWFDpK-4KFHR6 z#Z;%K5Gn9ablk@?hF=p6Y7>TYFT~+}PG80Xu(hE6>)zt_H-B~&Q+&dPbeu=0McUr} z$ukJY2TB!Y+&+Ngh*a8R=j(J!rBt=cGIHTVi}xyHn9Iy#=yQj4-)8NxnMl?pP*%%| zCnc?1o9QvN`z4`zQ^r)`jb>JMRUX5=4y=zpl*Uq|TGZ17gu7oSa4_ql=LyWZB&{%i zV0|rDaygdKrEc*zDj6o8^W_nDyQ$uDBgKFd0SXY#{ZTDJ6M9loK!q~=z7T=Hx?dzh zm_#@H2s=}R>?8pu?3l+Ru5X&tVo<_0$cK>>7y$n|x=*F`Dr3SzeP0ZZ z(@N7Pw6(s}73u7Bz4l9;AC5kvUueD~vDG4!vZ5c9r^O)KN zAn0{r2(q$0=p2>DdGg_mOv-IT13Ev9cFsJx*$*fFb%#aw)XnVQbO#S=zy~*MhwY)jvcFvf|jPcZ%$FHf|o0N5lk7(0qZrGNHD?@@na2O-F zV>$x}+&H0tgn%LGbn4O&Iek@S^><|WIsoyx?#{11JnqKlIOm{_w_bl+G$A9IrUsiWgU3vh@d+TIWa}S(L+8$>>$^$Frv*N4q^1ZC^ zTY}4;1P?jawj$Z$KYzu&lub|2mcQ*gAz%sf5FWbJik5d^cI>>!ocPMp->1T>6PXZWh<7+ z%lLTajSwXwY5XvA+tCL28YY&^W7y~kWI-vjbHMYf(i zQ{4-7L=Wk$pbzGoefNMPmn2F+7QS6!lAID!LXO=$+YD6Z#G#1{Aid<-D_a9`xXMx4QI$7Q$r6eMcVaGxt!(Uv8QJcVl(dBX#_m%**6G=*M4z9ptE3%c=4X~fj?BfrFRI7fQ zXC2rX^LVjAySbJh!Ogh|z`L{ky^lH73F*n(7a4ot@Gq$z?+T_d!*d!u0<6YO$dawkN;1(go^0Fo2ffdmob*hx#)5N$(+N_T9 zKm`A&y^7Y+Mr|QqKG?I>KlaGw^6!7jCLx>aKWTfTMZ36kpq6p9jgGvsELP!AB#BF!)?Z6 ziHwYt!-vz0%dgb$6zDmHY>2`K`Y2sLjrfoDlSGkoVWq18JP^@X@DqX4?%`N@)bL*)5)V`W5u-@Ws6>w8h~w@iDAk~=Y&Dj+al}|F=3<~6 zf5izR$#$rhj`sE5YMGAnZt0Qg$#72BOt&JVl(LXYk@G&`kEZussaRJS3pms3_^lua zk}O7D5EdQN=0z1Vsu`En&P$sVZ&Z~ zuik`VN|eO&Db7)6YtB{?Ouh_2NaXCku*)j)jev!p7~a3(Z>g5I~{f4I?|d7 zWt>u6pM}H+J{Mc+8R=B~J%i?J(msew+X@XuD>f-qNv@B;`t{?upw5a#2Q_3xRbIo3 zL&y+sPi#q++PvA&MX2dwTX%6o>s$A%O-J@s&I+TIKDcwY-Si#JpyMnyE+d;ImUVjf z7oV~-0eXpPrfEzl}FPi=k8FEdXH|ARpw5J_+V_9vTtP#b35y z-F`r>nXm_b8S!_)(Z4xgP0`q3MV8oLJ%FFZNS#<$E#k3D%SIzeG&J5gk%ZZ4tbBcc z{S3a+vP(i!LVda6u=R2hX;_g`RLg5w6VX;eBB2!JyhFMNhj+7P^L>PcTAzebQG`=E zIGl~XzW5!1sf_+_>yi_%0bITNZ4#FlEbvKZsM~aq;m+o@z*@iM(bJdOdH0yZ>(|HW z{O{iqMm~`4u4hZ^5zxr>g<)URP_!;*&2~`4QPBNIG!5y~4Y@KHkOxO0^{TyqSZ&ri zh+m`#w!eUO*k2Nl6L4vpAP&X!U^Wf}(}Kz%>@{ge!}^~(-@!m_;;lID43G(S zmMc7-3+4RkO_d4+Gx5f#R-6^Sgg?BWo+#}z_!hmUY6y}~Bb|gE?`~)Ncj*lF zxm~F{8QZkI#ynizt0&GOr3J(}{8!NjeJFxG+nTDl{j&V%&?{!Y}a4 z-k=?%dL%~3X|3!Ujizd0W49PgiW@dx&<&#sMhU;gwznSSmAL~oaagI^4iJ_vZf^ZZ zsR0fNiWz>Db3GTbD&9y4I5pbR11{945~N_e8*j5t?oZva8-QS^LzL=H(f5#6=K}I2 ztzfJQ5;F7qR&6kT+_XISl_s1wWe`W!56|(zm_*%I@9z`)h5E=Nkn#DVYOdSj>~#@xg1do>VbZ3I&YPiX=G zsF3stE0q~1#!aADQwS@(`{X?%sFXa~U?8wU)0t)5N)?%+FT3YI9uz<^C?oak4+>pK zta-`Z!I7VJ6sgs_`A%m877UL*aw2|-BgADd8Ie@6qVTI&um?2X=y#4@YlUDj zNdUPKY@qT<86Qy2H?f){XVWtPDqj4Mk2STiQn>SRX5NzXpVV`uOR2Mv(A9vXiL9gKK&|P}GAM=|0^Aas_|a1xvpUdfwD!d|-FEB;lV|Fpu7>qR}qU$cKyILbUUp>{m5#j-_t zX!@`9!3)7e?1)FmT>xHZZ1KO560#`|moyt<&P5o}n_P8n=y)8xj+z&~H6iw$M+fzA zd(4!_%^U~?;a1v`KQX)tRl2PipwR<5lp}Rh*S7BtkZ4Hwp`uPKg^p9sdqtj zL(-LK9GOj7v+8(m3c*Kv`eXHq{Pw%}K6nY2SLxk3=<2rn;toGa&HB?Xqy0yveNuMd z`0^}zC`rQ*sAA`mNlEUT`BV8wF?3=$Ofh2<1@J--CF9(bjP4w8-39tdO=lK6;Zhtr zc+$o-)Nbzq&C^Or!x( z8A*)EpHX`0UDyRat$#0i{`QqD`Zv;4ix4$&O_J3OxABRpnF~06X=-K{Wc;)(bbR^K zzl}s1h+jIw9~_r}u_}l4+IBC)hNh;9V~$%S)6F;~iUV=&{M4g>9+@bf!G?uf*(^w0 zhGN=>#};(&jw>mE;1q$5z-7^^DCpeZ+tMPPDy!4&pMTmERlA_#U~|M#0S#tZPD$qz z6BrvLt@%(Y1&05;su^M?G7)l&p|KS?6w&Etwkz7{N^7Ti>3scv6`hGc6aF8^UBx#_ zCCa&!tCF))WGh1CsN99g8Oa>EXH#TuIYx+8lB-C`S(|(A$z6`wm}_E(W7Ce`exJYL z^LTtd@AvC?uC}?z!xkmbYed%L7^70p18+^m_q(UM#nKW%-OT>n+Bb+l zSqH8|`QAur+(M-);uX>tGc|kis&JCVLCiFTcIM*wLY%(W#b3b1A(PkVD65)K756nZ zU!1QDD_T(#ojel4xaZ=|lnA2wdcIZqO_-UrL~QZFOjIuJ=a4CWL+<4QMr#Lb=G>r} za}UK&8?CNGz1K^f!ekRokg5?WhAa*EQLe@kU$}BRBle zl~PIZkT17oV7f;I@M%24qOn&T#%ZhjPw0jl$xH3&1x5sALWow&=#7V%$|iVNEQO5p z4LqBiwQ&839J^6njLC@)M&JB)*hQr1dF<4ckKyN~1foa7T)D+A&o$9&94Y+h*=~x@ z%Hks#N{-F*wd0&ON;QE|2u(KiE8yby>4YE5&N$D|BXF_KlYo55o*(+2bx2|I4LB~^ z?5FKhc*p7S1e)v6Uy3V~x&nX&>BuW0ARwK5fJL9vPRPjbRbE|Ra*&*Ts-Ylh8sI^X zr9a8Sjk^6c^+DjZt=6CSeiMAPb}$oR6K{YWK2Q-qOU-;B4YhktnZHXPgXvpBeN^)^5%}xrU_rdc%d33*q;Y20HZM&X0bm zJO(=|)FlC&4kyHGrYO&qQ%GkcSR^c`9UIE@a&8g&rXT?Mm70nBFOpIC4Ila78t!Lrq{E!Q#_v*6R__?`ZP-ZeUz8`VfE{dGtsw#QMg;-0?0H%LxEK6Nt`L@w4?%v%Y=A~fpKd# zF@^&oS2_Jc#&&4l{aSvq-Yq({;}!Vx^8NV;pkgF#kiD8YREuKq*yTFv_#>$uRW=pU zjs6ku^j~5Z2{|^MN+M$%cg{<&9V`Gw60eyyf>9JT0q{M?J44f}8|zzX2BOWQU#jjZ zB|5_0pjSU-kG*~F#e#VC+6^e^FkE`V45_yi3TkvcnDI|#e4*6e*=pr$npT26OV;; zGS?{NSCyn1Zh!e;`expBc6$a~E;o63zh|YEaX{ixwL5FU_#t}BhAE>7bSv29=Dj6t z#O$Y|?9BgL2aqJR{Z~TWnY*W5sv;Rr4=TSMHuwnM;ST5jsN-2%ddJWIu+8{Bk$6S^ z5_Y#~rQQcf)|MCnZ{8HVUtRBU*uDLrdr@Skvl<@YL9;w=DwlVJ#;CqnPrzc2NtsoP zH=GQacFI{CS`dc6i8?w`Z2B3h_r=R=Z7eD8Umwa?I^W0M(72{;AX9NroIOx$J-avr z3D}0M39HmE%>&R&Mc|d$V{B3QMxV$WQPtcb`ZMSJ7MmfF18xNsRAHPfp3b*p7&*Ro zMN}7QMXfURQxwV$TNL>GLRc?+i3~Smjo99t80Ffn=MMKZ?9VnWTd&dYhy66ayIFY) z+=%5P4WG-Q<=}k^1N;BAtI|${GL#rSkb4uTFedDTJp78JN;b}Xy?!$ z_8rsf9Kt?ghHm#EMGY=|eHL8EIYn*925V#!w_+K(KezLZrq>}Svl%M|e_ z+2yZ3ak4Z&d?KjQzauYB0|ef0?|ty<4moc5Tf|7N(zpN9SdDl8@N!qF90VGQ8|yzK zd5hPFE@AOHJZ|{*q-aV$)O3-j2}|31_uf75-w$4bQpzvzCbi4iMtC^7Cn=>Gy!^#G z4^aK8RPL=auT;#@St{gdl%cUWXl^4!VG*@5_VMXn?=@RJ$zl=xNH4wcovlDccc#*8 zb=#*nMKzMh(w=y?!DqN7uR^Wp8S7;63ZEIv+S6(ZO{IQ8DV^D}jwueTTtE$N;LufxV^OO+#+psO~ocX-5I93%G6mctSgcFPGgxBzwLYI5NM1w_~nX{A%- zQ~=hgA4ezp@&>B)N8%dXPMo`!EA+VX8YxrY?LyLm5k|R7Q;J&c%a8+He}}Y*d+7ot z3jm=ZNO5QRf+MK_3&U9h!ZqQu;(&A7wl}{Fe^n91bm|caHnK^A4akvWjmIw- zR>sehuo(GwESIH_SFPuRA`b^K7W5VJZ6cUi4e!X-WiK9hBCHFF|Gk=*bQOK?{Dr{p#W(XqZOk*8qrS>u z=a;5ZQ9DH_5r&de032c*a?-p7T6f`b9elxdonok5a6mu#RJd4)vgSlZ`Td=nHyxP6 z*_#KuQqrJ9kiH}ES)RHw@yeYEJ7g!A+;4LN%5mv9^=Z?Qv+d7V7Q-ABzB_zFrRR$XL;n*&xnB?%ty0QwqX8=6`=H97Add5 zgEhoA+cZXOo_Rr4E#}}EZGF>C2PRo{4Zu~+J1M_6 z+B|+8Jhpp248{tsGq3Y>pI)@V>; zn&kyfS7nZdJPeDd1v%9~SaTIr=2<`o!O@uM!(F0RBCM#=>0R=5Nm;rzvuj5^YidNF zR``BOU+00>{Eb!e!mcB5>#Gp68Od{|L5Z^aqVUT<8SabV_M>tJuJE)WP7dbDL1ONc zVrhMivCHag8PMlW$Tz(z4(CqBszunvuvkSD?%TVrM2XFYhbQI!`?&Yd(^WH7>d)!< z{nN-d#(qJd$V1mT9cFja#ZgNe&LIl$?+Nu#BM8v!;>SfU5iv=uhBI!-aZ>>^(A&U$ zHh&XKymV0>zYo?0R)&CSuY~j#cxv) zI9T@!Jw=tz?c=Szwvt53?o_uPjImq+t2~L48}ewuEXCV%0ZgRBE|^l}vZI2)d7pXt z9%rO;7gnwd%f3oGaOd1+fcc5Zrpv-tC#><20gn{Or+$3Vv9rF|j1_?Aeg#6WO!RUd z>+nUWHMda35L=2@S%G)_nl!mh|FWTrHisA%6RK}J9SMXYVkR`s?l1D*oumUChlgSr z87&u&&8+F6UA5d9`kmOKK4Fxd^77`nwmOcJN2~vKy6J}4bbl4Q!#8;XVdJMp1;!H= zlbbX&P^%=tQ4^8*7-?N+G<}NRJyp>=+Yxm8r}NQ1cdRf-kaajIMtE*W9u%mj1bZCV58=2k zE_ORNGYs`vC#>wgbSV_ZlOPO&UMj~%5e<1LsXu|*=|qfOymXIPRHu7kQn?H?J*Fo6 zmF2{h2I}8NlEo4;4THSQ}dFv3UkI?<)NqdlxK@_#9ti2PrKLi%2 zaO*zEQiWN>(O=fO{uF#=(YIAyJrwNVslH3hQFi<*pKE7?MU1TBV%)U$E=R=V#n_m; z$i7*Vo}QqVOJ&#Mqk0TY7cUxfzg6OyLa*}UQc+A{e2C*w$h}KiFY)>QB#VSZ0wrgG z;>i+3J!SO(9#C%Qsi1E0A@JdR1W^P17T2A|*;3Fq=H1s52*~M|OZ(}ydlZ}ZUZn!` z5F5&xsid-4*m*Dz*lieL8WJg{6>kIlYlr4|@DMluPQzK2;5~`H8=nWtH&5}3OYWSj zXc4BFp+z&`D-p&{s;a*Z=rnB`IFBnk*MjD0FDg4@aQrdWGAYjj9$1Xu#pNiawx%+) z72r+Tv>&Yk$i)z9x(hlQ#QY&iLNk$Yy8Sn(l3m!Q(sqC6`s=g>beQXeXvB+Hbrdoc zyhm8{^D5Oj=PN^d=DrcE*LJDq&uc=fKJI(oYW`r{fJ=>s2MR9uZlp^l4#0C(w0qF<3R$nCK;ldd{ zlP=_V)gQ@d$EF&IRls|+6<}&70V>5YYmGBL32tu#`!&IjD+D-&05g~7bGQ$KOJfDc zz8}HR6%D6Wr-G<6Uwokb@(9NkYE%+;wik0!TSQdQ#MhSg8)WcVvb-kZgMR+EvtTx1 z=rU{5g=y$Us(m=sX>%UkT1^6TY(_HB6u~&HRp5ma;R4gfg9}kWj_h{A;>E+bznO;% z#LOz0{rRc%?ug%?91W~E6kU59#om^aM_;y)&mEXhS=KEZn{TaP?0=ZA`9y2flXk#B zWqmjV&|1>$Z?#XbEEF{V#h&B~BzQm0J!{M5PC!fX(0X_6UZ^IDa#t}F;4Zx5N;GQ` z-sXCBVR*&*N}_rZ$^}e|GWszC51zdRwJF`z9yDVT=^BEni%HT(76@%nv`2lO>kn=a z$tBk=3=Xx|XfnSCEK?Q*b+x^=j#{i?E|>c6NQhvHwRZ`)%&WcK{l0~<6CZL_ zBDeE#$JH3kt2Tpk;HpLYj%ui78J$s@f|>wxB; zV!n?%v@;e4kNmEKwod3BDn)&KN^wls}WE98?}`ogG~W7%*AbR-Xt7jhfh z#SZhfOyVPYs*AqSg?BQvajV2uHQmw_{XMbau*^&<$fJ#GM&Gowk*KWJdT3@}`F$qY zcOShO9^A252-M?~mBO|gXFI1FPtUyP5C={U zr9)lL_vbJvs)8-94qU%-fy3#QN2&nm3n$?cc0y&!gBLDfXy(T+|FG1R`FXi%WAxnH z-aknn@`?cS^&nt4KM}uRBU7;Fgr;uyJwXAIKY9HzOt^lVi;7`_E{&aB;uZgUdwm>}*NAV4eKUxa}N8$*BzCE}DS3MX>>eMm>eeYEy}#QXlt zX#Y-;I-odap3l4-13llvCJ6FP44l!i>s?B~Xxth_72%pV(}+y!p$8nGsyIz>sXE`2 zsbL=P%ssO1GLXRL!nVO7BZ;|V{eENNehua4>#T#1Y}!^B29^U%9z1yvkl#LhMGTZa z&rz0ARdx~F6zstom)bLkc4{6DbXh85}FxVEdkLi z$&Z_E!$W6Nxa})i>;>^%qF}fFbfT6#5720~gTxR{yR|%7m?!hX+T4Sf1Kb1Lvzc>& zfKX6;q)Bgq!#E9#{s2!dhkM7NyedKEh~fb~Y;y2Jx5a?)h*+zb_a6hV*c)x`;Q1#w z3xJ56(Thc9qEygNA%C!{`z+OlzSo;v0G3r3-5A8zt)@26_A}r>sl1)8n1%x_X+x?CwjqDxeM_(>kwQ?t zckV}7=1c^~J^588R}Yp}4M4jApk6l1qYv;FWwW93p6V})%ixtad8WyhYqet~1Gze~ z-tyxnHlIp#r#^oN1g}D_%%=DS%RY)@-3r~NPw+$kWIO+!f&R0I?>bH;3d468s({1B zXr@3jzvZZlCd}va-txmQ#mS?*+%=J;8yQy+ODkHXNTM4f38%IZ)hKKzkGPv^6r~^`$$~7=Cv38mE@XnbOb-2psK<3!<4&L|O{_KdwXGc%4-3eqSPFI>e zbKSrNYy76<*wnj%8JhrK%_RWj$LnccB>%+M*IQ(rY37Dw&lvoZNQ}~|Fkps(^Ouy- zc0*+%G#^z<8yYAdf?f6s@t#^S=KAKrhoZQ5GEN}DC%iOuZX*XDXp}u@u0xsYxW_ouBxwM}`0H_=wyA| zE8)_i>OKbmw$;eho9to8`su9p#>P@i{m>v!HYrMx`by5{s2fgqV%IN2u``G2{;S#} z7(C_JHL#g4!TVKzH-;cqyTWYUbYJYD51;o&OW{neeF^8u{&=>3MOrA~?FdpJV zSYd`@e7yIF=r>t}q62JMgr{OifCEZ+OqL@U0qnPCM~vzAVAWSinbTGsoAj%8aAv*o zuWD3^SdZJGJp`)nD#ZmjSqj)I^?gr($f>AJ$#J))lJ(;mu}!}FFX04CDff;uyZT$@ z44yzaWcc(;REg2B-keS7+|){0hao1Ky6u~P!(lZL$EGcIp3i^I>#mUn%_C6l5a^P! z>!#Rsp#cEt6KG$x)xQV)s9bQ9Udl5Q!j2ysPa78L&HdLqdHuyUL@dr}NJnn_or0#u z)ho3h3FLS-gf8mRizhfvtzM0;@IyPk-^a6h9oP}I+0o=6~N{Rb6BX3y4 z5iV4cW^ZW|en}IQMT+TnetP+OC=>YD9ENf2e>0Cg{8J!oHPOl6dW}=^aM*Unss)1+rbRF+Sba7% zS^dsY{r8^f?G9m8-(u)oUlX_hU>wvBfuHDZcJ$scFzxx_sGe>&>$_MnNuJCsS&yi* z?S#{Ys<=ZKzX4zFL(&!$TFy;eGq<}lHtC1pKHZ{AsJ|Suh|q}G&Hj5`YQ6kg>-TLH z@Kyi8(;^duC=6+%3mPF4l)6`@ir!|39??Zz7I ztV%vhgYW=#7VO2Wemv>Gq}*g@;q;+w3>`V;kYxK;6FPKtq`3YYe^ONz(}&E_>Aq4d zi=*$Z4@FD3K~IDg#yC21E&p50#uK=4t=!6S^zF}6jtF|OY2C#@@z}oC8anXk#M0LC zd+<`)JID$k59QE^GI&PGf^LN=Mk)-?G zAp#plve>m9P|9#iZEcyjfDFB2Y_A!F^9a*j3Pm!I-(LKYNI0 A4*&oF literal 0 HcmV?d00001 diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/foreground.png b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/foreground.png new file mode 100644 index 0000000000000000000000000000000000000000..4483ddad1f079e1089d685bd204ee1cfe1d01902 GIT binary patch literal 12430 zcmeHuS6EX)+pUO#NL3(IK|}&d7YKwF5CM@UBE5tjTBw4Q5KwvxB2pw25vBJIB27p@ zOaSQt5eZd#CxmkF|4+F-=Q)?(#XNgvmzlk1)~tDFz3+~Fs;5bRo%8yoOPA=i9zS|^ z=@P~5f9V?4rAwDs!Yjfq4p(5Rx~i8hRVUG&*j~LT%Q>2AIqB+Nx_^yhg70E+c&i!%2~zqE0}mxIX= zz1$7|sWj&3yL#7D|4uLjQqV+x(Rz4WC{A9|^m@1A6`BNi38Cf3B^aJyqxF{TjS&2q=3$BC zB1Fu04C;%o9V_Yg;Ed;xpmge>%b<|5q52W_pTd9o;Qty2mQ+-Peu)^(K)RH^d5byH z>AGB-I7$|~9l)J0H_LPDsUUL#brIHpjO1>dJ9@_5&W zLV)s!AVn7*Hy{o<1zLA_Ky-TWzJ_^1=W=Gfyc#1ssqeY_2ww>;ANX%JT)(9uNHOtU zeqU2_{Wu6pLvCMBLgy+dx=13ZG-+cMrBf;#8KezD^}_F2x>_Nob0^iXEv>aML;8RQ@@sN(#bq~VsOa>) zW9RDe#_!zLkj)PyQ<05AjbPk5yJ^|B6q=sMX2L0JE|(P%=v2$6+4QL)cu$c*yt`EC z?)p#@xE12zK?QF2u^(xb0>KieYWS%DH`?=eOiFd!6)WRmCo6Joq6}7e=Nl_;oNJ{1 zu&szm^c0s*wAxfHSlk^+hb)aB<&B?9+_YvxC1LEy$(dDJ8J)d!>rwz?q zGTpJ5&uVwR#t4%B`T{*~RAd_Unnf&`*9c^zbZfsVc;v*@=BHOCX7VbyhnS5G*Pik} z@`U!W&dq$A-&GCYAWg@rG3W6ANL_2a)|;&HJSig{zyfyO87W{;ej&@-)yx~eu|G6S zO)U5U?QD)!ey@XcxEKX?m{R4VZN!*V9gT}6_lv@YD^}}y4OM(*#%kMMBij<9x4*by zCkGRQ3vqoZ)HvQ4oY~=kh{c09u`@Lzqk8)3R+$+hcYuhqajQqgq8qWy8X_QMy@1+T z0&yU)D$XzuW+GZpAB%%|^3*{x!r`8nOWhu6>t(2mvERH# zwD(@F(UyHL)A@d0q#?|SOaIrK7`~^_KhtD69y6E{G70hSpvkOuvhEmR1(|2efAmi@Xw9*}m%vZb>kVqe?t6*aL%179k2-;CD<(T2&{-rQ;%g&4b= zStwf@&UH8&T6lBt>jybuLy}~>HTF7(kmQuR6(8*l&xSQq79o~y=t@1Z0aSiA&-LWp z0NQ{@*q$n1m#1Z}?sFj0=6jxX!@eHh_D<=qD}vOG`kCQ^44In=iDu`srXYt8{4c&) z7G9;S9(*ydG({X#u#N%3l}&Yaq*lzrY-E%htNRQTrjCrX1NMi~a!soU$|=0*dXokbDxSFnm6OHLV@%5(K&ZQB%e+ZFne-TrP|veCOrVj;0pG zdbMMl{Z%MBfVA6b>SKLi zXyRQXFc}Krl(owbvDh?Um&9l0#P)rbdiZxK)8=RY8XvSG1@0=@vGxtW|3E{`T&9Zk zC0==A6=d?8`t>?}z3d12SZ$YU4KZHQPf~|w zJD7n^6bjSS+&0Kq6nxhj*9}9qDZC~A`nzEz{<+9lxx)v#qaCsGWko<{ahFVncU-R|715> z33|Jp;8Iq?Z)NXe;h$K{z8#lRB#JC*XUod!9+#hCfkg#-^FD5Jq@>Dt!SzYr@q0(& z;I!1>qg(PU*HMX7>G-#T5V;IOw~4L@XQ&5le>B4Va!sx0P1pm1PMa!%L##WB{CukUKwQLR#mw_r{d1DneIIJT(j#O#-det^FD zbdwZ-8R%84+Bo+g5iyd(a6x;*5F0xuclibP*ff{7PNPESiBNJu^Q2?h!4}38?XKcb z1cb%?RlBpM10D9~`7(D`#uzQxY}K)shcU_}%#WJZ`~FU)C1j&^b5i=Wc7uJW8^-NB z(rs3^Wms@#S~)+us~_(~uocjV^vU^euJHB^upc~CY%6gqBXHR3{FJ}D^V0uB8xrdo z%j>^}CvVUV6jaGJf5i$e;gXng&>{)uK?nWhEUaVrv+x8njtfCz>cqP8uUTn1`McQ;CD+jm zGle#Cefq~0!!v@W2XnNsA~8j@Gaaj+fT)QzP<&gR$L=bGEJ8^z*tHxS)sZ=vZPV!4 zw*)4rK3To_7<;de8PvEPu4Q5d;D=g00$bPnaG|sEP6(kDsxwc2+y=l@=8Gy3^DW?X z$=3@Y|B6^8mUadWxX-6z(Oh@9|3%Nv*Hz=bA3)}AiK3MrA@eOvp)YSd(Nf|v;6dz-v zI5xYnKImXz)PTM}jxK=GJh_OrE2HXqKgh*KB!U~;4W!DpXN6A98^kNt%~i7+I+`g5 zW}~Qod0A;Lw*Q@m73+!Rfuir!WXqcTd5mXE^DWV3AUSVk>5EA&b6Svd&!yh*!z+6( zh^>CvoV~2?y`UJ#Jho<+PlUEw=Y?Hyd8C#Oj$c!5d!Du*w4OQ9G&OxhDmQ=)tzD()srM-?#=f>aw-$x}3Z?qLOIJ{gnZu zd`Y3Pu@-6CD7)$*a6189&`vfy%c7^DmCj90Mw>5FgU_yh15-*dsMPOLpn%G&Gbq@c z)NN;i4jF!g3-}@w-}i(YUbp4WY;xYi8`sa3ep2V_UXf_!7A{;Fhp25CGF=6{xLd&d z!Mvrklt74KI=0hsCRMYBXM0Z?v1sDfN=Y&W2dW!hUyqiiU@A}R-XCxbIudes32?<&DQ!Hr>qn`aYQ?jSq?4X|x(CCDAB;b=wcWVCH1CfwqU1di z!|LlwpE@R5*{9XlM;`OM$(VZBN$c{`%$ZT3S3aYJwVO}kw)@4_EyP4SXgXkd)Q z7PtWeexnE98(N{TMKt-aG+YpQs`a~e_Y;}upm;CRXlTWI->sMI?cj%D`$7K@mQ<-e z6c3=23v>}kQ!+Z{G2&KQ99s+el!e053~lQJc`8%`$;xt_RQ&16M-jjl$HK)VZG-0esPL)%m(*xgTxhvj>YKkE?dOv3G%g-W9;dgR&pG1FoW|wrm7v|b_Y-VU zKV&S7NcSkHSjm4nrPIy#Wvwp8(lbN>^x7o60ICQ5m?QwOuUY9q(q~<6`0+a7 z_`Zhdli4>YUiT%XT1&z74m|S7pZ;||I*2@$Zd5=|9{V~xFLGS|sAE`ZQ=toXwPUzSz%(Ar!@#M}4%I2r*Ca<9 ze?7@cjo0^QC6zocYls~PXjm{I-w|^|?Hpmvl_!6;&?vERiS^(A2e-)2qxQ#IfuJ_M zgEhyUo8K;fE}w8OE$6nq26w$M-YgMyeYnhwguXF-@5ca=0xYn%I)Rl=_lZaUn5tgl zq{GPw`_E=ilA8s)Jy=%ks{*^ijmr0SqHYg5D%zYfzlqy~#fp6GHI7wm_SN!mo*B=(4jED535Cy$0WQgpMk_!VjQ zhjwgVnse1csNUVP_rkF)3q*bk`=D| zRm=kyT3qxBA7a}d4b433h)JR1r_zBVy6)DMRyM?5%=@^}YMnjurETi?w8)8Y2lox+B2Mc9(WcW709kmg&QO^PydT;QZ_K7tmYO8aA8M?Y);N zSn^>S4^jpy!tF}ZAn_;hcCNY$eyakky`&>*Nh{Yf8H17GR#{9&%f^ps6IAlo`0a7| z-5WT~hwWze!uONxb4D$Was0UyM#f|Al`@rMWg(+oyWOL{(2>P6$`ht&d;q3uD6W+D zQQKN!nzWpx$Ya8CUKa3dgn={(ad!Lm7qDcu`SB#dKHvAM#GW}Z>EZmS6yG22dWcVi zef}3H%>*xQE6XidovM|h{PD;~31ijm0ia9g=-tnlFk!0PDn12luSSt7gWP{nbUK-G z_;*xp66cFpR2OkYg+1wGZF$3SCHuNOh~T{QxmE}&DI?a%s+Q&BqRkJ^37TgbKmAKA z-lXW9)FAv@J#Z=C2lSk4@W5q7S0~BpAs>m(p{^)b2MCFka=_0~yTtPvSKJEH%6&GW zKv;f{iTBYXA0^wmTAmssRXI(3556s-FYRfgXSs2F7D?)Muw3X(n96>Fe~#_y!;5dQ zdOQ?Kp<{m8r8ee4PPIETr3Sr=L{BgNp=Hl~>nSiYS!vY-rs7>zJE&K9>k00!&bs>P zD`CMT*(GNFuh#^fdZE?R`V};&3K^rq3z5UT^^KE~V+Yq@nxU<{+Ug^t(FEIk@f~5* zgnEN(6_Zcdmg55!i|T1Xn2NBcinnnFghvgYxT5oG<#r&$ky|k5SaFs(+Vr@W6W!wc zhr8=;xACvw0kVQ6m+uK@w0M_|3*`l1D1SbQ1B%k-HMIa!=~kGkCfuQ8^C^ZQ&7xn%?zUs@ zJv~f?$}gE-(aEgrt|vKx z;}Q@0S-w8jTszP4_+Em>MvCg@+IT%eNk_MIr)gA`;*lhuP%vm}{=>pIah-$r^3{Da zp;l8BZIY#N3v`sN%POMh>Q=e-o^BM2OK_7-ztamrbZ{m49XWXIgg1Gqa+C!XfX?gxVvl@Yc z?lm`jKKariU3($HdVP4LPtp4+4mV=+tw*rjI~_q%R6DfIW|6`<`}My)W_VK!6c^i* zIvi5RI=c%+#{fOc1^%pnKBkmGk{n2 zC<)woa7^dmGd|$2v77jNVg{v9cP;?R<5Hz&w)i1YTrbpNc6%p0{Khx8hi!J94klTx zC9LuDS+2u)()U%ug}~voR<>Cq}#OQfXF2)TCm)4nk4dkJK<{Ji<% zcP30SBMi`eN&Lves%5zi8b`z0j<83Tc~cBqc7F%;N9zZcNAe!JR3!n;@j1h z1lCS;R&Xw6EFbwYNCw_`r4_DiPb}ogRDYy^watxfz7Xy(zQ=RKaRMV#RY}`WgLrrF zVY?S>T2T_0_gmfEc1P>euBpQk$h-TAw(GijhS$+YK=Tg$zQ6?>D}F1vFkHMoukc{a zEy_ED8Uf0r#&yr0HH7|2|B-{vV9-6x6%+AEp3Hd}4fvb`f5|t#1a^r!L``xWv0pYp zK_sWYo?M7Ka~?Ti?_2#VSWzD;+NOTq_0`+=>-+<27aH>r;wtxc2mAJdsVzr(62hGT z)&mW2D1I;#ot)2O9iIWid6J}Na=-qm<@K(sk9ppYVwcO*IkP(P8P9ER7!PsMfNBn& za^K3zdtRPHN^c^l9lmBs5m>rjxgOV7Io|5p!v}X)j;Ax&u7K?;q%XjX_~o%@lPr_8 z*9Uqq$6~D2?gL>l^=mP&+~8z3yT!99Io|+z9QCQwYR2S? z(t}t86UG(B`86l3E&Y`O1p($K!sj_~Szh|(peg0h(+?ymZ?)sk6C*iUD89q@SVAIS z4_&>H|FtF3pZ<_*-;w|rv%!y93`xISUXVWp-T~!8n*#@16?Q}v>{P^~9I69_ z%n*6qXY%Yy!%fWkW5OADjlkEKjP5d$8>`wRrhp=ra6@iEL)prjHQ=o3@+N$WN7maZarII1Zz-rqUrBVRY znukG8!4Q$))$$`IcgoPA;izr~)m2%Wl&%&EHeRmOXUJsiSwge{CQ5;l6K*f{(Y$dK zr+Ms$jZr918R?`Rysv0Z+#6wT~L%t0b;+Q^{rT$Y_J%=|3^Wd zt6$*epNax{<>cRLLyEm2t&MjM8j1U)pYxwc-MDWDwN~$V|G#;ney}e?-YB~f0-n-M zw?G0{JBvufZPvKoY*5O85X8y3)1IFwLkMFr+5G1knQdDje8Y{BGoelP12*9EUN%KY zxk|^L1xHs)rNCp_@p0*`=#9{%r)_7IsX3T&x{b&X;mgnjUOMtgKs#ylC}%kSdtkjl z8!FE;zg-elNMzzYzDjZ0)^Ieq?HW_G)|Sg=4mBA1EloCGZTG(+tr)OPwRZ{J7OY5O z-u^rg$|QACu3Cq*Al+><3gPrW!35XM#YAriTfXw+!m_NkpMN$HY+wKfNr4L9PYUX6 zzlS_jplR*TFaNt8ide7lbsipOGdSE!+zhi$@D8y%FCwjQ$r9L{z>FOk9`c^?Kjmj` zMuYzJ3lU=4n6Q;tr@a$L?%8~af{fraE2*s=hn>Cp;YCQ#>re~C6xoCO7}(mj#Xh*k zba*^&l5yo%qnHQd!W*<-IXZ+8vnMb>c^cM={07F5{v1ulw!aVecf>C42Ir44Vz);s zT-%=b<-{YEZ*nD{U;m4uIi#wyf4G^ggB0@5%#DRIbN7hz&!Bb!hl?A6#(~|dZ%%iN z%o^Sc0oq?wn5_;1HQ*s%km5+`HK!Bq9^dL$ZL7!o2j@&piKs-)bi>dGD9BCC4PSIk zrGJIk0P-Fv?{`4G0`eU>*i`V_XN2xXw%*xTUlVENh%_|iZDkl5p@Y866#=@Xg{cbE zjZtS75AB(^xEogv2B)1x^m!0XZdCqOZ~=~2%7kuI!6E74!u_j2iau*{do^aD^2Vk^O2eW~KSv(BzRD>xw` z&*Gb6ksujl^_Fg<9{Nxn%B8jSv6jcmU+Kw5-Q&psk7EU|G|_)%rogKwNzemwy6QX^ z@ujX`ZkT$alQ%3oWJ2VOJGz{G(ukN|LF&Ga)nKml$M>IY@1F)}2mL&m6~?A)CN|YS zLi^lZj;aN$DQnmlc~AgqcDB7)?<<0=D*JMD zM3%;`BX_AsO%3+;YjwAbOnkT+m^;*q5X>@S2hO@Aa1J zJCCx~6B|ewT}HQECVls)>JqY95!(x8tJTl^D9t}c_G8p6;&167Z{2*+*qbjZdPBKR zwYTwFdQwnL?Q_fZ1S5+O2`Bi&@(s_P_cQY7?>NOU&FL}U5YmlM6yw@TASK}~;pon& z&{?aE)kw+rf)rVR1R!KIA&R@6^&5tt+oJ8h+P)7GWpbZ0xhG1hCCSz8pFjdYT5mJUum4y`e6ST z&@%+@8U+Bx-^#X6vpu~G2`=~;;97zryltTvX_;q&`r%A)oV7(xhxX1-Obw!r%_aBq zXumue@LLi`iFY=9t~-zHYJC&!zW;W6TKK3YgAe-4E5@wu_HwjtlH4Ep5vqLS-2C5$ zSxHdkc#a7g$_vSgCJ_dxxPL&~SeaPflc=j>z18KsBxhHfhSRvim6wzyuJBI@*m2g@ zc2$Hh#1|Nide`x;s zFEY{lfS)AO1(&M2`md$eil6mNBxu2_M(#la)vUt>ub2uO+!3=jb#6Ic2xq$*jBF`n z%L9sP{NK&^17myQl!*yca`I%e*{%{^D5ld#5&5Dbmw2He%xl{Z?Bv@+UmIbjXEHB5 zH5Sh@UPidw19)2ZMmXkn`O@)IsF`Fbj+RLtb$qTJ#B-vXrZ?7??}cA6N56t|TzFj4 z=rAukcL+Zk?vE$J3_QP=HeaZiJ>sPUrar&8Ao}%X-FpDz+o?UsRbtr6!(ES)@vCo94^P>R%u%q(-9wy%Duenrn)jXuW z+2hV;WWLbrH-awRI4^BBwkb{USY=a|U+=L6IJbHc+!%aSb|KB}H$ z?;wmaMfCf`2o^LLsVRHayM++C2aVlLWRbMjawRSh!|`u4I8tjLx>H>?ZR&ba(LJXj z?DRP5gyUNUnznwc)C%qsQ!aTlw6i(@viQ+~|0fLN?FR=&Mz z!m?8%ms9Zm`@?A{S+a>p-JQ}TICnZa{gktp_;s>#3Wv_=7#GC;f$M! z&TRADKS2F7Grq42P=N2(^g3PHSv9Sr5khe~OZap~yE3UUWM-{Fh{H-BGK9MOV3L#y zw*TZQX^enrYRj7iXkEaCLTZF5z%T)MU*{_RxA-*;G{sl{7ry_e1h+X~HM>NyBnnV6 zzcFEEZvv5PId&nY^VG0nqu!l%4Ln9L8OVmkfQi1}=-j_u=t%I1_~|`SZ_zv+SV@2>e1;w+Y$vY75F((`NKQU2vax&tTw!~HE>c2M3z3d>g zk@W;ee$-qtx3IgJ&cQ;-5AmGPIIdtV0YQvcV7G)N!(PWkx#qq=;AiOzb$C@x+Z zu##CR=Q`hVF-LGTr?w9-umq+&6PrkTr)T1CJ!@XV9i+em9sS#E=UO}BNMwuBrCayH zAub{V#`%5ecrycz1$eSV8<2Ikv6CQ5E=h^K%3m6h74APzqFYP{oejD^Y7o_E2b3p| zeA*LbkS?zNs8`f>wX`CuZF=Vcnc?D9l|P;QF8KedIQiHkm!f>Y3}# zl9AL|w=FC#e&CG1Vj1SX@K&6z&wEdwI}i+9}=0 zD)hP8t2qSqGq-zz1>nRbHpsOX+Ou&rc&B>1K5Z`l|60?OVRG!%y@dyXhC`Y)1x&pBnbuTa%|7f^nM;OIHu%(W6&Ci`84e(2e5z z*ThM)rgG_sjP#cQ+Xs8;_5jS%p3?)1Cd0epUI+qH6)RAoaWyIr#O{wWN#wI+_de=e zPHAv`+(8DcYwZezvF?o<#{{xGw05-!dGx*J-i6B-YsG?>W6ke;g4Hg#P+$=@?s0UEI-*Bw6RE<{1I7> zjBlz61z%K{w(Fbs@*+5i`|zyRlh@qP_iu#(*1Wcpz$is&$q|YHc+dRFT7N)#@B@znBGn$2wXOi+ggc5BJ<+2( zlI3ksg*I$2(gaUp4h9pJY${1?hgh6#mU-3e=N{4cTb2V_4R`HbSASd)X&1AJD{hd8 z^}36_R=S?hhh>k{b|Q{V4g^$!<)__{4ZCIAOzE}*nn%8FpA_Bmaub%88)q94qdSj& zU&K}EwoAH(N;V`V{ZfKgP}7P8xX{2STb>)D)y3#SF&&=+6Jz=_o8pqGbBI1lUdL(1 zD2L567hm`YXfrYLV3fz4yv?7yE!3uaicqZ7ufRny<0U&B6qh8bcqsL`r9)-JOxkXy z+l@a1(ptpJ`{M2l$g!g@DX;KZcoPP93JT=vi}|dQ!tn5*k@U)brT5a*!NEAJ2Apj0 z3jNsKvYjiiy-sUG06+A3T)f+N_X|`ZAX$1+M8W1ZaK3Nm6Dd}Xw#CnL+A?Xi*n>}B z+g^J-yeBCQ;(6yjA1~5bLwIzXXp>6syw2d^&DXBrf$G@}~y*QOne;u_UdZD^Cl zXxza$QKpgXzp22W4GZI|8N{0M2?78Z`$wi+S>waN@uSr9`u5+ghvrjfhcjQNuoDp; zk9szfi0j_VBAd2M+55}LBoF!BASF5?QV6q5zf94lQ$2goh8#I@&N4tiMK&5WOgt0H zRiGPL-7G)N zj%2#teK$kweDwBL1+DK?B#>r?tjR02JIr zUq=)|zME?3CA9?-DRGfqM+;h7w&xgGmLjhTAOdy`b%#?iM;>=l7v)^GADOA64 zy}x#1eDIpJ^iQ-mHzp5#R2_{6(~wo;npi>z4tuCy@Z6Ovw1EGFOaCWi{Qog*{?+*F cSLciz6Px$lu1NER7gwh)v-%fVHC&l&zl7b6%NtT(%2MaqeK!>l+aSqV1iK%L9>lwOZ@{i zB&gLAQfst?5HvK^U*Hfp3Q+Y*%qXaWyQfavFY>}j$;-Z;KO zK&NmZotK*-oo?P|n${dko0X=5gZNwm#Ca^XSY{t_Hg{2cA4<{Jm)zm({XX2pz0H-j z4O(Be=oEfn4tKDW0ruxDiz=ehxIGuU-MC(pyaSWCja@Aod`X+%GGDi^Srw6OXmA;a z((zr28F&7}zb?Ct$^6Whn#8Kt^|k>5^d_hJXz5q^CmPho>S|Zr%fF@k0%3JU8OM7| QB>(^b07*qoM6N<$g8z5cQ~&?~ literal 0 HcmV?d00001 diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/ic_public_input_voice_default.png b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/ic_public_input_voice_default.png new file mode 100644 index 0000000000000000000000000000000000000000..f842d57073c509ffc004a2a5e669b8b313935289 GIT binary patch literal 438 zcmV;n0ZIOeP)Px$a7jc#R7gwh)k|wsK@bJtFD4q0fRFe5)Sl)VN6fA#MxPJ7k8$+&Z$#X=T4b$OiehyUVu4V#(W|48Sn6|MX(1T$?xIF zNU|L~Yk~9t+{C#G(~npmS49uN158(v{Kk{R0z7Q!{OCWxKbPnN&Xnm$6MFz=a08#P zIoy+jrsy_KV+~tarn->Xg~vFBJNSaO_VqkK45BsBdjrXIK)&TPRtQg+<)dr5Gt_|w%=X{IM~{So85DF0nTlGM&c zjyDG}A0YLZUD&1vnXKbO5~Z1WB4agr0ERCQmTs)U^)$K0XyO3*vZd;$4GV9MY482! gxxei}GvVlgJPx#;Ymb6R7gwhl`ReeF%X5nRd5E*fFUSQ1dkv=;yC~p-~vbl9*M&f6pnBU+@K^mlhib?^G#=(zKoV5bL3wGBVb&Dy#h`loeh`+JKzl50|!-rLPZ22HsAR5}?FS^eag-h_)@Y=>Hb!pML_&Xezhh0CWNQ#3$^}&J5`WTq;5)Z+{-z thUGK;DKMxAkcZ35<%r*K0V?JV%U^bWOBqFh{fYnp002ovPDHLkV1mz-az+3E literal 0 HcmV?d00001 diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_doc_default.png b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_doc_default.png new file mode 100644 index 0000000000000000000000000000000000000000..3a6e5f2c6cec2451a16613b462927b7d75981f2c GIT binary patch literal 295 zcmV+?0oeYDP)Px#;Ymb6R7gwhl`ReeF%X5nRd5E*fFUSQ1dkv=;yC~p-~vbl9*M&f6pnBU+@K^mlhib?^G#=(zKoV5bL3wGBVb&Dy#h`loeh`+JKzl50|!-rLPZ22HsAR5}?FS^eag-h_)@Y=>Hb!pML_&Xezhh0CWNQ#3$^}&J5`WTq;5)Z+{-z thUGK;DKMxAkcZ35<%r*K0V?JV%U^bWOBqFh{fYnp002ovPDHLkV1mz-az+3E literal 0 HcmV?d00001 diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/info_circle.png b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/info_circle.png new file mode 100644 index 0000000000000000000000000000000000000000..1bb7ed86d79442dc9cffeced41d3909e4bdd4f14 GIT binary patch literal 636 zcmV-?0)zdDP)Px%Hc3Q5R7gv;mA`5gQ4q$zIZq%6dS=B!3&F;aSXkI3s1XC^0WOlpprsNJ%fwOy z(NfVyo&Xh&UfawJ2U4yg#Ua| zTmCG|t`tRa8NgWp=c9fTz!&G-TAt_cn{w0wy!T5)Gz(zd%svoNPDHyQnVB6ovy6x? z0oXRPWmR3O0#yQ-c^$wcfR|?WT2(iy7JKg}iRcD^n|t~V5t-`O9RM)1MbTXmdD>RO z5NGB)0G@@EdQ4zVC01BWhA`2q|JO93MxHP>=tB7Aw+H`~2osNmW)c}Io_1Jc;T{oECf!xXDLZH#E)5>Jq zLmoIiHl1<}b7cMk5O#doGBZ1N=p*yr8U&{WFl}b%0F1@hXCm4GuzKi2zhy@c9R3Ri W?Hed!u$H6%0000Px%Hc3Q5R7gv;mA`5gQ4q$zIZq%6dS=B!3&F;aSXkI3s1XC^0WOlpprsNJ%fwOy z(NfVyo&Xh&UfawJ2U4yg#Ua| zTmCG|t`tRa8NgWp=c9fTz!&G-TAt_cn{w0wy!T5)Gz(zd%svoNPDHyQnVB6ovy6x? z0oXRPWmR3O0#yQ-c^$wcfR|?WT2(iy7JKg}iRcD^n|t~V5t-`O9RM)1MbTXmdD>RO z5NGB)0G@@EdQ4zVC01BWhA`2q|JO93MxHP>=tB7Aw+H`~2osNmW)c}Io_1Jc;T{oECf!xXDLZH#E)5>Jq zLmoIiHl1<}b7cMk5O#doGBZ1N=p*yr8U&{WFl}b%0F1@hXCm4GuzKi2zhy@c9R3Ri W?Hed!u$H6%0000AsI{U0tD9;7S&f z3`9H(<`G*WCN>bN493AFOi{!!!L|afI7%o`6&6lXK&2`L1YumJiZTQ+5doQ^Fu|gz zI6Nvw1cME>!8`;4iI*N+z3;u_gZtzG5&vyF~^*1 z?S1yyXYbweAFzGO*PdLxe&gE9j&{c{J=rY}9i1#6cCzdq+ASx~UzXhiC(H6orN{Ar zj;qq$yDTU7NWP@ws1J2_*G}Ykx7%{iE$G@-7-eF^Y3#}`(v#ySiIZdTj}`y+a>=Im9Vq=f1W5yxR*!@kj+Rxz&v=+4_?qb>2v z^P8^zTt$BB=j8B|JpIS7`QY>Jz4z#w<>ZT>lB09T6nS2-t-LNa`Yg!ixr}^gvZsB` z{B;rQ@uVEqwOt7oA8%Sn=e2VBs;^`dNc~|xx$^LKH+*6BuO8<1`K9&UDuw8t_%!FY zoV0NZ!^eH~qhBH?uakr4K4~ZC5VHnAA|L9#J5r^|-)7;Y zUl$mM>pDMqeipwr+7#N+YO&F-3t!twD#tH9_S*S{wQ+C`@f*(uNuw}s=xXMh&DI;Q z;_u$0c(3`5*FEq(O?pz@6#ee_pZMDAFS)(D{hdnlGw+UhHaZ&vMC3y~_HorR=oT!) zD&Jv0*w5!@vBS?MX~$>r(d*!xjZ=9%U3__Gl0?W|%cDAF&TIVSk@)+3cqc!3boGhhYzil=`)k_5%wL2pqQz`Ju@50G)sNfVj zoXGZ|Q(f3+@xx0`O2~K<`L6lJ-SXStp$#*Nk@$Du%RKJ9@n>4_fX zCq4RXG{SB86?4nquk-Hy-E#B;AN86?zpBs|J16`d(I5ZXNB^!~KL7eV0uKN-_1L$Q zfhXMkzP+y=*8|%=cJL*vJ8JS$i*h!V@e z?gp)OZL3q^qPRQ$mTS*l z!1Lo9sgwA)pzOQd7ry0nSAP)8dF^z>J#;@|{wb*sK5UU+HV4!!`0VEJLKou6^E1;q z{-F(t{g8gMTs+F%4CL8B(dE++Be1u} zQa1d_@^?2B{4?(K#G2gBZ2YKxYj^wS1vv8wb2h-K`rtLS+C4j5oS5zZQT6pjk(( zJ4B5)x)C<~DS-Jn#3lX27u>p0yp_M+jn)mGYaUy>+T%Nnb1#0!>tbyAQ%)nklRSgJ z&7=Ic?ks-hoA@5fJ^x~JiY`PYkDmW0C(plGd!Q$Ex;t|N@d~qieC9rdJUa(Jbmg%% zxJoLcUW^RY7oUugb$iXkOVyLI8AJG+ zNchYly!4G7Y^6~5nrXo&e$8p}lUVB0m<1UOEOBY-ht5+)-??6hPx|GZjRV(b``>-$ zM|{PjUt-09)0*964ZWy4qG3A!iZuCL5J4vSq$?ol?wO2=1e&!;9t z{HK#&d2T{`aKZSSV$8nw`5IF+b?d?_&_RB2Nn@S=KEJHRZ&{wfFD-HANt+d!8=g@V${FeVy<@Q=p|RCl}k1iW;RIY+rXYw+ro1J ztScYrS3bq4R+FlcH(!!*-yB2t`NcV#59x0CP?FiqC-VdG1vMIuAg3o=Td=#P|3Z0B%|-@17rLGk-6p<6~!$6~POh1kU3(XXZO`=|>$d z!lw$=5_RyEi#Jr~RP#^%iC^4A^2m;K+VClBHe2;z6Z14*Mk&|$%X0f<_lmdugY8>E zPThfcKaZ0b)2b2Pn1`Dkmvb_pUZ*zC08jjo)ep|hccB`;;R{6kL;Ts-DL%Zk@M}Ec zYe??S-~5VIlRb~$9A!25WQb$>P5#6re$4=RZ7!m^$ICJHQwLq8^3qO zSIW*0ziJfhY2#Np#+5qaD29V6USiSHHu0r%dVQte1>d!Te30L9h<8T(gM1~;2HMmK zAIaG=K2h~u$+A`Ao#yL~^C@rnmi3*Dn>*0%_Q|VFij#Is9D-CUfq|-t52LPSO>Mf;|h8QzG9r>i*kxj)D&%wf12-@hxpQE(boL;`OLW% z&4ra*97R9KXL{m{MVR>LH~jeO-Z?hkb&`yq#K-O6lT$@0DD?-g)^Uzc7T&5n8gw__ z0DpXP`45D@vQE5>CYLA9MXJba02$ioVhjTWVS5bZ6(4zN`ENe`p5>!H^k})NKh(Lb zKhik@lUA-Xx~smjY)TJqEB4J>%kshNC(AGX&hhfC|NQ3id+))>f~iYr%eBS5L6diS z0c(T7VNUk2yzB*+mM{H`dzO#=6GzJf`m=$1G@nblG}%hD(09V$W~@UCQLSS;5BqEV zWae*vfSYo>EH@?Gc;aOFp#GTWmw)f}@_j#ZYkBJ*Le`;RxE%9>G%3oHFxKHSfF_;E zFF&fw_1jO}dg1SWTfI@g(_fZ9_1ee&mj2x4J1a|pX>wLqgaW;Whu>GnNZR9Y^4s;%W zx4i1NzvUU8TZ6Uq$a?oX>%J5^9jAU9em|0;-_C;e(1}uEYG}e zr$t+qTP`-spu!U-M~AgevS79|o^g>`wAc>y@e7Vk`?z91a^qxq>GOBXzxbc8ET8gX z-7Xxv6CigTGJZUUv*`9=vmA1gzg4h49N+Y^ODZ8#@KI9`q-_X zaPu5;fuSS!*@le$mhP;#HK&jK(B1NbUvXvmPhY0_kiYDk{5AHRoIkT@vw@Z8z;F1q z7l7fCCi(MA@@nf@5q}|i{jv8-IsM&M6%o3LI{BfEQREKp4HG$@wUJ1eYx}Q!%BAIh z`K$LWk8838tEq&7|H$p$UeKq__MwZg*U!9Rnw3=(J#1>imzU))z3%$*uKvrZuZ{Wd>ES!5dgNmrfBPTZ zSl;rks&UNFhD?$g9J)KT33%MPXFTyAfBeSP=e+&fch`Iedi2_(FPHhgB&G`tFhZFY^iGZTPO8%A6S;JedWE&6Z7VgKJMLTtbV@Au;oe}a$|fo@8QFpeTE;~ z=(!{4cwATZ_x+vv)3p?oK6COMai}`b-FNw9`G;R}pRW2^Ajgt*_)SjojgA<};ZV-D zH)q&q4iEL*eWU|BFmM=S?>NY;&)5I;`<6?(5sl{jyXGx}^8>dxQX%Vtv5PEo8w6JK zToHH6efQkYp6Q3Mqvhz+s$i(tXF7XpLn?CV%Z6Oqu_p_+nw!5{zT;K*3%heMNzF;f zzun5oTzGVll(CU?9of+U+nP1y(OpU zvv~w9Sr;nLG5?3p<|70ueyyDbUY}Yd!E0=`V+1F2S@%7DUU z!+3G5v_Yp@FhhD(9o{OXys6YM@?dLP0LotS!( zZ~o{ThY!62s*m!Sg&e-XdU0#<$S=0*Pb|w{eYqaXoLkS+K6Rp~Y^EN+{G*Qi6P;tq z8XuKI#YV0>%Nz^2?6yhv9fh2b=evx?JV#`6&=bQOMZM+dz(~P{OOO4g=JV%2_LA3t zIWdLGe~6_L*6U?ZoidN$t=;E~mp$XEY0L*5)a)#9%C_**_ejXj1}SaGL~lF&7ro-L z5_Il{V)fCw*fu?YZqYMj%cgB7z3S~eAahn{_@cQMlFic3)%3UY#Noj!JH4cEvRr#S z^9EDCiHH1&FTSjo9Q4r{^K&2ha-QnFK^=vKuFYqvdxW=7K2uz)M)&XO4}*2S)oU;32*?s`tzhPoNdy zMK~{~T*=4;PVlC()T`0MfB8pTs;kbv+GgKHr(Rq!;3+S|5(B&y+n5*@z^5dLrcGjDVs3` zF=w9B8T=Q$;LA>~9`X4+qVFJ-liI=f8qb5;adlP9$i*t%;M>z~dBL;M7jh(|v1O@a za}jzx7Y{1+b#a=fVe#WfJ$C)~F&^GD!hg8&3xD97hwY{wLOxnA2;wJqo|?br07>n| zdc9}P-SQkmio~mhtX%z&MJycY7!O^|^}~~L*w+vLY!DscBm0>6jPaAr#6u#lPtl}a zn^g8A4RF_SY<9BpclX?P?PZtsH(oFGD^X@u>A2cxb^Xba#{f#>E7Bp? ztFxkR`P@dmpq)Vyx9`@uFnA8e#&tpr-DGb_G^IYIlqLQGW*i-bW1&6e29O6Y4AR#5 zvw3QcRQo|aIrZklmvExE$M4X$oUyA07_9mhM=sXuWE_~5;nT=?xmN7c}VZTZ(}?rL~jVuDCHDd zW0I>4RkJL)P{rpZ{mdS{51lA{3Pf+T`jPlbs|k>vbZN6ZbRkPI+fmPp0DeI6t7Nc~ z$NhZ%nT)>k;6(Zz50&~yf1iG^fs4sKviK#}-Dl{r>Bu~hY2DR;F}T*pmL9|4wUTbw z@xnlPQdFhr&E%R&<~6QfTI+#VgCJrYF+`(acGqTfD_@rASLH)IiT<#`a<+xCqjpL` z>#D>_%Q%UnL=``~nBcrnhfBLfp$0UGM~}`pY-%%xL2Su?1!0>O+=jhV^Q|SHHsi~S zD~0ov1zlYjfNIlt^GFNNb-;qpg1EPAM(ME^ps)?4i@M~QXic5q&!wGA8~zyJ#}kr& z^`4JJ%2R4dCKVL9!V%6$c5)Gv^*q_xt7|K06))bGDUPP7^FtSfX;?h<0|XKb062A zIY|b0!pj0C)Y$7;i^P=d-~9Mh&zQKh^`h&1%>hsw!5hUsnpx4t z<}nU3;cAnu{B7X&Vn5^sgN95?k&<*Nw-dMSz$p_Pc^$xvIFk*X^*T}DEO_*uml7(B z&nEcAJ#m?Xu}#P#5u(vuOElFSM`G;J(?_?d0s0skGYz4+p=0BMwY@=f?C04B`6n16 z7Y+?9wH$J zAxS-==YiY@80*`{n1+s)KEk056AV77g?$%2H0xq(Q))9XS&VWbRL_G=l_J9>UJl0D zL}N3`NDj2QCw^L+J)AKpGPZ04N*&EdoH2o<_uVvg5ExqK?h8cD!pAn(v{$fP*#~QU zh>wrmGmlPAjvv4qPUcCCWLhX|Ka2&~1>W*WY1;yK(tBoXnGCEf#s(&kaR8=O7&`Rb z4)NokexjR!kF~8MOFmU5aQ$lW3aOlWOo#8pn)8ot^lQLVQZO5XoZ}x``u%x;$Cmjs zwt{}jE1RV@QuzczTVvNF(%{QMY#aX3$pievr_W(l1ZA{3C6z9Llh!WOKW`#3*AYhq z-tucRhL5MYjUq^yq;P4yz(j=;Uhu<*6tg}0;12PFp$~4~hxPm_+Zg8Ct>f7*BneZNsSb8?%&Jh@KlZTTrOg zc*d4a&)A=--&QSt^&=aCKtMfi2RM(tjY0_3lN)$zC%(pMOo(G{xaW#VQD)ml*8}*( zn%f398D{+~2NGYgRbLr0gOY-ta%{uQ8}bVGoMs=E!xb*`2zR1d+}H1qgGY~B`-@YJ z>*a;j$od&444i_t&M>U#WibY2>CmtI+6%Qc>JFq&fKMxFac!J|LFhSyp@oAfvh|$Q!ky#K zhS(4BtuuI=bE{5uez>A2b4!3M+hm`g$1$&w|CB6iS~rUj(~}eO8bJK3dJ?_67ebx{ zSHS|R%y8%`=YQMnAR>?_}JgGOix59Mum~lwBBOj7l{Dr%(^B9~CeuB#Ukb0`^qvuU*Y(62BICR)&Tg!A&&-M+!2eTcS zQp|kcb?_I5@TRuW`$zm0SeN?*o>tHfJx!tLIT3p}glz!EcCx$YvH;wLhF24aiOPLh zoyM4vMhXD7pn%KA%I|SJ3pjFVbc&HshPKa%R-zM#w$p3fhA+q*C$x=DN^`o8SMD%{ zlYy6XyKVf(AvWYbX0=U|B7A&%L$qy^lSpgCbq?mNVK#inCYah3&VIO?=1DXw=#`qC zbt3TAho;;JwjNhLV1kW_T;f+5&f5zw$zb{>8{!V`+%h~%KVy-DqlO+=H=VZ=FkY%TPJGOKbO-eUMZb@k`Qw5*kXQI4 zNn-VY-V}k{dvi=NgDj)aFv2b;9&Lhj62jH0Xgt5%4NV`a$nS9VFeZ8jwL3ZT-35mn zvUwAUQ9a=cgBJ%U^%9B`*>UXEt~NPJ9a#K=jILPgIq5_LF4);`bivL2J}%hVmz_pI z&(zfWn4ASNsVrtA?CTky6@SLgnCP>dnQ&s$k2bCduV@v=0M<$2v&?X_w&f?0 zdVL4q!ob4O|06wo;ixOrj>l#y;~Gg=-=WAx*pV-hTSqte=+)3!U&FCJJ(R7IGj_tH zSk_m_@)csRD}7KQl3@|As*N?`C_c!U@vo=O(oUUM9HYTXr$fev>%5uanu%NzjR zCb4pse%58Ff_FbT99ZTs=22SCWBp8Il>D>{j4u>gKeWxhWg0&$HJ{gkdPXCf61P@& ztiI#OvjYd~D)hvhL4pdPanYqKH?T(AS0xsJjcpoa4(T1TJw`VIoTCqRpI?P*;>dsN z5f0BOf=znyxkaZ2tJWn8N$N>lK}c;lWS?W5vOBR=JKko}KC|$3Z%PH$J5|jKJ-NqE z_ZknrZ7W~D$^f(y8P~onU3Oty2J4NY*@llDx%i|JpU9&wHDK(xtG@VU#^kYat*h>i zdSLC^jL7(-#cz$a=M=p%&kPDtW4)wR`B-^()-G4{E(m^LY+5LRq%6%7l<6vOPNhVCyvY=4yUI zIx&MxLE28(nmXlm7viLOLSs$b4|GCD7I{^>sJ)bo<7qB^r=YAS^^JFY6;xwEh zZpDM~;ZEeb0~BvkTQTEG0U3VZL5j9H_mXvxdHwoPMGk8H%GZ$DSUoG};o!Bp*+kXX z`qy7&0LlzDGC5UnIv&!hC5g%LKEG*AaEI$`J|`zF9*~_UC6v2ef%Yt=w?iGS=`x{m`*tc1v}Pz zf~slY{K=p-7He#u7L@_cNMwKhd*f^(-Vaneam*r{gTf>LelwEqaEL>^IXTI3UTi}^ zZkltHCYX)!fRgkGlZFWF0F?CZ*bebcbNh5(fov2_4=P{4lkUMPb=`l~2uhFxu>7&DseW}mFpI(L7m<98w3m<&s^gYwzKLS`@ ziH2UU5yjHI=Sa0E5;z6n)mm>R$Iaaa0HpF2H=cyKrST)6aY5j>Y2EFa4KyaOJpi`Y z0cR0NFVNX;eH&s&2RLs_Wk`!X1Ktl5EXMuVY^M5^Na4ay{PgzMr(hU*GqwVm<`|tx zHqpMHc}$IYj}CnPhO8RSa9ryZ-xY7p0CWe2u`wOua|f#J0CPySsjO015zUoj^|=$R z&P!8a>m2?Q`plg2TfXWox!mch;lqB)b!%4}(i&%-8hjt^C)?8v8krgXwGp&JSbXUmUuKNKj;seLQ@+i{*gD4%I@RALNg?5Nv zHQN3d?-dcg{ZuEQo!};N-E}JHlr|#Z=D+=Y^?ah~?(8cL)5{VsbD?G)a@Zyct*NHxP>~FNNVt39Nz-u{udkt;$vC~g<^Q~(o z@!$ErW946qkAsrqYR=YH5b{$F!kam>41*1>C($G?Qu;QuA8=!KcHIVdWNDr-8-7uK zNuNiULdrZEx{d!~v71dXW?a|C=vhDe#uyuYWb4hW)6k0ypF8ER{BAwTAx;YE-wb!) zU;16Was^(;$OUp5dXvkJY0hDAS|8fn=gyP6&xSuan8cZ0vW)z(=x@DiJPDG%HphC= z- zpYdSh-(EFF=R=BYI@>x#_%jYWdLEjhM|USaBzVpNLG3+y_(R$BD_RmMas$MWs~oG^0ClV~+&9ED$w?cD|Yz+=nu2k$xd2U}uu6PP0V zCo+iBf#`{lqWxs#{-;()(J&9)cV& z*MIxg+j{>(@hd`~jcXbH;1z zth?n%0u(-3tD58KJI#tQPuPp_{T#@NnLsv#(utmIWON>=r)G}FN{F5lNBD@6U;Bn9 z>MqnKn+0+&Jbe!0Sg#XY1|IL>WT_VXUT;oA+Kv6ir{@DlMjpC8`1rDX*N^ifn3Oa- zP>v=r{|3wSjsMrp<+?rvZ1#&IQ%o*?Q%fUy9{OfIvd7w82leqs-`IVe19y5!^8?p+ z%lE(O);9mymq@O`lr{MH-Gap%a!lvK(+9_5!wv_d}s`<0wzR2F;-6sG^f)1 zfAhBE<$Hhn)^a}|--)B-fGBwkg|A}DfUPxB;ADB-k7x(+!4Wu(Z^V|l+qB6&n>1q*9dcD_jHBlT z*vR|+hTp{?KmT(AyX9Nn__#hpI{B~9Yw%ik6(uW2wP}cuI}>`1H0k-6=fBTqX`C$v zyXpzH+GeRX%|8xjW>_S<&=S+Pnr``~H$Jia)W5&2PruNUE@20Cie;tIvIjt59r&b0 zjV=c|+__#ALk??qI+k=+1B_gv^QeSsUl&j? z;p|tZ|KgJ`FMscq_bfcG=0&dhz{tYj7c4!e`8Av9+C(?nNM0J_+A`~hL2+5Y%lGV- zcj`{^cVGXwo}+cX;<;dQvT7u2?0R+qYFq{XM198e*L=}E%d_>lL3~zo=0om&Voy%^ z%h9>f^lD0ytPpr zg~{1jZAiO~^T97J@yeh09w`1xwSh24F`NSEhCjRLSXJn`%mH@4#+$x@;up2ebwIl&_3snm%EJ(YEoj{-clclgY{Q#$UL- z{G^^VuQM1Gu)n(U2vif97a;}2J2D&cm4Ei0<mZtf?9#n|`tkjxXn6KX&EI1=R@*$+Kyw>;|^ zN6TfsKa#H^pu#R*_}$O*#n-X_6q!ggu8IzGT!q@a0d4&GoYsxW{s08 zxcb6`!zl91*VjDiv#}r4pKJ1goci!UFDRc`2%OJ$tT_0@2dCnL<$j-qr9L&M`lL5D z(Jg%h*(2AFmk(S^Onhux>cB?H;>YJE=cKZwR~3}pmJcYob}zo~KupBx=(Nh~M4*nz zFreXsw&7fy?>G)Rb7uLh_>fd0az4fHf;q3Jlg~yVw=Ucr;=5V{Uqw2b-#L3OowL9U z9j+Ix`1q<;8v}WtQ-xXig+I)9(3;nXc|pGNB1^pvR0~0A$kl-?YrweTR}h1GVi

c)ijgxDm}8EsRXFt3h@+Ufr7@DN z^55r2UpdZvo*$)c`MJ_3zXBARbH%T}ifygzYy6g*WBtspGU<*Ccb`wpyW!Ui$gZ}y zo>MwK`K>f-62KfvO2{S zXF|ni6T=gB=C>=mF~5ojWS?I%DBt!ouB^&}v*S8G>5&(6>bM<0W9)PIeSXbv;v2lq zgZx&0)nJZqzUPEz=3RZouldy~VSciFe9|fxrs_KoD#u$hYz3BTu8Twxs@yt>*lp{< zm_XbpVEfL5#v}%x;+@AY<0*cV$ZF-248A&7CXCUG-9e@z7Va=V8J*&{q4I$n{~M-~K{qUmg-Y{N~tC__Y!6wZ`uS zAN=8SKnb`wARia}P{>}4q*mFJ2rt$xz9z}40>2@prKgMpJ4y?1MK zsu;8LLY(s8tNKp-L`??i35r}^567PuI=u8S&*EdFoy9Nf;48%{S#m8d=h|q*N!*Hw zE&QzCc2jn4u4(uar*pTPKCQ7DC)&Cs49?>3$7+X~)XJA`!=HT>p7`~r%@S~FvIWT% zL)t28t$h|BY!xpHnSQNXihG*>p${(0U;hi2mrwZcOUrZh0ee^UiT1oYO{3$5Hop*u zLXEN0l1qM=vD`rN)XOLJdon_5oHz3`AzpsrE1f=|*Mk1={U^)6{EcJ3kodUYZmX=p z&l4~2a)h&L*mG4|<3d+3_?Prr)`vgu$Y1U7EWIl2?@iUEd5K>;n9zxxlFNU^0vTLl zH@o9AcfQkuuVr{d?>6N1tv`70$?|*eKGqA1!uC8^rS(s+P1LOQ9lYFac+7nk_^^=}_9|LQHrRm;gm z#jgtmwd-2xd;fSm;rGSZd-@wbDeXS|)%sP&lv@b1qs`Sf43!0V?3qvsHeeF4^Q(*h z^}o7zxuRcU@`@_U0N4FIMxo}rPTLvJc{K#}XhYWmowJJ2$Yjbl`u)zkPnNIv?#GvR zeQ>x@oZ)FOm|m&l>_ivC(ek;URCk@4f5BINBIPcJedSknv#$7sL09O4r%@qb_M zz2et2d?)PSD|vhJv?jf^coe^7;*5D_(i{GoNjc@GFgNZjMJ5=HK91L-#6s_k5ZsDS zGS%RQ&sF+5eNE*3{W~3);ByDsjH9O)4$S@$?yR>?gy?){V`EPI$n>{$7kZJt&E|jq z@9tl&>KhB0wjiX?fvux_ph<@^P`xU#l~@YcVmvoP|52 zFCDST=db-|m-UT`(xE24+%n&4gZ%FnLi&Yo)!)!<`8*?XqEn@~PlG4oI{hPQc|SBA-3UqQo@Ok7n} zIAZ21l@78Rn`X^sw|ukiJP&AnypS?sjm)BYgRrvd_2vm*-zj>cKd@`Ab&91Yp=>6{)F%4)7auKu@lUJhnvWozKNZb^uG+`E@Y3=U zeK~|@uUf1nf;jWRpXQgYuqA_|MTZQJmcB;TNR^GlS{T8}iC6rO{IH|tWqO{uY5h}C zK^05FmfvX7IMk$1hE*ehH{+tKyHIa1DdB;;rJvHi z@XysN8q8vy7k-&z&tLr~zqICPT-#vO+|kk)bI{UP%}!$rHS^6TDD1uXt~a|@W*~+c z8vo^wJW;Rw34f4ZJkG`2_D~Yj%WRNd2O^Mwn=s<$0*s{9@EYCPT5v)bA~e(n|~6M0EUxGtnrcN&$s(s zzN8S(XWAcol9+ za@NCPqQw`HsBTqo#8>DWj&U^~+CTP~&69^IHqX$ty#E|%_>m7|XO7~asM|V+|Xy_l(fh&fm#RNST>VcoN?=6S_DPi%0~BG=sQt4-78)-@|b)lahBHa~PL<9jHj zNE~dl9PG02qUPM@QPu+cEDu-Af8%z}zB%Ihfge*{9Wd$&G+)E(=&9+o!^CjO`cwNdjVRH+WU`h_MXAOitJp5x3ifW{$igPf9iBj$(b=HI#x==`-hy-E&gI#->XR(BW&pMdcoR19-nNcPkY4s2bR7uK27u z;T-wi{Jv$d3tg^Khr|3zu!D-f$3GV1rd-BjB{h8+psmB&uHFO}3e<>-KnIym}P_oSC zslstp61Dm&1NiV|^pEbaNt}ZX!rh1GA<@OoA~K`yhAgd{@foOROsg!`F}gM(u1!jB zP-&PeM7Vk8W1#d^)-p1e`o(13g|c~w?dj`;4_bZu^_E|g3d=E{cLES;rdxmDH283uG=7WUKG<2~ea{IxU4q0( zBCeM((XD0e;O571>R|^u&Ev*jpsQGwzvm-2(K$^ICifY)?_e`E(umG-isbY(H;sFS z_TV{-u;uIR9OWMt?$V=eCxZbQ9k$3lC>2^A@xz~@XvD&(_uWN31AO=Zpf(=jB!lHh zOT3|j8)NsuFr00(J`~5*Aa@-yCcZDeY#2MK^7+byjE?yuYo4B|14zoWZPTeh8BIOF zi#LZ9-0pPpQq1&2arSg`YF@vQoGhb26RLwnlb*1L_^M-Vlx>giHItHpV-y+pt6ZEK z556G7lZ4?GS?qbNp_S;OAM&IlDs9+mIL@;^vinA)D6z3H9OHAVWxzHP_n^luSJ#<< zbsIty2lS^g(Tp%sL>_Jx%DMrbLPR&IRuN*2au@Mv3b3wQaDyVnmOp4Ma3Q*l1@}l- z7!@6xqcC>X;&3#^WC@2>d~Pt-WCFI;DSS*he8-yHfN>hl!&k7gZRoJWX*}IU_<3Dv zFh%O=_d;$wPTu#$88_QzeaYlJH`gOD^~u}%0AtVi0{v!P<5awgzdH2uJ`V|wUL*2lawezA2~fq&{P;mfB?8T6HUC*4h6A&Uoa8O-j$RT~z$aZBVg6 zzF?cyl6N zdHw?sJ7Tp$XXHMr#>SS7hWS(q4Vv|F6FxR`qoAKa__u1W&%AQI4T^VKan^IyU>zfs zE|$R$NQPNwnbWKcmi{dLjG5%b9r@2i8f!K??SvY4H+*lPY@EblJRiC1P#E;CqroIW z@amJ2xy(A56v{9|GuaTpMMj+DK>H#%Xah4-!k=}#^ zneQH-ALI49-brtya+(0Rs?MoH;W4xa=7q~HKFb7Z1nBuy5&@vrkTKXDY=saRII;oP z3R%&P2^nF-NYearIVR*J3O2Ys934KH3%!qF8Ezacu`vg0S*Oab^yt!p+xLq-xy5gM z#Kw5jI=`XA!CkZ&zAqE&VEj1=NFmPhl*4MSO=PEas`~e2-T71-1sApc|fu*Q}= zsYFnC_DZcy+zSDb@&j)&>t^-n;oK7;%>Y=GI zf;q6^#lf=W>#ky4S#ll)lVVQT_DO*_|C(c%5cIB9nT$1w zdZdwu#x~{=-+@S!Al?*`YqRX_$W)w|mL<42l`iKk-%cwYqIN?eH8`i)kL=}d1?JZx ztLCs2KGwvGug#(X==ud4yo;s5T!B+uNNV9YMyc!;d~C+efEeaJa{IVw7aDzJFOkR6 zSlJt<<>?A3vyx@)YW!;#RD~3cJ<+yt$FWi*K*_8K6|i@y5t3Ja zJ+H|ads>I+vjj95MRGK=^x>=qv2joEMXBp_IFN4`AdHaye#ZCSN+T3ki zEEWhGJ-%>&Q^eAnKgqhuJba{|Jl+AxddOr{Cxi+(@50!IbHi4?hjyY5LQ=XVPTEpb zyqVjwx1@vOf~d3GC@cCi=V6PSGqd|Ua>`SZ|JP5mkUUL?=|EPi{@-nlH?JLkAw z*sMbLgtgvL+o_1?*wJfZjcXpC5>GR~M4yu?y`l7N54Pg1hB01ME2+8Z!14qfU-Yz@ zpP&@C_lf&Q^@(4j;1EbkPV$`KhCay2t@XoalE&DO(HG;)bGsV$(1$|8a365@r{WKw zNW$FkEp^Sm<|7b9uV3Ad{N#D~L@0goVuYqx6L^T_<{Zg#=0otZT7J0Sg93< zJ_mX2IquB#Bm6s#^rsweb>du#$y5q2icb}=oNpi;{UA7T{^iK)*yGw5d6=pq_?*D>mRC&iQRDaItw;A9 zUwyN}YMcO55)^&3H9%p>YklyFuHBgRqrZ5o{^}Fg-RyE2Q&BkPr4P7!;2dsBBY5kZ z6MOo=-HSke#!JD&S`O^!e_!8v^T8YV)+p1?{L!gB{K1puy1vT%sWe=-JBLXqC(&~o zh8QdS8g_rYT88wPo<6+$(H>5CKO8#&q^#c>*j4hprAvR9e{%Kyt8YGf`?u>?8Tz14 zS1k!Et{sV(!ehcu#U^0M9yMmukRS`=W<1D5*Xuj%0?f#3B#i1AuV%Dk0a#p(np`Z z@Ny<>{{ZDV5+@v)mOs>&&;9Vv>-)pHaOkS3YygE%;ePHnZ!h`bKx(H9HZuLnZ`piM z2ii=ClLN3rsu>=c{+jNjKd(=0rLpid^!u4*y(mWJPG6kjm0Yv8i=0jt@0q$c?3SO6 zo`T_+i0(Myt98b;JQvD(PJ8@c_^spR4R6xbATVp;gA^fWJoolt6Viy=aHkR(bL6>a z0*u#QIOR-CHs#1eI_@gp{LgMJH~1i?ZcMM{ufkCb2He+@V%l*Br$@ccN`(OGk)9u)8Cl^IS$70>cnNtJOD;^adIv1mfzOH@{j*A zpUGT+)Iu&-&YD8$81J|E-`Afpo?Sod(=~-f1KG?W4N<>A4H|trX(W)6k{Oa&+m(#9NV~FpO<-jgq5FpLo=R80h%`t-tc094&kfl2?<-(g>J|r?=r^r}OA> zmp&f(`pX~wSI3@L@|*kMoPV!t)up3lQ3afNHGkNJ?ukAA%&S+P!*d|=aQo0Nz5YfK zKR4s_UId|>uzYyqbjJt5=GTt(Ez-yS$U9G{Cqm(9+ajN> zgT~ide(a0*RMefm>R_qQXttNTKUJiWa#G(o>gibbxL(-&eO>l^>-4Yw{;}#f=Ndog zTpjgwLr5GKkp=Bm^VjU9%39U~*@|iCk3RCfSN<|`f4G7d?}tSDTy`AIwQL?;#$97+ ztSvnwvYK=4p}Io0?fv>@g@5oyeJpBc$rtZF^xS26hCWZ4#Yok->p2VeHu^YSPUGG2k^A|XtmgmW>+a9E=9)4OCk5TSW^(Rd;pI_JfySLre zQLOv*sbCN46V?6wuS}=FN|eBT_p(bFq*`MXpIA`Vg(EMp(umI{;a4t?=!xmyYV?&H2P7PMKv=d+vjRBWh(As6Lj0Qcn$#3?!%y6`&&<3aj!!;n$@xk0 z*`QFf2~yb7*ZgYBR84)J;s=KZ&x_vE!tWtII60`G5(@|IFyHPr=5zVG<@(X_<1hTc z_kGCwAo)o&!Uw+XL*A!{f;S*LxN;y5=0e-ZrK)pdNED2liw(!iVbw-%n7!XMpG8kA zGUJMmr0RBj5-MyJddQOpL{O*s7%s{`6u+WXrgQwlI?smCIg$&Q{AYgqCt0wKb7$_% zm%{TugWsEv_{Fa|uJO;}cZ_9uLpG0)>jq*Vhu`WPlbLjiH(IU~Fm-o{X+n|rIebs+ zBK*FBMohVN%r4@=_@qH>4)KXqe5CL#cK)Tu;+Dei@z-rsKEYOe;uO{W-~*^lGv{e} zg4af91r84J?WZul<4pXy&Q9bMAD7uEiayKu@j6WtFdw~+#;%<5b$dDfR;X#?4us;} z-~EhV6zs>~=Rof`?o~=VM~9%M_?8J+n!&AcCV)?AP=;fE71{~UeEA>#S{QucDki=r zzHybu$j{hvT>Nr&n2+r=zY;+&dlw*cHh$KbFJ$UN=-6jIG7AR2vDH_c$iN1FmhpRt z?{%2s!?BZglURd~-k|DP8~&9Flv)o?mLI$Jz3h>-Z8i{UeJRS<(K9vL#!-~$F*1Sp z9>4-|wb7EC2gB>kF9$2`EI#_O(HBeOdGZy+=Ze2BPH_+Mi?qgP47=j(>kB=mJ%oMS z9r<0iE@an9F`Z)KGra&4x%#2EIrCiSSMf=2pI?~4w>$UPbpC{gT;8zlrl=Bb2 zc!MuoiVfHWSDf^|NDlF(^ZW;&*`LSHX6X1EeyW$cIeN{P*pA<}=H;OUB#~>P2l%!Y z!u69#KlsSz*U2UJ{M*;+{q-Mwz4pdlJGFtZ-+TGiS1Ql<#B&y|xO2F8BP#-G95X!= zS3AtF&0v5*jT?Lk8~!j1%0_T}otooBko6is#Sgz&6@Aj7$ONp`$^7Ks*zOGN$=Vl+ z!3WfQyRB%BY(65Ff(S*v1=yWtyJ{I0gB$4W-~OP!g>&~BlI$ss{JeWJ0Y~lvE4La}LgwmJ{B^=-^LrxrR*K+!NY34Y z%M z<9FfUS32e(gAJbEtbl5ub8iasSIo+HYW6cI2(;PPCVrX9hj6>)HIID%gYPzH@6^%v zv^{*@-@5)2n!;y#NN$bBu|)+fn^0}89(_q=8AGE|lG!A3qm}-*G$sPd@g2 zSN`*ry_F8$fdaX8yu3>5_^=Mm3a>SxDq|(W496V3gthog+!l-+gI^0x3>K~U0B9_I z@g1v9#%%cbQY(J<)|7{e%NhR$c6@0R)3;{wt|Y5hT-qAn?23((Ie*Is_;P_4Gx3j1 z3^!RMCcZ=O#~*wM_}}BBm6H6+W|(D1K9`SA_)O&v{7zZehxLm7tBQH}eC`H%|3AL+ zwv$WC=ZSiwBbOHn*aasRMW->jDp-wcQfvqt$sDPv&GGOq`KuGkd^o;c>O`@?JJE_` zdU788%6;TNa;;()znFK!uf=i(n|UXb!}$}T5F5S&N6!Fu`(`Au^2Zij=Z|V?HNBZ# z{Jg_J&>P3Qlh3>HhAVHIXs5)?*?J{TB9TPPY-Gp32p`^F3!lv=`TY2MT!#Dn_EX5YDwXjm4@%zo zyA%j0dpPZ8aUi>rp!dHqyG~d+l6Q>+x9T-*oC&4dQmFv;TYcH~Spj>DJ0esIt zzWNO+#A`{>E5i(Xk;Z0`sjgNLsQM^ePYfMu`tZTDpWqGSgiZetwnduxeT7P8ynTsi zel~9SC}kpn5&t6m<~Z?*-@e9Xw_7%@1cxGiwOUv!*ZAgV{^YpI;WyoHSsAi`#H6j9 zt$aSe;%xY&tQ7Q@%CCLw|GfH*c7B0V=63;TLHuy07aBFXpK@e@kz6>#YSGcv3{ghz zzVXF3=^Q@()T&z5KP7&Q>i!XZTNu&$kfkNQnO!8-_aDL+?R~C8sjF4t! z6x@c9tB)3F@nK85F<=By?G&Gi4}X@LiXJ2XmM&tvDMDVeZJcH{s6W+y1bgFn`9~ZXTFjEjziZ(}(o3vn z`%X>ZGshK%2W48h%Jnqix>9=bSGbGC-{Va~Hp{r_k-l2)R5e=9GXJFTue#GuTPtHLO_kpoE;{;<|N8ou=yCIP zN<{A~WY5T@7mLhsKlK)EER*b9LF?v{dT-&+=Hpvd_~PVB{13->Hs|DD_AU++MKR^? zVbs#s_)ceV^X6!`7vaB08NBAP@4xarcZzYI{jMLv_MN@||G4r!x9+?3(b^}k&qm0m zIJo%3!Mf<)XVROminu6NX7e>E)#+h2O$}L)eu$)~=3}XaGUgyZ_V8KMnK#)7zjPHp z_Ts=j%wK(OAJ%4maf|Pa51wLAKZDR6(r+-k<@J}An;-pDHxE9y+0Rj)g#6$aUwirP zX!kYxQ0mVy-QN2yL-92;)+QS*i|kvrv|fAPK+-?Jmin%y1ZS6N0LGw(w2!|y(vgZ*y#F}>^b>-1db)Nj=f;xC|Ft8@YI zMIq1nn~#0+?)d1{!hey9e+8a5izk@{Oplez2GHqrSUlSN&@^wrvVyP!giSlmuO%9r zW`jOGD83?gYTjdlCEZT%G_f_YKb`yp!)N?Qcc8y6-5c~LFW-9YpKRX@b^v?Vs?#fW z*DlT`JnOH$|Jl3C_q|fP=kqnu&(d`7^YSrkS5(VraZMu&zIv_2t3qXyto_-1d=_pk z^vbJk!~$p|XLVszAW2V_Pv+Y=r{jaEb~--#@C&o@YkYyT{(x!uak=@SdyXFer}KN5 zFTlMk$hvZOMZ0@2f4q3@#*LTjFKs?eK|fUioJEMtmjUO-<02&yOE|p|V-%X=6Xv@X(oCxjr1jf2;npdQ$tQM<2QW z=azp~pZ|S`@O0`r&8O4l#eLPLy7n@?{`u15<>(>(HP?sj)ax^gp0C0^Q@=iWK*f2c zD)fL#sXs~F-K&MVM;neWi6M8@tERwteOT%%cv{JMqtu2a&-F?ld~arKwAH@y=LKKw z#h-2EA?L&VSjQ(K-_mq$Dl8u&b4}hKRXUGo8jtD{dqj15STlZy(C<7sI)2CQ_~fnE k9@EG3{4s5ok?kb>|H;3ubeVRY^#A|>07*qoM6N<$f~C=$asU7T literal 0 HcmV?d00001 diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/profile/backup_config.json b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/profile/backup_config.json new file mode 100644 index 000000000..78f40ae7c --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/profile/backup_config.json @@ -0,0 +1,3 @@ +{ + "allowToBackupRestore": true +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/profile/main_pages.json b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/profile/main_pages.json new file mode 100644 index 000000000..1898d94f5 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/profile/main_pages.json @@ -0,0 +1,5 @@ +{ + "src": [ + "pages/Index" + ] +} diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/en_US/element/string.json b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/en_US/element/string.json new file mode 100644 index 000000000..f94595515 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/en_US/element/string.json @@ -0,0 +1,16 @@ +{ + "string": [ + { + "name": "module_desc", + "value": "module description" + }, + { + "name": "EntryAbility_desc", + "value": "description" + }, + { + "name": "EntryAbility_label", + "value": "label" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/zh_CN/element/string.json b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/zh_CN/element/string.json new file mode 100644 index 000000000..597ecf95e --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/zh_CN/element/string.json @@ -0,0 +1,16 @@ +{ + "string": [ + { + "name": "module_desc", + "value": "模块描述" + }, + { + "name": "EntryAbility_desc", + "value": "description" + }, + { + "name": "EntryAbility_label", + "value": "label" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/ohosTest/ets/test/Ability.test.ets b/harmony-os/SherpaOnnxVadAsr/entry/src/ohosTest/ets/test/Ability.test.ets new file mode 100644 index 000000000..8aa374977 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/ohosTest/ets/test/Ability.test.ets @@ -0,0 +1,35 @@ +import hilog from '@ohos.hilog'; +import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from '@ohos/hypium'; + +export default function abilityTest() { + describe('ActsAbilityTest', () => { + // Defines a test suite. Two parameters are supported: test suite name and test suite function. + beforeAll(() => { + // Presets an action, which is performed only once before all test cases of the test suite start. + // This API supports only one parameter: preset action function. + }) + beforeEach(() => { + // Presets an action, which is performed before each unit test case starts. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: preset action function. + }) + afterEach(() => { + // Presets a clear action, which is performed after each unit test case ends. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: clear action function. + }) + afterAll(() => { + // Presets a clear action, which is performed after all test cases of the test suite end. + // This API supports only one parameter: clear action function. + }) + it('assertContain', 0, () => { + // Defines a test case. This API supports three parameters: test case name, filter parameter, and test case function. + hilog.info(0x0000, 'testTag', '%{public}s', 'it begin'); + let a = 'abc'; + let b = 'b'; + // Defines a variety of assertion methods, which are used to declare expected boolean conditions. + expect(a).assertContain(b); + expect(a).assertEqual(a); + }) + }) +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/ohosTest/ets/test/List.test.ets b/harmony-os/SherpaOnnxVadAsr/entry/src/ohosTest/ets/test/List.test.ets new file mode 100644 index 000000000..794c7dc4e --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/ohosTest/ets/test/List.test.ets @@ -0,0 +1,5 @@ +import abilityTest from './Ability.test'; + +export default function testsuite() { + abilityTest(); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/ohosTest/module.json5 b/harmony-os/SherpaOnnxVadAsr/entry/src/ohosTest/module.json5 new file mode 100644 index 000000000..55725a929 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/ohosTest/module.json5 @@ -0,0 +1,13 @@ +{ + "module": { + "name": "entry_test", + "type": "feature", + "deviceTypes": [ + "phone", + "tablet", + "2in1" + ], + "deliveryWithInstall": true, + "installationFree": false + } +} diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/test/List.test.ets b/harmony-os/SherpaOnnxVadAsr/entry/src/test/List.test.ets new file mode 100644 index 000000000..bb5b5c373 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/test/List.test.ets @@ -0,0 +1,5 @@ +import localUnitTest from './LocalUnit.test'; + +export default function testsuite() { + localUnitTest(); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/test/LocalUnit.test.ets b/harmony-os/SherpaOnnxVadAsr/entry/src/test/LocalUnit.test.ets new file mode 100644 index 000000000..165fc1615 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/test/LocalUnit.test.ets @@ -0,0 +1,33 @@ +import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from '@ohos/hypium'; + +export default function localUnitTest() { + describe('localUnitTest', () => { + // Defines a test suite. Two parameters are supported: test suite name and test suite function. + beforeAll(() => { + // Presets an action, which is performed only once before all test cases of the test suite start. + // This API supports only one parameter: preset action function. + }); + beforeEach(() => { + // Presets an action, which is performed before each unit test case starts. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: preset action function. + }); + afterEach(() => { + // Presets a clear action, which is performed after each unit test case ends. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: clear action function. + }); + afterAll(() => { + // Presets a clear action, which is performed after all test cases of the test suite end. + // This API supports only one parameter: clear action function. + }); + it('assertContain', 0, () => { + // Defines a test case. This API supports three parameters: test case name, filter parameter, and test case function. + let a = 'abc'; + let b = 'b'; + // Defines a variety of assertion methods, which are used to declare expected boolean conditions. + expect(a).assertContain(b); + expect(a).assertEqual(a); + }); + }); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/hvigor/hvigor-config.json5 b/harmony-os/SherpaOnnxVadAsr/hvigor/hvigor-config.json5 new file mode 100644 index 000000000..06b278367 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/hvigor/hvigor-config.json5 @@ -0,0 +1,22 @@ +{ + "modelVersion": "5.0.0", + "dependencies": { + }, + "execution": { + // "analyze": "normal", /* Define the build analyze mode. Value: [ "normal" | "advanced" | false ]. Default: "normal" */ + // "daemon": true, /* Enable daemon compilation. Value: [ true | false ]. Default: true */ + // "incremental": true, /* Enable incremental compilation. Value: [ true | false ]. Default: true */ + // "parallel": true, /* Enable parallel compilation. Value: [ true | false ]. Default: true */ + // "typeCheck": false, /* Enable typeCheck. Value: [ true | false ]. Default: false */ + }, + "logging": { + // "level": "info" /* Define the log level. Value: [ "debug" | "info" | "warn" | "error" ]. Default: "info" */ + }, + "debugging": { + // "stacktrace": false /* Disable stacktrace compilation. Value: [ true | false ]. Default: false */ + }, + "nodeOptions": { + // "maxOldSpaceSize": 8192 /* Enable nodeOptions maxOldSpaceSize compilation. Unit M. Used for the daemon process. Default: 8192*/ + // "exposeGC": true /* Enable to trigger garbage collection explicitly. Default: true*/ + } +} diff --git a/harmony-os/SherpaOnnxVadAsr/hvigorfile.ts b/harmony-os/SherpaOnnxVadAsr/hvigorfile.ts new file mode 100644 index 000000000..f3cb9f1a8 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/hvigorfile.ts @@ -0,0 +1,6 @@ +import { appTasks } from '@ohos/hvigor-ohos-plugin'; + +export default { + system: appTasks, /* Built-in plugin of Hvigor. It cannot be modified. */ + plugins:[] /* Custom plugin to extend the functionality of Hvigor. */ +} diff --git a/harmony-os/SherpaOnnxVadAsr/oh-package-lock.json5 b/harmony-os/SherpaOnnxVadAsr/oh-package-lock.json5 new file mode 100644 index 000000000..f538ae290 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/oh-package-lock.json5 @@ -0,0 +1,19 @@ +{ + "meta": { + "stableOrder": true + }, + "lockfileVersion": 3, + "ATTENTION": "THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.", + "specifiers": { + "@ohos/hypium@1.0.19": "@ohos/hypium@1.0.19" + }, + "packages": { + "@ohos/hypium@1.0.19": { + "name": "@ohos/hypium", + "version": "1.0.19", + "integrity": "sha512-cEjDgLFCm3cWZDeRXk7agBUkPqjWxUo6AQeiu0gEkb3J8ESqlduQLSIXeo3cCsm8U/asL7iKjF85ZyOuufAGSQ==", + "resolved": "https://ohpm.openharmony.cn/ohpm/@ohos/hypium/-/hypium-1.0.19.har", + "registryType": "ohpm" + } + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/oh-package.json5 b/harmony-os/SherpaOnnxVadAsr/oh-package.json5 new file mode 100644 index 000000000..b7ec6952c --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/oh-package.json5 @@ -0,0 +1,14 @@ +{ + "modelVersion": "5.0.0", + "description": "Please describe the basic information.", + "dependencies": { + + // You can download sherpa_onnx-v1.10.32.har + // from + // https://huggingface.co/csukuangfj/sherpa-onnx-harmony-os/tree/main/har + "sherpa_onnx": "file:./entry/sherpa_onnx-v1.10.32.har" + }, + "devDependencies": { + "@ohos/hypium": "1.0.19" + } +} From be159f943ed83c607940958c654bde1c5fa82bec Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 29 Nov 2024 12:20:12 +0800 Subject: [PATCH 084/183] Fix publishing har packages for HarmonyOS (#1576) --- .github/workflows/har.yaml | 33 +++++++++++++++++++ harmony-os/SherpaOnnxHar/README.md | 19 +++++++++++ .../sherpa_onnx/BuildProfile.ets | 6 ++-- .../SherpaOnnxHar/sherpa_onnx/README.md | 30 +++++++++++++++++ .../sherpa_onnx/oh-package.json5 | 7 ++-- new-release.sh | 5 +++ 6 files changed, 95 insertions(+), 5 deletions(-) diff --git a/.github/workflows/har.yaml b/.github/workflows/har.yaml index afe99a1be..7b5b2e514 100644 --- a/.github/workflows/har.yaml +++ b/.github/workflows/har.yaml @@ -92,6 +92,12 @@ jobs: x86_64-unknown-linux-ohos-clang++ --version x86_64-unknown-linux-ohos-clang --version + - name: Install tree + shell: bash + run: | + sudo apt-get update -q + sudo apt-get install -y -q tree + - name: Build libraries shell: bash run: | @@ -113,6 +119,9 @@ jobs: pushd harmony-os/SherpaOnnxHar + cp -fv ../../LICENSE ./sherpa_onnx + cp -fv ../../CHANGELOG.md ./sherpa_onnx + hvigorw --mode module -p product=default -p module=sherpa_onnx@default assembleHar --analyze=normal --parallel --incremental --no-daemon ls -lh ./sherpa_onnx/build/default/outputs/default/sherpa_onnx.har cp -v ./sherpa_onnx/build/default/outputs/default/sherpa_onnx.har ../../ @@ -121,6 +130,30 @@ jobs: ls -lh *.har + - name: View Har + shell: bash + run: | + file sherpa_onnx.har + tar xvf sherpa_onnx.har + + cd package + ls -lh + + ls -lh libs + echo "---libs/x86_64---" + ls -lh libs/x86_64 + + echo "---libs/arm64-v8a---" + ls -lh libs/arm64-v8a + + echo "---src/main/ets/components---" + ls -lh src/main/ets/components/ + + echo "---src/main/cpp/types/libsherpa_onnx/---" + ls -lh src/main/cpp/types/libsherpa_onnx/ + + tree . + - name: Collect result shell: bash run: | diff --git a/harmony-os/SherpaOnnxHar/README.md b/harmony-os/SherpaOnnxHar/README.md index 936315564..e775cedee 100644 --- a/harmony-os/SherpaOnnxHar/README.md +++ b/harmony-os/SherpaOnnxHar/README.md @@ -16,3 +16,22 @@ hvigorw --mode module -p product=default -p module=sherpa_onnx@default assembleH ls -lh ./sherpa_onnx/build/default/outputs/default/sherpa_onnx.har ``` + +Pre-built `har` packages can be found at + + +You can also download it using +``` +wget https://ohpm.openharmony.cn/ohpm/sherpa_onnx/-/sherpa_onnx-1.10.32.har + +# Please replace the version 1.10.32 if needed. +``` + +You can also use +``` +ohpm install sherpa_onnx +``` +to install it. + +See also + diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets index 3a501e5dd..502c150fd 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets @@ -1,9 +1,9 @@ /** * Use these variables when you tailor your ArkTS code. They must be of the const type. */ -export const HAR_VERSION = '1.0.0'; -export const BUILD_MODE_NAME = 'debug'; -export const DEBUG = true; +export const HAR_VERSION = '1.10.32'; +export const BUILD_MODE_NAME = 'release'; +export const DEBUG = false; export const TARGET_NAME = 'default'; /** diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md b/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md index 2690bf624..8f8243981 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md @@ -10,3 +10,33 @@ It also supports embedded systems, Android, iOS, HarmonyOS, Raspberry Pi, RISC-V, x86_64 servers, websocket server/client, C/C++, Python, Kotlin, C#, Go, NodeJS, Java, Swift, Dart, JavaScript, Flutter, Object Pascal, Lazarus, Rust, etc. + + +# Installation + +To use `sherpa-onnx` in your project, please either use + +``` +ohpm install sherpa_onnx +``` +or update your `oh-package.json5` to include the following: + +``` + "dependencies": { + "sherpa_onnx": "1.10.32", + }, +``` + +Note that we recommend always using the latest version. + +# Examples + +| Demo | URL | Description| +|------|-----|------------| +|SherpaOnnxVadAsr|[Address](https://github.com/k2-fsa/sherpa-onnx/tree/master/harmony-os/SherpaOnnxVadAsr)|It shows how to use VAD with a non-streaming ASR model for speech recognition| + +# Documentation + +If you have any issues, please either look at our doc at + or create an issue at + diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 index cc2260a23..4d622d6e4 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 @@ -1,16 +1,19 @@ { "name": "sherpa_onnx", - "version": "1.0.0", + "version": "1.10.32", "description": "Speech-to-text, text-to-speech, and speaker diarization using Next-gen Kaldi without internet connection", "main": "Index.ets", "author": "The next-gen Kaldi team", "license": "Apache-2.0", "homepage": "https://github.com/k2-fsa/sherpa-onnx", - "repository": "https://github.com/k2-fsa/sherpa-onnx/tree/master/harmonyos-SherpaOnnxHar", + "repository": "https://github.com/k2-fsa/sherpa-onnx/tree/master/harmony-os/SherpaOnnxHar", "dependencies": { "libsherpa_onnx.so": "file:./src/main/cpp/types/libsherpa_onnx" }, "keywords": [ + "语音识别", + "语音合成", + "新一代Kaldi", "tts", "asr", "locally", diff --git a/new-release.sh b/new-release.sh index eaf868971..2fc30371f 100755 --- a/new-release.sh +++ b/new-release.sh @@ -1,8 +1,13 @@ #!/usr/bin/env bash +set -ex + sed -i.bak 's/1\.10\.31/1\.10\.32/g' ./build-ios-shared.sh find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.31/1\.10\.32/g' {} \; find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.31/1\.10\.32/g' {} \; find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.31/1\.10\.32/g' {} \; find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.31/1\.10\.32/g' {} \; find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.31/1\.10\.32/g' {} \; + +find harmony-os -name "README.md" -type f -exec sed -i.bak 's/1\.10\.31/1\.10\.32/g' {} \; +find harmony-os -name oh-package.json5 -type f -exec sed -i.bak 's/1\.10\.31/1\.10\.32/g' {} \; From 299f2392e289a55b9db7f9315c7fe0c53bf29091 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 29 Nov 2024 21:13:01 +0800 Subject: [PATCH 085/183] Add CI to build HAPs for HarmonyOS (#1578) --- .github/workflows/hap-vad-asr.yaml | 173 ++++++++++++++++++ .../entry/oh-package-lock.json5 | 29 +++ .../SherpaOnnxVadAsr/entry/oh-package.json5 | 5 +- .../main/ets/pages/NonStreamingAsrModels.ets | 2 +- .../workers/NonStreamingAsrWithVadWorker.ets | 41 +++-- .../entry/src/main/resources/rawfile/.gitkeep | 0 harmony-os/SherpaOnnxVadAsr/oh-package.json5 | 5 - scripts/apk/generate-vad-asr-apk-script.py | 28 ++- scripts/hap/.gitignore | 1 + scripts/hap/build-hap-vad-asr.sh.in | 115 ++++++++++++ scripts/hap/generate-vad-asr-hap-script.py | 1 + 11 files changed, 376 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/hap-vad-asr.yaml create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/oh-package-lock.json5 create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/rawfile/.gitkeep create mode 100644 scripts/hap/.gitignore create mode 100644 scripts/hap/build-hap-vad-asr.sh.in create mode 120000 scripts/hap/generate-vad-asr-hap-script.py diff --git a/.github/workflows/hap-vad-asr.yaml b/.github/workflows/hap-vad-asr.yaml new file mode 100644 index 000000000..9e64a9ab1 --- /dev/null +++ b/.github/workflows/hap-vad-asr.yaml @@ -0,0 +1,173 @@ +name: hap-vad-asr + +on: + push: + branches: + - hap + - hap-ci + + workflow_dispatch: + +concurrency: + group: hap-vad-asr-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: write + +jobs: + hap_vad_asr: + if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' + runs-on: ${{ matrix.os }} + name: Haps for vad asr ${{ matrix.index }}/${{ matrix.total }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + total: ["10"] + index: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # https://github.com/actions/setup-java + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' # See 'Supported distributions' for available options + java-version: '17' # it requires jdk 17 to sigh the hap + + - name: Show java version + shell: bash + run: | + which java + java --version + + - name: cache-toolchain + id: cache-toolchain-ohos + uses: actions/cache@v4 + with: + path: command-line-tools + key: commandline-tools-linux-x64-5.0.5.200.zip + + - name: Download toolchain + if: steps.cache-toolchain-ohos.outputs.cache-hit != 'true' + shell: bash + run: | + curl -SL -O https://huggingface.co/csukuangfj/harmonyos-commandline-tools/resolve/main/commandline-tools-linux-x64-5.0.5.200.zip + unzip commandline-tools-linux-x64-5.0.5.200.zip + rm commandline-tools-linux-x64-5.0.5.200.zip + + - name: Set environment variable + shell: bash + run: | + echo "$GITHUB_WORKSPACE/command-line-tools/sdk/default/openharmony/native/build-tools/cmake/bin" >> "$GITHUB_PATH" + which cmake + + cmake --version + + - name: Install Python dependencies + shell: bash + run: | + python3 -m pip install --upgrade pip jinja2 + + - name: Generate build script + shell: bash + run: | + cd scripts/hap + + total=${{ matrix.total }} + index=${{ matrix.index }} + + ./generate-vad-asr-hap-script.py --total $total --index $index + ls -lh + + chmod +x build-hap-vad-asr.sh + mv -v ./build-hap-vad-asr.sh ../.. + + - name: Generate secrets + shell: bash + run: | + echo "${{ secrets.HAP_SHERPA_ONNX_CER }}" > /tmp/sherpa_onnx.cer + shasum -a 256 /tmp/sherpa_onnx.cer + ls -lh /tmp/sherpa_onnx.cer + + # macos + # base64 -i sherpa_onnx_profileRelease.p7b -o sherpa_onnx_profileRelease.p7b.base64 + # + # linux + # base64 -w 0 sherpa_onnx_profileRelease.p7b > sherpa_onnx_profileRelease.p7b.base64 + # + # cat sherpa_onnx_profileRelease.p7b.base64 | base64 --decode > sherpa_onnx_profileRelease.p7b + # + echo "${{ secrets.HAP_SHERPA_ONNX_PROFILE }}" | base64 --decode > /tmp/sherpa_onnx_profileRelease.p7b + echo "${{ secrets.HAP_SHERPA_ONNX_KEY_STORE }}" > ./sherpa_onnx_ohos_key.p12.base64 + echo "${{ secrets.HAP_SHERPA_ONNX_KEY_STORE }}" | base64 --decode > /tmp/sherpa_onnx_ohos_key.p12 + + ls -l /tmp/sherpa_onnx_profileRelease.p7b + ls -l /tmp/sherpa_onnx_ohos_key.p12 + + ls -lh ./sherpa_onnx_ohos_key.p12.base64 + shasum -a 256 ./sherpa_onnx_ohos_key.p12.base64 + wc ./sherpa_onnx_ohos_key.p12.base64 + rm ./sherpa_onnx_ohos_key.p12.base64 + + shasum -a 256 /tmp/sherpa_onnx_profileRelease.p7b + shasum -a 256 /tmp/sherpa_onnx_ohos_key.p12 + + - name: build HAP + env: + HAP_KEY_ALIAS: ${{ secrets.HAP_KEY_ALIAS }} + HAP_KEY_PWD: ${{ secrets.HAP_KEY_PWD }} + HAP_KEY_STORE_PWD: ${{ secrets.HAP_KEY_STORE_PWD }} + shell: bash + run: | + export COMMANDLINE_TOOLS_DIR=$GITHUB_WORKSPACE/command-line-tools + ./build-hap-vad-asr.sh + + # remove secrets + rm /tmp/sherpa_onnx.cer + rm /tmp/sherpa_onnx_profileRelease.p7b + rm /tmp/sherpa_onnx_ohos_key.p12 + + - name: Display HAPs + shell: bash + run: | + ls -lh ./haps/ + du -h -d1 . + + - name: Publish to huggingface + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v3 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + + rm -rf huggingface + export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false + + SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" + + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-harmony-os huggingface + cd huggingface + du -h -d1 . + git fetch + git pull + git merge -m "merge remote" --ff origin main + + d=hap/vad-asr/$SHERPA_ONNX_VERSION + mkdir -p $d + cp -v ../haps/*.hap $d/ + git status + git lfs track "*.hap" + git add . + git commit -m "add more HAPs" + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-harmony-os main diff --git a/harmony-os/SherpaOnnxVadAsr/entry/oh-package-lock.json5 b/harmony-os/SherpaOnnxVadAsr/entry/oh-package-lock.json5 new file mode 100644 index 000000000..debb8e01e --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/oh-package-lock.json5 @@ -0,0 +1,29 @@ +{ + "meta": { + "stableOrder": true + }, + "lockfileVersion": 3, + "ATTENTION": "THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.", + "specifiers": { + "libsherpa_onnx.so@../oh_modules/.ohpm/sherpa_onnx@1.10.32/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx": "libsherpa_onnx.so@../oh_modules/.ohpm/sherpa_onnx@1.10.32/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx", + "sherpa_onnx@1.10.32": "sherpa_onnx@1.10.32" + }, + "packages": { + "libsherpa_onnx.so@../oh_modules/.ohpm/sherpa_onnx@1.10.32/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx": { + "name": "libsherpa_onnx.so", + "version": "1.0.0", + "resolved": "../oh_modules/.ohpm/sherpa_onnx@1.10.32/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx", + "registryType": "local" + }, + "sherpa_onnx@1.10.32": { + "name": "sherpa_onnx", + "version": "1.10.32", + "integrity": "sha512-yHYmWoeqhrunOqGr9gxPJJH/8+rdwcKFOW6onYByVObQVpbqypslg301IjGm9xpnc5bJEkO3S9sra2zQTpPA/w==", + "resolved": "https://ohpm.openharmony.cn/ohpm/sherpa_onnx/-/sherpa_onnx-1.10.32.har", + "registryType": "ohpm", + "dependencies": { + "libsherpa_onnx.so": "file:./src/main/cpp/types/libsherpa_onnx" + } + } + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 b/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 index 248c3b754..78a4d6b5b 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 @@ -5,6 +5,9 @@ "main": "", "author": "", "license": "", - "dependencies": {} + "dependencies": { + // please see https://ohpm.openharmony.cn/#/cn/detail/sherpa_onnx + "sherpa_onnx": "1.10.32", + } } diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/NonStreamingAsrModels.ets b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/NonStreamingAsrModels.ets index 3dc945025..a65242484 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/NonStreamingAsrModels.ets +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/NonStreamingAsrModels.ets @@ -4,7 +4,7 @@ import { OfflineModelConfig } from 'sherpa_onnx'; export function getOfflineModelConfig(type: number): OfflineModelConfig { - const c = new OfflineModelConfig(); + const c: OfflineModelConfig = new OfflineModelConfig(); switch (type) { case 0: { const modelDir = 'sherpa-onnx-paraformer-zh-2023-09-14' diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/workers/NonStreamingAsrWithVadWorker.ets b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/workers/NonStreamingAsrWithVadWorker.ets index 3ffe18671..346be5954 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/workers/NonStreamingAsrWithVadWorker.ets +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/workers/NonStreamingAsrWithVadWorker.ets @@ -2,8 +2,11 @@ import { ErrorEvent, MessageEvents, ThreadWorkerGlobalScope, worker } from '@kit import { OfflineRecognizer, OfflineRecognizerConfig, + OfflineStream, + OnlineRecognizerResult, readWaveFromBinary, SileroVadConfig, + SpeechSegment, Vad, VadConfig, } from 'sherpa_onnx'; @@ -18,7 +21,7 @@ let vad: Vad; // vad for decoding files function initVad(context: Context): Vad { let mgr = context.resourceManager; - const config = new VadConfig( + const config: VadConfig = new VadConfig( new SileroVadConfig( 'silero_vad.onnx', 0.5, @@ -37,7 +40,7 @@ function initVad(context: Context): Vad { function initNonStreamingAsr(context: Context): OfflineRecognizer { let mgr = context.resourceManager; - const config = new OfflineRecognizerConfig(); + const config: OfflineRecognizerConfig = new OfflineRecognizerConfig(); // Note that you can switch to a new model by changing type // @@ -61,7 +64,13 @@ function initNonStreamingAsr(context: Context): OfflineRecognizer { const type = 2; config.modelConfig = getOfflineModelConfig(type); config.modelConfig.debug = true; - return new OfflineRecognizer(config, mgr) + config.ruleFsts = ''; + return new OfflineRecognizer(config, mgr); +} + +interface Wave { + samples: Float32Array; + sampleRate: number; } function decode(filename: string): string { @@ -71,44 +80,44 @@ function decode(filename: string): string { const stat = fileIo.statSync(fp.fd); const arrayBuffer = new ArrayBuffer(stat.size); fileIo.readSync(fp.fd, arrayBuffer); - const data = new Uint8Array(arrayBuffer); + const data: Uint8Array = new Uint8Array(arrayBuffer); - const wave = readWaveFromBinary(data); + const wave: Wave = readWaveFromBinary(data); console.log(`sample rate ${wave.sampleRate}`); console.log(`samples length ${wave.samples.length}`); const resultList: string[] = []; - const windowSize = vad.config.sileroVad.windowSize; + const windowSize: number = vad.config.sileroVad.windowSize; for (let i = 0; i < wave.samples.length; i += windowSize) { - const thisWindow = wave.samples.subarray(i, i + windowSize) + const thisWindow: Float32Array = wave.samples.subarray(i, i + windowSize) vad.acceptWaveform(thisWindow); if (i + windowSize >= wave.samples.length) { vad.flush(); } while (!vad.isEmpty()) { - const segment = vad.front(); - const _startTime = (segment.start / wave.sampleRate); - const _endTime = _startTime + segment.samples.length / wave.sampleRate; + const segment: SpeechSegment = vad.front(); + const _startTime: number = (segment.start / wave.sampleRate); + const _endTime: number = _startTime + segment.samples.length / wave.sampleRate; if (_endTime - _startTime < 0.2) { vad.pop(); continue; } - const startTime = _startTime.toFixed(2); - const endTime = _endTime.toFixed(2); + const startTime: string = _startTime.toFixed(2); + const endTime: string = _endTime.toFixed(2); - const progress = (segment.start + segment.samples.length) / wave.samples.length * 100; + const progress: number = (segment.start + segment.samples.length) / wave.samples.length * 100; workerPort.postMessage({ 'msgType': 'non-streaming-asr-vad-decode-progress', progress }); - const stream = recognizer.createStream(); + const stream: OfflineStream = recognizer.createStream(); stream.acceptWaveform({ samples: segment.samples, sampleRate: wave.sampleRate }); recognizer.decode(stream); - const result = recognizer.getResult(stream); + const result: OnlineRecognizerResult = recognizer.getResult(stream); - const text = `${startTime} -- ${endTime} ${result.text}` + const text: string = `${startTime} -- ${endTime} ${result.text}` resultList.push(text); console.log(`partial result ${text}`); diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/rawfile/.gitkeep b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/rawfile/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/harmony-os/SherpaOnnxVadAsr/oh-package.json5 b/harmony-os/SherpaOnnxVadAsr/oh-package.json5 index b7ec6952c..a79d5300e 100644 --- a/harmony-os/SherpaOnnxVadAsr/oh-package.json5 +++ b/harmony-os/SherpaOnnxVadAsr/oh-package.json5 @@ -2,11 +2,6 @@ "modelVersion": "5.0.0", "description": "Please describe the basic information.", "dependencies": { - - // You can download sherpa_onnx-v1.10.32.har - // from - // https://huggingface.co/csukuangfj/sherpa-onnx-harmony-os/tree/main/har - "sherpa_onnx": "file:./entry/sherpa_onnx-v1.10.32.har" }, "devDependencies": { "@ohos/hypium": "1.0.19" diff --git a/scripts/apk/generate-vad-asr-apk-script.py b/scripts/apk/generate-vad-asr-apk-script.py index a111ff610..fe253841b 100755 --- a/scripts/apk/generate-vad-asr-apk-script.py +++ b/scripts/apk/generate-vad-asr-apk-script.py @@ -2,6 +2,7 @@ import argparse from dataclasses import dataclass +from pathlib import Path import jinja2 @@ -34,6 +35,7 @@ class Model: # e.g., zh, en, zh_en lang: str + lang2: str # e.g., whisper, paraformer, zipformer short_name: str = "" @@ -51,6 +53,7 @@ def get_models(): model_name="sherpa-onnx-whisper-tiny.en", idx=2, lang="en", + lang2="English", short_name="whisper_tiny", cmd=""" pushd $model_name @@ -71,6 +74,7 @@ def get_models(): model_name="sherpa-onnx-paraformer-zh-2023-09-14", idx=0, lang="zh_en", + lang2="Chinese,English", short_name="paraformer", rule_fsts="itn_zh_number.fst", cmd=""" @@ -92,6 +96,7 @@ def get_models(): model_name="sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17", idx=15, lang="zh_en_ko_ja_yue", + lang2="中英粤日韩", short_name="sense_voice", cmd=""" pushd $model_name @@ -109,6 +114,7 @@ def get_models(): model_name="sherpa-onnx-paraformer-zh-small-2024-03-09", idx=14, lang="zh_en", + lang2="Chinese,English", short_name="small_paraformer", rule_fsts="itn_zh_number.fst", cmd=""" @@ -132,6 +138,7 @@ def get_models(): model_name="icefall-asr-zipformer-wenetspeech-20230615", idx=4, lang="zh", + lang2="Chinese", short_name="zipformer", rule_fsts="itn_zh_number.fst", cmd=""" @@ -159,6 +166,7 @@ def get_models(): model_name="sherpa-onnx-nemo-fast-conformer-ctc-be-de-en-es-fr-hr-it-pl-ru-uk-20k", idx=7, lang="be_de_en_es_fr_hr_it_pl_ru_uk", + lang2="be_de_en_es_fr_hr_it_pl_ru_uk", short_name="fast_conformer_ctc_20k", cmd=""" pushd $model_name @@ -174,6 +182,7 @@ def get_models(): model_name="sherpa-onnx-nemo-fast-conformer-ctc-en-24500", idx=8, lang="en", + lang2="English", short_name="fast_conformer_ctc_24500", cmd=""" pushd $model_name @@ -188,7 +197,8 @@ def get_models(): Model( model_name="sherpa-onnx-nemo-fast-conformer-ctc-en-de-es-fr-14288", idx=9, - lang="en_des_es_fr", + lang="en_de_es_fr", + lang2="English,German,Spanish,French", short_name="fast_conformer_ctc_14288", cmd=""" pushd $model_name @@ -204,6 +214,7 @@ def get_models(): model_name="sherpa-onnx-nemo-fast-conformer-ctc-es-1424", idx=10, lang="es", + lang2="Spanish", short_name="fast_conformer_ctc_1424", cmd=""" pushd $model_name @@ -219,6 +230,7 @@ def get_models(): model_name="sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04", idx=11, lang="zh", + lang2="Chinese", short_name="telespeech", rule_fsts="itn_zh_number.fst", cmd=""" @@ -239,6 +251,7 @@ def get_models(): model_name="sherpa-onnx-zipformer-thai-2024-06-20", idx=12, lang="th", + lang2="Thai", short_name="zipformer", cmd=""" pushd $model_name @@ -260,6 +273,7 @@ def get_models(): model_name="sherpa-onnx-zipformer-korean-2024-06-24", idx=13, lang="ko", + lang2="Korean", short_name="zipformer", cmd=""" pushd $model_name @@ -281,6 +295,7 @@ def get_models(): model_name="sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01", idx=16, lang="ja", + lang2="Japanese", short_name="zipformer_reazonspeech", cmd=""" pushd $model_name @@ -300,6 +315,7 @@ def get_models(): model_name="sherpa-onnx-zipformer-ru-2024-09-18", idx=17, lang="ru", + lang2="Russian", short_name="zipformer", cmd=""" pushd $model_name @@ -320,6 +336,7 @@ def get_models(): model_name="sherpa-onnx-small-zipformer-ru-2024-09-18", idx=18, lang="ru", + lang2="Russian", short_name="small_zipformer", cmd=""" pushd $model_name @@ -340,6 +357,7 @@ def get_models(): model_name="sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24", idx=19, lang="ru", + lang2="Russian", short_name="nemo_ctc_giga_am", cmd=""" pushd $model_name @@ -358,6 +376,7 @@ def get_models(): model_name="sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24", idx=20, lang="ru", + lang2="Russian", short_name="nemo_transducer_giga_am", cmd=""" pushd $model_name @@ -376,6 +395,7 @@ def get_models(): model_name="sherpa-onnx-moonshine-tiny-en-int8", idx=21, lang="en", + lang2="English", short_name="moonshine_tiny_int8", cmd=""" pushd $model_name @@ -391,6 +411,7 @@ def get_models(): model_name="sherpa-onnx-moonshine-base-en-int8", idx=22, lang="en", + lang2="English", short_name="moonshine_base_int8", cmd=""" pushd $model_name @@ -436,9 +457,14 @@ def main(): filename_list = [ "./build-apk-vad-asr.sh", + "./build-hap-vad-asr.sh", ] for filename in filename_list: environment = jinja2.Environment() + if not Path(f"{filename}.in").is_file(): + print(f"skip {filename}") + continue + with open(f"{filename}.in") as f: s = f.read() template = environment.from_string(s) diff --git a/scripts/hap/.gitignore b/scripts/hap/.gitignore new file mode 100644 index 000000000..3153c7c06 --- /dev/null +++ b/scripts/hap/.gitignore @@ -0,0 +1 @@ +!build-*.in diff --git a/scripts/hap/build-hap-vad-asr.sh.in b/scripts/hap/build-hap-vad-asr.sh.in new file mode 100644 index 000000000..d4fabf060 --- /dev/null +++ b/scripts/hap/build-hap-vad-asr.sh.in @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# +# Auto generated! Please DO NOT EDIT! + +# Please set the environment variable COMMANDLINE_TOOLS_DIR +# before running this script + +# Inside the $COMMANDLINE_TOOL_DIR directory, you can find the following: +# +# command-line-tools fangjun$ ls +# LICENSE.txt NOTICE.txt bin codelinter hstack hvigor ohpm sdk tool + +set -ex + +log() { + # This function is from espnet + local fname=${BASH_SOURCE[1]##*/} + echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" +} + +SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + +log "Building streaming VAD + ASR Hap for sherpa-onnx v${SHERPA_ONNX_VERSION}" + +export SHERPA_ONNX_ENABLE_TTS=OFF + +if [ ! -f $COMMANDLINE_TOOLS_DIR/bin/hvigorw ]; then + echo "Please first download Command Line Tools for HarmonyOS" + echo "See https://developer.huawei.com/consumer/cn/download/" + echo "or" + echo "https://hf-mirror.com/csukuangfj/harmonyos-commandline-tools/tree/main" + exit 1 +fi + +jar=$COMMANDLINE_TOOLS_DIR/sdk/default/openharmony/toolchains/lib/hap-sign-tool.jar + +export PATH=$COMMANDLINE_TOOLS_DIR/bin:$PATH + +mkdir -p haps + +{% for model in model_list %} +pushd ./harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/rawfile +model_name={{ model.model_name }} +type={{ model.idx }} +lang={{ model.lang }} +lang2={{ model.lang2 }} +short_name={{ model.short_name }} + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/${model_name}.tar.bz2 +tar xvf ${model_name}.tar.bz2 + +{{ model.cmd }} + +rm -rf *.tar.bz2 +ls -lh $model_name + +if [ ! -f ./silero_vad.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx +fi + +popd +# Now we are at the project root directory + +git checkout . +pushd harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/workers/ +sed -i.bak s/"const type = 2/const type = $type/" ./NonStreamingAsrWithVadWorker.ets + +{% if model.rule_fsts %} + rule_fsts={{ model.rule_fsts }} + sed -i.bak s%"ruleFsts = ''"%"ruleFsts = \"$rule_fsts\""% ./NonStreamingAsrWithVadWorker.ets +{% endif %} + +git diff +popd + +pushd harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages +sed -i.bak s/English/$lang2/ ./Index.ets +popd + +pushd harmony-os/SherpaOnnxVadAsr + +git diff + +cd entry +ohpm install +cd .. + +hvigorw clean --no-daemon +hvigorw assembleHap --mode module -p product=default -p buildMode=release --no-daemon + +ls -lh ./entry/build/default/outputs/default/entry-default-unsigned.hap + +in_file=./entry/build/default/outputs/default/entry-default-unsigned.hap +out_file=$PWD/entry/build/default/outputs/default/entry-default-signed.hap + +java -jar $jar sign-app -keyAlias "$HAP_KEY_ALIAS" -signAlg "SHA256withECDSA" -mode "localSign" \ + -appCertFile "/tmp/sherpa_onnx.cer" -profileFile "/tmp/sherpa_onnx_profileRelease.p7b" \ + -inFile $in_file -keystoreFile "/tmp/sherpa_onnx_ohos_key.p12" \ + -outFile $out_file -keyPwd "$HAP_KEY_PWD" -keystorePwd "$HAP_KEY_STORE_PWD" -signCode "1" + +ls -l $in_file $out_file +ls -lh $in_file $out_file +rm $in_file +rm -rf ./entry/src/main/resources/rawfile/$model_name +popd + +mv $out_file ./haps/sherpa-onnx-${SHERPA_ONNX_VERSION}-vad_asr-$lang-$short_name.hap + +ls -lh haps + +{% endfor %} + +git checkout . + +ls -lh haps/ diff --git a/scripts/hap/generate-vad-asr-hap-script.py b/scripts/hap/generate-vad-asr-hap-script.py new file mode 120000 index 000000000..617339772 --- /dev/null +++ b/scripts/hap/generate-vad-asr-hap-script.py @@ -0,0 +1 @@ +../apk/generate-vad-asr-apk-script.py \ No newline at end of file From c9d3b6cd8cb69806666b1755eb38f445bc0d4934 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sat, 30 Nov 2024 15:23:45 +0800 Subject: [PATCH 086/183] Add microphone demo about VAD+ASR for HarmonyOS (#1581) --- .../entry/src/main/ets/pages/Index.ets | 177 ++++++++++++++++-- .../main/ets/pages/NonStreamingAsrModels.ets | 5 +- .../entry/src/main/ets/pages/Permission.ets | 26 +++ .../workers/NonStreamingAsrWithVadWorker.ets | 68 ++++++- .../entry/src/main/module.json5 | 12 ++ .../main/resources/base/element/string.json | 4 + scripts/hap/build-hap-vad-asr.sh.in | 7 +- 7 files changed, 276 insertions(+), 23 deletions(-) create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Permission.ets diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets index e32b4eb76..5aef4d9a8 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets @@ -3,47 +3,127 @@ import worker, { MessageEvents } from '@ohos.worker'; import { BusinessError } from '@kit.BasicServicesKit'; import { picker } from '@kit.CoreFileKit'; +import { Permissions } from '@kit.AbilityKit'; +import { allAllowed, requestPermissions } from './Permission'; +import { audio } from '@kit.AudioKit'; + @Entry @Component struct Index { @State currentIndex: number = 0; - @State resultFromFile: string = ''; + @State resultForFile: string = ''; @State progressForFile: number = 0; @State selectFileBtnEnabled: boolean = false; - @State message: string = 'To be implemented'; @State lang: string = 'English'; + @State resultForMic: string = ''; + @State micStarted: boolean = false; + @State message: string = 'Start recording'; + @State micInitDone: boolean = false; private controller: TabsController = new TabsController(); private workerInstance?: worker.ThreadWorker private readonly scriptURL: string = 'entry/ets/workers/NonStreamingAsrWithVadWorker.ets' + private mic?: audio.AudioCapturer; + private sampleList: Float32Array[] = [] + + flatten(samples: Float32Array[]): Float32Array { + let n = 0; + for (let i = 0; i < samples.length; ++i) { + n += samples[i].length; + } + + const ans: Float32Array = new Float32Array(n); + let offset: number = 0; + for (let i = 0; i < samples.length; ++i) { + ans.set(samples[i], offset); + offset += samples[i].length; + } + + return ans; + } + + async initMic() { + const permissions: Permissions[] = ["ohos.permission.MICROPHONE"]; + let allowed: boolean = await allAllowed(permissions); + if (!allowed) { + requestPermissions(permissions); + console.log("request to access the microphone"); + + allowed = await allAllowed(permissions); + if (!allowed) { + console.error('failed to get microphone permission'); + this.resultForMic = "Failed to get microphone permission. Please retry"; + return; + } + } else { + console.log("allowed to access microphone"); + } - aboutToAppear(): void { + const audioStreamInfo: audio.AudioStreamInfo = { + samplingRate: audio.AudioSamplingRate.SAMPLE_RATE_16000, + channels: audio.AudioChannel.CHANNEL_1, + sampleFormat: audio.AudioSampleFormat.SAMPLE_FORMAT_S16LE, + encodingType: audio.AudioEncodingType.ENCODING_TYPE_RAW, + }; + + const audioCapturerInfo: audio.AudioCapturerInfo = { + source: audio.SourceType.SOURCE_TYPE_MIC, + capturerFlags: 0 + }; + + const audioCapturerOptions: audio.AudioCapturerOptions = { + streamInfo: audioStreamInfo, + capturerInfo: audioCapturerInfo + + }; + audio.createAudioCapturer(audioCapturerOptions, (err, data) => { + if (err) { + console.error(`error code is ${err.code}, error message is ${err.message}`); + this.resultForMic = 'Failed to init microphone'; + } else { + console.info(`init mic successfully`); + this.mic = data; + this.mic.on('readData', this.micCallback); + + if (this.workerInstance) { + this.workerInstance.postMessage({ msgType: 'init-vad-mic', context: getContext() }); + } + } + }); + } + + async aboutToAppear() { this.workerInstance = new worker.ThreadWorker(this.scriptURL, { name: 'NonStreaming ASR worker' }); this.workerInstance.onmessage = (e: MessageEvents) => { const msgType = e.data['msgType'] as string; - console.log(`received data ${msgType}`); + console.log(`received msg from worker: ${msgType}`); + + if (msgType == 'init-vad-mic-done') { + this.micInitDone = true; + } if (msgType == 'init-non-streaming-asr-done') { this.selectFileBtnEnabled = true; + this.resultForFile = `Initializing done.\n\nPlease select a wave file of 16kHz in language ${this.lang}`; } if (msgType == 'non-streaming-asr-vad-decode-done') { - this.resultFromFile = e.data['text'] as string + '\n'; + this.resultForFile = e.data['text'] as string + '\n'; } if (msgType == 'non-streaming-asr-vad-decode-partial') { - if (this.resultFromFile == '') { - this.resultFromFile = e.data['text'] as string; + if (this.resultForFile == '') { + this.resultForFile = e.data['text'] as string; } else { - this.resultFromFile += '\n\n' + e.data['text'] as string; + this.resultForFile += '\n\n' + e.data['text'] as string; } } if (msgType == 'non-streaming-asr-vad-decode-error') { - this.resultFromFile = e.data['text'] as string; + this.resultForFile = e.data['text'] as string; } if (msgType == 'non-streaming-asr-vad-decode-progress') { @@ -51,11 +131,26 @@ struct Index { this.selectFileBtnEnabled = this.progressForFile >= 100; } + + if (msgType == 'non-streaming-asr-vad-mic-partial') { + if (this.resultForMic == '') { + this.resultForMic = e.data['text'] as string; + } else { + this.resultForMic += '\n\n' + e.data['text'] as string; + } + } + + if (msgType == 'non-streaming-asr-vad-mic-error') { + this.resultForMic = e.data['text'] as string; + } } const context = getContext(); + this.resultForFile = 'Initializing models'; this.workerInstance.postMessage({ msgType: 'init-vad', context }); this.workerInstance.postMessage({ msgType: 'init-non-streaming-asr', context }); + + await this.initMic(); } @Builder @@ -86,13 +181,13 @@ struct Index { .lineHeight(41) .fontWeight(500) - Button('Select .wav file ') + Button('Select .wav file (16kHz) ') .enabled(this.selectFileBtnEnabled) .fontSize(13) .width(296) .height(60) .onClick(() => { - this.resultFromFile = ''; + this.resultForFile = ''; this.progressForFile = 0; const documentSelectOptions = new picker.DocumentSelectOptions(); @@ -103,7 +198,7 @@ struct Index { console.log(`Result: ${result}`); if (!result[0]) { - this.resultFromFile = 'Please select a file to decode'; + this.resultForFile = 'Please select a file to decode'; this.selectFileBtnEnabled = true; return; } @@ -135,7 +230,7 @@ struct Index { }.width('100%').justifyContent(FlexAlign.Center) } - TextArea({ text: this.resultFromFile }).width('100%').lineSpacing({ value: 10, unit: LengthUnit.VP }); + TextArea({ text: this.resultForFile }).width('100%').lineSpacing({ value: 10, unit: LengthUnit.VP }); } .alignItems(HorizontalAlign.Center) @@ -144,10 +239,50 @@ struct Index { TabContent() { Column() { - Text(this.message) - .fontSize(50) - .fontWeight(FontWeight.Bold); + Button(this.message) + .enabled(this.micInitDone) + .onClick(() => { + console.log('clicked mic button'); + this.resultForMic = ''; + if (this.mic) { + if (this.micStarted) { + this.mic.stop(); + this.message = "Start recording"; + this.micStarted = false; + console.log('mic stopped'); + + const samples = this.flatten(this.sampleList); + let s = 0; + for (let i = 0; i < samples.length; ++i) { + s += samples[i]; + } + console.log(`samples ${samples.length}, sum: ${s}`); + + if (this.workerInstance) { + console.log('decode mic'); + this.workerInstance.postMessage({ + msgType: 'non-streaming-asr-vad-mic', + samples, + }); + } else { + console.log(`this worker instance is undefined ${this.workerInstance}`); + } + } else { + this.sampleList = []; + this.mic.start(); + this.message = "Stop recording"; + this.micStarted = true; + console.log('mic started'); + } + } + }); + + Text(`Supported languages: ${this.lang}`) + + TextArea({ text: this.resultForMic }).width('100%').lineSpacing({ value: 10, unit: LengthUnit.VP }); } + .alignItems(HorizontalAlign.Center) + .justifyContent(FlexAlign.Start) } .tabBar(this.TabBuilder('From mic', 1, $r('app.media.ic_public_input_voice'), $r('app.media.ic_public_input_voice_default'))) @@ -170,4 +305,14 @@ struct Index { .width('100%') .justifyContent(FlexAlign.Start) } + + private micCallback = (buffer: ArrayBuffer) => { + const view: Int16Array = new Int16Array(buffer); + + const samplesFloat: Float32Array = new Float32Array(view.length); + for (let i = 0; i < view.length; ++i) { + samplesFloat[i] = view[i] / 32768.0; + } + this.sampleList.push(samplesFloat); + } } \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/NonStreamingAsrModels.ets b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/NonStreamingAsrModels.ets index a65242484..4a1af6466 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/NonStreamingAsrModels.ets +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/NonStreamingAsrModels.ets @@ -229,9 +229,10 @@ export function getOfflineModelConfig(type: number): OfflineModelConfig { break; } + default: { + console.log(`Please specify a supported type. Given type ${type}`); + } } - console.log(`Please specify a supported type. Given type ${type}`); - return c; } diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Permission.ets b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Permission.ets new file mode 100644 index 000000000..40ef391ad --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Permission.ets @@ -0,0 +1,26 @@ +// This file is modified from +// https://gitee.com/ukSir/hmchat2/blob/master/entry/src/main/ets/utils/permissionMananger.ets +import { abilityAccessCtrl, bundleManager, common, Permissions } from '@kit.AbilityKit'; + +export function allAllowed(permissions: Permissions[]): boolean { + if (permissions.length == 0) { + return false; + } + + const mgr: abilityAccessCtrl.AtManager = abilityAccessCtrl.createAtManager(); + + const bundleInfo = bundleManager.getBundleInfoForSelfSync(bundleManager.BundleFlag.GET_BUNDLE_INFO_WITH_APPLICATION); + + let tokenID: number = bundleInfo.appInfo.accessTokenId; + + return permissions.every(permission => abilityAccessCtrl.GrantStatus.PERMISSION_GRANTED == + mgr.checkAccessTokenSync(tokenID, permission)); +} + +export async function requestPermissions(permissions: Permissions[]): Promise { + const mgr: abilityAccessCtrl.AtManager = abilityAccessCtrl.createAtManager(); + const context: Context = getContext() as common.UIAbilityContext; + + const result = await mgr.requestPermissionsFromUser(context, permissions); + return result.authResults.length > 0 && result.authResults.every(authResults => authResults == 0); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/workers/NonStreamingAsrWithVadWorker.ets b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/workers/NonStreamingAsrWithVadWorker.ets index 346be5954..3076183d0 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/workers/NonStreamingAsrWithVadWorker.ets +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/workers/NonStreamingAsrWithVadWorker.ets @@ -13,11 +13,13 @@ import { import { Context } from '@kit.AbilityKit'; import { fileIo } from '@kit.CoreFileKit'; import { getOfflineModelConfig } from '../pages/NonStreamingAsrModels'; +import { BusinessError } from '@kit.BasicServicesKit'; const workerPort: ThreadWorkerGlobalScope = worker.workerPort; let recognizer: OfflineRecognizer; let vad: Vad; // vad for decoding files +let vadMic: Vad; // vad for mic function initVad(context: Context): Vad { let mgr = context.resourceManager; @@ -73,7 +75,7 @@ interface Wave { sampleRate: number; } -function decode(filename: string): string { +function decodeFile(filename: string): string { vad.reset(); const fp = fileIo.openSync(filename); @@ -83,6 +85,9 @@ function decode(filename: string): string { const data: Uint8Array = new Uint8Array(arrayBuffer); const wave: Wave = readWaveFromBinary(data); + if (wave.sampleRate != 16000) { + return `the sample rate in ${filename} is not 16000Hz. Given: ${wave.sampleRate}Hz.\nPlease select a wav file of 16kHz.`; + } console.log(`sample rate ${wave.sampleRate}`); console.log(`samples length ${wave.samples.length}`); @@ -130,6 +135,47 @@ function decode(filename: string): string { return resultList.join('\n\n'); } +function decodeMic(samples: Float32Array) { + const resultList: string[] = []; + + const windowSize: number = vad.config.sileroVad.windowSize; + for (let i = 0; i < samples.length; i += windowSize) { + const thisWindow: Float32Array = samples.subarray(i, i + windowSize) + vad.acceptWaveform(thisWindow); + if (i + windowSize >= samples.length) { + vad.flush(); + } + while (!vad.isEmpty()) { + const segment: SpeechSegment = vad.front(); + const _startTime: number = (segment.start / 16000); + const _endTime: number = _startTime + segment.samples.length / 16000; + + if (_endTime - _startTime < 0.2) { + vad.pop(); + continue; + } + + const startTime: string = _startTime.toFixed(2); + const endTime: string = _endTime.toFixed(2); + + const stream: OfflineStream = recognizer.createStream(); + stream.acceptWaveform({ samples: segment.samples, sampleRate: 16000 }); + recognizer.decode(stream); + const result: OnlineRecognizerResult = recognizer.getResult(stream); + + const text: string = `${startTime} -- ${endTime} ${result.text}` + resultList.push(text); + console.log(`partial result ${text}`); + + workerPort.postMessage({ 'msgType': 'non-streaming-asr-vad-mic-partial', text }); + + vad.pop(); + } + } + + return resultList.join('\n\n'); +} + /** * Defines the event handler to be called when the worker thread receives a message sent by the host thread. * The event handler is executed in the worker thread. @@ -146,6 +192,13 @@ workerPort.onmessage = (e: MessageEvents) => { workerPort.postMessage({ 'msgType': 'init-vad-done' }); } + if (msgType == 'init-vad-mic' && !vadMic) { + const context = e.data['context'] as Context; + vadMic = initVad(context); + console.log('init vad mic done'); + workerPort.postMessage({ 'msgType': 'init-vad-mic-done' }); + } + if (msgType == 'init-non-streaming-asr' && !recognizer) { const context = e.data['context'] as Context; recognizer = initNonStreamingAsr(context); @@ -157,7 +210,7 @@ workerPort.onmessage = (e: MessageEvents) => { const filename = e.data['filename'] as string; console.log(`decoding ${filename}`); try { - const text = decode(filename); + const text = decodeFile(filename); workerPort.postMessage({ msgType: 'non-streaming-asr-vad-decode-done', text }); } catch (e) { workerPort.postMessage({ msgType: 'non-streaming-asr-vad-decode-error', text: `Failed to decode ${filename}` }); @@ -165,6 +218,17 @@ workerPort.onmessage = (e: MessageEvents) => { workerPort.postMessage({ 'msgType': 'non-streaming-asr-vad-decode-progress', progress: 100 }); } + + if (msgType == 'non-streaming-asr-vad-mic') { + const samples: Float32Array = e.data['samples'] as Float32Array; + vadMic.reset(); + try { + const text = decodeMic(samples); + workerPort.postMessage({ msgType: 'non-streaming-asr-vad-mic-done', text }); + } catch (e) { + workerPort.postMessage({ msgType: 'non-streaming-asr-vad-mic-error', text: `Failed to decode` }); + } + } } /** diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/module.json5 b/harmony-os/SherpaOnnxVadAsr/entry/src/main/module.json5 index a1cea8b6a..e8c24aeba 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/src/main/module.json5 +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/module.json5 @@ -47,6 +47,18 @@ } ], } + ], + "requestPermissions": [ + { + "name": "ohos.permission.MICROPHONE", + "reason": "$string:mic_reason", + "usedScene": { + "abilities": [ + "FormAbility", + ], + "when": "always", + } + } ] } } \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/element/string.json b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/element/string.json index 3b3015be7..09e201b54 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/element/string.json +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/element/string.json @@ -11,6 +11,10 @@ { "name": "EntryAbility_label", "value": "VAD_ASR" + }, + { + "name": "mic_reason", + "value": "access the microhone for speech recognition" } ] } \ No newline at end of file diff --git a/scripts/hap/build-hap-vad-asr.sh.in b/scripts/hap/build-hap-vad-asr.sh.in index d4fabf060..7020d927d 100644 --- a/scripts/hap/build-hap-vad-asr.sh.in +++ b/scripts/hap/build-hap-vad-asr.sh.in @@ -90,7 +90,7 @@ hvigorw assembleHap --mode module -p product=default -p buildMode=release --no-d ls -lh ./entry/build/default/outputs/default/entry-default-unsigned.hap -in_file=./entry/build/default/outputs/default/entry-default-unsigned.hap +in_file=$PWD/entry/build/default/outputs/default/entry-default-unsigned.hap out_file=$PWD/entry/build/default/outputs/default/entry-default-signed.hap java -jar $jar sign-app -keyAlias "$HAP_KEY_ALIAS" -signAlg "SHA256withECDSA" -mode "localSign" \ @@ -100,11 +100,12 @@ java -jar $jar sign-app -keyAlias "$HAP_KEY_ALIAS" -signAlg "SHA256withECDSA" -m ls -l $in_file $out_file ls -lh $in_file $out_file -rm $in_file rm -rf ./entry/src/main/resources/rawfile/$model_name popd -mv $out_file ./haps/sherpa-onnx-${SHERPA_ONNX_VERSION}-vad_asr-$lang-$short_name.hap +# Use unsigned hap +mv $in_file ./haps/sherpa-onnx-${SHERPA_ONNX_VERSION}-vad_asr-$lang-$short_name.hap +# mv $out_file ./haps/sherpa-onnx-${SHERPA_ONNX_VERSION}-vad_asr-$lang-$short_name.hap ls -lh haps From a3d6e1acc7e65d2bc9eda56d5548eeff295e9349 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sat, 30 Nov 2024 16:24:22 +0800 Subject: [PATCH 087/183] Fix getting microphone permission for HarmonyOS VAD+ASR example (#1582) --- .../SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets index 5aef4d9a8..b0695f3a0 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets @@ -46,8 +46,14 @@ struct Index { const permissions: Permissions[] = ["ohos.permission.MICROPHONE"]; let allowed: boolean = await allAllowed(permissions); if (!allowed) { - requestPermissions(permissions); console.log("request to access the microphone"); + const status: boolean = await requestPermissions(permissions); + + if (!status) { + console.error('access to microphone is denied') + this.resultForMic = "Failed to get microphone permission. Please retry"; + return; + } allowed = await allAllowed(permissions); if (!allowed) { From dc3287f3a8b60a1b020dd5bc33187f8e40926701 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sun, 1 Dec 2024 21:43:34 +0800 Subject: [PATCH 088/183] Add HarmonyOS support for text-to-speech. (#1584) --- .github/workflows/lazarus.yaml | 2 +- .github/workflows/test-nodejs-npm.yaml | 2 +- .../sherpa_onnx/BuildProfile.ets | 4 +- .../SherpaOnnxHar/sherpa_onnx/Index.ets | 9 +++ .../sherpa_onnx/src/main/cpp/macros.h | 2 +- .../src/main/cpp/non-streaming-asr.cc | 5 +- .../src/main/cpp/non-streaming-tts.cc | 20 ++++++ .../sherpa_onnx/src/main/cpp/streaming-asr.cc | 5 +- .../main/cpp/types/libsherpa_onnx/Index.d.ts | 5 ++ .../sherpa_onnx/src/main/cpp/vad.cc | 21 ++++-- .../sherpa_onnx/src/main/cpp/wave-reader.cc | 13 ++-- .../main/ets/components/NonStreamingAsr.ets | 2 +- .../main/ets/components/NonStreamingTts.ets | 66 +++++++++++++++++++ .../src/main/ets/components/StreamingAsr.ets | 4 +- .../entry/src/main/module.json5 | 6 +- scripts/check_style_cpplint.sh | 3 +- sherpa-onnx/c-api/c-api.cc | 43 ++++++++++-- sherpa-onnx/c-api/c-api.h | 3 + sherpa-onnx/csrc/lexicon.cc | 27 ++++++-- sherpa-onnx/csrc/lexicon.h | 14 ++-- sherpa-onnx/csrc/offline-ctc-model.cc | 2 - .../csrc/offline-tts-character-frontend.cc | 36 +++++++--- .../csrc/offline-tts-character-frontend.h | 10 +-- sherpa-onnx/csrc/offline-tts-impl.cc | 22 ++++++- sherpa-onnx/csrc/offline-tts-impl.h | 10 +-- sherpa-onnx/csrc/offline-tts-vits-impl.h | 18 ++--- sherpa-onnx/csrc/offline-tts-vits-model.cc | 29 ++++++-- sherpa-onnx/csrc/offline-tts-vits-model.h | 11 +--- sherpa-onnx/csrc/offline-tts.cc | 24 ++++++- sherpa-onnx/csrc/offline-tts.h | 10 +-- sherpa-onnx/csrc/piper-phonemize-lexicon.cc | 24 +++++-- sherpa-onnx/csrc/piper-phonemize-lexicon.h | 10 +-- sherpa-onnx/csrc/silero-vad-model.cc | 4 +- 33 files changed, 333 insertions(+), 133 deletions(-) create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets diff --git a/.github/workflows/lazarus.yaml b/.github/workflows/lazarus.yaml index e7cef16ef..f327c99e2 100644 --- a/.github/workflows/lazarus.yaml +++ b/.github/workflows/lazarus.yaml @@ -56,7 +56,7 @@ jobs: key: ${{ matrix.os }} # See https://github.com/gcarreno/setup-lazarus - - uses: gcarreno/setup-lazarus@v3 + - uses: gcarreno/setup-lazarus@v3.3.1 with: lazarus-version: "stable" with-cache: true diff --git a/.github/workflows/test-nodejs-npm.yaml b/.github/workflows/test-nodejs-npm.yaml index cc49ac0c4..e1358fd8d 100644 --- a/.github/workflows/test-nodejs-npm.yaml +++ b/.github/workflows/test-nodejs-npm.yaml @@ -26,7 +26,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-2019] - python-version: ["3.8"] + python-version: ["3.10"] steps: - uses: actions/checkout@v4 diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets index 502c150fd..c6564edc7 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets @@ -2,8 +2,8 @@ * Use these variables when you tailor your ArkTS code. They must be of the const type. */ export const HAR_VERSION = '1.10.32'; -export const BUILD_MODE_NAME = 'release'; -export const DEBUG = false; +export const BUILD_MODE_NAME = 'debug'; +export const DEBUG = true; export const TARGET_NAME = 'default'; /** diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets index 185aa51fd..959b6ba02 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets @@ -38,3 +38,12 @@ export { OnlineRecognizerResult, OnlineRecognizer, } from './src/main/ets/components/StreamingAsr'; + +export { + OfflineTtsVitsModelConfig, + OfflineTtsModelConfig, + OfflineTtsConfig, + OfflineTts, + TtsOutput, + TtsInput, +} from './src/main/ets/components/NonStreamingTts'; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/macros.h b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/macros.h index 7a6cc93e6..fc70abb01 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/macros.h +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/macros.h @@ -8,8 +8,8 @@ #include #if __OHOS__ -#include "rawfile/raw_file_manager.h" #include "hilog/log.h" +#include "rawfile/raw_file_manager.h" #undef LOG_DOMAIN #undef LOG_TAG diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-asr.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-asr.cc index c7d9560a2..a34139aa0 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-asr.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-asr.cc @@ -236,7 +236,10 @@ CreateOfflineRecognizerWrapper(const Napi::CallbackInfo &info) { SHERPA_ONNX_ASSIGN_ATTR_FLOAT(blank_penalty, blankPenalty); #if __OHOS__ - std::unique_ptr mgr (OH_ResourceManager_InitNativeResourceManager(env, info[1]), &OH_ResourceManager_ReleaseNativeResourceManager); + std::unique_ptr + mgr(OH_ResourceManager_InitNativeResourceManager(env, info[1]), + &OH_ResourceManager_ReleaseNativeResourceManager); const SherpaOnnxOfflineRecognizer *recognizer = SherpaOnnxCreateOfflineRecognizerOHOS(&c, mgr.get()); diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc index 70d97cddb..da70e662c 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc @@ -63,6 +63,17 @@ static SherpaOnnxOfflineTtsModelConfig GetOfflineTtsModelConfig( static Napi::External CreateOfflineTtsWrapper( const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); +#if __OHOS__ + // the last argument is the NativeResourceManager + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } +#else if (info.Length() != 1) { std::ostringstream os; os << "Expect only 1 argument. Given: " << info.Length(); @@ -71,6 +82,7 @@ static Napi::External CreateOfflineTtsWrapper( return {}; } +#endif if (!info[0].IsObject()) { Napi::TypeError::New(env, "Expect an object as the argument") @@ -90,7 +102,15 @@ static Napi::External CreateOfflineTtsWrapper( SHERPA_ONNX_ASSIGN_ATTR_INT32(max_num_sentences, maxNumSentences); SHERPA_ONNX_ASSIGN_ATTR_STR(rule_fars, ruleFars); +#if __OHOS__ + std::unique_ptr + mgr(OH_ResourceManager_InitNativeResourceManager(env, info[1]), + &OH_ResourceManager_ReleaseNativeResourceManager); + SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTtsOHOS(&c, mgr.get()); +#else SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTts(&c); +#endif if (c.model.vits.model) { delete[] c.model.vits.model; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/streaming-asr.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/streaming-asr.cc index ffe562d79..59ad5ce52 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/streaming-asr.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/streaming-asr.cc @@ -211,7 +211,10 @@ static Napi::External CreateOnlineRecognizerWrapper( c.ctc_fst_decoder_config = GetCtcFstDecoderConfig(o); #if __OHOS__ - std::unique_ptr mgr (OH_ResourceManager_InitNativeResourceManager(env, info[1]), &OH_ResourceManager_ReleaseNativeResourceManager); + std::unique_ptr + mgr(OH_ResourceManager_InitNativeResourceManager(env, info[1]), + &OH_ResourceManager_ReleaseNativeResourceManager); const SherpaOnnxOnlineRecognizer *recognizer = SherpaOnnxCreateOnlineRecognizerOHOS(&c, mgr.get()); diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts index 10ff7745c..f44ade356 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts @@ -33,3 +33,8 @@ export const decodeOnlineStream: (handle: object, streamHandle: object) => void; export const isEndpoint: (handle: object, streamHandle: object) => boolean; export const reset: (handle: object, streamHandle: object) => void; export const getOnlineStreamResultAsJson: (handle: object, streamHandle: object) => string; + +export const createOfflineTts: (config: object, mgr?: object) => object; +export const getOfflineTtsNumSpeakers: (handle: object) => number; +export const getOfflineTtsSampleRate: (handle: object) => number; +export const offlineTtsGenerate: (handle: object, input: object) => object; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/vad.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/vad.cc index b505c53d2..b1defcac0 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/vad.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/vad.cc @@ -70,8 +70,10 @@ static void CircularBufferPushWrapper(const Napi::CallbackInfo &info) { #if __OHOS__ // Note(fangjun): Normally, we don't need to divied it by sizeof(float). - // However, data.ElementLength() here returns number of bytes, not number of elements. - SherpaOnnxCircularBufferPush(buf, data.Data(), data.ElementLength() / sizeof(float)); + // However, data.ElementLength() here returns number of bytes, not number of + // elements. + SherpaOnnxCircularBufferPush(buf, data.Data(), + data.ElementLength() / sizeof(float)); #else SherpaOnnxCircularBufferPush(buf, data.Data(), data.ElementLength()); #endif @@ -353,10 +355,14 @@ CreateVoiceActivityDetectorWrapper(const Napi::CallbackInfo &info) { float buffer_size_in_seconds = info[1].As().FloatValue(); #if __OHOS__ - std::unique_ptr mgr(OH_ResourceManager_InitNativeResourceManager(env, info[2]), &OH_ResourceManager_ReleaseNativeResourceManager); + std::unique_ptr + mgr(OH_ResourceManager_InitNativeResourceManager(env, info[2]), + &OH_ResourceManager_ReleaseNativeResourceManager); SherpaOnnxVoiceActivityDetector *vad = - SherpaOnnxCreateVoiceActivityDetectorOHOS(&c, buffer_size_in_seconds, mgr.get()); + SherpaOnnxCreateVoiceActivityDetectorOHOS(&c, buffer_size_in_seconds, + mgr.get()); #else SherpaOnnxVoiceActivityDetector *vad = SherpaOnnxCreateVoiceActivityDetector(&c, buffer_size_in_seconds); @@ -410,9 +416,10 @@ static void VoiceActivityDetectorAcceptWaveformWrapper( Napi::Float32Array samples = info[1].As(); #if __OHOS__ - // Note(fangjun): For unknown reasons, we need to use `/sizeof(float)` here for Huawei - SherpaOnnxVoiceActivityDetectorAcceptWaveform(vad, samples.Data(), - samples.ElementLength() / sizeof(float)); + // Note(fangjun): For unknown reasons, we need to use `/sizeof(float)` here + // for Huawei + SherpaOnnxVoiceActivityDetectorAcceptWaveform( + vad, samples.Data(), samples.ElementLength() / sizeof(float)); #else SherpaOnnxVoiceActivityDetectorAcceptWaveform(vad, samples.Data(), samples.ElementLength()); diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-reader.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-reader.cc index 2973c6169..23b3a7242 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-reader.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-reader.cc @@ -102,10 +102,11 @@ static Napi::Object ReadWaveFromBinaryWrapper(const Napi::CallbackInfo &info) { return {}; } - + Napi::Uint8Array data = info[0].As(); int32_t n = data.ElementLength(); - const SherpaOnnxWave *wave = SherpaOnnxReadWaveFromBinaryData(reinterpret_cast(data.Data()), n); + const SherpaOnnxWave *wave = SherpaOnnxReadWaveFromBinaryData( + reinterpret_cast(data.Data()), n); if (!wave) { std::ostringstream os; os << "Failed to read wave"; @@ -113,7 +114,7 @@ static Napi::Object ReadWaveFromBinaryWrapper(const Napi::CallbackInfo &info) { return {}; } - + bool enable_external_buffer = true; if (info.Length() == 2) { if (info[1].IsBoolean()) { @@ -165,7 +166,7 @@ static Napi::Object ReadWaveFromBinaryWrapper(const Napi::CallbackInfo &info) { void InitWaveReader(Napi::Env env, Napi::Object exports) { exports.Set(Napi::String::New(env, "readWave"), Napi::Function::New(env, ReadWaveWrapper)); - + exports.Set(Napi::String::New(env, "readWaveFromBinary"), - Napi::Function::New(env, ReadWaveFromBinaryWrapper)); -} \ No newline at end of file + Napi::Function::New(env, ReadWaveFromBinaryWrapper)); +} diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingAsr.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingAsr.ets index 0cc8466a9..d3f849cca 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingAsr.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingAsr.ets @@ -79,7 +79,7 @@ export class OfflineModelConfig { public tokens: string = ''; public numThreads: number = 1; public debug: boolean = false; - public provider: string = "cpu"; + public provider: string = 'cpu'; public modelType: string = ''; public modelingUnit: string = "cjkchar"; public bpeVocab: string = ''; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets new file mode 100644 index 000000000..c568b9990 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets @@ -0,0 +1,66 @@ +import { + createOfflineTts, + getOfflineTtsNumSpeakers, + getOfflineTtsSampleRate, + offlineTtsGenerate, +} from "libsherpa_onnx.so"; + +export class OfflineTtsVitsModelConfig { + public model: string = ''; + public lexicon: string = ''; + public tokens: string = ''; + public dataDir: string = ''; + public dictDir: String = ''; + public noiseScale: number = 0.667; + public noiseScaleW: number = 0.8; + public lengthScale: number = 1.0; +} + +export class OfflineTtsModelConfig{ + public vits: OfflineTtsVitsModelConfig = new OfflineTtsVitsModelConfig(); + public numThreads: number = 1; + public debug: boolean = false; + public provider: string = 'cpu'; +} + +export class OfflineTtsConfig{ + public model: OfflineTtsModelConfig = new OfflineTtsModelConfig(); + public ruleFsts: string = ''; + public ruleFars: string = ''; + public maxNumSentences: number = 1; +} + +export class TtsOutput { + public samples: Float32Array = new Float32Array(0); + public sampleRate: number = 0; +} + +export class TtsInput { + public text: string = ''; + public sid: number = 0; + public speed: number = 1.0; +} + +export class OfflineTts { + private handle: object; + public config: OfflineTtsConfig; + public numSpeakers: number; + public sampleRate: number; + constructor(config: OfflineTtsConfig, mgr?: object) { + this.handle = createOfflineTts(config, mgr); + this.config = config; + + this.numSpeakers = getOfflineTtsNumSpeakers(this.handle); + this.sampleRate = getOfflineTtsSampleRate(this.handle); + } + + /* + input obj: {text: "xxxx", sid: 0, speed: 1.0} + where text is a string, sid is a int32, speed is a float + + return an object {samples: Float32Array, sampleRate: } + */ + generate(input: TtsInput): TtsOutput { + return offlineTtsGenerate(this.handle, input) as TtsOutput; + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/StreamingAsr.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/StreamingAsr.ets index 7ecc552ca..3b2985771 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/StreamingAsr.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/StreamingAsr.ets @@ -52,7 +52,7 @@ export class OnlineModelConfig { public zipformer2_ctc: OnlineZipformer2CtcModelConfig = new OnlineZipformer2CtcModelConfig(); public tokens: string = ''; public numThreads: number = 1; - public provider: string = "cpu"; + public provider: string = 'cpu'; public debug: boolean = false; public modelType: string = ''; public modelingUnit: string = "cjkchar"; @@ -67,7 +67,7 @@ export class OnlineCtcFstDecoderConfig { export class OnlineRecognizerConfig { public featConfig: FeatureConfig = new FeatureConfig(); public modelConfig: OnlineModelConfig = new OnlineModelConfig(); - public decodingMethod: string = "greedy_search"; + public decodingMethod: string = 'greedy_search'; public maxActivePaths: number = 4; public enableEndpoint: boolean = false; public rule1MinTrailingSilence: number = 2.4; diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/module.json5 b/harmony-os/SherpaOnnxVadAsr/entry/src/main/module.json5 index e8c24aeba..660c2bb47 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/src/main/module.json5 +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/module.json5 @@ -54,11 +54,11 @@ "reason": "$string:mic_reason", "usedScene": { "abilities": [ - "FormAbility", + "EntryAbility", ], - "when": "always", + "when": "inuse", } } ] } -} \ No newline at end of file +} diff --git a/scripts/check_style_cpplint.sh b/scripts/check_style_cpplint.sh index ea419242a..dcadcd991 100755 --- a/scripts/check_style_cpplint.sh +++ b/scripts/check_style_cpplint.sh @@ -103,6 +103,7 @@ function do_check() { 2) echo "Check all files" files=$(find $sherpa_onnx_dir/cxx-api-examples $sherpa_onnx_dir/c-api-examples $sherpa_onnx_dir/sherpa-onnx/csrc $sherpa_onnx_dir/sherpa-onnx/python $sherpa_onnx_dir/scripts/node-addon-api/src $sherpa_onnx_dir/sherpa-onnx/jni $sherpa_onnx_dir/sherpa-onnx/c-api -name "*.h" -o -name "*.cc") + files2=$(find $sherpa_onnx_dir/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/ -name "*.cc") ;; *) echo "Check last commit" @@ -110,7 +111,7 @@ function do_check() { ;; esac - for f in $files; do + for f in $files $files2; do need_check=$(is_source_code_file $f) if $need_check; then [[ -f $f ]] && check_style $f diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index a6ad0772e..166430da4 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -485,9 +485,9 @@ static sherpa_onnx::OfflineRecognizerConfig GetOfflineRecognizerConfig( if (config->model_config.debug) { #if __OHOS__ - SHERPA_ONNX_LOGE("%{public}s", recognizer_config.ToString().c_str()); + SHERPA_ONNX_LOGE("%{public}s\n", recognizer_config.ToString().c_str()); #else - SHERPA_ONNX_LOGE("%s", recognizer_config.ToString().c_str()); + SHERPA_ONNX_LOGE("%s\n", recognizer_config.ToString().c_str()); #endif } @@ -967,9 +967,9 @@ sherpa_onnx::VadModelConfig GetVadModelConfig( if (vad_config.debug) { #if __OHOS__ - SHERPA_ONNX_LOGE("%{public}s", vad_config.ToString().c_str()); + SHERPA_ONNX_LOGE("%{public}s\n", vad_config.ToString().c_str()); #else - SHERPA_ONNX_LOGE("%s", vad_config.ToString().c_str()); + SHERPA_ONNX_LOGE("%s\n", vad_config.ToString().c_str()); #endif } @@ -1053,7 +1053,7 @@ struct SherpaOnnxOfflineTts { std::unique_ptr impl; }; -SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTts( +static sherpa_onnx::OfflineTtsConfig GetOfflineTtsConfig( const SherpaOnnxOfflineTtsConfig *config) { sherpa_onnx::OfflineTtsConfig tts_config; @@ -1084,9 +1084,20 @@ SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTts( tts_config.max_num_sentences = SHERPA_ONNX_OR(config->max_num_sentences, 2); if (tts_config.model.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s\n", tts_config.ToString().c_str()); +#else SHERPA_ONNX_LOGE("%s\n", tts_config.ToString().c_str()); +#endif } + return tts_config; +} + +SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTts( + const SherpaOnnxOfflineTtsConfig *config) { + auto tts_config = GetOfflineTtsConfig(config); + if (!tts_config.Validate()) { SHERPA_ONNX_LOGE("Errors in config"); return nullptr; @@ -1908,6 +1919,7 @@ SherpaOnnxOfflineSpeakerDiarizationProcessWithCallbackNoArg( return ans; } +#endif #ifdef __OHOS__ @@ -1959,6 +1971,23 @@ SherpaOnnxVoiceActivityDetector *SherpaOnnxCreateVoiceActivityDetectorOHOS( return p; } -#endif -#endif +#if SHERPA_ONNX_ENABLE_TTS == 1 +SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTtsOHOS( + const SherpaOnnxOfflineTtsConfig *config, NativeResourceManager *mgr) { + if (!mgr) { + return SherpaOnnxCreateOfflineTts(config); + } + + auto tts_config = GetOfflineTtsConfig(config); + + SherpaOnnxOfflineTts *tts = new SherpaOnnxOfflineTts; + + tts->impl = std::make_unique(mgr, tts_config); + + return tts; +} + +#endif // #if SHERPA_ONNX_ENABLE_TTS == 1 + +#endif // #ifdef __OHOS__ diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index 5251413a8..e9cd5be0a 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -1558,6 +1558,9 @@ SHERPA_ONNX_API SherpaOnnxVoiceActivityDetector * SherpaOnnxCreateVoiceActivityDetectorOHOS( const SherpaOnnxVadModelConfig *config, float buffer_size_in_seconds, NativeResourceManager *mgr); + +SHERPA_ONNX_API SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTtsOHOS( + const SherpaOnnxOfflineTtsConfig *config, NativeResourceManager *mgr); #endif #if defined(__GNUC__) diff --git a/sherpa-onnx/csrc/lexicon.cc b/sherpa-onnx/csrc/lexicon.cc index 499f17f68..fe5e595b9 100644 --- a/sherpa-onnx/csrc/lexicon.cc +++ b/sherpa-onnx/csrc/lexicon.cc @@ -7,17 +7,19 @@ #include #include #include +#include #include +#include #include #if __ANDROID_API__ >= 9 -#include - #include "android/asset_manager.h" #include "android/asset_manager_jni.h" #endif -#include +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/onnx-utils.h" @@ -110,8 +112,8 @@ Lexicon::Lexicon(const std::string &lexicon, const std::string &tokens, InitPunctuations(punctuations); } -#if __ANDROID_API__ >= 9 -Lexicon::Lexicon(AAssetManager *mgr, const std::string &lexicon, +template +Lexicon::Lexicon(Manager *mgr, const std::string &lexicon, const std::string &tokens, const std::string &punctuations, const std::string &language, bool debug /*= false*/ ) @@ -132,7 +134,6 @@ Lexicon::Lexicon(AAssetManager *mgr, const std::string &lexicon, InitPunctuations(punctuations); } -#endif std::vector Lexicon::ConvertTextToTokenIds( const std::string &text, const std::string & /*voice*/ /*= ""*/) const { @@ -371,4 +372,18 @@ void Lexicon::InitPunctuations(const std::string &punctuations) { } } +#if __ANDROID_API__ >= 9 +template Lexicon::Lexicon(AAssetManager *mgr, const std::string &lexicon, + const std::string &tokens, + const std::string &punctuations, + const std::string &language, bool debug = false); +#endif + +#if __OHOS__ +template Lexicon::Lexicon(NativeResourceManager *mgr, + const std::string &lexicon, const std::string &tokens, + const std::string &punctuations, + const std::string &language, bool debug = false); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/lexicon.h b/sherpa-onnx/csrc/lexicon.h index fb60cdb7f..39329694b 100644 --- a/sherpa-onnx/csrc/lexicon.h +++ b/sherpa-onnx/csrc/lexicon.h @@ -13,11 +13,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/offline-tts-frontend.h" namespace sherpa_onnx { @@ -31,11 +26,10 @@ class Lexicon : public OfflineTtsFrontend { const std::string &punctuations, const std::string &language, bool debug = false); -#if __ANDROID_API__ >= 9 - Lexicon(AAssetManager *mgr, const std::string &lexicon, - const std::string &tokens, const std::string &punctuations, - const std::string &language, bool debug = false); -#endif + template + Lexicon(Manager *mgr, const std::string &lexicon, const std::string &tokens, + const std::string &punctuations, const std::string &language, + bool debug = false); std::vector ConvertTextToTokenIds( const std::string &text, const std::string &voice = "") const override; diff --git a/sherpa-onnx/csrc/offline-ctc-model.cc b/sherpa-onnx/csrc/offline-ctc-model.cc index daff5654a..6ca5f0054 100644 --- a/sherpa-onnx/csrc/offline-ctc-model.cc +++ b/sherpa-onnx/csrc/offline-ctc-model.cc @@ -136,7 +136,6 @@ std::unique_ptr OfflineCtcModel::Create( switch (model_type) { case ModelType::kEncDecCTCModelBPE: - return std::make_unique(config); case ModelType::kEncDecCTCModel: return std::make_unique(config); case ModelType::kEncDecHybridRNNTCTCBPEModel: @@ -187,7 +186,6 @@ std::unique_ptr OfflineCtcModel::Create( switch (model_type) { case ModelType::kEncDecCTCModelBPE: - return std::make_unique(mgr, config); case ModelType::kEncDecCTCModel: return std::make_unique(mgr, config); case ModelType::kEncDecHybridRNNTCTCBPEModel: diff --git a/sherpa-onnx/csrc/offline-tts-character-frontend.cc b/sherpa-onnx/csrc/offline-tts-character-frontend.cc index 72481e094..0806a9fa2 100644 --- a/sherpa-onnx/csrc/offline-tts-character-frontend.cc +++ b/sherpa-onnx/csrc/offline-tts-character-frontend.cc @@ -2,20 +2,24 @@ // // Copyright (c) 2023 Xiaomi Corporation -#if __ANDROID_API__ >= 9 -#include - -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif #include #include #include #include #include #include +#include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/offline-tts-character-frontend.h" #include "sherpa-onnx/csrc/onnx-utils.h" @@ -82,9 +86,9 @@ OfflineTtsCharacterFrontend::OfflineTtsCharacterFrontend( token2id_ = ReadTokens(is); } -#if __ANDROID_API__ >= 9 +template OfflineTtsCharacterFrontend::OfflineTtsCharacterFrontend( - AAssetManager *mgr, const std::string &tokens, + Manager *mgr, const std::string &tokens, const OfflineTtsVitsModelMetaData &meta_data) : meta_data_(meta_data) { auto buf = ReadFile(mgr, tokens); @@ -92,8 +96,6 @@ OfflineTtsCharacterFrontend::OfflineTtsCharacterFrontend( token2id_ = ReadTokens(is); } -#endif - std::vector OfflineTtsCharacterFrontend::ConvertTextToTokenIds( const std::string &_text, const std::string & /*voice = ""*/) const { // see @@ -189,4 +191,18 @@ std::vector OfflineTtsCharacterFrontend::ConvertTextToTokenIds( return ans; } +#if __ANDROID_API__ >= 9 +template OfflineTtsCharacterFrontend::OfflineTtsCharacterFrontend( + AAssetManager *mgr, const std::string &tokens, + const OfflineTtsVitsModelMetaData &meta_data); + +#endif + +#if __OHOS__ +template OfflineTtsCharacterFrontend::OfflineTtsCharacterFrontend( + NativeResourceManager *mgr, const std::string &tokens, + const OfflineTtsVitsModelMetaData &meta_data); + +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-tts-character-frontend.h b/sherpa-onnx/csrc/offline-tts-character-frontend.h index ffd2bb5f4..fcd2f6dd5 100644 --- a/sherpa-onnx/csrc/offline-tts-character-frontend.h +++ b/sherpa-onnx/csrc/offline-tts-character-frontend.h @@ -9,11 +9,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/offline-tts-frontend.h" #include "sherpa-onnx/csrc/offline-tts-vits-model-metadata.h" @@ -24,11 +19,10 @@ class OfflineTtsCharacterFrontend : public OfflineTtsFrontend { OfflineTtsCharacterFrontend(const std::string &tokens, const OfflineTtsVitsModelMetaData &meta_data); -#if __ANDROID_API__ >= 9 - OfflineTtsCharacterFrontend(AAssetManager *mgr, const std::string &tokens, + template + OfflineTtsCharacterFrontend(Manager *mgr, const std::string &tokens, const OfflineTtsVitsModelMetaData &meta_data); -#endif /** Convert a string to token IDs. * * @param text The input text. diff --git a/sherpa-onnx/csrc/offline-tts-impl.cc b/sherpa-onnx/csrc/offline-tts-impl.cc index 063730db8..62b6eebba 100644 --- a/sherpa-onnx/csrc/offline-tts-impl.cc +++ b/sherpa-onnx/csrc/offline-tts-impl.cc @@ -6,6 +6,15 @@ #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/offline-tts-vits-impl.h" namespace sherpa_onnx { @@ -16,12 +25,21 @@ std::unique_ptr OfflineTtsImpl::Create( return std::make_unique(config); } -#if __ANDROID_API__ >= 9 +template std::unique_ptr OfflineTtsImpl::Create( - AAssetManager *mgr, const OfflineTtsConfig &config) { + Manager *mgr, const OfflineTtsConfig &config) { // TODO(fangjun): Support other types return std::make_unique(mgr, config); } + +#if __ANDROID_API__ >= 9 +template std::unique_ptr OfflineTtsImpl::Create( + AAssetManager *mgr, const OfflineTtsConfig &config); +#endif + +#if __OHOS__ +template std::unique_ptr OfflineTtsImpl::Create( + NativeResourceManager *mgr, const OfflineTtsConfig &config); #endif } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-tts-impl.h b/sherpa-onnx/csrc/offline-tts-impl.h index 3c9e27b1b..db8b7162d 100644 --- a/sherpa-onnx/csrc/offline-tts-impl.h +++ b/sherpa-onnx/csrc/offline-tts-impl.h @@ -8,11 +8,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/offline-tts.h" namespace sherpa_onnx { @@ -23,10 +18,9 @@ class OfflineTtsImpl { static std::unique_ptr Create(const OfflineTtsConfig &config); -#if __ANDROID_API__ >= 9 - static std::unique_ptr Create(AAssetManager *mgr, + template + static std::unique_ptr Create(Manager *mgr, const OfflineTtsConfig &config); -#endif virtual GeneratedAudio Generate( const std::string &text, int64_t sid = 0, float speed = 1.0, diff --git a/sherpa-onnx/csrc/offline-tts-vits-impl.h b/sherpa-onnx/csrc/offline-tts-vits-impl.h index 72b21a3b5..972303dd4 100644 --- a/sherpa-onnx/csrc/offline-tts-vits-impl.h +++ b/sherpa-onnx/csrc/offline-tts-vits-impl.h @@ -6,16 +6,10 @@ #include #include +#include #include #include -#if __ANDROID_API__ >= 9 -#include - -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "fst/extensions/far/far.h" #include "kaldifst/csrc/kaldi-fst-io.h" #include "kaldifst/csrc/text-normalizer.h" @@ -82,8 +76,8 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { } } -#if __ANDROID_API__ >= 9 - OfflineTtsVitsImpl(AAssetManager *mgr, const OfflineTtsConfig &config) + template + OfflineTtsVitsImpl(Manager *mgr, const OfflineTtsConfig &config) : config_(config), model_(std::make_unique(mgr, config.model)) { InitFrontend(mgr); @@ -130,7 +124,6 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { } // for (const auto &f : files) } // if (!config.rule_fars.empty()) } -#endif int32_t SampleRate() const override { return model_->GetMetaData().sample_rate; @@ -297,8 +290,8 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { } private: -#if __ANDROID_API__ >= 9 - void InitFrontend(AAssetManager *mgr) { + template + void InitFrontend(Manager *mgr) { const auto &meta_data = model_->GetMetaData(); if (meta_data.frontend == "characters") { @@ -323,7 +316,6 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { meta_data.punctuations, meta_data.language, config_.model.debug); } } -#endif void InitFrontend() { const auto &meta_data = model_->GetMetaData(); diff --git a/sherpa-onnx/csrc/offline-tts-vits-model.cc b/sherpa-onnx/csrc/offline-tts-vits-model.cc index 90e0993d6..38efc6204 100644 --- a/sherpa-onnx/csrc/offline-tts-vits-model.cc +++ b/sherpa-onnx/csrc/offline-tts-vits-model.cc @@ -9,6 +9,15 @@ #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/onnx-utils.h" #include "sherpa-onnx/csrc/session.h" @@ -26,8 +35,8 @@ class OfflineTtsVitsModel::Impl { Init(buf.data(), buf.size()); } -#if __ANDROID_API__ >= 9 - Impl(AAssetManager *mgr, const OfflineTtsModelConfig &config) + template + Impl(Manager *mgr, const OfflineTtsModelConfig &config) : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), @@ -35,7 +44,6 @@ class OfflineTtsVitsModel::Impl { auto buf = ReadFile(mgr, config.vits.model); Init(buf.data(), buf.size()); } -#endif Ort::Value Run(Ort::Value x, int64_t sid, float speed) { if (meta_data_.is_piper || meta_data_.is_coqui) { @@ -336,11 +344,10 @@ class OfflineTtsVitsModel::Impl { OfflineTtsVitsModel::OfflineTtsVitsModel(const OfflineTtsModelConfig &config) : impl_(std::make_unique(config)) {} -#if __ANDROID_API__ >= 9 -OfflineTtsVitsModel::OfflineTtsVitsModel(AAssetManager *mgr, +template +OfflineTtsVitsModel::OfflineTtsVitsModel(Manager *mgr, const OfflineTtsModelConfig &config) : impl_(std::make_unique(mgr, config)) {} -#endif OfflineTtsVitsModel::~OfflineTtsVitsModel() = default; @@ -359,4 +366,14 @@ const OfflineTtsVitsModelMetaData &OfflineTtsVitsModel::GetMetaData() const { return impl_->GetMetaData(); } +#if __ANDROID_API__ >= 9 +template OfflineTtsVitsModel::OfflineTtsVitsModel( + AAssetManager *mgr, const OfflineTtsModelConfig &config); +#endif + +#if __OHOS__ +template OfflineTtsVitsModel::OfflineTtsVitsModel( + NativeResourceManager *mgr, const OfflineTtsModelConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-tts-vits-model.h b/sherpa-onnx/csrc/offline-tts-vits-model.h index 543963c9d..a880934ef 100644 --- a/sherpa-onnx/csrc/offline-tts-vits-model.h +++ b/sherpa-onnx/csrc/offline-tts-vits-model.h @@ -8,11 +8,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/offline-tts-model-config.h" #include "sherpa-onnx/csrc/offline-tts-vits-model-metadata.h" @@ -24,9 +19,9 @@ class OfflineTtsVitsModel { ~OfflineTtsVitsModel(); explicit OfflineTtsVitsModel(const OfflineTtsModelConfig &config); -#if __ANDROID_API__ >= 9 - OfflineTtsVitsModel(AAssetManager *mgr, const OfflineTtsModelConfig &config); -#endif + + template + OfflineTtsVitsModel(Manager *mgr, const OfflineTtsModelConfig &config); /** Run the model. * diff --git a/sherpa-onnx/csrc/offline-tts.cc b/sherpa-onnx/csrc/offline-tts.cc index 12feda0b7..ec2c69523 100644 --- a/sherpa-onnx/csrc/offline-tts.cc +++ b/sherpa-onnx/csrc/offline-tts.cc @@ -7,6 +7,15 @@ #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/file-utils.h" #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/offline-tts-impl.h" @@ -78,10 +87,9 @@ std::string OfflineTtsConfig::ToString() const { OfflineTts::OfflineTts(const OfflineTtsConfig &config) : impl_(OfflineTtsImpl::Create(config)) {} -#if __ANDROID_API__ >= 9 -OfflineTts::OfflineTts(AAssetManager *mgr, const OfflineTtsConfig &config) +template +OfflineTts::OfflineTts(Manager *mgr, const OfflineTtsConfig &config) : impl_(OfflineTtsImpl::Create(mgr, config)) {} -#endif OfflineTts::~OfflineTts() = default; @@ -95,4 +103,14 @@ int32_t OfflineTts::SampleRate() const { return impl_->SampleRate(); } int32_t OfflineTts::NumSpeakers() const { return impl_->NumSpeakers(); } +#if __ANDROID_API__ >= 9 +template OfflineTts::OfflineTts(AAssetManager *mgr, + const OfflineTtsConfig &config); +#endif + +#if __OHOS__ +template OfflineTts::OfflineTts(NativeResourceManager *mgr, + const OfflineTtsConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-tts.h b/sherpa-onnx/csrc/offline-tts.h index 03a13a159..8399443c0 100644 --- a/sherpa-onnx/csrc/offline-tts.h +++ b/sherpa-onnx/csrc/offline-tts.h @@ -10,11 +10,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/offline-tts-model-config.h" #include "sherpa-onnx/csrc/parse-options.h" @@ -69,9 +64,8 @@ class OfflineTts { ~OfflineTts(); explicit OfflineTts(const OfflineTtsConfig &config); -#if __ANDROID_API__ >= 9 - OfflineTts(AAssetManager *mgr, const OfflineTtsConfig &config); -#endif + template + OfflineTts(Manager *mgr, const OfflineTtsConfig &config); // @param text A string containing words separated by spaces // @param sid Speaker ID. Used only for multi-speaker models, e.g., models diff --git a/sherpa-onnx/csrc/piper-phonemize-lexicon.cc b/sherpa-onnx/csrc/piper-phonemize-lexicon.cc index de753db61..298274654 100644 --- a/sherpa-onnx/csrc/piper-phonemize-lexicon.cc +++ b/sherpa-onnx/csrc/piper-phonemize-lexicon.cc @@ -11,16 +11,19 @@ #include // NOLINT #include #include +#include #include #include #if __ANDROID_API__ >= 9 -#include - #include "android/asset_manager.h" #include "android/asset_manager_jni.h" #endif +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "espeak-ng/speak_lib.h" #include "phoneme_ids.hpp" #include "phonemize.hpp" @@ -196,9 +199,9 @@ PiperPhonemizeLexicon::PiperPhonemizeLexicon( InitEspeak(data_dir); } -#if __ANDROID_API__ >= 9 +template PiperPhonemizeLexicon::PiperPhonemizeLexicon( - AAssetManager *mgr, const std::string &tokens, const std::string &data_dir, + Manager *mgr, const std::string &tokens, const std::string &data_dir, const OfflineTtsVitsModelMetaData &meta_data) : meta_data_(meta_data) { { @@ -212,7 +215,6 @@ PiperPhonemizeLexicon::PiperPhonemizeLexicon( // data_dir. InitEspeak(data_dir); } -#endif std::vector PiperPhonemizeLexicon::ConvertTextToTokenIds( const std::string &text, const std::string &voice /*= ""*/) const { @@ -255,4 +257,16 @@ std::vector PiperPhonemizeLexicon::ConvertTextToTokenIds( return ans; } +#if __ANDROID_API__ >= 9 +template PiperPhonemizeLexicon::PiperPhonemizeLexicon( + AAssetManager *mgr, const std::string &tokens, const std::string &data_dir, + const OfflineTtsVitsModelMetaData &meta_data); +#endif + +#if __OHOS__ +template PiperPhonemizeLexicon::PiperPhonemizeLexicon( + NativeResourceManager *mgr, const std::string &tokens, + const std::string &data_dir, const OfflineTtsVitsModelMetaData &meta_data); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/piper-phonemize-lexicon.h b/sherpa-onnx/csrc/piper-phonemize-lexicon.h index 34922de29..ccd790a96 100644 --- a/sherpa-onnx/csrc/piper-phonemize-lexicon.h +++ b/sherpa-onnx/csrc/piper-phonemize-lexicon.h @@ -9,11 +9,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/offline-tts-frontend.h" #include "sherpa-onnx/csrc/offline-tts-vits-model-metadata.h" @@ -24,11 +19,10 @@ class PiperPhonemizeLexicon : public OfflineTtsFrontend { PiperPhonemizeLexicon(const std::string &tokens, const std::string &data_dir, const OfflineTtsVitsModelMetaData &meta_data); -#if __ANDROID_API__ >= 9 - PiperPhonemizeLexicon(AAssetManager *mgr, const std::string &tokens, + template + PiperPhonemizeLexicon(Manager *mgr, const std::string &tokens, const std::string &data_dir, const OfflineTtsVitsModelMetaData &meta_data); -#endif std::vector ConvertTextToTokenIds( const std::string &text, const std::string &voice = "") const override; diff --git a/sherpa-onnx/csrc/silero-vad-model.cc b/sherpa-onnx/csrc/silero-vad-model.cc index 80f0cbd65..1b281e5db 100644 --- a/sherpa-onnx/csrc/silero-vad-model.cc +++ b/sherpa-onnx/csrc/silero-vad-model.cc @@ -51,11 +51,11 @@ class SileroVadModel::Impl { : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), - allocator_{} { + allocator_{}, + sample_rate_(config.sample_rate) { auto buf = ReadFile(mgr, config.silero_vad.model); Init(buf.data(), buf.size()); - sample_rate_ = config.sample_rate; if (sample_rate_ != 16000) { SHERPA_ONNX_LOGE("Expected sample rate 16000. Given: %d", config.sample_rate); From 0d6bf528446bbd8cca996cf6cdb1fa6c7790309c Mon Sep 17 00:00:00 2001 From: JiayuXu <84259897+JiayuXu0@users.noreply.github.com> Date: Tue, 3 Dec 2024 17:22:12 +0800 Subject: [PATCH 089/183] fix: support both old and new websockets request headers format (#1588) Co-authored-by: xujiayu --- python-api-examples/non_streaming_server.py | 6 +++++- python-api-examples/streaming_server.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/python-api-examples/non_streaming_server.py b/python-api-examples/non_streaming_server.py index 3dd12564e..3ffa8b7d5 100755 --- a/python-api-examples/non_streaming_server.py +++ b/python-api-examples/non_streaming_server.py @@ -641,7 +641,11 @@ async def process_request( path: str, request_headers: websockets.Headers, ) -> Optional[Tuple[http.HTTPStatus, websockets.Headers, bytes]]: - if "sec-websocket-key" not in request_headers: + if "sec-websocket-key" not in ( + request_headers.headers # For new request_headers + if hasattr(request_headers, "headers") + else request_headers # For old request_headers + ): # This is a normal HTTP request if path == "/": path = "/index.html" diff --git a/python-api-examples/streaming_server.py b/python-api-examples/streaming_server.py index c7c1b8de6..5691f67c3 100755 --- a/python-api-examples/streaming_server.py +++ b/python-api-examples/streaming_server.py @@ -584,7 +584,11 @@ async def process_request( path: str, request_headers: websockets.Headers, ) -> Optional[Tuple[http.HTTPStatus, websockets.Headers, bytes]]: - if "sec-websocket-key" not in request_headers: + if "sec-websocket-key" not in ( + request_headers.headers # For new request_headers + if hasattr(request_headers, "headers") + else request_headers # For old request_headers + ): # This is a normal HTTP request if path == "/": path = "/index.html" From 47a2dd4cf889c4cc6ec9784452fb5dee5859aa60 Mon Sep 17 00:00:00 2001 From: goddamnVincent <84380030+goddamnVincent@users.noreply.github.com> Date: Wed, 4 Dec 2024 09:22:24 +0800 Subject: [PATCH 090/183] 'update20241203' (#1589) add '--modeling-unit' and "--bpe-vocab" to /sherpa-onnx/python-api-examples/streaming_server.py make it specifiable. --- python-api-examples/streaming_server.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/python-api-examples/streaming_server.py b/python-api-examples/streaming_server.py index 5691f67c3..ef6bac8e3 100755 --- a/python-api-examples/streaming_server.py +++ b/python-api-examples/streaming_server.py @@ -229,6 +229,28 @@ def add_hotwords_args(parser: argparse.ArgumentParser): --hotwords-file is given. """, ) + parser.add_argument( + "--modeling-unit", + type=str, + default='cjkchar', + help=""" + The modeling unit of the used model. Current supported units are: + - cjkchar(for Chinese) + - bpe(for English like languages) + - cjkchar+bpe(for multilingual models) + """, + ) + parser.add_argument( + "--bpe-vocab", + type=str, + default='', + help=""" + The bpe vocabulary generated by sentencepiece toolkit. + It is only used when modeling-unit is bpe or cjkchar+bpe. + if you can’t find bpe.vocab in the model directory, please run: + python script/export_bpe_vocab.py --bpe-model exp/bpe.model + """, + ) def add_modified_beam_search_args(parser: argparse.ArgumentParser): @@ -409,6 +431,8 @@ def create_recognizer(args) -> sherpa_onnx.OnlineRecognizer: rule2_min_trailing_silence=args.rule2_min_trailing_silence, rule3_min_utterance_length=args.rule3_min_utterance_length, provider=args.provider, + modeling_unit=args.modeling_unit, + bpe_vocab=args.bpe_vocab ) elif args.paraformer_encoder: recognizer = sherpa_onnx.OnlineRecognizer.from_paraformer( From 74a8735f7a563b072292e54aea9cf4bcbe696a37 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 4 Dec 2024 14:27:12 +0800 Subject: [PATCH 091/183] Add on-device tex-to-speech (TTS) demo for HarmonyOS (#1590) --- harmony-os/.gitignore | 1 + .../SherpaOnnxHar/sherpa_onnx/Index.ets | 6 +- .../sherpa_onnx/build-profile.json5 | 2 +- .../sherpa_onnx/src/main/cpp/CMakeLists.txt | 5 + .../src/main/cpp/non-streaming-tts.cc | 260 ++++++++++- .../main/cpp/sherpa-onnx-node-addon-api.cc | 12 + .../main/cpp/types/libsherpa_onnx/Index.d.ts | 11 +- .../sherpa_onnx/src/main/cpp/utils.cc | 76 ++++ .../main/ets/components/NonStreamingTts.ets | 18 +- .../src/main/ets/components/Vad.ets | 1 - harmony-os/SherpaOnnxTts/.gitignore | 12 + harmony-os/SherpaOnnxTts/AppScope/app.json5 | 10 + .../resources/base/element/string.json | 8 + .../resources/base/media/app_icon.png | Bin 0 -> 2777 bytes harmony-os/SherpaOnnxTts/build-profile.json5 | 40 ++ harmony-os/SherpaOnnxTts/code-linter.json5 | 20 + harmony-os/SherpaOnnxTts/entry/.gitignore | 6 + .../SherpaOnnxTts/entry/build-profile.json5 | 33 ++ harmony-os/SherpaOnnxTts/entry/hvigorfile.ts | 6 + .../SherpaOnnxTts/entry/obfuscation-rules.txt | 23 + .../SherpaOnnxTts/entry/oh-package-lock.json5 | 29 ++ .../SherpaOnnxTts/entry/oh-package.json5 | 12 + .../main/ets/entryability/EntryAbility.ets | 43 ++ .../entrybackupability/EntryBackupAbility.ets | 12 + .../entry/src/main/ets/pages/Index.ets | 409 ++++++++++++++++++ .../ets/workers/NonStreamingTtsWorker.ets | 284 ++++++++++++ .../SherpaOnnxTts/entry/src/main/module.json5 | 52 +++ .../main/resources/base/element/color.json | 8 + .../main/resources/base/element/string.json | 16 + .../main/resources/base/media/background.png | Bin 0 -> 57364 bytes .../main/resources/base/media/foreground.png | Bin 0 -> 12430 bytes .../src/main/resources/base/media/home.svg | 1 + .../src/main/resources/base/media/info.svg | 1 + .../resources/base/media/layered_image.json | 7 + .../main/resources/base/media/startIcon.png | Bin 0 -> 20093 bytes .../resources/base/profile/backup_config.json | 3 + .../resources/base/profile/main_pages.json | 5 + .../main/resources/en_US/element/string.json | 16 + .../entry/src/main/resources/rawfile/.gitkeep | 0 .../main/resources/zh_CN/element/string.json | 16 + .../src/ohosTest/ets/test/Ability.test.ets | 35 ++ .../entry/src/ohosTest/ets/test/List.test.ets | 5 + .../entry/src/ohosTest/module.json5 | 13 + .../entry/src/test/List.test.ets | 5 + .../entry/src/test/LocalUnit.test.ets | 33 ++ .../SherpaOnnxTts/hvigor/hvigor-config.json5 | 22 + harmony-os/SherpaOnnxTts/hvigorfile.ts | 6 + .../SherpaOnnxTts/oh-package-lock.json5 | 19 + harmony-os/SherpaOnnxTts/oh-package.json5 | 9 + .../entry/src/main/ets/pages/Index.ets | 148 +++---- .../main/resources/base/element/string.json | 8 +- .../main/resources/en_US/element/string.json | 10 +- .../main/resources/zh_CN/element/string.json | 10 +- sherpa-onnx/c-api/c-api.cc | 11 + sherpa-onnx/c-api/c-api.h | 11 + sherpa-onnx/csrc/circular-buffer.cc | 16 +- sherpa-onnx/csrc/lexicon.cc | 47 +- sherpa-onnx/csrc/melo-tts-lexicon.cc | 118 ++++- sherpa-onnx/csrc/melo-tts-lexicon.h | 10 + sherpa-onnx/csrc/offline-tts-vits-impl.h | 43 +- sherpa-onnx/csrc/offline-tts-vits-model.cc | 4 + 61 files changed, 1930 insertions(+), 117 deletions(-) create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/utils.cc create mode 100644 harmony-os/SherpaOnnxTts/.gitignore create mode 100644 harmony-os/SherpaOnnxTts/AppScope/app.json5 create mode 100644 harmony-os/SherpaOnnxTts/AppScope/resources/base/element/string.json create mode 100644 harmony-os/SherpaOnnxTts/AppScope/resources/base/media/app_icon.png create mode 100644 harmony-os/SherpaOnnxTts/build-profile.json5 create mode 100644 harmony-os/SherpaOnnxTts/code-linter.json5 create mode 100644 harmony-os/SherpaOnnxTts/entry/.gitignore create mode 100644 harmony-os/SherpaOnnxTts/entry/build-profile.json5 create mode 100644 harmony-os/SherpaOnnxTts/entry/hvigorfile.ts create mode 100644 harmony-os/SherpaOnnxTts/entry/obfuscation-rules.txt create mode 100644 harmony-os/SherpaOnnxTts/entry/oh-package-lock.json5 create mode 100644 harmony-os/SherpaOnnxTts/entry/oh-package.json5 create mode 100644 harmony-os/SherpaOnnxTts/entry/src/main/ets/entryability/EntryAbility.ets create mode 100644 harmony-os/SherpaOnnxTts/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets create mode 100644 harmony-os/SherpaOnnxTts/entry/src/main/ets/pages/Index.ets create mode 100644 harmony-os/SherpaOnnxTts/entry/src/main/ets/workers/NonStreamingTtsWorker.ets create mode 100644 harmony-os/SherpaOnnxTts/entry/src/main/module.json5 create mode 100644 harmony-os/SherpaOnnxTts/entry/src/main/resources/base/element/color.json create mode 100644 harmony-os/SherpaOnnxTts/entry/src/main/resources/base/element/string.json create mode 100644 harmony-os/SherpaOnnxTts/entry/src/main/resources/base/media/background.png create mode 100644 harmony-os/SherpaOnnxTts/entry/src/main/resources/base/media/foreground.png create mode 100644 harmony-os/SherpaOnnxTts/entry/src/main/resources/base/media/home.svg create mode 100644 harmony-os/SherpaOnnxTts/entry/src/main/resources/base/media/info.svg create mode 100644 harmony-os/SherpaOnnxTts/entry/src/main/resources/base/media/layered_image.json create mode 100644 harmony-os/SherpaOnnxTts/entry/src/main/resources/base/media/startIcon.png create mode 100644 harmony-os/SherpaOnnxTts/entry/src/main/resources/base/profile/backup_config.json create mode 100644 harmony-os/SherpaOnnxTts/entry/src/main/resources/base/profile/main_pages.json create mode 100644 harmony-os/SherpaOnnxTts/entry/src/main/resources/en_US/element/string.json create mode 100644 harmony-os/SherpaOnnxTts/entry/src/main/resources/rawfile/.gitkeep create mode 100644 harmony-os/SherpaOnnxTts/entry/src/main/resources/zh_CN/element/string.json create mode 100644 harmony-os/SherpaOnnxTts/entry/src/ohosTest/ets/test/Ability.test.ets create mode 100644 harmony-os/SherpaOnnxTts/entry/src/ohosTest/ets/test/List.test.ets create mode 100644 harmony-os/SherpaOnnxTts/entry/src/ohosTest/module.json5 create mode 100644 harmony-os/SherpaOnnxTts/entry/src/test/List.test.ets create mode 100644 harmony-os/SherpaOnnxTts/entry/src/test/LocalUnit.test.ets create mode 100644 harmony-os/SherpaOnnxTts/hvigor/hvigor-config.json5 create mode 100644 harmony-os/SherpaOnnxTts/hvigorfile.ts create mode 100644 harmony-os/SherpaOnnxTts/oh-package-lock.json5 create mode 100644 harmony-os/SherpaOnnxTts/oh-package.json5 diff --git a/harmony-os/.gitignore b/harmony-os/.gitignore index ddb010f66..dd2f4066e 100644 --- a/harmony-os/.gitignore +++ b/harmony-os/.gitignore @@ -1 +1,2 @@ !build-profile.json5 +*.har diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets index 959b6ba02..14dff071e 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets @@ -1,4 +1,8 @@ -export { readWave, readWaveFromBinary } from "libsherpa_onnx.so"; +export { + listRawfileDir, + readWave, + readWaveFromBinary, +} from "libsherpa_onnx.so"; export { CircularBuffer, diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/build-profile.json5 b/harmony-os/SherpaOnnxHar/sherpa_onnx/build-profile.json5 index 8f789fb2a..905c57127 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/build-profile.json5 +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/build-profile.json5 @@ -4,7 +4,7 @@ "externalNativeOptions": { "path": "./src/main/cpp/CMakeLists.txt", "arguments": "", - "cppFlags": "", + "cppFlags": "-std=c++17", "abiFilters": [ "arm64-v8a", "x86_64", diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/CMakeLists.txt b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/CMakeLists.txt index e131b21da..26dda1789 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/CMakeLists.txt +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/CMakeLists.txt @@ -2,6 +2,10 @@ cmake_minimum_required(VERSION 3.13.0) project(myNpmLib) +if (NOT CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD 17 CACHE STRING "The C++ version to use") +endif() + # Disable warning about # # "The DOWNLOAD_EXTRACT_TIMESTAMP option was not given and policy CMP0135 is @@ -46,6 +50,7 @@ add_library(sherpa_onnx SHARED speaker-identification.cc spoken-language-identification.cc streaming-asr.cc + utils.cc vad.cc wave-reader.cc wave-writer.cc diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc index da70e662c..67f348e9b 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc @@ -213,12 +213,13 @@ static Napi::Number OfflineTtsNumSpeakersWrapper( return Napi::Number::New(env, num_speakers); } +// synchronous version static Napi::Object OfflineTtsGenerateWrapper(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); if (info.Length() != 2) { std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); + os << "Expect only 2 arguments. Given: " << info.Length(); Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); @@ -298,8 +299,8 @@ static Napi::Object OfflineTtsGenerateWrapper(const Napi::CallbackInfo &info) { int32_t sid = obj.Get("sid").As().Int32Value(); float speed = obj.Get("speed").As().FloatValue(); - const SherpaOnnxGeneratedAudio *audio = - SherpaOnnxOfflineTtsGenerate(tts, text.c_str(), sid, speed); + const SherpaOnnxGeneratedAudio *audio; + audio = SherpaOnnxOfflineTtsGenerate(tts, text.c_str(), sid, speed); if (enable_external_buffer) { Napi::ArrayBuffer arrayBuffer = Napi::ArrayBuffer::New( @@ -334,6 +335,256 @@ static Napi::Object OfflineTtsGenerateWrapper(const Napi::CallbackInfo &info) { } } +struct TtsCallbackData { + std::vector samples; + float progress; + bool processed = false; + bool cancelled = false; +}; + +// see +// https://github.com/nodejs/node-addon-examples/blob/main/src/6-threadsafe-function/typed_threadsafe_function/node-addon-api/clock.cc +void InvokeJsCallback(Napi::Env env, Napi::Function callback, + Napi::Reference *context, + TtsCallbackData *data) { + if (env != nullptr) { + if (callback != nullptr) { + Napi::ArrayBuffer arrayBuffer = + Napi::ArrayBuffer::New(env, sizeof(float) * data->samples.size()); + + Napi::Float32Array float32Array = + Napi::Float32Array::New(env, data->samples.size(), arrayBuffer, 0); + + std::copy(data->samples.begin(), data->samples.end(), + float32Array.Data()); + + Napi::Object arg = Napi::Object::New(env); + arg.Set(Napi::String::New(env, "samples"), float32Array); + arg.Set(Napi::String::New(env, "progress"), data->progress); + + auto v = callback.Call(context->Value(), {arg}); + data->processed = true; + if (v.IsNumber() && v.As().Int32Value()) { + data->cancelled = false; + } else { + data->cancelled = true; + } + } + } +} + +using TSFN = Napi::TypedThreadSafeFunction, + TtsCallbackData, InvokeJsCallback>; + +class TtsGenerateWorker : public Napi::AsyncWorker { + public: + TtsGenerateWorker(const Napi::Env &env, TSFN tsfn, SherpaOnnxOfflineTts *tts, + const std::string &text, float speed, int32_t sid, + bool use_external_buffer) + : tsfn_(tsfn), + Napi::AsyncWorker{env, "TtsGenerateWorker"}, + deferred_(env), + tts_(tts), + text_(text), + speed_(speed), + sid_(sid), + use_external_buffer_(use_external_buffer) {} + + Napi::Promise Promise() { return deferred_.Promise(); } + + ~TtsGenerateWorker() { + for (auto d : data_list_) { + delete d; + } + } + + protected: + void Execute() override { + auto callback = [](const float *samples, int32_t n, float progress, + void *arg) -> int32_t { + TtsGenerateWorker *_this = reinterpret_cast(arg); + + for (auto d : _this->data_list_) { + if (d->cancelled) { + OH_LOG_INFO(LOG_APP, "TtsGenerate is cancelled"); + return 0; + } + } + + auto data = new TtsCallbackData; + data->samples = std::vector{samples, samples + n}; + data->progress = progress; + _this->data_list_.push_back(data); + + _this->tsfn_.NonBlockingCall(data); + + return 1; + }; + audio_ = SherpaOnnxOfflineTtsGenerateWithProgressCallbackWithArg( + tts_, text_.c_str(), sid_, speed_, callback, this); + + tsfn_.Release(); + } + + void OnOK() override { + Napi::Env env = deferred_.Env(); + Napi::Object ans = Napi::Object::New(env); + if (use_external_buffer_) { + Napi::ArrayBuffer arrayBuffer = Napi::ArrayBuffer::New( + env, const_cast(audio_->samples), sizeof(float) * audio_->n, + [](Napi::Env /*env*/, void * /*data*/, + const SherpaOnnxGeneratedAudio *hint) { + SherpaOnnxDestroyOfflineTtsGeneratedAudio(hint); + }, + audio_); + Napi::Float32Array float32Array = + Napi::Float32Array::New(env, audio_->n, arrayBuffer, 0); + + ans.Set(Napi::String::New(env, "samples"), float32Array); + ans.Set(Napi::String::New(env, "sampleRate"), audio_->sample_rate); + } else { + // don't use external buffer + Napi::ArrayBuffer arrayBuffer = + Napi::ArrayBuffer::New(env, sizeof(float) * audio_->n); + + Napi::Float32Array float32Array = + Napi::Float32Array::New(env, audio_->n, arrayBuffer, 0); + + std::copy(audio_->samples, audio_->samples + audio_->n, + float32Array.Data()); + + ans.Set(Napi::String::New(env, "samples"), float32Array); + ans.Set(Napi::String::New(env, "sampleRate"), audio_->sample_rate); + SherpaOnnxDestroyOfflineTtsGeneratedAudio(audio_); + } + + deferred_.Resolve(ans); + } + + private: + TSFN tsfn_; + Napi::Promise::Deferred deferred_; + SherpaOnnxOfflineTts *tts_; + std::string text_; + float speed_; + int32_t sid_; + bool use_external_buffer_; + + const SherpaOnnxGeneratedAudio *audio_; + + std::vector data_list_; +}; + +static Napi::Object OfflineTtsGenerateAsyncWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be an offline tts pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + SherpaOnnxOfflineTts *tts = + info[0].As>().Data(); + + if (!info[1].IsObject()) { + Napi::TypeError::New(env, "Argument 1 should be an object") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Object obj = info[1].As(); + + if (!obj.Has("text")) { + Napi::TypeError::New(env, "The argument object should have a field text") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Get("text").IsString()) { + Napi::TypeError::New(env, "The object['text'] should be a string") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Has("sid")) { + Napi::TypeError::New(env, "The argument object should have a field sid") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Get("sid").IsNumber()) { + Napi::TypeError::New(env, "The object['sid'] should be a number") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Has("speed")) { + Napi::TypeError::New(env, "The argument object should have a field speed") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Get("speed").IsNumber()) { + Napi::TypeError::New(env, "The object['speed'] should be a number") + .ThrowAsJavaScriptException(); + + return {}; + } + + bool enable_external_buffer = true; + if (obj.Has("enableExternalBuffer") && + obj.Get("enableExternalBuffer").IsBoolean()) { + enable_external_buffer = + obj.Get("enableExternalBuffer").As().Value(); + } + + Napi::String _text = obj.Get("text").As(); + std::string text = _text.Utf8Value(); + int32_t sid = obj.Get("sid").As().Int32Value(); + float speed = obj.Get("speed").As().FloatValue(); + + Napi::Function cb; + if (obj.Has("callback") && obj.Get("callback").IsFunction()) { + cb = obj.Get("callback").As(); + } + + auto context = + new Napi::Reference(Napi::Persistent(info.This())); + + TSFN tsfn = TSFN::New( + env, + cb, // JavaScript function called asynchronously + "TtsGenerateFunc", // Name + 0, // Unlimited queue + 1, // Only one thread will use this initially + context, + [](Napi::Env, void *, Napi::Reference *ctx) { delete ctx; }); + + const SherpaOnnxGeneratedAudio *audio; + TtsGenerateWorker *worker = new TtsGenerateWorker( + env, tsfn, tts, text, speed, sid, enable_external_buffer); + worker->Queue(); + return worker->Promise(); +} + void InitNonStreamingTts(Napi::Env env, Napi::Object exports) { exports.Set(Napi::String::New(env, "createOfflineTts"), Napi::Function::New(env, CreateOfflineTtsWrapper)); @@ -346,4 +597,7 @@ void InitNonStreamingTts(Napi::Env env, Napi::Object exports) { exports.Set(Napi::String::New(env, "offlineTtsGenerate"), Napi::Function::New(env, OfflineTtsGenerateWrapper)); + + exports.Set(Napi::String::New(env, "offlineTtsGenerateAsync"), + Napi::Function::New(env, OfflineTtsGenerateAsyncWrapper)); } diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/sherpa-onnx-node-addon-api.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/sherpa-onnx-node-addon-api.cc index 3f0affd79..54f0350fe 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/sherpa-onnx-node-addon-api.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/sherpa-onnx-node-addon-api.cc @@ -27,6 +27,10 @@ void InitKeywordSpotting(Napi::Env env, Napi::Object exports); void InitNonStreamingSpeakerDiarization(Napi::Env env, Napi::Object exports); +#if __OHOS__ +void InitUtils(Napi::Env env, Napi::Object exports); +#endif + Napi::Object Init(Napi::Env env, Napi::Object exports) { InitStreamingAsr(env, exports); InitNonStreamingAsr(env, exports); @@ -41,7 +45,15 @@ Napi::Object Init(Napi::Env env, Napi::Object exports) { InitKeywordSpotting(env, exports); InitNonStreamingSpeakerDiarization(env, exports); +#if __OHOS__ + InitUtils(env, exports); +#endif + return exports; } +#if __OHOS__ +NODE_API_MODULE(sherpa_onnx, Init) +#else NODE_API_MODULE(addon, Init) +#endif diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts index f44ade356..057d5af25 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts @@ -1,3 +1,5 @@ +export const listRawfileDir: (mgr: object, dir: string) => Array; + export const readWave: (filename: string, enableExternalBuffer: boolean = true) => {samples: Float32Array, sampleRate: number}; export const readWaveFromBinary: (data: Uint8Array, enableExternalBuffer: boolean = true) => {samples: Float32Array, sampleRate: number}; export const createCircularBuffer: (capacity: number) => object; @@ -37,4 +39,11 @@ export const getOnlineStreamResultAsJson: (handle: object, streamHandle: object) export const createOfflineTts: (config: object, mgr?: object) => object; export const getOfflineTtsNumSpeakers: (handle: object) => number; export const getOfflineTtsSampleRate: (handle: object) => number; -export const offlineTtsGenerate: (handle: object, input: object) => object; + +export type TtsOutput = { + samples: Float32Array; + sampleRate: number; +}; + +export const offlineTtsGenerate: (handle: object, input: object) => TtsOutput; +export const offlineTtsGenerateAsync: (handle: object, input: object) => Promise; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/utils.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/utils.cc new file mode 100644 index 000000000..33b8f2a29 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/utils.cc @@ -0,0 +1,76 @@ +// Copyright (c) 2024 Xiaomi Corporation + +#include +#include +#include +#include + +#include "macros.h" // NOLINT +#include "napi.h" // NOLINT + +static std::vector GetFilenames(NativeResourceManager *mgr, + const std::string &d) { + std::unique_ptr raw_dir( + OH_ResourceManager_OpenRawDir(mgr, d.c_str()), + &OH_ResourceManager_CloseRawDir); + int count = OH_ResourceManager_GetRawFileCount(raw_dir.get()); + std::vector ans; + ans.reserve(count); + for (int32_t i = 0; i < count; ++i) { + std::string filename = OH_ResourceManager_GetRawFileName(raw_dir.get(), i); + bool is_dir = OH_ResourceManager_IsRawDir( + mgr, d.empty() ? filename.c_str() : (d + "/" + filename).c_str()); + if (is_dir) { + auto files = GetFilenames(mgr, d.empty() ? filename : d + "/" + filename); + for (auto &f : files) { + ans.push_back(std::move(f)); + } + } else { + if (d.empty()) { + ans.push_back(std::move(filename)); + } else { + ans.push_back(d + "/" + filename); + } + } + } + + return ans; +} + +static Napi::Array ListRawFileDir(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + std::unique_ptr + mgr(OH_ResourceManager_InitNativeResourceManager(env, info[0]), + &OH_ResourceManager_ReleaseNativeResourceManager); + + if (!info[1].IsString()) { + Napi::TypeError::New(env, "Argument 1 should be a string") + .ThrowAsJavaScriptException(); + + return {}; + } + + std::string dir = info[1].As().Utf8Value(); + + auto files = GetFilenames(mgr.get(), dir); + Napi::Array ans = Napi::Array::New(env, files.size()); + for (int32_t i = 0; i != files.size(); ++i) { + ans[i] = Napi::String::New(env, files[i]); + } + return ans; +} +void InitUtils(Napi::Env env, Napi::Object exports) { + exports.Set(Napi::String::New(env, "listRawfileDir"), + Napi::Function::New(env, ListRawFileDir)); +} diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets index c568b9990..a60a0e748 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets @@ -3,6 +3,7 @@ import { getOfflineTtsNumSpeakers, getOfflineTtsSampleRate, offlineTtsGenerate, + offlineTtsGenerateAsync, } from "libsherpa_onnx.so"; export class OfflineTtsVitsModelConfig { @@ -16,14 +17,14 @@ export class OfflineTtsVitsModelConfig { public lengthScale: number = 1.0; } -export class OfflineTtsModelConfig{ +export class OfflineTtsModelConfig { public vits: OfflineTtsVitsModelConfig = new OfflineTtsVitsModelConfig(); public numThreads: number = 1; public debug: boolean = false; public provider: string = 'cpu'; } -export class OfflineTtsConfig{ +export class OfflineTtsConfig { public model: OfflineTtsModelConfig = new OfflineTtsModelConfig(); public ruleFsts: string = ''; public ruleFars: string = ''; @@ -35,17 +36,24 @@ export class TtsOutput { public sampleRate: number = 0; } +interface TtsCallbackData { + samples: Float32Array; + progress: number; +} + export class TtsInput { public text: string = ''; public sid: number = 0; public speed: number = 1.0; + public callback?: (data: TtsCallbackData) => number; } export class OfflineTts { - private handle: object; public config: OfflineTtsConfig; public numSpeakers: number; public sampleRate: number; + private handle: object; + constructor(config: OfflineTtsConfig, mgr?: object) { this.handle = createOfflineTts(config, mgr); this.config = config; @@ -63,4 +71,8 @@ export class OfflineTts { generate(input: TtsInput): TtsOutput { return offlineTtsGenerate(this.handle, input) as TtsOutput; } + + generateAsync(input: TtsInput): Promise { + return offlineTtsGenerateAsync(this.handle, input); + } } \ No newline at end of file diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/Vad.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/Vad.ets index 155eac680..8f1bf18d6 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/Vad.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/Vad.ets @@ -57,7 +57,6 @@ export class CircularBuffer { // samples is a float32 array push(samples: Float32Array) { - console.log(`here samples: ${samples}`); circularBufferPush(this.handle, samples); } diff --git a/harmony-os/SherpaOnnxTts/.gitignore b/harmony-os/SherpaOnnxTts/.gitignore new file mode 100644 index 000000000..d2ff20141 --- /dev/null +++ b/harmony-os/SherpaOnnxTts/.gitignore @@ -0,0 +1,12 @@ +/node_modules +/oh_modules +/local.properties +/.idea +**/build +/.hvigor +.cxx +/.clangd +/.clang-format +/.clang-tidy +**/.test +/.appanalyzer \ No newline at end of file diff --git a/harmony-os/SherpaOnnxTts/AppScope/app.json5 b/harmony-os/SherpaOnnxTts/AppScope/app.json5 new file mode 100644 index 000000000..e5d0228ac --- /dev/null +++ b/harmony-os/SherpaOnnxTts/AppScope/app.json5 @@ -0,0 +1,10 @@ +{ + "app": { + "bundleName": "com.k2fsa.sherpa.onnx.tts", + "vendor": "next-gen Kaldi", + "versionCode": 1000000, + "versionName": "1.0.0", + "icon": "$media:app_icon", + "label": "$string:app_name" + } +} diff --git a/harmony-os/SherpaOnnxTts/AppScope/resources/base/element/string.json b/harmony-os/SherpaOnnxTts/AppScope/resources/base/element/string.json new file mode 100644 index 000000000..2db317614 --- /dev/null +++ b/harmony-os/SherpaOnnxTts/AppScope/resources/base/element/string.json @@ -0,0 +1,8 @@ +{ + "string": [ + { + "name": "app_name", + "value": "SherpaOnnxTts" + } + ] +} diff --git a/harmony-os/SherpaOnnxTts/AppScope/resources/base/media/app_icon.png b/harmony-os/SherpaOnnxTts/AppScope/resources/base/media/app_icon.png new file mode 100644 index 0000000000000000000000000000000000000000..a39445dc87828b76fed6d2ec470dd455c45319e3 GIT binary patch literal 2777 zcmV;~3MTc5P)9*YHQQH znh@I(s7WDIN`nJ+5@|<)iZcg=qN74U#DNnD1Se7u4fs(|1ivr?9ayP|B3iYCD$mfQ zCQ{S1n2)}^yxe#1J=_0pt-a1UPwQ^Z*?X_`Uu*sM+8<}X+baE^a`3seUF}?bEaiMO zrD`Qrd5@qw^epHZ>Df|p-qKBUEB%*?!m0{PHC6j|RplEgR~PkM5a^}N)Sfwi>W;Uz zdhwo_4HXBU%kRl^w@&7iKPx$e-n9%#IU!&oMI~iNsw0n19qSX;dS>I`G_G=WdcN9r z;_Rtv9XC<7kbL+HHxJ782T~pg05t)tf^>2vNJqfYt{YmqQDoBxkv+ra*BxxhcuK2v zm5%@Y)biQz)R8O%e=o%n${;ojY;EUP>`Qj6Cq)7GHm)C%2%^+hI;Z4T#a|oKIvshv z5H%!I+|I4PEXaXj04%ybsVolr%vhKnW7AEhC?eP!o1{y;8m2R#;}{6VZPc!+)ou0C zVWz$|1#2(|L5z%EYRxOzP+uLB>qYGuajX-<#^u;Kw&2uh&93)h>nHaFA%{&2PW=Nn zr?*a;gk3xvRhQIRa1de-!r(ss&?tRmZ=L2FMkhxI3lK6Jn<>5c*ID|@KU#^MCIo6> zpFA{|R(4fsBwHIW z9v!7G|7enadv4}~*8q_h%tD^j$7=PCnn0=dR0GKA(fgb9`2IRg6ksBIo+Gdw#|-3eSe=3tmDe zIqVN)tScM`0W#Z>2wc>~2Uv=3L)~D4gXqZtPQ8rifbYJqwkG>bv}95G7+};9Br?hF zWSa3b)X}z#79W9kukM%6-b_54WDJm~Ub=gsrJ0lz-8&lrQ7zfK1qzuZQkZvcE3|~S zZWmk0ETaNIHnMALn>akuvHLf5c4`y%!f+u>ZGp%@q_;T!`76_snc_?K;Wx%YpF;5K zw^F+BCYUPy`fpRif@5O@Im5cf?evD$>KlAgX;D0*HiO0`Yg3j;R4jT(9h(L_TsY6yxk*@ZBe%+dMqY=cB5oGs{D$QwOFbH)G$iVf<3Olcd7^#fr- zM{!ILWt#coT)s9ySkwDCPHv0oww8g8K%Yr{aR}msELVX(}JQr%F4Q8=KKn*OjSO*uSp;JK%GwhRF_K??vGC$ZqmJX z@+}8sQ)9Z}3*DiWl+L_7OXn_^{SW~2&C*b^;%IP!j$lkre7H&bMR1}7aTT*G8P}|G zHM1)hZDe{r_E3{{Y=d}}_PxJO_w4MaE4)$<<3JwzPdwPzfNemK(-X;{UCzmVr0zu5 zEnT}fzx)oVd!*W77`1Ig`DFcZ6TkPaI$hO1+`cGb$({ukz&{p4Ic-Xnwrg-KEkDqW zW3l$7Q`V$!1T(=QL1jgjIachdr75>-8>1A^h+;rTrD^nnwf?bw(Rang!*16Odj$Pn z@)JN5&5w~}ae6d};oa|&G>sT!)ixE#5;QW(u(=bqYHXcOflE%@t4A?n5fTUm0F~8_ zwpoz9rrU`@G=vsNjDRY(CrF(jIjqg8bd|CP02>eFag7T?u;C^ir+Z7YKmBYw;%%XdT2T}a$X4yR7EI;zaof3a)5Z;`OwVi%D?gbkBj!{;z2tOBSFk&E1DeiZXD**uvNqL}+|pO{ ztO$}2NMRit2ddU?)7Prq&*&H3X>&=E{-+j4iUz zrvL;?0$^@lyl=LHz9G^$SJV6ID__@7z->Bh>Vm=6AK&5bP%@heveHja5F@agGgUsY z@L@W2+^*NVoId0!kS~4XkWb%y;f}XBf>S+NIw9aHK;vN+4mJ|em)_QjIVfb2$;bwv zDKmoq6AThgKydS6Hs+UpKPWq|UA}s=UOEBZNM3oNT5qTAabY)X>L6jxfGDuu7&GD_ z=@@m?sJ-o2GS}&hNRW}-zHkr>o4&138@a8IC-FjSBxzjx?(*3@YmdmWGAd%0QvXzS zJ53JpX%Fp!=>v&`Hd7F@+Atw2vx9%^2M-APg0Jd|ePsRn3*B$#9Z5hCou4fo7W#SN z#}-@-N=##yQDh26pNzr9f*Q88krhI5@DHcf{dU-~PLSs}MvI4s1i|<=qxD~9`7>*~ znlw5lr$_6mTG4XbBNF_79BzvZ!TeIP)exdk3)kSHjYdW1P10ZJ_NCJSlrCuIU#gqw f88(SSw!Z%ZUzhC#9QlKF00000NkvXXu0mjfG$}gK literal 0 HcmV?d00001 diff --git a/harmony-os/SherpaOnnxTts/build-profile.json5 b/harmony-os/SherpaOnnxTts/build-profile.json5 new file mode 100644 index 000000000..8e63d9768 --- /dev/null +++ b/harmony-os/SherpaOnnxTts/build-profile.json5 @@ -0,0 +1,40 @@ +{ + "app": { + "signingConfigs": [], + "products": [ + { + "name": "default", + "signingConfig": "default", + "compatibleSdkVersion": "4.0.0(10)", + "runtimeOS": "HarmonyOS", + "buildOption": { + "strictMode": { + "caseSensitiveCheck": true, + } + } + } + ], + "buildModeSet": [ + { + "name": "debug", + }, + { + "name": "release" + } + ] + }, + "modules": [ + { + "name": "entry", + "srcPath": "./entry", + "targets": [ + { + "name": "default", + "applyToProducts": [ + "default" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxTts/code-linter.json5 b/harmony-os/SherpaOnnxTts/code-linter.json5 new file mode 100644 index 000000000..77b31b517 --- /dev/null +++ b/harmony-os/SherpaOnnxTts/code-linter.json5 @@ -0,0 +1,20 @@ +{ + "files": [ + "**/*.ets" + ], + "ignore": [ + "**/src/ohosTest/**/*", + "**/src/test/**/*", + "**/src/mock/**/*", + "**/node_modules/**/*", + "**/oh_modules/**/*", + "**/build/**/*", + "**/.preview/**/*" + ], + "ruleSet": [ + "plugin:@performance/recommended", + "plugin:@typescript-eslint/recommended" + ], + "rules": { + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxTts/entry/.gitignore b/harmony-os/SherpaOnnxTts/entry/.gitignore new file mode 100644 index 000000000..e2713a277 --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/.gitignore @@ -0,0 +1,6 @@ +/node_modules +/oh_modules +/.preview +/build +/.cxx +/.test \ No newline at end of file diff --git a/harmony-os/SherpaOnnxTts/entry/build-profile.json5 b/harmony-os/SherpaOnnxTts/entry/build-profile.json5 new file mode 100644 index 000000000..554d19f3b --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/build-profile.json5 @@ -0,0 +1,33 @@ +{ + "apiType": "stageMode", + "buildOption": { + "sourceOption": { + "workers": [ + "./src/main/ets/workers/NonStreamingTtsWorker.ets" + ] + } + }, + "buildOptionSet": [ + { + "name": "release", + "arkOptions": { + "obfuscation": { + "ruleOptions": { + "enable": false, + "files": [ + "./obfuscation-rules.txt" + ] + } + } + } + }, + ], + "targets": [ + { + "name": "default" + }, + { + "name": "ohosTest", + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxTts/entry/hvigorfile.ts b/harmony-os/SherpaOnnxTts/entry/hvigorfile.ts new file mode 100644 index 000000000..c6edcd904 --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/hvigorfile.ts @@ -0,0 +1,6 @@ +import { hapTasks } from '@ohos/hvigor-ohos-plugin'; + +export default { + system: hapTasks, /* Built-in plugin of Hvigor. It cannot be modified. */ + plugins:[] /* Custom plugin to extend the functionality of Hvigor. */ +} diff --git a/harmony-os/SherpaOnnxTts/entry/obfuscation-rules.txt b/harmony-os/SherpaOnnxTts/entry/obfuscation-rules.txt new file mode 100644 index 000000000..272efb6ca --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/obfuscation-rules.txt @@ -0,0 +1,23 @@ +# Define project specific obfuscation rules here. +# You can include the obfuscation configuration files in the current module's build-profile.json5. +# +# For more details, see +# https://developer.huawei.com/consumer/cn/doc/harmonyos-guides-V5/source-obfuscation-V5 + +# Obfuscation options: +# -disable-obfuscation: disable all obfuscations +# -enable-property-obfuscation: obfuscate the property names +# -enable-toplevel-obfuscation: obfuscate the names in the global scope +# -compact: remove unnecessary blank spaces and all line feeds +# -remove-log: remove all console.* statements +# -print-namecache: print the name cache that contains the mapping from the old names to new names +# -apply-namecache: reuse the given cache file + +# Keep options: +# -keep-property-name: specifies property names that you want to keep +# -keep-global-name: specifies names that you want to keep in the global scope + +-enable-property-obfuscation +-enable-toplevel-obfuscation +-enable-filename-obfuscation +-enable-export-obfuscation \ No newline at end of file diff --git a/harmony-os/SherpaOnnxTts/entry/oh-package-lock.json5 b/harmony-os/SherpaOnnxTts/entry/oh-package-lock.json5 new file mode 100644 index 000000000..debb8e01e --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/oh-package-lock.json5 @@ -0,0 +1,29 @@ +{ + "meta": { + "stableOrder": true + }, + "lockfileVersion": 3, + "ATTENTION": "THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.", + "specifiers": { + "libsherpa_onnx.so@../oh_modules/.ohpm/sherpa_onnx@1.10.32/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx": "libsherpa_onnx.so@../oh_modules/.ohpm/sherpa_onnx@1.10.32/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx", + "sherpa_onnx@1.10.32": "sherpa_onnx@1.10.32" + }, + "packages": { + "libsherpa_onnx.so@../oh_modules/.ohpm/sherpa_onnx@1.10.32/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx": { + "name": "libsherpa_onnx.so", + "version": "1.0.0", + "resolved": "../oh_modules/.ohpm/sherpa_onnx@1.10.32/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx", + "registryType": "local" + }, + "sherpa_onnx@1.10.32": { + "name": "sherpa_onnx", + "version": "1.10.32", + "integrity": "sha512-yHYmWoeqhrunOqGr9gxPJJH/8+rdwcKFOW6onYByVObQVpbqypslg301IjGm9xpnc5bJEkO3S9sra2zQTpPA/w==", + "resolved": "https://ohpm.openharmony.cn/ohpm/sherpa_onnx/-/sherpa_onnx-1.10.32.har", + "registryType": "ohpm", + "dependencies": { + "libsherpa_onnx.so": "file:./src/main/cpp/types/libsherpa_onnx" + } + } + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxTts/entry/oh-package.json5 b/harmony-os/SherpaOnnxTts/entry/oh-package.json5 new file mode 100644 index 000000000..daff21b30 --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/oh-package.json5 @@ -0,0 +1,12 @@ +{ + "name": "entry", + "version": "1.0.0", + "description": "Please describe the basic information.", + "main": "", + "author": "", + "license": "", + "dependencies": { + "sherpa_onnx": "1.10.32", + } +} + diff --git a/harmony-os/SherpaOnnxTts/entry/src/main/ets/entryability/EntryAbility.ets b/harmony-os/SherpaOnnxTts/entry/src/main/ets/entryability/EntryAbility.ets new file mode 100644 index 000000000..679d91453 --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/src/main/ets/entryability/EntryAbility.ets @@ -0,0 +1,43 @@ +import AbilityConstant from '@ohos.app.ability.AbilityConstant'; +import hilog from '@ohos.hilog'; +import UIAbility from '@ohos.app.ability.UIAbility'; +import Want from '@ohos.app.ability.Want'; +import window from '@ohos.window'; + +export default class EntryAbility extends UIAbility { + onCreate(want: Want, launchParam: AbilityConstant.LaunchParam): void { + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onCreate'); + } + + onDestroy(): void { + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onDestroy'); + } + + onWindowStageCreate(windowStage: window.WindowStage): void { + // Main window is created, set main page for this ability + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onWindowStageCreate'); + + windowStage.loadContent('pages/Index', (err) => { + if (err.code) { + hilog.error(0x0000, 'testTag', 'Failed to load the content. Cause: %{public}s', JSON.stringify(err) ?? ''); + return; + } + hilog.info(0x0000, 'testTag', 'Succeeded in loading the content.'); + }); + } + + onWindowStageDestroy(): void { + // Main window is destroyed, release UI related resources + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onWindowStageDestroy'); + } + + onForeground(): void { + // Ability has brought to foreground + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onForeground'); + } + + onBackground(): void { + // Ability has back to background + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onBackground'); + } +} diff --git a/harmony-os/SherpaOnnxTts/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets b/harmony-os/SherpaOnnxTts/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets new file mode 100644 index 000000000..d2c48b421 --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets @@ -0,0 +1,12 @@ +import hilog from '@ohos.hilog'; +import BackupExtensionAbility, { BundleVersion } from '@ohos.application.BackupExtensionAbility'; + +export default class EntryBackupAbility extends BackupExtensionAbility { + async onBackup() { + hilog.info(0x0000, 'testTag', 'onBackup ok'); + } + + async onRestore(bundleVersion: BundleVersion) { + hilog.info(0x0000, 'testTag', 'onRestore ok %{public}s', JSON.stringify(bundleVersion)); + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxTts/entry/src/main/ets/pages/Index.ets b/harmony-os/SherpaOnnxTts/entry/src/main/ets/pages/Index.ets new file mode 100644 index 000000000..45927b772 --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/src/main/ets/pages/Index.ets @@ -0,0 +1,409 @@ +import { CircularBuffer } from 'sherpa_onnx'; +import worker, { MessageEvents } from '@ohos.worker'; +import { audio } from '@kit.AudioKit'; +import picker from '@ohos.file.picker'; +import fs from '@ohos.file.fs'; +import systemTime from '@ohos.systemTime'; + + +function savePcmToWav(filename: string, samples: Int16Array, sampleRate: number) { + const fp = fs.openSync(filename, fs.OpenMode.READ_WRITE | fs.OpenMode.CREATE); + + const header = new ArrayBuffer(44); + const view = new DataView(header); + + // http://soundfile.sapp.org/doc/WaveFormat/ + // F F I R + view.setUint32(0, 0x46464952, true); // chunkID + view.setUint32(4, 36 + samples.length * 2, true); // chunkSize // E V A W + view.setUint32(8, 0x45564157, true); // format // // t m f + view.setUint32(12, 0x20746d66, true); // subchunk1ID + view.setUint32(16, 16, true); // subchunk1Size, 16 for PCM + view.setUint32(20, 1, true); // audioFormat, 1 for PCM + view.setUint16(22, 1, true); // numChannels: 1 channel + view.setUint32(24, sampleRate, true); // sampleRate + view.setUint32(28, sampleRate * 2, true); // byteRate + view.setUint16(32, 2, true); // blockAlign + view.setUint16(34, 16, true); // bitsPerSample + view.setUint32(36, 0x61746164, true); // Subchunk2ID + view.setUint32(40, samples.length * 2, true); // subchunk2Size + + fs.writeSync(fp.fd, new Uint8Array(header).buffer, { length: header.byteLength }); + fs.writeSync(fp.fd, samples.buffer, { length: samples.buffer.byteLength }); + + fs.closeSync(fp.fd); +} + +function toInt16Samples(samples: Float32Array): Int16Array { + const int16Samples = new Int16Array(samples.length); + for (let i = 0; i < samples.length; ++i) { + let s = samples[i] * 32767; + s = s > 32767 ? 32767 : s; + s = s < -32768 ? -32768 : s; + int16Samples[i] = s; + } + + return int16Samples; +} + + +@Entry +@Component +struct Index { + @State currentIndex: number = 0; + @State title: string = 'Next-gen Kaldi: Text-to-speech'; + @State info: string = ''; + @State btnStartCaption: string = 'Start'; + @State btnStartEnabled: boolean = false; + @State btnStopCaption: string = 'Stop'; + @State btnStopEnabled: boolean = false; + @State btnSaveCaption: string = 'Save'; + @State btnSaveEnabled: boolean = false; + @State progress: number = 0; + @State sid: string = '0'; + @State speechSpeed: string = '1.0'; + @State isGenerating: boolean = false; + @State initTtsDone: boolean = false; + @State ttsGeneratedDone: boolean = true; + @State numSpeakers: number = 1; + @State initAudioDone: boolean = false; + private controller: TabsController = new TabsController(); + private cancelled: boolean = false; + private sampleRate: number = 0; + private startTime: number = 0; + private stopTime: number = 0; + private inputText: string = ''; + // it specifies only the initial capacity. + private workerInstance?: worker.ThreadWorker + private readonly scriptURL: string = 'entry/ets/workers/NonStreamingTtsWorker.ets' + // note that circular buffer can automatically resize. + private sampleBuffer: CircularBuffer = new CircularBuffer(16000 * 5); + private finalSamples: Float32Array | null = null; + private audioRenderer: audio.AudioRenderer | null = null; + + initAudioRenderer() { + if (this.audioRenderer) { + console.log(`Audio renderer has already been created. Skip creating`); + return; + } // see // https://developer.huawei.com/consumer/cn/doc/harmonyos-guides-V5/using-audiorenderer-for-playback-V5 + console.log('Initializing audio renderer'); + const audioStreamInfo: audio.AudioStreamInfo = { + samplingRate: this.sampleRate, + channels: audio.AudioChannel.CHANNEL_1, // 通道 + sampleFormat: audio.AudioSampleFormat.SAMPLE_FORMAT_S16LE, + encodingType: audio.AudioEncodingType.ENCODING_TYPE_RAW + }; + + const audioRendererInfo: audio.AudioRendererInfo = { + usage: audio.StreamUsage.STREAM_USAGE_MUSIC, rendererFlags: 0 + }; + + const audioRendererOptions: audio.AudioRendererOptions = { + streamInfo: audioStreamInfo, rendererInfo: audioRendererInfo + }; + + audio.createAudioRenderer(audioRendererOptions, (err, renderer) => { + if (!err) { + console.log('audio renderer initialized successfully'); + this.initAudioDone = true; + if (renderer) { + this.audioRenderer = renderer; + this.audioRenderer.on("writeData", this.audioPlayCallback); + if (this.sampleBuffer.size()) { + this.audioRenderer.start(); + } + } else { + console.log(`returned audio renderer is ${renderer}`); + } + } else { + console.log(`Failed to initialize audio renderer. error message: ${err.message}, error code: ${err.code}`); + } + }); + } + + async aboutToAppear() { + this.initAudioRenderer(); + + this.workerInstance = new worker.ThreadWorker(this.scriptURL, { + name: 'NonStreaming TTS worker' + }); + this.workerInstance.onmessage = (e: MessageEvents) => { + const msgType = e.data['msgType'] as string; + console.log(`received msg from worker: ${msgType}`); + + if (msgType == 'init-tts-done') { + this.info = 'Model initialized!\nPlease enter text and press start.'; + this.sampleRate = e.data['sampleRate'] as number; + this.numSpeakers = e.data['numSpeakers'] as number; + + this.initTtsDone = true; + } + + if (msgType == 'tts-generate-partial') { + if (this.cancelled) { + return; + } + + const samples: Float32Array = e.data['samples'] as Float32Array; + const progress: number = e.data['progress'] as number; + this.progress = progress; + + this.sampleBuffer.push(samples); + + if (!this.initAudioDone) { + this.initAudioRenderer(); + } + + if (this.audioRenderer && this.audioRenderer?.state != audio.AudioState.STATE_RUNNING) { + this.audioRenderer.start(); + } + } + + if (msgType == 'tts-generate-done') { + this.isGenerating = false; + const samples: Float32Array = e.data['samples'] as Float32Array; + + systemTime.getRealTime((err, data) => { + + if (err) { + console.log(`Failed to get stop time`) + } else { + this.stopTime = data; + + const audioDuration = samples.length / this.sampleRate; + const elapsedSeconds = (this.stopTime - this.startTime) / 1000; + const RTF = elapsedSeconds / audioDuration; + + this.info = `Audio duration: ${audioDuration} s +Elapsed: ${elapsedSeconds} s +RTF = ${elapsedSeconds.toFixed(2)}/${audioDuration.toFixed(2)} = ${RTF.toFixed(3)} +`; + if (this.cancelled) { + this.info += '\nCancelled.'; + } + } + }); + + this.finalSamples = samples; + this.ttsGeneratedDone = true; + this.btnSaveEnabled = true; + + this.ttsGeneratedDone = true; + + if (this.audioRenderer && this.audioRenderer?.state != audio.AudioState.STATE_RUNNING && + this.sampleBuffer.size() == 0) { + this.sampleBuffer.push(samples); + this.progress = 1; + this.audioRenderer.start(); + } + + if (!this.initAudioDone) { + this.btnStartEnabled = true; + this.btnStopEnabled = false; + this.info += '\nAudio renderer is not initialized. Disable playing audio.'; + } + } + } + + this.info = 'Initializing TTS model ...'; + this.workerInstance.postMessage({ msgType: 'init-tts', context: getContext() }); + } + + @Builder + TabBuilder(title: string, targetIndex: number, selectedImg: Resource, normalImg: Resource) { + Column() { + Image(this.currentIndex == targetIndex ? selectedImg : normalImg).size({ width: 25, height: 25 }) + Text(title).fontColor(this.currentIndex == targetIndex ? '#28bff1' : '#8a8a8a') + }.width('100%').height(50).justifyContent(FlexAlign.Center).onClick(() => { + this.currentIndex = targetIndex; + this.controller.changeIndex(this.currentIndex); + }) + } + + build() { + Column() { + Tabs({ barPosition: BarPosition.End, controller: this.controller }) { + TabContent() { + Column({ space: 10 }) { + Text(this.title).fontSize(20).fontWeight(FontWeight.Bold); + if (this.numSpeakers > 1) { + Row({ space: 10 }) { + Text(`Speaker ID (0-${this.numSpeakers - 1})`).width('60%') + + TextInput({ text: this.sid }).onChange((text) => { + this.sid = text.trim(); + }).width('20%') + }.justifyContent(FlexAlign.Center) + } + + Row() { + Text('Speech speed').width('60%'); + + TextInput({ text: this.speechSpeed }).onChange((text) => { + this.speechSpeed = text.trim(); + }).width('20%') + } + + Row({ space: 10 }) { + Button(this.btnStartCaption).enabled(this.btnStartEnabled).onClick(async () => { + let sid = parseInt(this.sid); + if (sid.toString() != this.sid) { + this.info = 'Please input a valid speaker ID'; + return; + } + + let speed = parseFloat(this.speechSpeed); + if (isNaN(speed)) { + this.info = 'Please enter a valid speech speed'; + return; + } + + if (speed <= 0) { + this.info = 'Please enter a positive speech speed'; + return; + } + + if (this.workerInstance && this.initTtsDone) { + this.info = 'Generating...'; + this.cancelled = false; + this.finalSamples = null; + this.sampleBuffer.reset(); + this.ttsGeneratedDone = false; + this.progress = 0; + + this.btnStartEnabled = false; + this.btnStopEnabled = true; + this.btnSaveEnabled = false; + console.log(`sending ${this.inputText}`) + this.ttsGeneratedDone = false; + this.startTime = await systemTime.getRealTime(); + this.workerInstance?.postMessage({ + msgType: 'tts-generate', + text: this.inputText, + sid: sid, + speed: speed, + }); + this.isGenerating = true; + this.info = ''; + } else { + this.info = 'Failed to initialize tts model'; + this.btnStartEnabled = false; + } + }); + + Button(this.btnStopCaption).enabled(this.btnStopEnabled).onClick(() => { + this.ttsGeneratedDone = true; + this.btnStartEnabled = true; + this.btnStopEnabled = false; + this.sampleBuffer.reset(); + this.cancelled = true; + this.isGenerating = false; + + if (this.workerInstance && this.initTtsDone) { + this.workerInstance.postMessage({ msgType: 'tts-generate-cancel' }); + } + this.audioRenderer?.stop(); + }) + + Button(this.btnSaveCaption).enabled(this.btnSaveEnabled).onClick(() => { + if (!this.finalSamples || this.finalSamples.length == 0) { + + this.btnSaveEnabled = false; + return; + } + + let uri: string = ''; + + const audioOptions = new picker.AudioSaveOptions(); // audioOptions.newFileNames = ['o.wav']; + + const audioViewPicker = new picker.AudioViewPicker(); + + audioViewPicker.save(audioOptions).then((audioSelectResult: Array) => { + uri = audioSelectResult[0]; + if (this.finalSamples) { + savePcmToWav(uri, toInt16Samples(this.finalSamples), this.sampleRate); + console.log(`Saved to ${uri}`); + this.info += `\nSaved to ${uri}`; + } + }); + }); + } + + if (this.info != '') { + TextArea({ text: this.info }).focusable(false); + } + if (this.progress > 0) { + Row() { + Progress({ value: 0, total: 100, type: ProgressType.Capsule }) + .width('80%') + .height(20) + .value(this.progress * 100); + + Text(`${(this.progress * 100).toFixed(2)}%`).width('15%') + }.width('100%').justifyContent(FlexAlign.Center) + } + + TextArea({ placeholder: 'Input text for TTS and click the start button' }) + .width('100%') + .height('100%') + .focusable(this.isGenerating == false && this.initTtsDone) + .onChange((text) => { + this.inputText = text; + if (text.trim() == '') { + this.btnStartEnabled = false; + return; + } + this.btnStartEnabled = true; + }) + }.width('100%') + + // see https://composeicons.com/ + }.tabBar(this.TabBuilder('TTS', 0, $r('app.media.home'), $r('app.media.home'))) + + TabContent() { + Column({space: 10}) { + Text(this.title).fontSize(20).fontWeight(FontWeight.Bold); + TextArea({text: ` +Everyting is open-sourced. + +It runs locally, without accessing the network + +See also https://github.com/k2-fsa/sherpa-onnx + +新一代 Kaldi QQ 和微信交流群: 请看 + +https://k2-fsa.github.io/sherpa/social-groups.html + +微信公众号: 新一代 Kaldi + `}).width('100%') + .height('100%') + .focusable(false) + }.justifyContent(FlexAlign.Start) + }.tabBar(this.TabBuilder('Help', 1, $r('app.media.info'), $r('app.media.info'))) + }.scrollable(false) + } + } + + private audioPlayCallback = (buffer: ArrayBuffer) => { + const numSamples = buffer.byteLength / 2; + if (this.sampleBuffer.size() >= numSamples) { + const samples: Float32Array = this.sampleBuffer.get(this.sampleBuffer.head(), numSamples); + + const int16Samples = new Int16Array(buffer); + for (let i = 0; i < numSamples; ++i) { + let s = samples[i] * 32767; + s = s > 32767 ? 32767 : s; + s = s < -32768 ? -32768 : s; + int16Samples[i] = s; + } + this.sampleBuffer.pop(numSamples); + } else { + (new Int16Array(buffer)).fill(0); + if (this.ttsGeneratedDone) { + this.audioRenderer?.stop(); + this.btnStartEnabled = true; + this.btnStopEnabled = false; + } + } + }; +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxTts/entry/src/main/ets/workers/NonStreamingTtsWorker.ets b/harmony-os/SherpaOnnxTts/entry/src/main/ets/workers/NonStreamingTtsWorker.ets new file mode 100644 index 000000000..bd5c7a5b8 --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/src/main/ets/workers/NonStreamingTtsWorker.ets @@ -0,0 +1,284 @@ +import worker, { ThreadWorkerGlobalScope, MessageEvents, ErrorEvent } from '@ohos.worker'; + +import { fileIo as fs } from '@kit.CoreFileKit'; + +import {OfflineTtsConfig, OfflineTts, listRawfileDir, TtsInput, TtsOutput} from 'sherpa_onnx'; +import { buffer } from '@kit.ArkTS'; + +const workerPort: ThreadWorkerGlobalScope = worker.workerPort; + +let tts: OfflineTts; +let cancelled = false; + +function mkdir(context: Context, parts: string[]) { + const path = parts.join('/'); + if (fs.accessSync(path)) { + return; + } + + const sandboxPath: string = context.getApplicationContext().filesDir; + let d = sandboxPath + for (const p of parts) { + d = d + '/' + p; + + if (fs.accessSync(d)) { + continue; + } + + fs.mkdirSync(d); + } +} + +function copyRawFileDirToSandbox(context: Context, srcDir: string) { + let mgr = context.resourceManager; + const allFiles: string[] = listRawfileDir(mgr, srcDir); + for (const src of allFiles) { + const parts: string[] = src.split('/'); + if (parts.length != 1) { + mkdir(context, parts.slice(0, -1)); + } + + copyRawFileToSandbox(context, src, src); + } +} + +function copyRawFileToSandbox(context: Context, src: string, dst: string) { + // see https://blog.csdn.net/weixin_44640245/article/details/142634846 + // https://developer.huawei.com/consumer/cn/doc/harmonyos-guides-V5/rawfile-guidelines-V5 + let uint8Array: Uint8Array = context.resourceManager.getRawFileContentSync(src); + + // https://developer.huawei.com/consumer/cn/doc/harmonyos-references-V5/js-apis-file-fs-V5#fsmkdir + let sandboxPath: string = context.getApplicationContext().filesDir; + let filepath = sandboxPath + '/' + dst; + + if (fs.accessSync(filepath)) { + // if the destination exists and has the expected file size, + // then we skip copying it + let stat = fs.statSync(filepath); + if (stat.size == uint8Array.length) { + return; + } + } + + const fp = fs.openSync(filepath, fs.OpenMode.WRITE_ONLY | fs.OpenMode.CREATE | fs.OpenMode.TRUNC); + fs.writeSync(fp.fd, buffer.from(uint8Array).buffer) + fs.close(fp.fd); +} + +function initTts(context: Context): OfflineTts { + // Such a design is to make it easier to build flutter APPs with + // github actions for a variety of tts models + // + // See https://github.com/k2-fsa/sherpa-onnx/blob/master/scripts/flutter/generate-tts.py + // for details + + let modelDir = ''; + let modelName = ''; + let ruleFsts = ''; + let ruleFars = ''; + let lexicon = ''; + let dataDir = ''; + let dictDir = ''; + // You can select an example below and change it according to match your + // selected tts model + + // ============================================================ + // Your change starts here + // ============================================================ + + // Example 1: + // modelDir = 'vits-vctk'; + // modelName = 'vits-vctk.onnx'; + // lexicon = 'lexicon.txt'; + + // Example 2: + // https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models + // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 + // modelDir = 'vits-piper-en_US-amy-low'; + // modelName = 'en_US-amy-low.onnx'; + // dataDir = 'espeak-ng-data'; + + // Example 3: + // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-icefall-zh-aishell3.tar.bz2 + // modelDir = 'vits-icefall-zh-aishell3'; + // modelName = 'model.onnx'; + // ruleFsts = 'phone.fst,date.fst,number.fst,new_heteronym.fst'; + // ruleFars = 'rule.far'; + // lexicon = 'lexicon.txt'; + + // Example 4: + // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/vits.html#csukuangfj-vits-zh-hf-fanchen-c-chinese-187-speakers + // modelDir = 'vits-zh-hf-fanchen-C'; + // modelName = 'vits-zh-hf-fanchen-C.onnx'; + // lexicon = 'lexicon.txt'; + // dictDir = 'dict'; + + // Example 5: + // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-coqui-de-css10.tar.bz2 + // modelDir = 'vits-coqui-de-css10'; + // modelName = 'model.onnx'; + + // Example 6 + // https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models + // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-libritts_r-medium.tar.bz2 + // modelDir = 'vits-piper-en_US-libritts_r-medium'; + // modelName = 'en_US-libritts_r-medium.onnx'; + // dataDir = 'espeak-ng-data'; + + // Example 7 + // https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models + // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-melo-tts-zh_en.tar.bz2 + // modelDir = 'vits-melo-tts-zh_en'; + // modelName = 'model.onnx'; + // lexicon = 'lexicon.txt'; + // dictDir = 'dict'; + // ruleFsts = `date.fst,phone.fst,number.fst`; + + // ============================================================ + // Please don't change the remaining part of this function + // ============================================================ + + if (modelName == '') { + throw new Error('You are supposed to select a model by changing the code before you run the app'); + } + + modelName = modelDir + '/' + modelName; + + if (ruleFsts != '') { + let fsts = ruleFsts.split(',') + let tmp: string[] = []; + for (const f of fsts) { + tmp.push(modelDir + '/' + f); + } + ruleFsts = tmp.join(','); + } + + if (ruleFars != '') { + let fars = ruleFars.split(',') + let tmp: string[] = []; + for (const f of fars) { + tmp.push(modelDir + '/' + f); + } + ruleFars = tmp.join(','); + } + + if (lexicon != '') { + lexicon = modelDir + '/' + lexicon; + } + + if (dataDir != '') { + copyRawFileDirToSandbox(context, modelDir + '/' + dataDir) + let sandboxPath: string = context.getApplicationContext().filesDir; + dataDir = sandboxPath + '/' + modelDir + '/' + dataDir; + } + + if (dictDir != '') { + copyRawFileDirToSandbox(context, modelDir + '/' + dictDir) + let sandboxPath: string = context.getApplicationContext().filesDir; + dictDir = sandboxPath + '/' + modelDir + '/' + dictDir; + } + + const tokens = modelDir + '/tokens.txt'; + + const config: OfflineTtsConfig = new OfflineTtsConfig(); + config.model.vits.model = modelName; + config.model.vits.lexicon = lexicon; + config.model.vits.tokens = tokens; + config.model.vits.dataDir = dataDir; + config.model.vits.dictDir = dictDir; + config.model.numThreads = 2; + config.model.debug = true; + config.ruleFsts = ruleFsts; + config.ruleFars = ruleFars; + + return new OfflineTts(config, context.resourceManager); +} + +interface TtsCallbackData { + samples: Float32Array; + progress: number; +} + +function callback(data: TtsCallbackData): number { + workerPort.postMessage({ + 'msgType': 'tts-generate-partial', + samples: Float32Array.from(data.samples), + progress: data.progress, + }); + + // 0 means to stop generating in C++ + // 1 means to continue generating in C++ + return cancelled? 0 : 1; +} + +/** + * Defines the event handler to be called when the worker thread receives a message sent by the host thread. + * The event handler is executed in the worker thread. + * + * @param e message data + */ +workerPort.onmessage = (e: MessageEvents) => { + const msgType = e.data['msgType'] as string; + console.log(`msg-type: ${msgType}`); + if (msgType == 'init-tts' && !tts) { + const context = e.data['context'] as Context; + tts = initTts(context); + workerPort.postMessage({ 'msgType': 'init-tts-done', + sampleRate: tts.sampleRate, + numSpeakers: tts.numSpeakers, + }); + } + + if (msgType == 'tts-generate-cancel') { + cancelled = true; + } + + if (msgType == 'tts-generate') { + const text = e.data['text'] as string; + console.log(`recevied text ${text}`); + const input: TtsInput = new TtsInput(); + input.text = text; + input.sid = e.data['sid'] as number; + input.speed = e.data['speed'] as number; + input.callback = callback; + + cancelled = false; + if (true) { + tts.generateAsync(input).then((ttsOutput: TtsOutput) => { + console.log(`sampleRate: ${ttsOutput.sampleRate}`); + + workerPort.postMessage({ + 'msgType': 'tts-generate-done', + samples: Float32Array.from(ttsOutput.samples), + }); + + }); + } else { + const ttsOutput: TtsOutput = tts.generate(input); + workerPort.postMessage({ + 'msgType': 'tts-generate-done', + samples: Float32Array.from(ttsOutput.samples), + }); + } + + + } +} + +/** + * Defines the event handler to be called when the worker receives a message that cannot be deserialized. + * The event handler is executed in the worker thread. + * + * @param e message data + */ +workerPort.onmessageerror = (e: MessageEvents) => { +} + +/** + * Defines the event handler to be called when an exception occurs during worker execution. + * The event handler is executed in the worker thread. + * + * @param e error message + */ +workerPort.onerror = (e: ErrorEvent) => { +} diff --git a/harmony-os/SherpaOnnxTts/entry/src/main/module.json5 b/harmony-os/SherpaOnnxTts/entry/src/main/module.json5 new file mode 100644 index 000000000..a1cea8b6a --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/src/main/module.json5 @@ -0,0 +1,52 @@ +{ + "module": { + "name": "entry", + "type": "entry", + "description": "$string:module_desc", + "mainElement": "EntryAbility", + "deviceTypes": [ + "phone", + "tablet", + "2in1" + ], + "deliveryWithInstall": true, + "installationFree": false, + "pages": "$profile:main_pages", + "abilities": [ + { + "name": "EntryAbility", + "srcEntry": "./ets/entryability/EntryAbility.ets", + "description": "$string:EntryAbility_desc", + "icon": "$media:layered_image", + "label": "$string:EntryAbility_label", + "startWindowIcon": "$media:startIcon", + "startWindowBackground": "$color:start_window_background", + "exported": true, + "skills": [ + { + "entities": [ + "entity.system.home" + ], + "actions": [ + "action.system.home" + ] + } + ] + } + ], + "extensionAbilities": [ + { + "name": "EntryBackupAbility", + "srcEntry": "./ets/entrybackupability/EntryBackupAbility.ets", + "type": "backup", + "exported": false, + "metadata": [ + { + "name": "ohos.extension.backup", + "resource": "$profile:backup_config" + } + ], + } + ] + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/element/color.json b/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/element/color.json new file mode 100644 index 000000000..3c712962d --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/element/color.json @@ -0,0 +1,8 @@ +{ + "color": [ + { + "name": "start_window_background", + "value": "#FFFFFF" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/element/string.json b/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/element/string.json new file mode 100644 index 000000000..29b5d21cd --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/element/string.json @@ -0,0 +1,16 @@ +{ + "string": [ + { + "name": "module_desc", + "value": "On-device text-to-speech with Next-gen Kaldi" + }, + { + "name": "EntryAbility_desc", + "value": "On-device text-to-speech with Next-gen Kaldi" + }, + { + "name": "EntryAbility_label", + "value": "TTS" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/media/background.png b/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/media/background.png new file mode 100644 index 0000000000000000000000000000000000000000..f939c9fa8cc8914832e602198745f592a0dfa34d GIT binary patch literal 57364 zcmYIuc|6qL_rIk#Su&MMQlYU)cz{|$Qc0x~A^BEf( z`{n=HaSk>%wsfNM*uUkN^8dI{qxxW z*@b_`#>VlLWSG9 z0>QdPQ-&i_RCVdp2s$-u%S362^SHV0`EO6;@n(xK));G>#qwhPWrDXGk@OBMV}H!J za!48&`xhWJKj{_+f3ir<>Jg6Ax<&Xgn;)U7UJyAw{(u?zlf{oLsJTS-_o1?+lSg-j z8fcZj1*Ad(!X>WuuxM!H5t@V3*8vLL6`QnC!q!BwQjI{yk*;~@|3;B)`p$WYcDmnZ zt`R zr=oS6o-D$WZsYKh1PiOdhhX&YWGOzpc<6ITKzr^zi-#>z){t;yz3tu_a!>)(tTU9d zd}COuy~Tb}UIRNX@aVGJqEKUa)1#E-u}pl!sY)z4cu+Hu9==`6=0Ob#x-%q}t@jBp zmoiZDcfF1WL{PB0ZO**8yZ+%;LF6K*JDUoHrJkl0Wzak+Y%E( znUmuA^p@Jv6{%Y;MsiZ4O?#ID2b2ssEq6_KGL z8T%zdA3YhMnkBu19bNsa_$$_1^16jadx`0ZzPx`M%T>qZpYyNYOeDdmqLTNWpR5T% zOlRrW_xNCD+*3_WSxvt4P-@qQ9g_$aedDk-hcV~t>Oxw;UaAk1V?9m5<2k4%VrM$- z?{KH{)m_U~yJcBbX+vqVfq&4)Vf+FvAHd|s{V34=f#uJM!Tp?b32THmfzNn1unwY& zPNtaE{ZZ=OkZFh*xW2FT&fDF?64Q%l>dwdZ#Bg;^v;dAbU*QLEQG@_|ucNXFyx~H( z#h?kJKeI3jD^U~`e`*^zcm?PlIWj|tL_a8NC?HVl*gX%;5PW5Y%ZZ*G=jPn5#o+Sh zhnE>D@Wb!f*O>cZ0}ZT=HlEdoWVWk}5H1S;$vxe#Rv~;l5rJ=w--wPl621jCW}B|gxECKzT9 z3FKlD{=OfN5$J3?Ag0g4F5t8_D(RvO8W!*~?#q{Dhx(Sj=)^9ZlE|LyI?p1NXMWr| zGGbzFN^3)5?P^vfnD7XZo*8yf&O&>7XULUUvhJT@rHcF>PmjodH~u4RSmX4TH?v`IKg2cy7 z(T@e4&pPRHRczikEvwvO?jbblSVp z2qpyT+LHUFhHwcunP(^h)G#uA95vF`Gd&1k%F@wuCk3DnjNjw;b}*;dY{F5{7tNsg zLf4y|)RTV`PjQ^!NoWB3YA@S@Cw zUAr?mUcn7g)n!3J`D7*H`y{%TuT$wNY;))rP(y@kdFdPH#h|rjcW2#oRybxTchXlQ zwMW{bVcqRRc_2r^tI)Zav_+qLwdd|Bw=*pM!|pflbT%K&Eof^{6+|k{2_;HcrWd3? z#z;>@Y3dp#B^R5c9RhH8lT5MRr*;>xd<%C3sV2Y}>{On@a*oump`g#H<6V&DKeZ-?Zic$S$>ulEiZvJG8kHMeSzVE(R|E-<}cEG^n2E*Cp z-25-DQv_Mf+&WhT3r?23Phid$q`Z3HE($RgC{EJA0Yc1SP6(a(oZ4RU2L1~H6k0Q< zHY1Mj{)b(ll3Wr=HakbiEk13zYKN&f#9*}tMZiQ7h@Us+N(Jk`aWQHf)r!ObZAT>_STJuzjuO{qHMlTjN9^hPZ8sZBMl zl&MX}xk{d5VUEInRK9r^Tnx#HE2;hFoa7?NDufAxZV6Mj9B^NaAt4;oStAtWfVg8< zjQAfLPj#u>Xp*sALAi;M(f1>la|_-k(E*-1Sa_Vdt$KsCNAwAbm8CmvpDbwL$`Cx8 zkBC0&3#@q@7E3LVtGQcrGS=s-uh6FHuC)WTtU_@t5c_T~`Wv+F0Jd$a9s(?ucd&l{ zWThjQ*u4YqU6Wq{+^0sC%S;vXx~qO|+s%Am1m-N}zkd84>GT;5u}a1*p9&!g%3wk2 zl=rj+H9g>!z4_zdU1iItL}Zox?lwK^ykQ+_#Ym~p>s8CgcLQYV4wezL^K-_HzM$r! z1m$U&G13HqDckgHschNcoe73o=)$P$j46Y)SnaZK(U|F7d#{AGb%>@b+jX#5*Rf5x zq}@ejPTyyn&&@n|dDGl-o-=XF%6dndW+}@7JDd?6b}Mt-SX_GV^3{!3Yz5a~X@$Fw zyDIkaWq*rtn{8knumG6=yF(6lzQnq)&M@%8RzdC%{%-0Ey{v&0pp-aIPP$bTrF|=~!MvLftx2pd=0-86i#@A;;(b^r-TzBJn~W4d42|-V*)} zt}h95!TwDQ%xWD9TFS{BwGO@d9P>kia=+LQ@r>0>5VvEV8(&tEuw%+YP*Qm6KzZs9 z#qL6SPwl9DtPZ{0`)Vph`^ryNV|=I7r2Vf@LrX3<=$f6zv1^z*!<6j{f$|6Jw=%s2 zb)|d{?()1m_Xoab$B5r9#&taTI^E@0yTQ$UB1_f0nc<oQhFOi;b@!o=y6w&Tsrw|K5XXEJEA>@Eb?8hi( zlT-*bXZd6g*C+W9V6B5iF$2f(_-ek(ko^JW%$@}`#GJVV0S8A~FwzM(JdY)c1B&ls(qJ=bvy&S10cqD8@1Clbooq|3kmbD_she z@O#tu^ibgYfM#HD%WIF%%uf7+)sc&Dejs@WRQE+Q1jXlN2z>9dB;X9e>Y3a-&-A;T z>||D+o$j^$E>F`4y02DTELRMYH*biv(5+ed(cQq&82Gu z2~UNnOcNc&MwT3lD@S}nPJMsvOT%0L{`dN}DU&^Z#6?2^aE!5ulUV_Zct}2~K6R!_ z4ReuaX$@AP?M!XMpi&ZJwsY2up5F-xe0{ym`9#@pr%63v->d&@UoFthcC1`k$L=ze zYX1{xl49Q=z953h>NzyMc3UuH96t7)-k|lRw-P=T%Q`;dC7@r`uCOq8Eqi7gKC)~7 zb(*Q>H|T2(e>5DVf9nswM~C%V2G2 z#B|VOitZm{FlV>EydvsFF|Ue~ium0%0KOaFiMOLk(X}jHq@dI@*AM2G6XzCU zSpFR?#U4MPz~VZR>RA@a!CZu45#f<)^f#kJ+ULtRLJKzSj=cf+NxQ}Kw)Yme6wJz; zu3W=Jz<}rEm$g7sNy>yr-Z|OiI>qQ4m37~);`_~Xgr~N4wOAssk(HTh5er1XtFm+! zb`5FT&FoKA{ADaUP!Y#o^sGPb?mT2wBY9ZfQ}ujLk`C_dyTvT&)34sj!RXJcZ%lCzF?kE~i-xCSGh{ zy%iUR0+S_JP(#%W9!Npk=RL(8tFB7(up1ms-Q#8 z$-{dva97!EQB<5#@0KgW&2S|ddKN*<(?}37-=O@1bF668sG)3(D61=Ech&sJ;j|An zqu1a;`}bcMj;#tF3l~&Ue9ES7GRw~kIPKK&q&^No_3M#yjp?ygI;To&wcXbe%ju*z zpMI!gbi8@{AJVkgXR+py{dMSfko}H`^q^elZQ-5<2bG-K8tYq8Q@*4t)`Blvz!#v> zE;3kk_e^|Kew4?}eU;3n)q48yWgAm)d+F(;W(>jPB_glQLiH|IE=EDVFI*j_FBebS0vXyh5@x9LS?RNi7vXf?RckfXjvy^Pifki$9G zzwp&k7S+aNOI8%DUON~#xxv+a5rJDE+^6;@RcjnwKZ|%#%Ukq~@&vL#Pts;`f?jwYL)Y zDOROB^T8hlFfA@(=$bFYKWy{F^5$#{h*A1FG5GZZ1?>Y+!}UULap(oEekfHZCJkpC zppRS@+Uvrs>_Df!YT#HWpuaEwRq)V49)TgZ7Jf{A6@tpv&>tG)c9F&eZWo)(tDPDB z4Fkl6@ov*S4!gboeokhZ>My7@q%!Z93-zy>Y(_9axnH2W2Ie&#X2Z->o1A6ZoV(OgY z@PpdL`E%U}QN-vzdLCdkVX)Vp-z|CGg)^e06LvMfbj%1)ZdXNB>r>{Jk&ApwTkkLr z-2C5e31{3c{*xsm?)EItQ%pSW(%723B}AHgke#M{7KJW6TT*>9^+`FIe4;VHRwSF$ z9rBta7_>vwCuV;vFY=|NZ2KlX$A`EUk*phH=Pd~I8Kkr|v!j3sBAD^fPD!FoPpnHf zqP&jc&^s{jm0M&oBNXjUol2${7|G^u7UtOd2kxA0b?japS#xlwo_TaY+jh-`+$sfO zFLgfqb~kaemX{ErUn7}?_tb>g?G@UyT99HoY^;BG(5|gh>F3J!9J* zvrz6TP+;XdE$<41%Vony^Y}i*aCz@+4v^38p)5?Nhw`m%Cbg5Lpz%VOxaBnlA9P;N z9D=#{(>`$N_!?&CKf9eJGzIc>dhWes8XtpX`{gOhP;HMklZ8~@Yu~YT1bZZ{VwrAffDNiZ6Mh5vEzpq z=5A;0ff@>1MG@vbwRU!?7ZFD-SYng>JN(=>uwrkrl@4u6M^n6jl1shsk;DM`t#|F? z(H9W(@&~b(mmUR)30H=vAZdIrX%9iR7rMruZ_I4$Eq7YnBI4Z8T zj5;RTUu8?(ZsW>30%Hk#$^zfAtgZ&y!|p@5%e_4oe7)3{Y6c^x>zv=o_XPiF*wI1y zNe5L3p=L;8_D7-+5I+LfNgDYrOIUD_Iu_VJQD^=4v=Gd z_u%h$8{Lobyu6%VkeZI%T_vssgc#J4yD+&6pVkdLYl@3@NdcQbwl!J%4{RC80oF1z z`ksIXyrZT=Apq3kOR#m795+y}-8NizKBNESZCmBS#jqG`n4kCydp-4DZ^BF-zWD2# z1@F?p*^9m)EPrkd^E&cimk<1mN+iwSCVTHpqz^#`_Dj;-5xURqxK*!kp5asE##*=< zc{bFC-`m;q4VL3=| zKN6@)%XIu=yS*-K-9Bw`jN+-lWBttd77x>|g)~$UgPB_qH0h&bm}j3#sdLfV&xcR^ zQFk=d3;U8~YLqm@^61C zmaLbHw=dJ0oLP?>eyJ&=wgtZm!2mS9V!i~62x+n`%jyesf0bKruxRDH-)c2uF;&qT z4Z0drBbHg-G#ueH1vVaEJFTw$U))8mlUjFz?!PDqNpcIqZ%B6$Ju$CzrK@_na@?na5LpJODS}`)`8j7i#>C z0RNEb>nnQ8v$qXrgh)-(=VVRFwj4 zZKH}5T4rlZ$PiI2z3_*{`av5A0jPJY!Y*RQ?XbKPZmNdwp6ufAH4m~1%r{gYeOJBR zai+gl7I{Z35P0Q7EoGmkkLGHe5rR^{bdxWyMiC1~&kI@I-bYJrdGv{=O7!p&kKxN3 ztOoyzWj_asX!zA>`fa~&>#$n@3{c@VVcl3(1m5=dCI-~1uR+4s;@87ozKCU|Z(EhE z7~Csbr}e|&-zPK~*W}WcKqB+rv-rNRzvqfY299AvP zA5u^Rs->xN6b@MzP_f(M+}|~RxUHs#zO%D772V@B$F;5<%Jx|0#Oh_?#%yrHfV>}I z!Lfe59_VCjJ!pEQOWyUr;CdyL z-RzERMQjU_j%}N!Av?++44uVMc#r_KCTZxxSZL>4`xbm)#)*?4I#nFDOZLv10s^{6 zAyo6zfA)w8n^jk|KBb4J;|Gbx9)grFflY-Nyl_v8_@}gizDNn(Y2l6TqM&aN(+9Qg zTBo#J4N$h%f!;K&2NqBlT~J6aqHGy6HI`Xn*)UV$w2>iLk~P=l)VTdah9Ab`z%}dg zxIvG$xPG=H0NRw|6_-~Bzh+BPv9&C;z)58?`7t~$HupdHcF!F5dirrGrn3d}wAHr! z^@&!aoW@3sENjl#i@LzRYOZ4b#v|Jk_Mo$-VYlgbE3LQVKniS1mH)uO`90X{bc~{1 z*%Wm4$E_2-W__`4`mDu;Ld(wv8e147=mMu!AKSC=mw*4n^8S>~fm9mJgf4~8t(bb> z^_3WSK>aAZ6lK3OZ#_7g@)?z1#pZ zoR2>rm%_enbG!+Y34#Jmal)V9@-s8li+_Le^~z8cxHeF5vR%p~{93TJv%YmeTB|@^ zc=}q4Gofbju_Z#%Iv9|44|pawNvh^mFGBA_KZ5C^rx-l~Ytqf4;%SxezE8%O)aJh& z>2it7b`epB=P&=s^y`mJMjMq&9Jvpdhn}6sFHk)q%d zE_RV6%-}?H)w7yAW9TA)&7XbMyu=N}tRA-JTl2iG6u8;@?;!BW;ykyof{i+alo zJu1v~ITow6y^)5crWdi)&;yNs0d)3*vN+aSszJ%`1`(%9X-Hi}3gH#iRg@{Svm?cP zM}T*)U{A8FTQ7b@oc$7vr_EeTIj6N%Cr}VI5VcfZk+@1UFc>zpJkm3S%cb<~=~`BV ztbyjzOPJuDkTJJ!hL^nLk}*=2EXd?->%+3NWrq&5a$%1G{r2~cLQT2W>8!pd$9G;K ziQIDUErsVk$XQPRm)pDFYVuLFlx&eiFlnoixT|jvAoB)ryM_}euaYFXrdKLqi|4AL zG`rnvWi4Qa>Wvo=;Y+t@ecMjl{#37K;?VkYdoSbT(2m}8!k~RT{yv0l8cPp{jtiXr z$7KAJAvM_g4ak}0Yo*q!sO%PN_CK)Pv>lC7xoB~vG1hs?Wv>^kpOBU0WV@$|oL!cE z1FV3%^4Pjr5Fqc)|Sv+upxx8BCM z9*cYQYi3jY(^pUL8`I|3rHf+5>sq98e!hkPsfNMQ1@y7Tnf4{F2p zx9AO&@zYO;WpCQUf4G@!d<{t43@&RHh2Ukg^D-8_;De`dc{hz?yPS_7BzU!x^P-tj zBWt_uk{g94M1uo_&0l?m1qh!Q>=dKy5cx zRa7mv(}`xYKJOm)h3s8goQ*XK1OT<#&Ozf35uTB^VD8m)Z6Bnlal5r-bkso}J^TcM zo)ZSc#2@`h0Si}lrnCFt67JFa*e&}2avKCL|IIk<$R2*5sILkv4P( zesTX_tP#NqXN#>Q{4oe!N=G{SZ_I#~%^kq5ilGc=Q63_5uRt!D^j$k=&$`Ha&bGlAjZ2&hWa=M};Cw|5onME2e;8le z)-hK+mgNbGw-4puLN6g_q5p6T?0XM^dMo810rSBSw7Rrl(jt2JNVBwhB0o3``lZ1y zBr`Dy8LdVilxv`X5b0N8#{#(y<2vQrLj;qv`XA#RZ+@Q~*aYa^UY~;#F>6BL>75+E zeH2(L#HhLeI=Mz1#%^96zY$Se;@N)biYOvM6H1p6-4LcvA=&GP()#?u=_WXgAoZl* z+bR{6BA52?12Rex)v?(LMRsKvf9{KzP<^4&NISV{2!a;wEhr&E)EloHqSR9%ezb)? zl9X;qQSTg@es%UevGs9-KQk6RqJ;Ui(v@S0=JpkXQVYgXlRKQcfFLT2A%*#c?7(b} zjki==Q^Y#Qf}ZVpFtF6<4SbGKkkU>I6wY*Ps*EAzemS5Z0r!-oD>~r!<<+c~fHK+{ z`u4nWcW&4!()0%2>r>@zr$F6$;5*IAuq5bc>cn-IEZ+B|hkO&NPeBi&47YiU-<$w0 zq-j9aGH~K;Y%0{D&e90RZ(J_@o*`(e0TgqWM zz>V1_2|7MMg_6zbeK`A2oW6>`dUuDIll*?4hKaK{^>2t!B*N9o7_!iC51?A=hss#S zTOD48mGM}}JkMLeB>f0zNw|zPj8Efyx1Qh?QyT7Bp*PsC1%+$kgboSqDR=rTEs%8X z-t2|68n3XC`A-sBYO9tXuQqE7{}pE3mRASQTvScN7(%JH0{M|k4t%rE7xh`qUf4A- zgEE3f#zcuMyMYyiu;w=#PFC-_W0rb;u#{l@E}K0uMy~Ec1MBz-KglT}I_AG%m9nb!XAkpoW-`_85Umy)5g0j(3(>`;o1;w;CKp zLKdGc@@LrE*Y6B#H>jMeTcD6nZx;FZw zZ?8nd;T;sv#~t>9Stu`V2=$pLBHrDq3VNw9{KZU-50LlNLK@?o*hLF?1Kjl3op`;u z=nFLXc(CuUKp%gcxwwBm08`iDki>51cyobB9Eypc5@0Uv%$x+m$P}vtzJ@yXv2Y(6 z%G|Dfw#*GyPhoZ)9Obc;u$h*k0~W zv)EW8ChYvHNP~Ws5(MQk4JSGnG!l*4I-odrw$8E;u9uTN)1sDTSK-9%H|jqRi1XpO z_RLbdR5?V7FZiM9a@_RLzrIa?o8u(&ct}&dJFEmRO#py=1J(LW)$S@B$xLi6T)SOw|;fa7Myzv z?MOZ*b$o!rCg?J9&v6SsP#m&goHWvlC%0`IUKT~X&=s1cU$O`0Ea`_f|aU@(<=bXW{`6+7W#cu@H9t zagx-Usc&&vez&!Mjqpdk+Ol(}Uo_B;A&JhUaOe-iG9|*Z<)SYRZ;!m{$5X=V;9Cl+ zs(#H}WR`823f+9`wmRKF;(;wyt*?b3@Y`H^;&@1GipUF_{Gb_RzIV!3$qMq++{iyr8Th+msVi*eA69cY1K|TmaXNA-rCXT%k z%$21aDiQY_-+BI`52BI$rv}FI)tg7-CaaD7_O`l9ngVYH9#Xu44ly2flHy-xuzEyCWC^6c-^K*QrZW zNG1PL`B#xfh_CD57q**Q+=Ty9EEolHUwT`)Z`SWJPvsxa-f8_iHO;AQOj^^?v$Pd6 zy~3pjahT&?UwB@2zW1)s8+UfK$SFAL~tHHx3whuvPyW4mh3w z`_Q5~nHOsoDT0sx@+N~J<-Y&TvqV4MCkgXgo^ntecjdoSopR%@?wkEfAuHDOIVHQe z|K0}d$IAWT3jC{=QJCD$*L3=%k#f)T)tT7R=nTHqn)i5$Q)sm)53ZV1w&{swK_X#n zpD3;2Eb$r)$CDg__L8tv=0-5U5hB))B~SI2(6`QM95phAkktAVs0hU305vOGT{|^t zH`?>)3!24y5TBnjRfAJG|J9jjj_JYwB?gujfD3QwPf@~K(A2Z4KynC|m! zMt!}`yx4=~u?@-#ab5-T?In;dGAUlGajcN(yFF%ypy(av6(B6-=d(A}}k7wcgUJ%c_TA&p~<@ZA~EU-mvx5S_ykM?O8{R|mH|RE75BR5QQ#CTpy{;f{(N zFpFjUOJ}EEwov(%eX6wm&~H5dD|PO&*VQvG&6Br6eo1I>i7L)sk`T?{8}`lQfCB2R z@nDF(51Rl?^;uv9K%Wz-qpmyIoZjoO+tGhY)P>lU7U1Rpv;b{^)mu_I7=1e%POI7M zneWYe`!E(sG!D4Pm@9XD2!jhItDw15w=Vl)ioN}tjFK(3~fxy=!h!`6@!cQuCP6#aH;{{dyV2@O1#ZX{Zl4pLmD z7*{Ip)`V*gV-QVaE+>|4R`><5Z1*;n%pfkb3AiZ1s39)5f5khONJ{XZ5dEj{AwE^i zj6G1{WVlyMNlC3!_Nyk^Z0DjKo$ha)xbx}7UO*rnNj8he_fyO?v!so#$d4^uhxAXf zZNG(a)^5wM7^{-xB|`JITdre*!q^0$>^GMLKm@oauH?5G^;l>0Hp)xxzomAmYTE02 z+c%CPd*0$Be%v~(u%mMywt>EgIlKPOZH{Q%Y5c6=;F0usNLUPph9Xez1H1>s1YOPG zz|s4D9}W5qUuupaM_2#&;|@Jl=mK~Bc0i~OYb643=Gzqz>i%czm6IJ}e-nj~`8ZFe zGWf#c?5=VP0hlqMCIlRJj0p>6ob8O5e(*AYuP~QI>C$d^Yi`)_a|r1LwH(~NZ9P?Y ze?ts^N2upq=Br??YX8%HZ%xopU$9Z$(sjX zPFNIynnhW{IRi^L#G9#+Ew!gHJ%T1dibisJk2~6dM4r$&WR1@Yh3+PZbrp7G519Z>UKXw(mZMT+M-ozzkggshV_x`b zthj%~?f*E&m2-P{17aTUsk&fyuduoa3w}G`Ii-fByRE*XlORaY&Ax;2q^Y}9DeUiq zyMK?>G}eX;GkTjbS%GZr z5T&~;Y#yW|>Ep#W|B^P_r=X1$4uFNPGyw?zjr2lT?F6>ZQaaY;=%~?w4R^35Z=yWu z?(pW}`Hbg{7^L5u3abb48R>Wz-8&e~ld& zG34mkg*Nsz8LkANRe$e1~y0OAYcFkLVXfFw#0X3 z=EB)RkCjS-zhk?;_Eww$ZWCeYf2AIt@_v0+O&5H%+nUcKQQZ*-D#Mj9~nh zx&c!=`cApy)!}O~mTV6{@dbum`*7{`e8wKXQ$qf(L_&%pEl%&9Hz4Ua`%w=5(|{Fe zG=KtAxRHvVR%isJiC+qS)RMDX`xiqORyFg!x&NkABWs5}rYfi3W6r|&5P*I>{#$0n zSspPdl-FAPCWDVqU+`hp5SJ)}U4;QxQ*A|gM$`7-D_HnBBw1Px+%y8Fr*ZBkK&P(5 zLO)g}sM)3#vqJr|zOLiUYMzC)Ip0^+BMHE(YMU_d9|WolPeKCgmx*JYTE6;S>Wa~2 z4x7~9yMFQiL85QHvJtCUi;sWX->6#j?bP;4-B$$B=t*-7v~dwa7d_l5=?cxUgm6Cd zaZr_|B^X5;{k6{%BEZY5G{tgIXaw~PMvhi$_PDnHbyno3v;_gj5-=Qm12)lz+O@kglm5{q;M_RZxMCq-* znMrLfk)rYkS^lo@-6`Sd+^FUeRw9NYH^+}naYE(H+Zh38KI`SA9vUIYM`w7n(({Fc z<0<5oW06nE*}@UB$5AV7a^dI2srSJRcWrClmn7EQdBmJ6?_NrBl@wo_%pe-;K3ph= zN1j@y%^Bw-|7I#-OsQL<1zRV2i1N8h%Jz zJwR0GxN$z5cL7T2`h@=Nn-d!(GsG9!?+6zh=pQ$E{l5S3TiBHQ1&Bvy(*8{} z3j>EOJw+p*2|#VfcRh@u)N+@NMx-@QrQhRg>Tr5cY}aHl3CA*moGLkK0}rdRVR=E^ z{#;gyR7l*RccCrEo*H}%3X|@5YPQ+FM>u|=k#sp-M{J+EGRGl7LH4Z8UIUZqJ%O1S$-a-TXZC__K^ zV}HQ($I)a#fHDGwtEVN4+}*Rszq5|ewZGZEuA5Iw2OpA6%g^thr!`g2lSe?v{V!Zs zZR|Oezz_e)(WIs7nejBn3Q;m~{el(T15QaA3slu+pDiHa->pWfN1C6rVtf%}cuYmO zgKLKj2iNqdxC5nzUkN5bWkY7QyW{~Jm`(yqq=456x~COUo&to>DhmwrE0T1u8eLBX zmGKaO;crc6pm6&VjM@?bZCAXTbba*pRUvkbglVZYwEkF8YfO`T(Y8Hj5McaI z|C{H>yx3qKlRMuy-lc?Sc1!2)CVr8jr{HCfqbxH-_?m>w0h)fl`U3oh{a{=<4u=GG zzB1dSG{rJNtgG}nPU<2q1UPrW{mUkc8)_`L7OAnol7dZB_a(SX@-|yK8Wwm(0F1NEm_aN1wVsURw>% zPcJ-K`1h9E5@B%#7Tn`q0}2)m8v1N;72R}2#~JeoV=z!u6nMx5Hh%7WcQf@>B}s}j zpX2a$CtQcsC3W?=6QyG8m#bS^7MwKolNJR0blaxwZnvS?S;Zd`$Td4sdlY4B=DpVj z;GB--4WcwwL>bZgwia+-FoH)nTd?9m$)`kWfURntsPevI9OkDUq}At_Fhr2*m>J<7 z|K^#22*1UDq{{(|XIx*ulqtAAdQ3OrRygED^IBKe*@i}bZ9_@AZve0qu;T?J2LZ}j zw%cP}y=TD%H^Z>GUW2*063o&E!US9==;FnvZpXFNHRbelmmD_~T)}7{w z&e;xBEsak%$=pypJ3t9=dtnbS!6w40@X`hEdjEiR%*$gfB`8X5t54B?{Y@k+{O-C( zyWn|kD&H^1e6{Z}+mjH!-{_d1n-62-&sj0eAIe`j`?O4m+Khn*F7;(ko`grc}wJs-Gcu{X=-q9>JtlE}duQ+wL-kpryH@ zy?9QcUQwlU%a{$3@vO{6uEg-;vQ6$i3UQK;nO(8qR*T1<;wvvr-5aev6Kzq@WY?yI z8CkJ-_v2o5#Cy<>1tkp7W+umyd18ce*OX=Fs(i}ooB^lb_(Z+B(#0c+peWSQ7vamb z`z_V8WZ6ITb0VsHVCX3uI!$aMYq+2H_VJv|H+xOae}8%g0Ho5T!|3N(fPIQlqqpY} zehINqo%!U~bwZHmWWWQHbG6yOu;gWGMqLHRHz7_bwPG8clq4AvuJY+yO|fZb!!O?8 zu}-gsTJ7>_YGOwb9ZP{7Y~E_-54t0uZ3t;;kkys%#n||9@a5P2V=teS?-R*n9l4LU zX`b4bjK#bVZd&U8y01tpmu%od$DMxAMMv9l&MoL=#mqz@UrVGR_l0_DR1(?*60e1Gde-2*c+IsqkdsUBQplCu zbAh}kVEU~E+wWc#ljwacB1;-}=6;qO#+T9U6+R*7gTqwax52TW8BT?9baXZbe&3!{KI_6)y4?e%W{LkWI2jCl?{Trz8L**uH#O^Q>E0F; zvZVDQPmj+y3P_#pP5&8F;btP7L{R3-N@^b&z}P6C*IselB-bHG;@9&O))tmx7<0R@ zq~8V%kqZ)eZcoE~O~sQ8B8+i&1Ue*r4H|9dY8S&zqWooS;5LT2)V0emG9SEr9t7AM z08Kh_ER&MkZz||l>!~yU@mi`?QQ4AitwkZp6F1DCU$U*G8x922-bf6%3pYrD#i2*< zwpz(G$kV;(&?c|bI?kVkWtK(xu`&B#;UTMoJn+{-FXYMJH&~sfC%3D^A2%%pYB~Fx zYFb@KR!L)a;xpqnrzd^@O_;-5c!|es9)R%NkQ;Y{;h&+Ck8^jTn&jZ}P=M)n>!7A9 zbI=`ms%#Cn4 zcD|SP<@REH*!8~greM*drUAx|97aK~i?nk84xe+fW zZ{VZUt^WcR{^_IyCA?BgZ6gdxVu5?G1|-aEz1&EUsaWP+cJ~=7?fk17Km5W&X3{&= zr6*juZl+Xa>izM!qk7^<2X1*30KepqIdjyV2i+e+zNXSxbK7Tpa}Fm~tK0+5Cmz|g zd=qVePKdNVx^>DVw^plZ?2M6Lxb`!8Ti#RkyDG;^w5l=4mTJ7GuF?>G>j?|lQi82< zNSi&Ar21!4wJGm%haIm3(&qHRaalgKQ+Zo*VUmdvO3d*r$tQiZdevGg?sUI{@hBMB z#c4dG%$ziRt^bWNf~3wy9fsIN_Xz#^hwnqZ)3n%{%nU9mIShVGJbF@_aV%R@{2`Bd zRRV1z;iLf8vnhQhV!*)}h_XFiU+=HG5zruPk-I(^EEW2+SP43iUg88Ktt+fn{a3`C zxH5^rzt^)}NibifBptLnWW>O$q<;o81Ytp^|JHO2c^)R9nQizz@=pua-L?WcDwzsk zqLYg1NS}l0EoS1SEwfU_n>3wtIkq4r(>>1vzP9Z)u* z7!cFZk(y94Ta9;@KGI}VuVTz%OclFRP84+NBUYBAN9)j18h-Dk(N_YxRc+#$@;E!G zk3>;{dx`$+A4-y+OCDz=U?O~&oq10pF2=@SEP`h*hn*uC*BdqRBV;NUWL%7GQwvf+ zy^@Jg8oV=aF&&>FIZfBUhPx!`mVdKBuW_kcOjuX6o{4h~GUS(Oc#=*IhjnUUK6V>q z3|r^NJ1i%lyLPs-RMaW{5i$=F!>FC4M0Pj0<<@G%muXC?eGi&&ai*KS|^#9Ba>V z1r&49PJmi&clkkAhrws5!q)&@Ng2>63rG~VPQPfM6P3_7JQhw!k2;x7`97!rb;o&f zj*N+5e^fk>D^vzYxcBT!!vc`_!+5f!_>XV3z@oz}r2l;7v?ybOOoLg1yQEm1p==et z8!M{V&DaVz@Xg1^2sOzN<|B~4p!Qqom;IvMJuhY^iq(pcg1vcJBD)9j$F|MVwyRM%Pf=l_jD+NyPHL%YE6 z$(-O5y>IX=Oj2(?JA*YBgFzC#Ok z9`8k0Tqim&9(eUu$uOl3X@wSOFmmcm0q`1mIA64Ve_<%3$nNID@10j(FXICMN0-)z_1h!Y(wFt@%rzn&KWkzAN|(aV{DA=J;-G z#?ZdfVo{uhmv0)tmnXPt7NlYVPN%)+Ps(HATs zB#a;EeCAVi=f9W$o`(OvXpJzf;CLh}-04ibR;6BeF3%HSpb7P|@BS;Ns&;?bSOo4F z4DlH!B~h1(AX80$!u6fC-}OET`Dlw`(|?}OMDd~ z>qFr8tnPYIjcmoZtVUn^-ei%&OQA5Tc=Z`Iz9m6b8v)SNDYgGI z&ufpuaQSeQ_2BtH5K)eKXd4pr>O-P(?zf3-LUZVNwLsusL-~7SqM_*WS%%V#M4_TG z{P&M5x)q1sQS4zgx}C=u@Q?t@YU*P&n!}ih@#Hx{2kRN*I*QhP*keYtJ=k?c?y9!B$5bcgrQql3d(MDOE& z$&4)a62X+@f)63w)4wmU=x5`h3F6ai?c0HhJ~iZLYXK!aa#)hyA>2 z|mZaulq=2%a+*w}~-#`f<0;rmBC$8kUReVyk83I8Vz z9h*!SORnHE+X=(t1767g6#NDfz8iGC>whkQKj)G}l@4r;Kv22N_b&h+TX2u|j7#Oj z(K3uiNL1XY*yk@SMq0V^nF^C4tY7F%Xkl1!XVbIhi9k&fR@zT?lM-aSH@RdqE*fzT z0x=nU5YhN`oe2_Me7X&Slwrh-emZTam}o^KV=~utowP0%qBQVdeF^BBD(JrsnqT=g z0Kw~8J^_6p*PaLgV@w0$mjgf4%j*&bCxW;?u04g`wLQC{3<iiFFlUUNQ@-0`3U0PTr^* zMu`6+{ji*^jscj}HzT-Ix^mFBSE+}Zet434IpXr-z;GbHM|<6Z$ud>QLOHm$q>Yj? zi=X^?XVKh5dmh63E6q?c-(MkM>f(9y>kJ)X*W=($$*zh%V_IowxHcM_Px=q^tBS~D z^CNokYN*qIzqTFLw@*J|W1E6Y93dEjFM7bVH;omm!&C=Z%kF zDZ!5^rmEV)HlD6O6Tr*st_e4;^fb1cMxb2+e*K7{dMXd+lY~LT*&%qoG(^LQ;xu2U zlX&3i8OG86!Vntf_USh9iF4*U|J`}Z=mVM)PeAs{D4WZ*4$7P zB%t)P&$2Kr04o8Xy;J`g@KPzWe`1T}m6IZ9MOy`GPfato?=$ik(>JsouPv<{^B1k$GpotiH# zAFc}^jX-(p!24l8(M&7@pUe|Pfm=;J8d^`&7M`y}lC2ikiklLO3&7s(v`TZM_wLvp z)BGvu*V#(5myOg0-#f?hZM~gOm)pbI4r6l2`c;x+BoKN zlf8pTUa5LIE_z>y*IP(5Wwu|3hR`D}LJe2Z{OO%LwF75itx_bm2;*V*L_d!<^U`113iZ?AUR2fo{~@G!O7S z8ry*a+L@ya1s~1tUwKIw=9Y$~W4(^vWXYd@p8Pzd41rg5Et!ZFn)0i|BZzsFQS{Ma z45FpX$A2OpdxJDya+vhWuRX!EMr)~=G60EB#(9=Cm{yUH#1~9tH^>Jf<0R6m#c}G< zi(K*ezx7%l*|KrLE}7Nbi?ghND_o~9`pZ1q-*}Q*Q`{_{6rWZ;i3So3-$FI8e&&NC zWaY{pZS>)b>-mE2`c_1^jB#|!C|63e+q*hQFKyk1RQ#UTkJI!M6}>*G=VmpY(8bq{tn;^1f`?7^Zc-BLmxn4n zI7ms3JW&2@wCq%Iun#b{=0FF4fUU|6)~D`fAdrMDf-%qb7}(_}O-Q%nk`;V~i0&E` znTDr*@a5IOZ9_&vz`~lLmNpX8``JG1kxEJD;}0!4K|3<0TVqBa%r23*zlrBZWH4U0 z5PQ(DoTHN$fb7YEFYgjdU<)3`W~2TCFZR=#A)q&Z+nJ$iP35--s`>pS@B(Z1_+$t{8(iqnGXFSA(Eez$U z(rAcMIv(%#M&j7W?q4q*k#Rn$E zuip+NtT*wwH#{;4u5GD8u}hZ<6@&20Q`j4GxWAW}!MyTY;KIYKaj~9lLj|ADb-{w> zXQV5^!qH%Z=(nxMKm85L9tLs3cFQNel6fR6KmK|2x@yy>gzqqyx%l2?3(eDsLCocG zdslQ2dcLqbO%Nc`$|v^)KCTKql8YQ&?l90WQGtlNjj$*dWc`kau){M=;cMhq|fFjQ_6$TE)+((=L zN}9jU#9gO~MwryIRsj`Atd^e}?`()lD^;B%s>2xr9u$3Ux0maqBQ-M>|74?_%Xg7K z!Rj9hvpde``3walaYgh+!5Q07qw5!{qQ@py4<7ToKiaHbesEVf#mwc)!Ha{sUwaYR zYil{4w$X?jszTm52%aZddax+>6ZVji-I*L2fukc8YS$2F;Fp7qW|#QMx9#UKh&WC@ z@b|j|WKkGzxI%6W_|)$N(vBy^<2S&=M}T&+nZ~}8nxXRO<)lH7nb=UnCA)@o7GYXG zo3mta!~WY5Dh@By(QrLSG!7x6di% zS9=>}2G(da?F-j0X5}QM<)9<2P^&l*D$0iYCMgnRBFhgP;FHiQ{{xc#7njIn&F46G z?iOCDCSZ+j2-Bt2p^J`aBdnQ2?1U{L4m?WeF)8Z<2czjUtR`T$m;{Z_29g z>0R-hEnP?RcHD}C;UCvlJW`!Q#=eH%5m;&(#~y)~Xxx)!XmTP*e;VXL8x+aO(;`p| z^Y7W=lRA)%A&Qg4Ci82P=5l54I9(e#7KD~f&prgcc-_0=Y$*(6kGR#%a+Hj=nMsHH z{nStbI?Mq~mcO0m3g4GMOW%!sg=~(F zHo*;$bSAPDVg*dJd-V~f&<4;QrUGPQ6G10(WzW(3hbT`A_0#Y>R2$q%MZMcYywII% z>aI2%Lsu?S5d6~Z&+thwjJ}cHCua1T#4KIVsE)J)J~nf3t4Di|CU2=n)FGexBvJ*U zcqjy-l@EC24Xf1KX1_uW^(#D5hrp2oIs)xY*_=Xl}7sic0DaxuVQ;Vj(H8jl6{ ztl@;=7&sO8d1Gy79NJS|g5yuZoY}H4{hxfL0oDiPGb?VB&s?rXwe~sbb+Sdvx96Mi zf7XvCdY<~>#8qEs6=adRIh)T#cly&iVqloGZYgq2DE$sBY(0R;w#HyO5m{Xi|j`ryzeJhFvObXi}zQ$^dkUa z8-=*j7t{_XJ~$Hv+WXY=obm2O&HfejylNDi~KEqaO>WLW#z~4D&S_4?L?|I7O zd9bOA>y97h8sWz}k$zJxC8agx00PU z=&q>}m9ckFl0H+8hHU7@QXQTDL?Q5QW~dH6U!?M-P2yvDhHyR=*S$jlFb&0tEg}In&YcQjdt18>ST2pa1*s+G_eQ z$i_(cvP~<#>q^Bp?-6%4Xz=QHw?E&1dQfBsGqE1{N7)PW@SLg91&af=IdJ<2o23%I z=B3MHDwg?zEY+b7?2pWuog5RCD;Ts$p6L=wk|sWaAE$aA+6Z*uB?%5v$opCbw9)s| zLe|cu36WL79#gea+kAOY86xuP@wbA8`P>mQkI<_463)vU;mhz}ev%wYe9GJV8DG zsI*WsdD7gNyjS4W75N&vocg7{z5xOXo$IkwyV2@+8uJ0z_5FJ|yr3t0HolQ8DNX*! z@UtBrYSwpRoJm))>Ui-&I|GfHtg}9}+AglmSHBzP+5p0(>?gKNG`pAQ!o9wA#@CUV?kk=n|xk;NAC7^On%cCA6GUg(8h74Mx zmW0D{fTc@BUs1k3M=8z#svN%Ei)~)D$!SRh)g|_VkdkQiW;lkt?N}oDiND=P-Idjx zkXC>GUNXXJwB{;*6!`ng08u+T37|1I=G#2R0wvra0A!Sc!<9r=?}l{$d_EW{5PB5< zwUrHoXWjP(om^Xc&*V*LNj~HwO;dHpPQq`eu13BY+nHVMI=pjOlsk;VH~8AK#p3E# z1Ayw~&8+%!P<)FVQz)NqdGfTyNTcPU!_)~5lQhDRYkp zC_%1KG3Srg*YlBCiN@6Rz58(IAeQR&A_FooBDOZM83P*b{nB%0neKaT#g$Y7rGmbH zHMCz_Yq+w?u72_rRDz6F4}2GfvaFfx80_zu;fIdvk1$FYLSXCbPQ#V%gzb)_Nq(}y zU3ZOC)Aq>!)bT44i|W`IwFgrG;@_%k*I%D4G6?l|eYRk%UGdM|8h^+cnFz~LymyV5 z5h^5j|4ieG`CvT0^v)hdx>x$4e6v^czfVQlAfgj#Fy_(pxneG?yXsOU8$@^>PX-We zw`wab$am3g+C&Uz4)|>7a*fvwKsEZ&?Ybqt9)qDXf}-cC5E22Loax}F)rj@7O7$(2 z?!By3nfztcBnGSUa1VZ)041(8iYs;m!`C^1Tiyg?|0l^IwgFc*BSY;i+Ru*Uh}%B( zpGlO&;XTgsH^=xdf>7^jmsz*4(_pfM?Wj~cXnBx z$yXh{O^XBq{@qVmy!3{Fe;!W@={=aK2j2UzP5%pMBJj0CeFX*AMz0*|e5> z0wrQ0n97T;j_W9N+s3LX;fTC8`{qy)IZ0K9riL!D!5uE5b9WPVf&!-Q=RVOjTSwBi z;k8~2s=sRnuy~C3mJ|d`StNjPSpD|gN1T; zzn|xTg~NK#smNy7NR@gBtcTMt3~%0kdbzV9%NPq6P)tbZzz0`C{C#mdv%>;Ao>|XF z9T!uW%f{;V^q70#wi`Y&^GyCG4UkW@$`FG>2r$|+R>cng%Ay@aip@1NWmZ1+gcN$V zGh=iq+^Iy7a|>y}@#KfqSDsgM>yr($WF&@~n1*KGhMF{vmm|Fakd5mo!~zM$Gew zn{T}s^aD5dq_;fJQ%))f`$5s3r1`G7tNu9Cv_YzL=G)n86=SkQN(esj_>Q{^f$Q0l zj$sILcM@Rv$kp*t$s4ktEp{iiV&b;eWR+O7^3?$9y^dc_N(V^%wbpl*ZmZW}s~61t zC)3`KlBcpmunVa)|J8NwWr3e`izfB^AQkzeKpWXQY){k@)2p5_!R@8GcPFT#3p_sS zU2P7<-pWbsgYLk%M&LUO#ycYKV59bKe8nkHyyH-9+I^Gtsekp|x9$Vh6x$K2JW4MH z?B97keW}HJL>CBgaJvcIuqZwH&v0t{zp6rmOjcJdt=5#U0gz%O;r5BPbli`~bn-B~x)jPcuX;Qa4p=fVKCY!AcXB)_9R@svcMQ3a+3Qf#anpAW6c zy`hp8b*Np5O#tA*6rhnIK0?8wYULw21)NewAS@DQyw=aryfmQb0zC~6F(8jHAmH%yD&YeYF3g2R$mBpYO8RPkdMs{f+{XJILUCPEi(lE9^uM}al?6z}`_pj_)mbUDDEc^i26 z^#|94ClCxrF#PNB6U=hBSP%DQzhg!rc^sg`bNY4$x@IgCJ_Sk>1Ce0sp47kZzXIY9 z|7!cT`@e6#M>bl%n(^E0X@sPdj`Wk)&2m9A|eG&Uv*S&;NUT2*W&tD|}H=7Wpy5$Op4C z;lrxxFPj050yU58a@~5snJrO;gF|XTcxBFwrycmk?zoNvu6Cu}Gr@DrqBwXLlharC zl1vBO)RIe=mBUAV+QtI_*stF9v3zwjExdyrp!b|Em z^Qi{xZ+SxKi*%CxJR`=belBN2@N*NRaj@ydsNK{UIK2gkP!gwG=z;sfD^oQzTA#La zO5vBp_e3}q=cE4-Kbqa{n-PV-zF=n@csZ2&dJ< zfPr0T)65}Y8PR7?#2yb`jv;P)6TsvSoOqenNdzgKy#1i7h!>dojt|V;PIc}Z;55sXdP=l9(^p|759HpLCBthH#}Aa`oZ`9GAO=*n{lX#bRAm^gh`ld{8~~gycM6iYEUB7zn&$9I}i%`)4W;V0V(Jht>^f zV!k8yO{{Cv1jw`yBk8d85UqHM5mK#FpJ3fnn2WQtrDy9`CEQO68Kxw??(_}4`m&iQ zn>(Hh5S=F6y#FT24V9j|Trq(4`!-UVkr>`Hu!LD=3vz0ks3PQsHSoStgeYXiK=vGzZpKaR8a6rQN!4etGo|kBLTOdJzt8YADqF*68=L zY+4i#i9+9$xs`EF*s$V5G6!#;J-EZDvfDh2F4xfkUa^ny{IpzpCqRC?vPY5~C+HEo zw2A<6CfR4qiAr<&J`>#S`=sNLi@g%rg=i@z|;p+JN}{J+d~3!bwR|1_p_WZ*zFg8JdY2H&$(=>qm|h~`0d88 zWfyZh%%J_j4Dq6hl=rxTCAnU4frH$_ytGsCU*D1mn`Z+sw9>F*#!002LkOF@J|RgG z&VYXmonzYG{uD{CvS4 z2zvgHZG^kGrEZme_YMX^>Jp5Ekly?SG)UqM2$JF;2kQZuO3HlZJBAWt5XB?QAtk6p z;PZBUYmLv}O4#vA`t8Ta9W!j|LYfuO*R{kX~Gkj&k=x{OR zgyuxc7eyW4QKwM~Y;XaJ4k9|Rj;;=@E%@FF)P+@9Wx#6|HcbPs9Er>v%et4vJrx)Y z3O+mlAgaHtAg>Nf|0Z2za?+B6+hfpony5lDAE$d(o?L1}N0%V|tJR#e1J<;%&1W}W z4sdoDCj#!=VGrjHHMfK~!Aastb2s_g)o|qjTPwpxh%bS!912Ze_R1@tsT?0hUX>l= z0g~f3qq>IyyT|fEsc3UU%%e9f@6tYuSbu!PUgly3^o}%#>ptxjwWfP1pM1AwR0`_Q z%ul*q5UsD$nLPe0@(4Nfp56?GD!KCH8Cq7Ut-*bUr}KB^_liJCg=aP&2w@$IA|4wz z09gyWU?8N!5TMlMU;(rK)zk;6jObF@{cH>4aH;$*7AvDf@#!;Um?R*(8&!b z5TAj!VC4&7_>dCm<;$(+T{TeoPk0>2{Bi?uVfbTXN!yb(S#~8f2){1p713Ty*{jc_ zRf2HseOZT8+!fPXa&@%N3i994vCh!EtP(;}!4)kKE%-$Ir&(6wqjxugE|6~v?;rNi z^h=ZRn^;Nzm0U~}M7eO*=BYA-tWFv8ZnP1qe?Ete!mwVw)ZOGc|2qNyR1{vBFqdt9 zt8xG7xKiWPD||`~g42zB1A?)^}Kb zHZN&k&5<=QopZ~J#!ma`OZ1?J|EfUB-SQyjl4>N4fd(x7L!Tv?k{Xl|Zi zj!2NPdK#Lr$aN7wpAeRyx5Er=tJ$^W!M|(Z|tTlIzdC>lf3BIlUt5Nq<^Tm~-|%FF_W;5qeHfl!yrS z9V6$z>|&Do^kuvZw?FH)k}b0zXk(QJeS<=)fX#LP&{-( zR1mXZ<8?!2fYl{@0Ezi8RS2-g=bTa3d*Q&5p}B_RA`OEM>K{D%u@0Na==gQGyV{eE z-kFU(OR^Kv7pt2ORs?Lq@qv7IXi2vKqKf33 zR~4e`{tcY0mG_o&UQI&*yPiUi5dRcXr0|&)XZQi&;?5gVlgjsGONiCF!slVgk!>pJ ztZJM|yhmK~(d5AOK36q1cB9m~^hW}b?T;y(@{Wy2Pli96zt0DS-1xLeo%g87+w+(p z>nEs|=n}0MPb;Eh_?gkGvf)rv3^I(x!*_Q~yK^$LoJi7p0jnH_?F3AMe?u6qKfACz zxBXJe>2EQe*q$tu`?_BD9)1(HV@WigmKpH)8qa8vN?apP0c^wh78>C_RjVEiq^C_M ziLc~F=qyRnDrNWFk00VNCHidqC;&lO-YJo^ilZH&&-2-nnG7s%+mw0h_s~!K*O8R3 zdXceMp|+2$u<*a4dybOy{rsWgc1HcLhxIs2qQ3&MoFc#~p7=ka}> zSXC^xPkO?8?qUqhJM_C!S!&(m8G3Jwc`Rc0Lv(=16$e0NUMq zg&0AcMq)4ca){?MH15c7r++038WzbRm^di@BInT7Q-|RVTyl#F$ zN#cH-@iNC$)^ouQ!q6}$)J3U?09q+e;jv%7R-)S-Tg~Fv-s)g$Za{wkkBTK+0U;hs zJXGJte6PM&iTX!8$oZr`sB{db{2cefDoJ1AZ*D#m-oYZdmG{q?_rL4IK4v0^_kBK= z-j#xDpZt3e8`$7C&CK}3T!m8lU>~eN6kQ*41SgS%V5hKZw=j)Y0#FP)dY2(Th|uUH z*sKv>v8vZVEx?Sto1+TzzFaFnv5g#17WrL9fQ9+6OXt`vpdPYF5qWs`#godJitEns zqdqueW_c6LUNyQ!6e)bV(zIh${I@c-qB98Qqq!2VR${EvJCyR!=6RF<@y{hl_Qyl2 zRdh>gWyr&rj-TmBVa~l0g-EWuk#WqPgx0ure2V|klh;4=KQV%yBZ<&=`Hd`3vbOwb zM`EK7C~{MW#PqMwf&TJ@9#J1^mA=^L?)=LLp?z4} zz^fRs$dnB19)LxSBwkz09b)2&L~W|Jf5_!{@4+(syl>;jtxMRO)@!;>_C* zf|Li*srkh>E${4jGP6<;xw<_rokHRO<7G2pVd?P#keF5p9sPK4xZ#+U7-rMwnLkG= zQp}}lGrZ!*cZq-z186@_t{%;RgXMksAD(?aQ)6-CqZ=`L_M!Oh1Io|y@hP=8=Z;nE6WMYM!8hA-?f{1$b8cd%+$!rUIY(C?#tyd?@}8%cbPu%fuV zHmJ?qK(RGCn^1^sz0*lppm$UUzNT_2bypgib!{*TbgoE-8kMliGrE|*OR;L`nD~#8B-YU(wWNs_(+5Un**Ep zff5*To$NlVS%x59R8Luue(S12jXGt_L*fDL?dgaseG8>+IdO-~L@F|zkWY>U^Dh1x z0rk7Qi)kd!8?2c~1Fy)kWslqI^)fQSdt)j@1z`Z2M)M41OCzTRx}ZKg!ot(XDZH5;arI>LD3nB^1q++cv|OT~`i z8ZoAX%GydeBvt!>ee56IT-VRx%(otrPQUJ(00XuH?IE}$Y?tClldCSub+=SuqEB+D zkt!~vrgb*u#_nbS1i$a3D{OkQhQ9C*_ovEATl&}ISmP<2KAlQ_-Grxw;okhm`w5qK z$_!LEkAFQ2I`dNsF(z*}iya2}T2Gyy!JHg6a?(VNYQ-;G6|4Wf_7F}vyw!Qmqj_bZ z4>QdG;vN z=^|&NU-I7b*sajdJc@(!q=!6FXSTadlX49Q)nc-2%~l9^p=1bvHRosomH4qXkdb@k zwK%z;z?zgB&4?-P8#|sLzsT z%{Y;tU%0KwHCb3~$ktLakPPO$8i3d~dkjW@-}c&{roA_Xy008E#BLYgH~|6E5d|T5 z1-=~Mav%F2rjId+NmKW#&3}4tNTnvK&2WU!&Nh^Zcj&P(k)yJceJO~@ zoS%KO6uItbmOcCzhD!{lYhWV4@#fZO*oy7o-8*q#kz1lxvw;y#OF@^7UpH9N5Gr9D zYX;BMkr2>|+2vZuzwSUhgC&IIbE^sZG9UEj@$y~S&z<4_c`&!!@pbI=$YmMMAVTzP z!hhUsnCf~c_FROUC;_J{ehp==1oXfm^pPqb?6%TBxJWN{YB}-$xNgnc47!yy?)4~9 zW6^M%8DbP(-}y*_8Fcpo(^}Ga9~-mB)pA8)~?JOV4olI{h0(@B+Q$xC5d~le-8b& zY#`>{j%RNi=Y+3Q8JeK8lqc~AWDpn6ABE0bo)xBW^l5+iByDp*_AG z{a+ch7yxnh2-*Dy0ou!wH}(i)Tdy_C+LlrjNC}H6oR&W~t|{>)!iqZ@y6F z{Z9uEMXfon-58Px??G!D5oo{xn_qE58U8r<{UL@3iFJ7md=6aaM45`lyZE<6eG8P0 zM+Mung>esC$yKLmsfO4+x7~jV3cjMTb@*iwBQd_KiT~bVMD7G_Fp-i#3Ag3VvwvgJ zeDa^SDwA}O33bLZdDOqk{PT2>}^ZuiwC z;D=h{g{AxG60UoTEx_=y8X}RY`67bD=rAHwZ~`vs`Cl9+)W^D#c=^|MK^l0IzPS41 z>RH|V-K#!>g^OjYfWDh6G?-KFP~=n8*#jfad4nU}&x-_VP)ifu|NZ2NXLv%`xe)Rm zaN2*^Is&#*_a^vh`05^UOnY*g&NH5O**!7oW}4H9xfyUZnHgZ~0K+~v_b!(td%2#s zA|rICEg_#ru(Op_*H7m-p+vt=$fN zl0Qxne}1|j#4)x@(su-^ZXsUZ&0`U>#&wsB4sdxCkP>pfg9q8I)PzY^z-%`J?NJ5B#wAUF*E2Sh8%o4VuZNg zhn+rNdZLtMTj=$|uiVd*tJpT=#8*~vliD`09q3=`vI~SPiE2whwhMl##D7H+MK?>c z9qx91xPZQD#cTSpLwZk5pbp&Wau1%yZ&}IM+_TuhJ}t1BDZ>aUr;y5D*_dLM_>Nhu zW{83uG!i$muzqsesr7=fVVV|SlyYf&jCFxqiSH+5-I=A@KglOh93TnIQ06WWwkHLi z`0(;_E#OI;>y-BS` zRm|I);;aH=hTh%rn;-wey*2XFe+YF-UJX&cX5d(H!3o{=vw*t1xcbYe_}x`48RXm( z2qznisI9=Rd#nlMm0S%6sVZoNE5d{J7WmoU2tT+%aICh?!;F{08 zghazF>D0pG24#JQ)Ma6K)cNP>Qr8}e3zM4XO&dkAwC6^+Tqz0GK((Yks9PR52Y)ee zaK?{9Fh z1OzF{6Z6zi=_B4F_4tM&(p6ufcX59*0K|pS-EFRos`0#BxB7L5LxZ5_UPTdAX^u+4 zk$9hZ+`{9j{Wzi@62z>L9lE~Nu3YmmKinE@mFXWlux76q1Ml#$2J zy~IT%@vm!(DmvUe<1z?0uks9UEt46=ExfsnMMi5nUL=8;h@pbhLh_fZRqa!_-VAAd zZ4kcH@p+K$r|y5suWeCLiF|VN$gz@cGdn9NDaOHVBs;=*wIW}drsdk;6KY3lo`2{AI5+U$BDWJUFm)aqj6;(x(Lbi7|Yf6yphgBoS@~ z@&3jP+jYo3-s7Jh6Ll86nw__T=~6!L{6`!G;#on#%J<>gaa>pc!8nirBEEOvD83b2DkFGe}n&vL_Vt7~BYWb7J?oTY5-bIK) zp$Wj)JV^Tv$30cGG-B}zio@Xc`g9iODv@tv5F<*T9f*EXNsILj(&5p#`)vj&LmKE@ zJYK=(vAM@6xoIfSeNoq*%i(xKmjsrk_OgAueO~k`*L~Z7e zG3nQs*XWS(`E4m7!$u$_u$@tYTjlC(IjL@S==w_alVmiyuJ(^(Bk{5D*_u!pd?>(} z^uz1f=n5YEtRF!919q7GvVTZ946bY&zn`pou#&sWCoFn+UqEnf?{`r&uIVIm^~=t0jOnZog6W`^$>?)m1L z2WWq_QHkKRuh>q}4<3bzfY;F?HpDLG%OYwa7>9-nN+Ul$mb z)}d>ObXR{(Il?cG)(n0iFAyZ)9h^xvS4GnJ9BiMuw#9}|PnZ4``H#`sEItn+NY_H$ zMv-g$J)?uqt%56~B=5pwGp^d|uO2)V^?gePPWIHo$*p{ z6+>TaHo3+CrpMqvE_U%n%+Vyhm-mR_ATK2a?1MwQ%*mg=@YteVRT%l&W=yGK4z;hMYLiI-d7jH45`uo~Q7q7}y zfK7gF5dWbfX3pw)gOG;zXTO37mt-de`NkO^)!O{6<{4L)>i%1|53+~T9A(i`akJ^c zVFDALp43U8v>D_o9SpxwQi_`DP?%B&Ku-1){GRrlX=HAikQD)Me2ovR&?D%ca(EBy zc=&6#_LtuIsY!%%sA6fY@p~ziWhoQ=OCt;>AmG}gWuKyRHw+T%Zbbhx{2bgE2x;5! zB)Z951iOh|T-)vNQ3|j7e*I<$-p-u(XT(}{B8#*cX%1cNXeg+HS=?>T`tI0~hTw>N zhzHIt z-wJuuWFu!DV+jd3l5|wjKaQ|98RQ;JOz;H4ncj#z+^U` zrh{^b3RJ;17r6k%*gQr2UScJ8CD{Z1z(^5DtkdW}FR`S0=iBIWdp-)hfq8OYqaLfU z1j)d>Q8r|9uSww}e2xa&1zfFBm|-k`-&=jWhFe5At#mxI%{ zxjnzZQw#Kz8CyxCor{W>(GN?%*p)0Xv_PMTs$O2ZtL9|Ug4sOdsva*IZz%yyz6G$* z;-;YwJo=@9yjDSv?qfC`PdR~rF{7Wd);QPDwHYZ!7!Y7Gm~U! zPTv^s34I*{I?#&xv?sFNk?XNy@n%dg#LZ~za)Xn18G{%qTRd_Op)?D{3rivId@I6w zWO>o~SO{H*=eR5;{Z(3$xo3UK!SZcP9P99=JicQ3&^^Dw^?L%;Fj+G>Xe>|_dx)<~~ZxS{*H1P97@Za9mlfgC*wjU)~yV?`)M#>TrI1Q(tWCw*OwNV6^i5qdA5vX?j-LrqYfo7yX$8s?i zB&WcgzHzMi`pM*atDU{M*6tg4=^GUi0(f9>GJ;sxPN-fqYe^WAM3x@MzT=A*ViVp~YzR!-_9svJmMlBU;YuI& zB7T*I{Ix8mee5wL*+JO8dUtdMBbwX!t(~x2fO~qFx(8f*9Neeg4#bHB=YUKSmdzEziS6~iVSC^u(*farDs5R(tY^Xw6_y%; z^E>>!^z6x7;=2R?S(xHg#>*bjZ>y12AMNW>=vUWb> z{bfD^cEU>vj`kl$t;6MidWc4%E?U$wc+7wgbwC7g>^gFH1o2o@d(9PE>al6T6J;pAt)TKLm zG5w}$NZ@v)%JyIY?_6iiObOg2t$}0#g|R3~p0~x^h4LjU-918XT5Vz;XmRa@&Ycu3 z)(0M;zK)$F*|@oUcs1eSgQp#Fq&9Ykc^C_x)1XTA82F*U+S-Oo?Gl)RDsMpc70trd zg3{VgqdG=0Xlem!%O1q5_Fj|y<8stHbqkYdB(dUj%{tB8qLLJj^v^mPDp^~H?Yw_~ zkM}I-*RTA&g+nbnt+uww4yo;%)&wz0L)F6@1q$e>4xDKg-+Bjx9RRI7H`SOGIGhxG zD$V_3JanT!yi%WTyM-NfD8m|uru{+MME}-aT@wny`_(~~bd+yN1DR4@833DS?Yqm-|<5+gF7u)C>4f?f}&Xc{@vbRpcB?YG2!*^m1M)UieMh zw~N)&APr53HF6MxBukt?E$KQC zB6A}^=jseIY#R|bC#fB9q)U-tfj;U+X^&&GiiY3hT${ym`!k$>pSFA(8+*`kFHK2q zAzFTtdV4^C+7<0JROnyM>u0C_Dqx*`=y-KKDM-PGzwiTFX!XdJu=tEBfkT!=(Tl@2 zz!_e0q8m8?nYo!t_k9D{N*svv7bn9Y-9Y^K|9x=S6m#G$rc(wM0aXw+(%A(J6C`6S z+jY@&Q3v8v$9>(}aL&d)Mz+jc8?^qi8FJ|+3TS_^d-=vx zKFR8FKAp!#ex_PL&W?_3Fw~_S;9jSiqaVR=65uVF2ImC3+dre!&uGe7NGn>-_jI%g zj1)1_#*OVA*!_CK(Ido zaR)cL>XJ5VK%w3MpW!cuVY9{^!l)JzJDwr6Wt#I@(nF-1rw-P0a_b2_`=<8rYuS%R zn@fUwb*pJhgylPNKPBuoI=lT3=wNYD@S8PXU>Ng(7z5dny=~6v-k$-tPIftYNyJ>U z?xgCCsQddaz=^zurlg+=_-(qqp4(*B$J19*IALzYuZaQ`@11i_r(kQ$$XLPN?V5ul ztIh)9K-#Qb2YiJJQQ=e?GR;ixB86K%-GlKjt=0`kRqn(XMeM=VLhc}^&#Nrh!uS!Z z%=x8p;9w~NqLaz$`v-5wrJWwMoZfd%!M#ExN&m;a5sYxy|6BkR&5lBpR{mTh@@O&V_ar;XKeAZ*~?F4PEGzjal z(F_R1QT?90Le7%LUCR^%S*B;lk?&Xf}{r(5{mwO-Y zdtT=}pA~+SSKH!J@e;dPI{T-7&!;Mo) zhWCtZ*wr{k8#RuE|LSgxnf`TL;vhKSL}Fe|-fQT_#Hv^@r}wor1OAm;t{17?V|QkK!+JqCehFni7@_sOh_S3HiwgNHRV6>J%EwIQdXB>rIBo^_yCT zUx(?^>NTtUQtkCi*6#=vlTx4KDH0{p%lDMb9ehT3K$6PS-39q>{<>NR zm;Q?W6vAX|ck2|BQDgYMp<*klK(QoAYGrbq4=m$~a^5f-DqP;d0LZwv)>vdBEqUwF z?B35U0^_!80O1I<#q$a!MkU*&>y`J=Xe70qdF45 zLGzB#Blk3N57~M-L{F*;N60obdO(5`~06DL?qHL$^kx= zZ&>@B(*8Qimsl>B)(;P+#*q84%;u=Ek}`aI!aucI3mFLhzspI#YoT0@i0}~-nO3_E zDiu&ZT^j5Nw_7~R0Uc8X{;+!2{NSTvIC|ETwaxem?A9u;`||VXmc*7E#)F&*ATbHv zj?(kR-LL>|!!}D=?QFPEMFY&xYl<>o-kl9bfhoN-f55_9j3*M>KMa%&U+A6Q==?T8*J;%dbIRf-;pYA&M@X;-D*1i z7wouNogBnKFJa&IvY1vA|Np5K0%Y}@FW<8GM&%{p(haA776W?f?_Mv${1}+&Q zwqiY{_>6{XZd(sSnX*69BnIb?zu+cD?|-WnbeUiUiP=Cb7RpQ7%e7+5?s6eMIPGjU zMc(O&B1N##BW-b~)1~Ec+1X2sfFAAk)10mHJw|})SYZD6SK$eyt{$9OJ5RosaMzLJ z@qN0pgrW5!b4zH;U{o#0Oxkph2JD)ao%=C$+BD)s}q-aJI zRv_?_7i8^a!G8}&9D*%hrhKzbbt~5$gZ}tty!?XPp?@Ohg+sdgud6Z$evIBSgEkXT zFr1qTb2_M+kCX*=cE4qSxQO0Am%3QRI=FZmSq1WSmxnWwXg9UZ0pewPh_EQq!vT$B zr>S6+p;SF961n^rFJk%>Kj-21{K4c)iIG$o^~lR*fyyIkfmj4G*VJ3y?UlA;T)-*a zp=(PXBLDCBos+S9)o-U49|Q;`3cK>Etz7xJ!nSU!y1itzR) zcpaG+%B%9lU;Vz;WQ^FyHr(GW*FsyJg463D9G~_TC+so+tAqkWkS-!KHj40C#{`l* z@5g&wi85gFTWcxhtDn3UdjRJ}c5X`dE&Yc1j-vS8=yex>-1SUo&?YGzuD55o#H zqu;vsdRpMw`G`-_89A+FfdAZcJ#8dhXy?z`q?WOEW2f^zGR>T^p?i$2tA|TIzp;O|ZwINSoEoHpO z^E$(+rz@ycjUiyXPQaOd?C_wNPj;M@oP$EzWCn~|6`|sxu74>Hp}A~W7KefshCT8b zZY3YJ-}z8ieFhH&N5sk1=sqV?ZB@rFo&V9j>vNdAyGs^Q74Y-L^v3&7USa)(Vqo1c z*5zUw$Za=yStsg^)izn$fK4x%YT71W=E>mxKY;sf4vwrkY(SY|Fjp_e{IVOMcoOc4 zBYBhHpj_^?LjFoa*>utBiIsMyQ@V}ACt~Wz&p*Z=u2;$4=%K9uhU=K}T6fqD3qnt6 z_Ex4S8z@F5T&vv?+}y$Pn2+97bMc2P!)8rU9w8Cxm-=O^ca2HiO^SPZ^kHQ^N3RZ3 zn+W1i7W+E(TVr>>r?uQoQ+&+)4>A`&%0+8##oi0TZ_aEC^L|Y{j6LF*@&GQ_?5jab zrX%chQIWK&3O!ckoBz6*12;xW2*!MMe)utN14?lyz_flV^mn2PeyuvTZ{Pz~mkkIT zr1h;iH3P;wql4n|Ul-NJdh5LF(CquRW$szN&1zH7&!q73bRHo4>4p z_O*+feaIKIZv$l?2Gf&nBNkyB^&~l@1^Q3dG@yj|SgBE~sQi*olYapT+1;qP(E>bwc?=sSAhQrrN8%ey; zNyxa1bNH2;zzrQCM0=>y?ZDv?KUsMKm%@$IezQbo_@!-LrzN8t3G=a3T@0a zB$-^g`m+gnEBCoI_3mL7Ge;chmf}$BJqKzRDc}&e3`-1tvp#zpbex7`E>-kQ&?V5D zkWlr)w}l|sG0r8O`?1v#OT6>NiuRwlNoE}v9m?EtsD539S1<-JyAHOvGW(MOqtivR zUB4Q;sFYMLIFAKT=UC1#c(OsEMdN4}N(^Zq&Z8jZFUuikG9>Ico@N`*let@10Tl(Y zbC$~O7v0(M5vm4Z+oCkt{#_J(M)qFM`u(zL!U213*Zz$$hVRCbb0cVg#W#mI6)wKqz$W>3pn>%45liDw^ETFqD7 z546xl)PqV8>K3nyXIzRANr|LDRv#!*t^i_!J?iea6g7O!@%edv&-;)sX=PAuebbj` zqEpWYQty;ciJrz*|Kr#seFjl)C~TS#4Ih^8k$!_A#CeVY@@!>jZ)W&*(%Tsr zj}x5JkSy%X3G|Zv3HdEXj6+p>{_qyd{MmjZ&}@cJp*ncyy`D~b>q7W5c~WvGCw9fM zNaFDRu#5~pGjbzF*2{1>A|n}^zn6s)%u+y$fIS8t{yUziuPEmB=+Wsbg3aB z7EG(0D^^&jBrb;}6|ftWg^pzVYVDc%nzm8BlQE}zQ|mCG>KU!47Otu}X*KH-1R`I= z)4z;tRejDuKHRN1*B1fL1VwgZ1>nmmpSO?Uj~`49|M#bIj)$#W9C*c>`Gehk?07k3 z(78ie-MDA#y(o2*M|;+BX}7$By<(i*_Xa##+seuG+HG=eH~@&fcYSN5-FIlu17Y*E z2_$t8*(BR_X4rhuvp+MTs9+YP{dyvo@iNGa-Mj0JtCoB-U%~-nIqt-xB?*}=> z!Q#P-xyS<}D9beLe4L>Zi=$P4<WAFo; z1Ik5R)Fjxf^$CpT&ueiU_YIUm`pf}vDZx(8A?rVxK4=Z%cKEL`0Jb!>PqtJYjIaDU zKhpWjZNCpjXWg}=86)5t8vLDqA>N$7%Sv93V{7^s47ba;MVFoI!dtYzOY4lLLHraP z{Y=_C2O5OG>}6~fQ);n(y!*!8gOq}HM&!ixtpb$Ui+17W2$zX+P@)YbqD7#Z7Uli@ zrBaXv_3QPT8-_iLxvgY&SSEYQfAa%5S=n{6$~%?4+)tzrzwZw zT9oli5B}_tx8nw}EAYME$%7l6^~*guhP7_*+|&J@9zd?Oovw*1$7qxG=RtGV6y%}b6qBb!V$-MA|P^@|a`8a$7bdCBCyi!vY_bmgYLMRl- zC%-38_HuR~B;;GTrED8rcYHy6*lTVa5=s}rBqW=k4$G%54}G`g`D$(!UGVeLts>`b zX&YhX&u!-8X@r_$1o}hKG^WKrW+{s6UTu_zk{_)}+9&ZZBNJcpnF>HJ+NF+zPVTLe zC`gtFHJvxE2sR`!ej2t$xyiSg@JRH|BE{jX_t8Q(xkFmFyo|;i9QMH#1m1AM)~i*d zTIk_OMO#hM`sjLjqTltyON}R#ZZvArA>`cua+RDPrn%e+5=P(<;Ah-3Vz4Lp4N&LH zxFthC3Pd#R>3@5}O64(uVZdIEBcGWk?Am*;&Z*F>usHRkvBd0*jQpX1?*)E^vjYY= zYkft|Zv{4_FmNj5&HkCEYsu$5J_r{A>k~PO_(1dJ=7$%DC%FOgM1$sU>8Zo<+Fu~p z*Q=UeemyYo&W}*W8z@1xM?C8KxauaW<-h`Pe60YT8g1atirF9wY4CVa97`{%{wv=; z+1u@n&6OWdOYmOgoto`9nd0RuKd&>1RD4LX^hNVT`OKcfM`ZyXMh-4fLu=X}QIxi>8fhws)z>zwT2V&}Dp=ov zjwy#+!j2DK(OvKeb9YW=MOyD` zHn>&8`!8^(u#|n@{FCd6DQuAQf@-&t->L#BaUzQUxV@5`cr*+w1yMhf)*=x zoV}dHfw3C!V@7Bp$F7vZWsJ)HjZfH!C*S(Kb*aS}>Lp!YXOK!kJ0i_y`faDq(0{xD z2nKPgCy!f>tS;~fHvM>m#5OGT3{UYbx{Fk>IQ7+)$Du0qsu}JQUG(tfXy{piOu5-Z zkz?7d-zLm-Kx4tYk?-DXIZ15C5PGD`+vJw90ZrWZxLXgDeIEVWy`@oi_L45W?ta$< zBh=UUHB$jU0?W}v{okg+(3ZlKg*x%X zHC`?fE9u5v?B)a`JCmh5_IysX;t>_gig{wKP81wYO9{SBx$nUv9T}2xaDa9k!ka?4 z&DbUi4gv@;bRiJWVL>8jdxUYU;8Pfn1~cVN`R_?Xi*sJGfqsoCbiK(uHypUK1>z!A zzcac|az+3kG3G|YIh~iHUwuMQs#il7Q@XDR(`(c~9Ou#QwU7A)c>#D{mj$BI^UsQB z7xL;e-g|u2fw^<$3=5!k}S?Xg7AhdpF^JUM^F zOR=@eQ?P3G^fD@hAATp$c>}y|;(kFo=|N_TZQM!K*wUvt|5;ABU))UOa{#8T8=p!D_~U8%ME>V2Irm^m$HnxvYMmNC$e1*MOmbXBYvJt*bW`1 zZl%R~Z_QFf%3Y7re)wrsQgiulGeY6N<00;VjPvB;e+PpC|KLiUb1}b z`5L?bC0VV^IW?ALoblV0#V?F57jW(KJ=;y%-;bb&k6> z!0N^Gqu>83e#7WZ`$k6l-^*%8ft&a@uz!c;G_D;OsdUPuZW_44LXBQ__Q(5^QL|z` zWp=nMwRRArI5a*G1PRzqnKU?jGy=MOA_knp2fEImd2qC8-M1(B+qU9O?5FO@g~`q@ ziUEPRl!rvLu5hd`=J|ojU?xJ=48cAEcC|Hf09TKV^Gf?R((Vw{{i)&#Swe1@dF_ z8bF7y|FPH!Ep$bKrghtD#m02`dBkvBzdsx(W*XooPL!RJ!_^jDZTs&a*I7Gb9M)hs z+C!(PgGdydXSb=V;dd#1YTSeYb~XavtesuF`G()j_UAli_Q-qbh5glUxc|&{6hQ3r ziu39m5)Z6t@7`?stYxs<7WY~pqtLi#@IPZcv(q0}=kfO9b4hyKeyJRERpi3jWuj3Nkcbl$TzOQTl|+a_wH&*%phVtk^V1ad--#iLN77V8e-0e?YT^! zf-HP+q75i=@h@uR7aS)VE_}KBaxahk+X!O%uYwB^P94otejug)@7Z3Smk0BMn*B6v zpMV354hSh?c~e8_r?@Ejo{6}9f-5|!J>mlv-R*u)`J4n;0UmEd++l+HQ;B>mZ~mNFY%`>JuCWKvbnPFLrOAxRE)+Xt}yt4YA&DG`lK z`7y57u`AO?yx_);#vn&)v1!MO&1;9o=l0aOqYy5ZZ z1?$>YqV;%#ds``o!_hVxyXpE4JEWHC@kz#hhZ=;tt3%0+z@_d?|A=NJD&79wGWo%P z(%wYTgS3r(0p#bZS{*x`8XR_0`thirMoGNqs4H`L`5)xT!q;>7s9dL4xF;iAC0TT1 zfP|s#-gv}OAEIj?N;S^BZe_oQ_h$_6gddG{ndaFJ z{3p4o5Z?DIu-fPK8|mU4dE{&pq&$9x}{~okfwzMlJ+Tjnua5nC<(Ge85&_ z`64SI==z}c8cueu@#f|oSyG^N3$Z*1>-~;V3o7|LKNe0MKe6>STsPbFOuZRb!R}zz zcFz@_i*lB(^B|J6rrT@Ya8V-vq)2Z8opKVK%SxV@4qOB$aU7e~1|>Mrq)Wa2dn^4Y zm8tFab)!=tG_x3jYhEmbe+(G`QT}dF#Ib_W=%M`wM5y2}$XWzOR+r=3xSscSDy1VS zDMimsiD~n%qigf;X+yE6@gt_V4=(f55_A4Rmnnmf8;gu<3acYF1ky+6-Zngk4|cA2 zgyChD{@&=f@4)6atG(O8+w0Nk_yQW>Y0+t2cJu`UT%6RxzSLN`UK+No{D8}$MLe%5Z7xd$z7+H zq_va|EGiLjYcUH9xi5511H5|1&kfa(>s0t#1^eMm5GKyaD+bCw4xax^0m9a%1R|Dx zEd1+sv_CkVrIy+^Txtd5L(1wNn=$)c>tu4w8r|#J3dQK0&F{aK#t1+sat2(mH(;1Q z=zOg*e?=Bf-e6@4YPMFKD-$^Q3b89UL9_R&L9YmcuLzdv53gQJm9)qglViHSw&l#z+UO)(6kwwhneyUv$=c z4&H zwY{VMxu?@_;7*V#@Hh=vZCQaooPCl(v||t{?w>40S2k&S{SArw1YqczbymV#lKXp8 zO;TC^Am-wvjQs0`V5sUl1pWa6(N9_h5cXaCl0X|bH7VOGLpBu|aOXcb^mQZ7+-+O+ zWwZi4gZ&cX_w_olH|F?d*Hb|E#Gy?T0);5%b}ajZwBJS>ncnpO_Q~0L=a0qLSy%}6 zKkc>Y?byWMqTL(ATr`x@r>T2un1M1cX%EEnEFjYmBdkmmS(^Cx>j7!31XiitqVsOB znK0ILnxm(VD?VS(^6KJ7L{&UuPOlF8B2Xc6>l@8>FfMw~Uvb2lCe{AqC!Ooh5t5rw z?6#CBZdJhUx)B7p}ImJCvuH2<%YgQ3N zo3;Os4HJxYYtnS|nqq`9$%vK@+m|f!u`nE@_!nRDk6{iE<4Lln_nH_&dUJLNe^ zL;DS3P(xnN@w+W))Rb{=^V2_Wgn*P`Oc{ynf1NPseSdg(lk&Cq$u16Z{C6B}4U>3=a)uaH0tg_D4~#r!ql5;4_VtN_)sb_o6B0(t)Ip)X7Ov6~Dq6e|Fw zpYm&PP(C)k9UHm7pwz`QsMse}gOYyTPDS!=-)-zNft-h!2S@euiZm86!15SCeRqgi zAkLdX*>8Wb!fFq$uU!IE!FYLRwmBJy)UGoQI=ueX`R!K!#1H?To*UY^Ik_oELCR`bWUXv9zn_v)e@D^=;u0Ms9Y|P7MD&>*TsBrGq4f5OL)4i# za<~Qos`b*53M0X?HI$NQ_)#qByNegESw(?*Z%Redvh~ZU7g0#cDI!|kO^U&R=LX*= zTG+}T_B%aW@NOrL+x2`Bh@`rX5OjKM>X*evOD7%q`z6eZQ`95xMZO+mvc%^?7s2=+ z!->Ust<%q(IyNmoj7YCjk~I&ry+cA|ZVL@7r9>(`^UeL`qbxT7^y2LSD}RQfMNO`c z#C=y1FC}eK%I}%m?JBhm3KObP#m0}uF*F}I1WFWN=XPH!e-FF!W+ep-7Dv!#0PjVC zT><#uJsSup`*_0S$2BCogeM{au9gl!9Zx)o1ml%hpa0lQN{4Ix+Vz0K0`Mz6?3avC z>ly^H6DRA1-NqUA$~IB@9Y~D1zN!^nS|QBkxz*K$P5IuM>yqotF(dxh8LY3k$P~GC zJNQa~_+Jv;ALsBCMv{41_o~bJr1kzKu<+UsY#7$3PuDaIX$ljg1TP?&c8dun`b6f+fPmOfc3*voorAuD8!)ALz z9zmE=$M(#ucTl0&f)2S$r7i%;8K-AK7e{pAhX6C}_7JKR!Q>=*E zI>zmtr1{dOf&z64lKZJ(FOABJ;)6a+3FP~I1>%;DVV~|x*b@YHBXHT8xY8#0=_2|4#`FMq=gy>8??~k+8Sri<=(^<)lp~ z(x7CwP&6=LW~EkW(uA;#Ip)W4GFVCdNL+Q3??o6xP~>Ize#cgUbMRg&d~VEgZ>@8D zV(L#8Bhc`&8jhMSpM1rQNcvVm<^fNn(c$ZFC-Z^v6>d@A48ne63-!K&@ezQI0NjcM zIm4fR4GVL52{XdHDj*+Mi0hq&PoJWMUGxj7HFZVAh2mzd*24onvm)(=CwVs;vtHb! z8(Nivy(f5J`3QNSY_l+kQvB7(G}iQ}XWJw{Rh!dbV;UeCP(eyS67`9(AOJmjvm&>$ zlAFXdqog{#Zg&OlxK}*-bZC9|lgrsqFXM(dbfl$&EaITOcg2A1wRA9|>s;nH7B-A;3h7$0;GOCM$ke znTned0rm$g0EK;N zDLIeIf4j~~dU|lsmuP;r(3G|gn)sT}*`Ie{1`H*kkBYZo{Da0SjiJl}@#nQ4HCTB1 z*ev>vS@?e*4;J6$pUL4-F`U>sXSMh%;F!^83$qK*nu*H!Spn#m2K?M`f4VidAc z964PLdw}u+G{J)IihQ#->zC5Cz&0Sm4}6}{*YPi3uh?S!^rTi>QJdLk4=~-7{QmA} z4usypjbj8c)}WgdJTLz({aR44rW)!b=(}?l55%NpA?+XY-4xE%MgFjYyi~y_UIw_H z5f;U*%QgQZ#-w8p;=|WtO{BNd)`}++rUNwaSKbG&Uq?iAq6rm37QfK3Hf8u1>9F_H zlYwaAtw6VV1n%)D_54O9xasz%W13G#^IPnDh4W)$^XK&(Ev6=yoqx86hIr{(YcPjqnS0dIglTK*jWdpr!eLkr;J&p5gns&Hb zc`F#s{4_L?{o>36d(v#65)*xDXY-LoHT7<3=vBza)TTL!wa1d^=By(Cz%w;b;g1@kCc95U9Rn zzI~K%GFGB(eMqj~a2Qcv3U@wx$6heU2BCF-EJyNxnruGA;cvtJbL!tlfVM=#lN{#) z4NK}~@~oVa?IvH+2w=%!tB7+bc0Ee*R-HnwFCL5!!f)jKj##!_aB*J>ygA}LGXF%f zm=XTk={<~2?$JeLLi3HD@^Wr|%hso?!~gVcGA7=`l1|sItgZ>L3yXP8Nc+#4J6iXJ zsWA!cj3s*FHLRd{5VSdvK@CW8t@5YDi$txkKc5|{c6a>2`X01E~3MgRA3_ws31vt+DENJiEr8BW+} zv%`C)s0`sD&%b}}b6{5l48Ko^Zh%fS(lKeqLBrgy2^mt-T+2y*@(<3}+>2{?xG5DM zl;?E3zf_IlZYqD41VTr(;C)6-CQ6#s=#KRpn;D{z{zg3BuOx4NyF|>LU?^S$VXN>- zdX?KJMwNO6QJuj&m!|{tYVcod>XJWAmk%Qd<1UH3e z3yX0ru`B%}3b)_}wFbrGL}5hZ($ThKeV%>Ausf!PTlF-bto&kBN>u&Fn+@jK8Q`Bi zh>v(+Z<>M%m*Z3Mea=a?vKn_$s@RqKUf<~$?;eKRnQ9HnZ0sFa!>-JBuk4G?m90Ps zmS#h0s9c7=;?ab+m&LOS*PfgHK)>ZZrKfM|tgJ*70C&1t$SWOFxaPeaQZiW4^Ka8M zTEJtc2DL{C(F|^j5%Iss5ZM?>WSS1XfMRl7_RwT)BF8rWuaxl8t_;SO<7o*N-Q3X} zfEytr(d6EQpers`Lna?0+fgJ!GyPDmUu?q7{{@3EzvX(I)H{W9kwO+fW++hAtP7$`Y@-OyKm|JCJij8#Te4JE&w3oa+S1`XXN4^!2|7Wsq?~-;?vr=a7N|`_E-FE zEPE&={pK8g?mQ4v2GXJ{W&?+FOUA$Vj_rBh=H_%mg{v8p6!%D*2z3>!G*rJqni7A8z;wiCOhVZt;3!|9xfM-^RWFyi{)#7W_zr{q67dT1+DxI{BvNk%ok zo@Dd!DU`@dQZ}=Lr0kY3d;f{0EX&*+^g&uWFP%PCZJ1PlQ@G**JQmp`#Wh3Tu>ZwN zsXigqr9eOo7g?vBcP8B|Z22-m{hIlvsc-6xW4$@6{Fs z=eX>H3uwH*eUQjtLAm1cgY83?^BG#+@(*~RibD}UXfAp4(F4PvNukrBruIW22l-~v zd>6Bg56qE?YpbrcT%KPP%7Xz%WWjA;2O_ zzy0!a)Wkby1BaVnMdzVNz(TRWN9GO2E%WjB_8W|TxL|G(fjY<^1qm;4#Ci9(1a7}F z$qz(1QUUpOICJ_7R52-pMh6<93VAyj89U9(pc}4&nT?H~c#cy@ECDB_5||$G_#1L` z`{>zqRgXjx2+a!sQehS<8!*+oyt-=ESJU)=Xv_l{H-662Zj_NQfAV`Kmg?J*xPjXB z6ga{9RaE#UMt=Upy$J%3zq4<&r))&V=vd268jsvXDONCeRcq6{4k%0v>&7}vVvY8G zrvWEdqe^V9rEqzoiG%Z|1Rx}OsCtJL^u5-b8f}V4!P8EjDSpd-3-D_i`C4;P4pR7p zt4KrKxV^f#xB5dO!e>_%~x1xshps8f^f6`A1 zTP$J76FV&k@?A=>+lptg7~$S$;Mrzq?RJ+=nzCZ3rZwAtv>S7GQWA2m?tIcvk>WT_{TrDw+JD;PtZ$m!g7EYLiyx-oe z=3)h5oijW@*_^?OEaK!N=h~;WDdL9rviT=0aeU0oy-&fDO_Ol-!vOWFDpK-4KFHR6 z#Z;%K5Gn9ablk@?hF=p6Y7>TYFT~+}PG80Xu(hE6>)zt_H-B~&Q+&dPbeu=0McUr} z$ukJY2TB!Y+&+Ngh*a8R=j(J!rBt=cGIHTVi}xyHn9Iy#=yQj4-)8NxnMl?pP*%%| zCnc?1o9QvN`z4`zQ^r)`jb>JMRUX5=4y=zpl*Uq|TGZ17gu7oSa4_ql=LyWZB&{%i zV0|rDaygdKrEc*zDj6o8^W_nDyQ$uDBgKFd0SXY#{ZTDJ6M9loK!q~=z7T=Hx?dzh zm_#@H2s=}R>?8pu?3l+Ru5X&tVo<_0$cK>>7y$n|x=*F`Dr3SzeP0ZZ z(@N7Pw6(s}73u7Bz4l9;AC5kvUueD~vDG4!vZ5c9r^O)KN zAn0{r2(q$0=p2>DdGg_mOv-IT13Ev9cFsJx*$*fFb%#aw)XnVQbO#S=zy~*MhwY)jvcFvf|jPcZ%$FHf|o0N5lk7(0qZrGNHD?@@na2O-F zV>$x}+&H0tgn%LGbn4O&Iek@S^><|WIsoyx?#{11JnqKlIOm{_w_bl+G$A9IrUsiWgU3vh@d+TIWa}S(L+8$>>$^$Frv*N4q^1ZC^ zTY}4;1P?jawj$Z$KYzu&lub|2mcQ*gAz%sf5FWbJik5d^cI>>!ocPMp->1T>6PXZWh<7+ z%lLTajSwXwY5XvA+tCL28YY&^W7y~kWI-vjbHMYf(i zQ{4-7L=Wk$pbzGoefNMPmn2F+7QS6!lAID!LXO=$+YD6Z#G#1{Aid<-D_a9`xXMx4QI$7Q$r6eMcVaGxt!(Uv8QJcVl(dBX#_m%**6G=*M4z9ptE3%c=4X~fj?BfrFRI7fQ zXC2rX^LVjAySbJh!Ogh|z`L{ky^lH73F*n(7a4ot@Gq$z?+T_d!*d!u0<6YO$dawkN;1(go^0Fo2ffdmob*hx#)5N$(+N_T9 zKm`A&y^7Y+Mr|QqKG?I>KlaGw^6!7jCLx>aKWTfTMZ36kpq6p9jgGvsELP!AB#BF!)?Z6 ziHwYt!-vz0%dgb$6zDmHY>2`K`Y2sLjrfoDlSGkoVWq18JP^@X@DqX4?%`N@)bL*)5)V`W5u-@Ws6>w8h~w@iDAk~=Y&Dj+al}|F=3<~6 zf5izR$#$rhj`sE5YMGAnZt0Qg$#72BOt&JVl(LXYk@G&`kEZussaRJS3pms3_^lua zk}O7D5EdQN=0z1Vsu`En&P$sVZ&Z~ zuik`VN|eO&Db7)6YtB{?Ouh_2NaXCku*)j)jev!p7~a3(Z>g5I~{f4I?|d7 zWt>u6pM}H+J{Mc+8R=B~J%i?J(msew+X@XuD>f-qNv@B;`t{?upw5a#2Q_3xRbIo3 zL&y+sPi#q++PvA&MX2dwTX%6o>s$A%O-J@s&I+TIKDcwY-Si#JpyMnyE+d;ImUVjf z7oV~-0eXpPrfEzl}FPi=k8FEdXH|ARpw5J_+V_9vTtP#b35y z-F`r>nXm_b8S!_)(Z4xgP0`q3MV8oLJ%FFZNS#<$E#k3D%SIzeG&J5gk%ZZ4tbBcc z{S3a+vP(i!LVda6u=R2hX;_g`RLg5w6VX;eBB2!JyhFMNhj+7P^L>PcTAzebQG`=E zIGl~XzW5!1sf_+_>yi_%0bITNZ4#FlEbvKZsM~aq;m+o@z*@iM(bJdOdH0yZ>(|HW z{O{iqMm~`4u4hZ^5zxr>g<)URP_!;*&2~`4QPBNIG!5y~4Y@KHkOxO0^{TyqSZ&ri zh+m`#w!eUO*k2Nl6L4vpAP&X!U^Wf}(}Kz%>@{ge!}^~(-@!m_;;lID43G(S zmMc7-3+4RkO_d4+Gx5f#R-6^Sgg?BWo+#}z_!hmUY6y}~Bb|gE?`~)Ncj*lF zxm~F{8QZkI#ynizt0&GOr3J(}{8!NjeJFxG+nTDl{j&V%&?{!Y}a4 z-k=?%dL%~3X|3!Ujizd0W49PgiW@dx&<&#sMhU;gwznSSmAL~oaagI^4iJ_vZf^ZZ zsR0fNiWz>Db3GTbD&9y4I5pbR11{945~N_e8*j5t?oZva8-QS^LzL=H(f5#6=K}I2 ztzfJQ5;F7qR&6kT+_XISl_s1wWe`W!56|(zm_*%I@9z`)h5E=Nkn#DVYOdSj>~#@xg1do>VbZ3I&YPiX=G zsF3stE0q~1#!aADQwS@(`{X?%sFXa~U?8wU)0t)5N)?%+FT3YI9uz<^C?oak4+>pK zta-`Z!I7VJ6sgs_`A%m877UL*aw2|-BgADd8Ie@6qVTI&um?2X=y#4@YlUDj zNdUPKY@qT<86Qy2H?f){XVWtPDqj4Mk2STiQn>SRX5NzXpVV`uOR2Mv(A9vXiL9gKK&|P}GAM=|0^Aas_|a1xvpUdfwD!d|-FEB;lV|Fpu7>qR}qU$cKyILbUUp>{m5#j-_t zX!@`9!3)7e?1)FmT>xHZZ1KO560#`|moyt<&P5o}n_P8n=y)8xj+z&~H6iw$M+fzA zd(4!_%^U~?;a1v`KQX)tRl2PipwR<5lp}Rh*S7BtkZ4Hwp`uPKg^p9sdqtj zL(-LK9GOj7v+8(m3c*Kv`eXHq{Pw%}K6nY2SLxk3=<2rn;toGa&HB?Xqy0yveNuMd z`0^}zC`rQ*sAA`mNlEUT`BV8wF?3=$Ofh2<1@J--CF9(bjP4w8-39tdO=lK6;Zhtr zc+$o-)Nbzq&C^Or!x( z8A*)EpHX`0UDyRat$#0i{`QqD`Zv;4ix4$&O_J3OxABRpnF~06X=-K{Wc;)(bbR^K zzl}s1h+jIw9~_r}u_}l4+IBC)hNh;9V~$%S)6F;~iUV=&{M4g>9+@bf!G?uf*(^w0 zhGN=>#};(&jw>mE;1q$5z-7^^DCpeZ+tMPPDy!4&pMTmERlA_#U~|M#0S#tZPD$qz z6BrvLt@%(Y1&05;su^M?G7)l&p|KS?6w&Etwkz7{N^7Ti>3scv6`hGc6aF8^UBx#_ zCCa&!tCF))WGh1CsN99g8Oa>EXH#TuIYx+8lB-C`S(|(A$z6`wm}_E(W7Ce`exJYL z^LTtd@AvC?uC}?z!xkmbYed%L7^70p18+^m_q(UM#nKW%-OT>n+Bb+l zSqH8|`QAur+(M-);uX>tGc|kis&JCVLCiFTcIM*wLY%(W#b3b1A(PkVD65)K756nZ zU!1QDD_T(#ojel4xaZ=|lnA2wdcIZqO_-UrL~QZFOjIuJ=a4CWL+<4QMr#Lb=G>r} za}UK&8?CNGz1K^f!ekRokg5?WhAa*EQLe@kU$}BRBle zl~PIZkT17oV7f;I@M%24qOn&T#%ZhjPw0jl$xH3&1x5sALWow&=#7V%$|iVNEQO5p z4LqBiwQ&839J^6njLC@)M&JB)*hQr1dF<4ckKyN~1foa7T)D+A&o$9&94Y+h*=~x@ z%Hks#N{-F*wd0&ON;QE|2u(KiE8yby>4YE5&N$D|BXF_KlYo55o*(+2bx2|I4LB~^ z?5FKhc*p7S1e)v6Uy3V~x&nX&>BuW0ARwK5fJL9vPRPjbRbE|Ra*&*Ts-Ylh8sI^X zr9a8Sjk^6c^+DjZt=6CSeiMAPb}$oR6K{YWK2Q-qOU-;B4YhktnZHXPgXvpBeN^)^5%}xrU_rdc%d33*q;Y20HZM&X0bm zJO(=|)FlC&4kyHGrYO&qQ%GkcSR^c`9UIE@a&8g&rXT?Mm70nBFOpIC4Ila78t!Lrq{E!Q#_v*6R__?`ZP-ZeUz8`VfE{dGtsw#QMg;-0?0H%LxEK6Nt`L@w4?%v%Y=A~fpKd# zF@^&oS2_Jc#&&4l{aSvq-Yq({;}!Vx^8NV;pkgF#kiD8YREuKq*yTFv_#>$uRW=pU zjs6ku^j~5Z2{|^MN+M$%cg{<&9V`Gw60eyyf>9JT0q{M?J44f}8|zzX2BOWQU#jjZ zB|5_0pjSU-kG*~F#e#VC+6^e^FkE`V45_yi3TkvcnDI|#e4*6e*=pr$npT26OV;; zGS?{NSCyn1Zh!e;`expBc6$a~E;o63zh|YEaX{ixwL5FU_#t}BhAE>7bSv29=Dj6t z#O$Y|?9BgL2aqJR{Z~TWnY*W5sv;Rr4=TSMHuwnM;ST5jsN-2%ddJWIu+8{Bk$6S^ z5_Y#~rQQcf)|MCnZ{8HVUtRBU*uDLrdr@Skvl<@YL9;w=DwlVJ#;CqnPrzc2NtsoP zH=GQacFI{CS`dc6i8?w`Z2B3h_r=R=Z7eD8Umwa?I^W0M(72{;AX9NroIOx$J-avr z3D}0M39HmE%>&R&Mc|d$V{B3QMxV$WQPtcb`ZMSJ7MmfF18xNsRAHPfp3b*p7&*Ro zMN}7QMXfURQxwV$TNL>GLRc?+i3~Smjo99t80Ffn=MMKZ?9VnWTd&dYhy66ayIFY) z+=%5P4WG-Q<=}k^1N;BAtI|${GL#rSkb4uTFedDTJp78JN;b}Xy?!$ z_8rsf9Kt?ghHm#EMGY=|eHL8EIYn*925V#!w_+K(KezLZrq>}Svl%M|e_ z+2yZ3ak4Z&d?KjQzauYB0|ef0?|ty<4moc5Tf|7N(zpN9SdDl8@N!qF90VGQ8|yzK zd5hPFE@AOHJZ|{*q-aV$)O3-j2}|31_uf75-w$4bQpzvzCbi4iMtC^7Cn=>Gy!^#G z4^aK8RPL=auT;#@St{gdl%cUWXl^4!VG*@5_VMXn?=@RJ$zl=xNH4wcovlDccc#*8 zb=#*nMKzMh(w=y?!DqN7uR^Wp8S7;63ZEIv+S6(ZO{IQ8DV^D}jwueTTtE$N;LufxV^OO+#+psO~ocX-5I93%G6mctSgcFPGgxBzwLYI5NM1w_~nX{A%- zQ~=hgA4ezp@&>B)N8%dXPMo`!EA+VX8YxrY?LyLm5k|R7Q;J&c%a8+He}}Y*d+7ot z3jm=ZNO5QRf+MK_3&U9h!ZqQu;(&A7wl}{Fe^n91bm|caHnK^A4akvWjmIw- zR>sehuo(GwESIH_SFPuRA`b^K7W5VJZ6cUi4e!X-WiK9hBCHFF|Gk=*bQOK?{Dr{p#W(XqZOk*8qrS>u z=a;5ZQ9DH_5r&de032c*a?-p7T6f`b9elxdonok5a6mu#RJd4)vgSlZ`Td=nHyxP6 z*_#KuQqrJ9kiH}ES)RHw@yeYEJ7g!A+;4LN%5mv9^=Z?Qv+d7V7Q-ABzB_zFrRR$XL;n*&xnB?%ty0QwqX8=6`=H97Add5 zgEhoA+cZXOo_Rr4E#}}EZGF>C2PRo{4Zu~+J1M_6 z+B|+8Jhpp248{tsGq3Y>pI)@V>; zn&kyfS7nZdJPeDd1v%9~SaTIr=2<`o!O@uM!(F0RBCM#=>0R=5Nm;rzvuj5^YidNF zR``BOU+00>{Eb!e!mcB5>#Gp68Od{|L5Z^aqVUT<8SabV_M>tJuJE)WP7dbDL1ONc zVrhMivCHag8PMlW$Tz(z4(CqBszunvuvkSD?%TVrM2XFYhbQI!`?&Yd(^WH7>d)!< z{nN-d#(qJd$V1mT9cFja#ZgNe&LIl$?+Nu#BM8v!;>SfU5iv=uhBI!-aZ>>^(A&U$ zHh&XKymV0>zYo?0R)&CSuY~j#cxv) zI9T@!Jw=tz?c=Szwvt53?o_uPjImq+t2~L48}ewuEXCV%0ZgRBE|^l}vZI2)d7pXt z9%rO;7gnwd%f3oGaOd1+fcc5Zrpv-tC#><20gn{Or+$3Vv9rF|j1_?Aeg#6WO!RUd z>+nUWHMda35L=2@S%G)_nl!mh|FWTrHisA%6RK}J9SMXYVkR`s?l1D*oumUChlgSr z87&u&&8+F6UA5d9`kmOKK4Fxd^77`nwmOcJN2~vKy6J}4bbl4Q!#8;XVdJMp1;!H= zlbbX&P^%=tQ4^8*7-?N+G<}NRJyp>=+Yxm8r}NQ1cdRf-kaajIMtE*W9u%mj1bZCV58=2k zE_ORNGYs`vC#>wgbSV_ZlOPO&UMj~%5e<1LsXu|*=|qfOymXIPRHu7kQn?H?J*Fo6 zmF2{h2I}8NlEo4;4THSQ}dFv3UkI?<)NqdlxK@_#9ti2PrKLi%2 zaO*zEQiWN>(O=fO{uF#=(YIAyJrwNVslH3hQFi<*pKE7?MU1TBV%)U$E=R=V#n_m; z$i7*Vo}QqVOJ&#Mqk0TY7cUxfzg6OyLa*}UQc+A{e2C*w$h}KiFY)>QB#VSZ0wrgG z;>i+3J!SO(9#C%Qsi1E0A@JdR1W^P17T2A|*;3Fq=H1s52*~M|OZ(}ydlZ}ZUZn!` z5F5&xsid-4*m*Dz*lieL8WJg{6>kIlYlr4|@DMluPQzK2;5~`H8=nWtH&5}3OYWSj zXc4BFp+z&`D-p&{s;a*Z=rnB`IFBnk*MjD0FDg4@aQrdWGAYjj9$1Xu#pNiawx%+) z72r+Tv>&Yk$i)z9x(hlQ#QY&iLNk$Yy8Sn(l3m!Q(sqC6`s=g>beQXeXvB+Hbrdoc zyhm8{^D5Oj=PN^d=DrcE*LJDq&uc=fKJI(oYW`r{fJ=>s2MR9uZlp^l4#0C(w0qF<3R$nCK;ldd{ zlP=_V)gQ@d$EF&IRls|+6<}&70V>5YYmGBL32tu#`!&IjD+D-&05g~7bGQ$KOJfDc zz8}HR6%D6Wr-G<6Uwokb@(9NkYE%+;wik0!TSQdQ#MhSg8)WcVvb-kZgMR+EvtTx1 z=rU{5g=y$Us(m=sX>%UkT1^6TY(_HB6u~&HRp5ma;R4gfg9}kWj_h{A;>E+bznO;% z#LOz0{rRc%?ug%?91W~E6kU59#om^aM_;y)&mEXhS=KEZn{TaP?0=ZA`9y2flXk#B zWqmjV&|1>$Z?#XbEEF{V#h&B~BzQm0J!{M5PC!fX(0X_6UZ^IDa#t}F;4Zx5N;GQ` z-sXCBVR*&*N}_rZ$^}e|GWszC51zdRwJF`z9yDVT=^BEni%HT(76@%nv`2lO>kn=a z$tBk=3=Xx|XfnSCEK?Q*b+x^=j#{i?E|>c6NQhvHwRZ`)%&WcK{l0~<6CZL_ zBDeE#$JH3kt2Tpk;HpLYj%ui78J$s@f|>wxB; zV!n?%v@;e4kNmEKwod3BDn)&KN^wls}WE98?}`ogG~W7%*AbR-Xt7jhfh z#SZhfOyVPYs*AqSg?BQvajV2uHQmw_{XMbau*^&<$fJ#GM&Gowk*KWJdT3@}`F$qY zcOShO9^A252-M?~mBO|gXFI1FPtUyP5C={U zr9)lL_vbJvs)8-94qU%-fy3#QN2&nm3n$?cc0y&!gBLDfXy(T+|FG1R`FXi%WAxnH z-aknn@`?cS^&nt4KM}uRBU7;Fgr;uyJwXAIKY9HzOt^lVi;7`_E{&aB;uZgUdwm>}*NAV4eKUxa}N8$*BzCE}DS3MX>>eMm>eeYEy}#QXlt zX#Y-;I-odap3l4-13llvCJ6FP44l!i>s?B~Xxth_72%pV(}+y!p$8nGsyIz>sXE`2 zsbL=P%ssO1GLXRL!nVO7BZ;|V{eENNehua4>#T#1Y}!^B29^U%9z1yvkl#LhMGTZa z&rz0ARdx~F6zstom)bLkc4{6DbXh85}FxVEdkLi z$&Z_E!$W6Nxa})i>;>^%qF}fFbfT6#5720~gTxR{yR|%7m?!hX+T4Sf1Kb1Lvzc>& zfKX6;q)Bgq!#E9#{s2!dhkM7NyedKEh~fb~Y;y2Jx5a?)h*+zb_a6hV*c)x`;Q1#w z3xJ56(Thc9qEygNA%C!{`z+OlzSo;v0G3r3-5A8zt)@26_A}r>sl1)8n1%x_X+x?CwjqDxeM_(>kwQ?t zckV}7=1c^~J^588R}Yp}4M4jApk6l1qYv;FWwW93p6V})%ixtad8WyhYqet~1Gze~ z-tyxnHlIp#r#^oN1g}D_%%=DS%RY)@-3r~NPw+$kWIO+!f&R0I?>bH;3d468s({1B zXr@3jzvZZlCd}va-txmQ#mS?*+%=J;8yQy+ODkHXNTM4f38%IZ)hKKzkGPv^6r~^`$$~7=Cv38mE@XnbOb-2psK<3!<4&L|O{_KdwXGc%4-3eqSPFI>e zbKSrNYy76<*wnj%8JhrK%_RWj$LnccB>%+M*IQ(rY37Dw&lvoZNQ}~|Fkps(^Ouy- zc0*+%G#^z<8yYAdf?f6s@t#^S=KAKrhoZQ5GEN}DC%iOuZX*XDXp}u@u0xsYxW_ouBxwM}`0H_=wyA| zE8)_i>OKbmw$;eho9to8`su9p#>P@i{m>v!HYrMx`by5{s2fgqV%IN2u``G2{;S#} z7(C_JHL#g4!TVKzH-;cqyTWYUbYJYD51;o&OW{neeF^8u{&=>3MOrA~?FdpJV zSYd`@e7yIF=r>t}q62JMgr{OifCEZ+OqL@U0qnPCM~vzAVAWSinbTGsoAj%8aAv*o zuWD3^SdZJGJp`)nD#ZmjSqj)I^?gr($f>AJ$#J))lJ(;mu}!}FFX04CDff;uyZT$@ z44yzaWcc(;REg2B-keS7+|){0hao1Ky6u~P!(lZL$EGcIp3i^I>#mUn%_C6l5a^P! z>!#Rsp#cEt6KG$x)xQV)s9bQ9Udl5Q!j2ysPa78L&HdLqdHuyUL@dr}NJnn_or0#u z)ho3h3FLS-gf8mRizhfvtzM0;@IyPk-^a6h9oP}I+0o=6~N{Rb6BX3y4 z5iV4cW^ZW|en}IQMT+TnetP+OC=>YD9ENf2e>0Cg{8J!oHPOl6dW}=^aM*Unss)1+rbRF+Sba7% zS^dsY{r8^f?G9m8-(u)oUlX_hU>wvBfuHDZcJ$scFzxx_sGe>&>$_MnNuJCsS&yi* z?S#{Ys<=ZKzX4zFL(&!$TFy;eGq<}lHtC1pKHZ{AsJ|Suh|q}G&Hj5`YQ6kg>-TLH z@Kyi8(;^duC=6+%3mPF4l)6`@ir!|39??Zz7I ztV%vhgYW=#7VO2Wemv>Gq}*g@;q;+w3>`V;kYxK;6FPKtq`3YYe^ONz(}&E_>Aq4d zi=*$Z4@FD3K~IDg#yC21E&p50#uK=4t=!6S^zF}6jtF|OY2C#@@z}oC8anXk#M0LC zd+<`)JID$k59QE^GI&PGf^LN=Mk)-?G zAp#plve>m9P|9#iZEcyjfDFB2Y_A!F^9a*j3Pm!I-(LKYNI0 A4*&oF literal 0 HcmV?d00001 diff --git a/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/media/foreground.png b/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/media/foreground.png new file mode 100644 index 0000000000000000000000000000000000000000..4483ddad1f079e1089d685bd204ee1cfe1d01902 GIT binary patch literal 12430 zcmeHuS6EX)+pUO#NL3(IK|}&d7YKwF5CM@UBE5tjTBw4Q5KwvxB2pw25vBJIB27p@ zOaSQt5eZd#CxmkF|4+F-=Q)?(#XNgvmzlk1)~tDFz3+~Fs;5bRo%8yoOPA=i9zS|^ z=@P~5f9V?4rAwDs!Yjfq4p(5Rx~i8hRVUG&*j~LT%Q>2AIqB+Nx_^yhg70E+c&i!%2~zqE0}mxIX= zz1$7|sWj&3yL#7D|4uLjQqV+x(Rz4WC{A9|^m@1A6`BNi38Cf3B^aJyqxF{TjS&2q=3$BC zB1Fu04C;%o9V_Yg;Ed;xpmge>%b<|5q52W_pTd9o;Qty2mQ+-Peu)^(K)RH^d5byH z>AGB-I7$|~9l)J0H_LPDsUUL#brIHpjO1>dJ9@_5&W zLV)s!AVn7*Hy{o<1zLA_Ky-TWzJ_^1=W=Gfyc#1ssqeY_2ww>;ANX%JT)(9uNHOtU zeqU2_{Wu6pLvCMBLgy+dx=13ZG-+cMrBf;#8KezD^}_F2x>_Nob0^iXEv>aML;8RQ@@sN(#bq~VsOa>) zW9RDe#_!zLkj)PyQ<05AjbPk5yJ^|B6q=sMX2L0JE|(P%=v2$6+4QL)cu$c*yt`EC z?)p#@xE12zK?QF2u^(xb0>KieYWS%DH`?=eOiFd!6)WRmCo6Joq6}7e=Nl_;oNJ{1 zu&szm^c0s*wAxfHSlk^+hb)aB<&B?9+_YvxC1LEy$(dDJ8J)d!>rwz?q zGTpJ5&uVwR#t4%B`T{*~RAd_Unnf&`*9c^zbZfsVc;v*@=BHOCX7VbyhnS5G*Pik} z@`U!W&dq$A-&GCYAWg@rG3W6ANL_2a)|;&HJSig{zyfyO87W{;ej&@-)yx~eu|G6S zO)U5U?QD)!ey@XcxEKX?m{R4VZN!*V9gT}6_lv@YD^}}y4OM(*#%kMMBij<9x4*by zCkGRQ3vqoZ)HvQ4oY~=kh{c09u`@Lzqk8)3R+$+hcYuhqajQqgq8qWy8X_QMy@1+T z0&yU)D$XzuW+GZpAB%%|^3*{x!r`8nOWhu6>t(2mvERH# zwD(@F(UyHL)A@d0q#?|SOaIrK7`~^_KhtD69y6E{G70hSpvkOuvhEmR1(|2efAmi@Xw9*}m%vZb>kVqe?t6*aL%179k2-;CD<(T2&{-rQ;%g&4b= zStwf@&UH8&T6lBt>jybuLy}~>HTF7(kmQuR6(8*l&xSQq79o~y=t@1Z0aSiA&-LWp z0NQ{@*q$n1m#1Z}?sFj0=6jxX!@eHh_D<=qD}vOG`kCQ^44In=iDu`srXYt8{4c&) z7G9;S9(*ydG({X#u#N%3l}&Yaq*lzrY-E%htNRQTrjCrX1NMi~a!soU$|=0*dXokbDxSFnm6OHLV@%5(K&ZQB%e+ZFne-TrP|veCOrVj;0pG zdbMMl{Z%MBfVA6b>SKLi zXyRQXFc}Krl(owbvDh?Um&9l0#P)rbdiZxK)8=RY8XvSG1@0=@vGxtW|3E{`T&9Zk zC0==A6=d?8`t>?}z3d12SZ$YU4KZHQPf~|w zJD7n^6bjSS+&0Kq6nxhj*9}9qDZC~A`nzEz{<+9lxx)v#qaCsGWko<{ahFVncU-R|715> z33|Jp;8Iq?Z)NXe;h$K{z8#lRB#JC*XUod!9+#hCfkg#-^FD5Jq@>Dt!SzYr@q0(& z;I!1>qg(PU*HMX7>G-#T5V;IOw~4L@XQ&5le>B4Va!sx0P1pm1PMa!%L##WB{CukUKwQLR#mw_r{d1DneIIJT(j#O#-det^FD zbdwZ-8R%84+Bo+g5iyd(a6x;*5F0xuclibP*ff{7PNPESiBNJu^Q2?h!4}38?XKcb z1cb%?RlBpM10D9~`7(D`#uzQxY}K)shcU_}%#WJZ`~FU)C1j&^b5i=Wc7uJW8^-NB z(rs3^Wms@#S~)+us~_(~uocjV^vU^euJHB^upc~CY%6gqBXHR3{FJ}D^V0uB8xrdo z%j>^}CvVUV6jaGJf5i$e;gXng&>{)uK?nWhEUaVrv+x8njtfCz>cqP8uUTn1`McQ;CD+jm zGle#Cefq~0!!v@W2XnNsA~8j@Gaaj+fT)QzP<&gR$L=bGEJ8^z*tHxS)sZ=vZPV!4 zw*)4rK3To_7<;de8PvEPu4Q5d;D=g00$bPnaG|sEP6(kDsxwc2+y=l@=8Gy3^DW?X z$=3@Y|B6^8mUadWxX-6z(Oh@9|3%Nv*Hz=bA3)}AiK3MrA@eOvp)YSd(Nf|v;6dz-v zI5xYnKImXz)PTM}jxK=GJh_OrE2HXqKgh*KB!U~;4W!DpXN6A98^kNt%~i7+I+`g5 zW}~Qod0A;Lw*Q@m73+!Rfuir!WXqcTd5mXE^DWV3AUSVk>5EA&b6Svd&!yh*!z+6( zh^>CvoV~2?y`UJ#Jho<+PlUEw=Y?Hyd8C#Oj$c!5d!Du*w4OQ9G&OxhDmQ=)tzD()srM-?#=f>aw-$x}3Z?qLOIJ{gnZu zd`Y3Pu@-6CD7)$*a6189&`vfy%c7^DmCj90Mw>5FgU_yh15-*dsMPOLpn%G&Gbq@c z)NN;i4jF!g3-}@w-}i(YUbp4WY;xYi8`sa3ep2V_UXf_!7A{;Fhp25CGF=6{xLd&d z!Mvrklt74KI=0hsCRMYBXM0Z?v1sDfN=Y&W2dW!hUyqiiU@A}R-XCxbIudes32?<&DQ!Hr>qn`aYQ?jSq?4X|x(CCDAB;b=wcWVCH1CfwqU1di z!|LlwpE@R5*{9XlM;`OM$(VZBN$c{`%$ZT3S3aYJwVO}kw)@4_EyP4SXgXkd)Q z7PtWeexnE98(N{TMKt-aG+YpQs`a~e_Y;}upm;CRXlTWI->sMI?cj%D`$7K@mQ<-e z6c3=23v>}kQ!+Z{G2&KQ99s+el!e053~lQJc`8%`$;xt_RQ&16M-jjl$HK)VZG-0esPL)%m(*xgTxhvj>YKkE?dOv3G%g-W9;dgR&pG1FoW|wrm7v|b_Y-VU zKV&S7NcSkHSjm4nrPIy#Wvwp8(lbN>^x7o60ICQ5m?QwOuUY9q(q~<6`0+a7 z_`Zhdli4>YUiT%XT1&z74m|S7pZ;||I*2@$Zd5=|9{V~xFLGS|sAE`ZQ=toXwPUzSz%(Ar!@#M}4%I2r*Ca<9 ze?7@cjo0^QC6zocYls~PXjm{I-w|^|?Hpmvl_!6;&?vERiS^(A2e-)2qxQ#IfuJ_M zgEhyUo8K;fE}w8OE$6nq26w$M-YgMyeYnhwguXF-@5ca=0xYn%I)Rl=_lZaUn5tgl zq{GPw`_E=ilA8s)Jy=%ks{*^ijmr0SqHYg5D%zYfzlqy~#fp6GHI7wm_SN!mo*B=(4jED535Cy$0WQgpMk_!VjQ zhjwgVnse1csNUVP_rkF)3q*bk`=D| zRm=kyT3qxBA7a}d4b433h)JR1r_zBVy6)DMRyM?5%=@^}YMnjurETi?w8)8Y2lox+B2Mc9(WcW709kmg&QO^PydT;QZ_K7tmYO8aA8M?Y);N zSn^>S4^jpy!tF}ZAn_;hcCNY$eyakky`&>*Nh{Yf8H17GR#{9&%f^ps6IAlo`0a7| z-5WT~hwWze!uONxb4D$Was0UyM#f|Al`@rMWg(+oyWOL{(2>P6$`ht&d;q3uD6W+D zQQKN!nzWpx$Ya8CUKa3dgn={(ad!Lm7qDcu`SB#dKHvAM#GW}Z>EZmS6yG22dWcVi zef}3H%>*xQE6XidovM|h{PD;~31ijm0ia9g=-tnlFk!0PDn12luSSt7gWP{nbUK-G z_;*xp66cFpR2OkYg+1wGZF$3SCHuNOh~T{QxmE}&DI?a%s+Q&BqRkJ^37TgbKmAKA z-lXW9)FAv@J#Z=C2lSk4@W5q7S0~BpAs>m(p{^)b2MCFka=_0~yTtPvSKJEH%6&GW zKv;f{iTBYXA0^wmTAmssRXI(3556s-FYRfgXSs2F7D?)Muw3X(n96>Fe~#_y!;5dQ zdOQ?Kp<{m8r8ee4PPIETr3Sr=L{BgNp=Hl~>nSiYS!vY-rs7>zJE&K9>k00!&bs>P zD`CMT*(GNFuh#^fdZE?R`V};&3K^rq3z5UT^^KE~V+Yq@nxU<{+Ug^t(FEIk@f~5* zgnEN(6_Zcdmg55!i|T1Xn2NBcinnnFghvgYxT5oG<#r&$ky|k5SaFs(+Vr@W6W!wc zhr8=;xACvw0kVQ6m+uK@w0M_|3*`l1D1SbQ1B%k-HMIa!=~kGkCfuQ8^C^ZQ&7xn%?zUs@ zJv~f?$}gE-(aEgrt|vKx z;}Q@0S-w8jTszP4_+Em>MvCg@+IT%eNk_MIr)gA`;*lhuP%vm}{=>pIah-$r^3{Da zp;l8BZIY#N3v`sN%POMh>Q=e-o^BM2OK_7-ztamrbZ{m49XWXIgg1Gqa+C!XfX?gxVvl@Yc z?lm`jKKariU3($HdVP4LPtp4+4mV=+tw*rjI~_q%R6DfIW|6`<`}My)W_VK!6c^i* zIvi5RI=c%+#{fOc1^%pnKBkmGk{n2 zC<)woa7^dmGd|$2v77jNVg{v9cP;?R<5Hz&w)i1YTrbpNc6%p0{Khx8hi!J94klTx zC9LuDS+2u)()U%ug}~voR<>Cq}#OQfXF2)TCm)4nk4dkJK<{Ji<% zcP30SBMi`eN&Lves%5zi8b`z0j<83Tc~cBqc7F%;N9zZcNAe!JR3!n;@j1h z1lCS;R&Xw6EFbwYNCw_`r4_DiPb}ogRDYy^watxfz7Xy(zQ=RKaRMV#RY}`WgLrrF zVY?S>T2T_0_gmfEc1P>euBpQk$h-TAw(GijhS$+YK=Tg$zQ6?>D}F1vFkHMoukc{a zEy_ED8Uf0r#&yr0HH7|2|B-{vV9-6x6%+AEp3Hd}4fvb`f5|t#1a^r!L``xWv0pYp zK_sWYo?M7Ka~?Ti?_2#VSWzD;+NOTq_0`+=>-+<27aH>r;wtxc2mAJdsVzr(62hGT z)&mW2D1I;#ot)2O9iIWid6J}Na=-qm<@K(sk9ppYVwcO*IkP(P8P9ER7!PsMfNBn& za^K3zdtRPHN^c^l9lmBs5m>rjxgOV7Io|5p!v}X)j;Ax&u7K?;q%XjX_~o%@lPr_8 z*9Uqq$6~D2?gL>l^=mP&+~8z3yT!99Io|+z9QCQwYR2S? z(t}t86UG(B`86l3E&Y`O1p($K!sj_~Szh|(peg0h(+?ymZ?)sk6C*iUD89q@SVAIS z4_&>H|FtF3pZ<_*-;w|rv%!y93`xISUXVWp-T~!8n*#@16?Q}v>{P^~9I69_ z%n*6qXY%Yy!%fWkW5OADjlkEKjP5d$8>`wRrhp=ra6@iEL)prjHQ=o3@+N$WN7maZarII1Zz-rqUrBVRY znukG8!4Q$))$$`IcgoPA;izr~)m2%Wl&%&EHeRmOXUJsiSwge{CQ5;l6K*f{(Y$dK zr+Ms$jZr918R?`Rysv0Z+#6wT~L%t0b;+Q^{rT$Y_J%=|3^Wd zt6$*epNax{<>cRLLyEm2t&MjM8j1U)pYxwc-MDWDwN~$V|G#;ney}e?-YB~f0-n-M zw?G0{JBvufZPvKoY*5O85X8y3)1IFwLkMFr+5G1knQdDje8Y{BGoelP12*9EUN%KY zxk|^L1xHs)rNCp_@p0*`=#9{%r)_7IsX3T&x{b&X;mgnjUOMtgKs#ylC}%kSdtkjl z8!FE;zg-elNMzzYzDjZ0)^Ieq?HW_G)|Sg=4mBA1EloCGZTG(+tr)OPwRZ{J7OY5O z-u^rg$|QACu3Cq*Al+><3gPrW!35XM#YAriTfXw+!m_NkpMN$HY+wKfNr4L9PYUX6 zzlS_jplR*TFaNt8ide7lbsipOGdSE!+zhi$@D8y%FCwjQ$r9L{z>FOk9`c^?Kjmj` zMuYzJ3lU=4n6Q;tr@a$L?%8~af{fraE2*s=hn>Cp;YCQ#>re~C6xoCO7}(mj#Xh*k zba*^&l5yo%qnHQd!W*<-IXZ+8vnMb>c^cM={07F5{v1ulw!aVecf>C42Ir44Vz);s zT-%=b<-{YEZ*nD{U;m4uIi#wyf4G^ggB0@5%#DRIbN7hz&!Bb!hl?A6#(~|dZ%%iN z%o^Sc0oq?wn5_;1HQ*s%km5+`HK!Bq9^dL$ZL7!o2j@&piKs-)bi>dGD9BCC4PSIk zrGJIk0P-Fv?{`4G0`eU>*i`V_XN2xXw%*xTUlVENh%_|iZDkl5p@Y866#=@Xg{cbE zjZtS75AB(^xEogv2B)1x^m!0XZdCqOZ~=~2%7kuI!6E74!u_j2iau*{do^aD^2Vk^O2eW~KSv(BzRD>xw` z&*Gb6ksujl^_Fg<9{Nxn%B8jSv6jcmU+Kw5-Q&psk7EU|G|_)%rogKwNzemwy6QX^ z@ujX`ZkT$alQ%3oWJ2VOJGz{G(ukN|LF&Ga)nKml$M>IY@1F)}2mL&m6~?A)CN|YS zLi^lZj;aN$DQnmlc~AgqcDB7)?<<0=D*JMD zM3%;`BX_AsO%3+;YjwAbOnkT+m^;*q5X>@S2hO@Aa1J zJCCx~6B|ewT}HQECVls)>JqY95!(x8tJTl^D9t}c_G8p6;&167Z{2*+*qbjZdPBKR zwYTwFdQwnL?Q_fZ1S5+O2`Bi&@(s_P_cQY7?>NOU&FL}U5YmlM6yw@TASK}~;pon& z&{?aE)kw+rf)rVR1R!KIA&R@6^&5tt+oJ8h+P)7GWpbZ0xhG1hCCSz8pFjdYT5mJUum4y`e6ST z&@%+@8U+Bx-^#X6vpu~G2`=~;;97zryltTvX_;q&`r%A)oV7(xhxX1-Obw!r%_aBq zXumue@LLi`iFY=9t~-zHYJC&!zW;W6TKK3YgAe-4E5@wu_HwjtlH4Ep5vqLS-2C5$ zSxHdkc#a7g$_vSgCJ_dxxPL&~SeaPflc=j>z18KsBxhHfhSRvim6wzyuJBI@*m2g@ zc2$Hh#1|Nide`x;s zFEY{lfS)AO1(&M2`md$eil6mNBxu2_M(#la)vUt>ub2uO+!3=jb#6Ic2xq$*jBF`n z%L9sP{NK&^17myQl!*yca`I%e*{%{^D5ld#5&5Dbmw2He%xl{Z?Bv@+UmIbjXEHB5 zH5Sh@UPidw19)2ZMmXkn`O@)IsF`Fbj+RLtb$qTJ#B-vXrZ?7??}cA6N56t|TzFj4 z=rAukcL+Zk?vE$J3_QP=HeaZiJ>sPUrar&8Ao}%X-FpDz+o?UsRbtr6!(ES)@vCo94^P>R%u%q(-9wy%Duenrn)jXuW z+2hV;WWLbrH-awRI4^BBwkb{USY=a|U+=L6IJbHc+!%aSb|KB}H$ z?;wmaMfCf`2o^LLsVRHayM++C2aVlLWRbMjawRSh!|`u4I8tjLx>H>?ZR&ba(LJXj z?DRP5gyUNUnznwc)C%qsQ!aTlw6i(@viQ+~|0fLN?FR=&Mz z!m?8%ms9Zm`@?A{S+a>p-JQ}TICnZa{gktp_;s>#3Wv_=7#GC;f$M! z&TRADKS2F7Grq42P=N2(^g3PHSv9Sr5khe~OZap~yE3UUWM-{Fh{H-BGK9MOV3L#y zw*TZQX^enrYRj7iXkEaCLTZF5z%T)MU*{_RxA-*;G{sl{7ry_e1h+X~HM>NyBnnV6 zzcFEEZvv5PId&nY^VG0nqu!l%4Ln9L8OVmkfQi1}=-j_u=t%I1_~|`SZ_zv+SV@2>e1;w+Y$vY75F((`NKQU2vax&tTw!~HE>c2M3z3d>g zk@W;ee$-qtx3IgJ&cQ;-5AmGPIIdtV0YQvcV7G)N!(PWkx#qq=;AiOzb$C@x+Z zu##CR=Q`hVF-LGTr?w9-umq+&6PrkTr)T1CJ!@XV9i+em9sS#E=UO}BNMwuBrCayH zAub{V#`%5ecrycz1$eSV8<2Ikv6CQ5E=h^K%3m6h74APzqFYP{oejD^Y7o_E2b3p| zeA*LbkS?zNs8`f>wX`CuZF=Vcnc?D9l|P;QF8KedIQiHkm!f>Y3}# zl9AL|w=FC#e&CG1Vj1SX@K&6z&wEdwI}i+9}=0 zD)hP8t2qSqGq-zz1>nRbHpsOX+Ou&rc&B>1K5Z`l|60?OVRG!%y@dyXhC`Y)1x&pBnbuTa%|7f^nM;OIHu%(W6&Ci`84e(2e5z z*ThM)rgG_sjP#cQ+Xs8;_5jS%p3?)1Cd0epUI+qH6)RAoaWyIr#O{wWN#wI+_de=e zPHAv`+(8DcYwZezvF?o<#{{xGw05-!dGx*J-i6B-YsG?>W6ke;g4Hg#P+$=@?s0UEI-*Bw6RE<{1I7> zjBlz61z%K{w(Fbs@*+5i`|zyRlh@qP_iu#(*1Wcpz$is&$q|YHc+dRFT7N)#@B@znBGn$2wXOi+ggc5BJ<+2( zlI3ksg*I$2(gaUp4h9pJY${1?hgh6#mU-3e=N{4cTb2V_4R`HbSASd)X&1AJD{hd8 z^}36_R=S?hhh>k{b|Q{V4g^$!<)__{4ZCIAOzE}*nn%8FpA_Bmaub%88)q94qdSj& zU&K}EwoAH(N;V`V{ZfKgP}7P8xX{2STb>)D)y3#SF&&=+6Jz=_o8pqGbBI1lUdL(1 zD2L567hm`YXfrYLV3fz4yv?7yE!3uaicqZ7ufRny<0U&B6qh8bcqsL`r9)-JOxkXy z+l@a1(ptpJ`{M2l$g!g@DX;KZcoPP93JT=vi}|dQ!tn5*k@U)brT5a*!NEAJ2Apj0 z3jNsKvYjiiy-sUG06+A3T)f+N_X|`ZAX$1+M8W1ZaK3Nm6Dd}Xw#CnL+A?Xi*n>}B z+g^J-yeBCQ;(6yjA1~5bLwIzXXp>6syw2d^&DXBrf$G@}~y*QOne;u_UdZD^Cl zXxza$QKpgXzp22W4GZI|8N{0M2?78Z`$wi+S>waN@uSr9`u5+ghvrjfhcjQNuoDp; zk9szfi0j_VBAd2M+55}LBoF!BASF5?QV6q5zf94lQ$2goh8#I@&N4tiMK&5WOgt0H zRiGPL-7G)N zj%2#teK$kweDwBL1+DK?B#>r?tjR02JIr zUq=)|zME?3CA9?-DRGfqM+;h7w&xgGmLjhTAOdy`b%#?iM;>=l7v)^GADOA64 zy}x#1eDIpJ^iQ-mHzp5#R2_{6(~wo;npi>z4tuCy@Z6Ovw1EGFOaCWi{Qog*{?+*F cSLciz6 \ No newline at end of file diff --git a/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/media/info.svg b/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/media/info.svg new file mode 100644 index 000000000..2210223f4 --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/media/info.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/media/layered_image.json b/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/media/layered_image.json new file mode 100644 index 000000000..fb4992044 --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/media/layered_image.json @@ -0,0 +1,7 @@ +{ + "layered-image": + { + "background" : "$media:background", + "foreground" : "$media:foreground" + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/media/startIcon.png b/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/media/startIcon.png new file mode 100644 index 0000000000000000000000000000000000000000..205ad8b5a8a42e8762fbe4899b8e5e31ce822b8b GIT binary patch literal 20093 zcmV)JK)b(*P)AsI{U0tD9;7S&f z3`9H(<`G*WCN>bN493AFOi{!!!L|afI7%o`6&6lXK&2`L1YumJiZTQ+5doQ^Fu|gz zI6Nvw1cME>!8`;4iI*N+z3;u_gZtzG5&vyF~^*1 z?S1yyXYbweAFzGO*PdLxe&gE9j&{c{J=rY}9i1#6cCzdq+ASx~UzXhiC(H6orN{Ar zj;qq$yDTU7NWP@ws1J2_*G}Ykx7%{iE$G@-7-eF^Y3#}`(v#ySiIZdTj}`y+a>=Im9Vq=f1W5yxR*!@kj+Rxz&v=+4_?qb>2v z^P8^zTt$BB=j8B|JpIS7`QY>Jz4z#w<>ZT>lB09T6nS2-t-LNa`Yg!ixr}^gvZsB` z{B;rQ@uVEqwOt7oA8%Sn=e2VBs;^`dNc~|xx$^LKH+*6BuO8<1`K9&UDuw8t_%!FY zoV0NZ!^eH~qhBH?uakr4K4~ZC5VHnAA|L9#J5r^|-)7;Y zUl$mM>pDMqeipwr+7#N+YO&F-3t!twD#tH9_S*S{wQ+C`@f*(uNuw}s=xXMh&DI;Q z;_u$0c(3`5*FEq(O?pz@6#ee_pZMDAFS)(D{hdnlGw+UhHaZ&vMC3y~_HorR=oT!) zD&Jv0*w5!@vBS?MX~$>r(d*!xjZ=9%U3__Gl0?W|%cDAF&TIVSk@)+3cqc!3boGhhYzil=`)k_5%wL2pqQz`Ju@50G)sNfVj zoXGZ|Q(f3+@xx0`O2~K<`L6lJ-SXStp$#*Nk@$Du%RKJ9@n>4_fX zCq4RXG{SB86?4nquk-Hy-E#B;AN86?zpBs|J16`d(I5ZXNB^!~KL7eV0uKN-_1L$Q zfhXMkzP+y=*8|%=cJL*vJ8JS$i*h!V@e z?gp)OZL3q^qPRQ$mTS*l z!1Lo9sgwA)pzOQd7ry0nSAP)8dF^z>J#;@|{wb*sK5UU+HV4!!`0VEJLKou6^E1;q z{-F(t{g8gMTs+F%4CL8B(dE++Be1u} zQa1d_@^?2B{4?(K#G2gBZ2YKxYj^wS1vv8wb2h-K`rtLS+C4j5oS5zZQT6pjk(( zJ4B5)x)C<~DS-Jn#3lX27u>p0yp_M+jn)mGYaUy>+T%Nnb1#0!>tbyAQ%)nklRSgJ z&7=Ic?ks-hoA@5fJ^x~JiY`PYkDmW0C(plGd!Q$Ex;t|N@d~qieC9rdJUa(Jbmg%% zxJoLcUW^RY7oUugb$iXkOVyLI8AJG+ zNchYly!4G7Y^6~5nrXo&e$8p}lUVB0m<1UOEOBY-ht5+)-??6hPx|GZjRV(b``>-$ zM|{PjUt-09)0*964ZWy4qG3A!iZuCL5J4vSq$?ol?wO2=1e&!;9t z{HK#&d2T{`aKZSSV$8nw`5IF+b?d?_&_RB2Nn@S=KEJHRZ&{wfFD-HANt+d!8=g@V${FeVy<@Q=p|RCl}k1iW;RIY+rXYw+ro1J ztScYrS3bq4R+FlcH(!!*-yB2t`NcV#59x0CP?FiqC-VdG1vMIuAg3o=Td=#P|3Z0B%|-@17rLGk-6p<6~!$6~POh1kU3(XXZO`=|>$d z!lw$=5_RyEi#Jr~RP#^%iC^4A^2m;K+VClBHe2;z6Z14*Mk&|$%X0f<_lmdugY8>E zPThfcKaZ0b)2b2Pn1`Dkmvb_pUZ*zC08jjo)ep|hccB`;;R{6kL;Ts-DL%Zk@M}Ec zYe??S-~5VIlRb~$9A!25WQb$>P5#6re$4=RZ7!m^$ICJHQwLq8^3qO zSIW*0ziJfhY2#Np#+5qaD29V6USiSHHu0r%dVQte1>d!Te30L9h<8T(gM1~;2HMmK zAIaG=K2h~u$+A`Ao#yL~^C@rnmi3*Dn>*0%_Q|VFij#Is9D-CUfq|-t52LPSO>Mf;|h8QzG9r>i*kxj)D&%wf12-@hxpQE(boL;`OLW% z&4ra*97R9KXL{m{MVR>LH~jeO-Z?hkb&`yq#K-O6lT$@0DD?-g)^Uzc7T&5n8gw__ z0DpXP`45D@vQE5>CYLA9MXJba02$ioVhjTWVS5bZ6(4zN`ENe`p5>!H^k})NKh(Lb zKhik@lUA-Xx~smjY)TJqEB4J>%kshNC(AGX&hhfC|NQ3id+))>f~iYr%eBS5L6diS z0c(T7VNUk2yzB*+mM{H`dzO#=6GzJf`m=$1G@nblG}%hD(09V$W~@UCQLSS;5BqEV zWae*vfSYo>EH@?Gc;aOFp#GTWmw)f}@_j#ZYkBJ*Le`;RxE%9>G%3oHFxKHSfF_;E zFF&fw_1jO}dg1SWTfI@g(_fZ9_1ee&mj2x4J1a|pX>wLqgaW;Whu>GnNZR9Y^4s;%W zx4i1NzvUU8TZ6Uq$a?oX>%J5^9jAU9em|0;-_C;e(1}uEYG}e zr$t+qTP`-spu!U-M~AgevS79|o^g>`wAc>y@e7Vk`?z91a^qxq>GOBXzxbc8ET8gX z-7Xxv6CigTGJZUUv*`9=vmA1gzg4h49N+Y^ODZ8#@KI9`q-_X zaPu5;fuSS!*@le$mhP;#HK&jK(B1NbUvXvmPhY0_kiYDk{5AHRoIkT@vw@Z8z;F1q z7l7fCCi(MA@@nf@5q}|i{jv8-IsM&M6%o3LI{BfEQREKp4HG$@wUJ1eYx}Q!%BAIh z`K$LWk8838tEq&7|H$p$UeKq__MwZg*U!9Rnw3=(J#1>imzU))z3%$*uKvrZuZ{Wd>ES!5dgNmrfBPTZ zSl;rks&UNFhD?$g9J)KT33%MPXFTyAfBeSP=e+&fch`Iedi2_(FPHhgB&G`tFhZFY^iGZTPO8%A6S;JedWE&6Z7VgKJMLTtbV@Au;oe}a$|fo@8QFpeTE;~ z=(!{4cwATZ_x+vv)3p?oK6COMai}`b-FNw9`G;R}pRW2^Ajgt*_)SjojgA<};ZV-D zH)q&q4iEL*eWU|BFmM=S?>NY;&)5I;`<6?(5sl{jyXGx}^8>dxQX%Vtv5PEo8w6JK zToHH6efQkYp6Q3Mqvhz+s$i(tXF7XpLn?CV%Z6Oqu_p_+nw!5{zT;K*3%heMNzF;f zzun5oTzGVll(CU?9of+U+nP1y(OpU zvv~w9Sr;nLG5?3p<|70ueyyDbUY}Yd!E0=`V+1F2S@%7DUU z!+3G5v_Yp@FhhD(9o{OXys6YM@?dLP0LotS!( zZ~o{ThY!62s*m!Sg&e-XdU0#<$S=0*Pb|w{eYqaXoLkS+K6Rp~Y^EN+{G*Qi6P;tq z8XuKI#YV0>%Nz^2?6yhv9fh2b=evx?JV#`6&=bQOMZM+dz(~P{OOO4g=JV%2_LA3t zIWdLGe~6_L*6U?ZoidN$t=;E~mp$XEY0L*5)a)#9%C_**_ejXj1}SaGL~lF&7ro-L z5_Il{V)fCw*fu?YZqYMj%cgB7z3S~eAahn{_@cQMlFic3)%3UY#Noj!JH4cEvRr#S z^9EDCiHH1&FTSjo9Q4r{^K&2ha-QnFK^=vKuFYqvdxW=7K2uz)M)&XO4}*2S)oU;32*?s`tzhPoNdy zMK~{~T*=4;PVlC()T`0MfB8pTs;kbv+GgKHr(Rq!;3+S|5(B&y+n5*@z^5dLrcGjDVs3` zF=w9B8T=Q$;LA>~9`X4+qVFJ-liI=f8qb5;adlP9$i*t%;M>z~dBL;M7jh(|v1O@a za}jzx7Y{1+b#a=fVe#WfJ$C)~F&^GD!hg8&3xD97hwY{wLOxnA2;wJqo|?br07>n| zdc9}P-SQkmio~mhtX%z&MJycY7!O^|^}~~L*w+vLY!DscBm0>6jPaAr#6u#lPtl}a zn^g8A4RF_SY<9BpclX?P?PZtsH(oFGD^X@u>A2cxb^Xba#{f#>E7Bp? ztFxkR`P@dmpq)Vyx9`@uFnA8e#&tpr-DGb_G^IYIlqLQGW*i-bW1&6e29O6Y4AR#5 zvw3QcRQo|aIrZklmvExE$M4X$oUyA07_9mhM=sXuWE_~5;nT=?xmN7c}VZTZ(}?rL~jVuDCHDd zW0I>4RkJL)P{rpZ{mdS{51lA{3Pf+T`jPlbs|k>vbZN6ZbRkPI+fmPp0DeI6t7Nc~ z$NhZ%nT)>k;6(Zz50&~yf1iG^fs4sKviK#}-Dl{r>Bu~hY2DR;F}T*pmL9|4wUTbw z@xnlPQdFhr&E%R&<~6QfTI+#VgCJrYF+`(acGqTfD_@rASLH)IiT<#`a<+xCqjpL` z>#D>_%Q%UnL=``~nBcrnhfBLfp$0UGM~}`pY-%%xL2Su?1!0>O+=jhV^Q|SHHsi~S zD~0ov1zlYjfNIlt^GFNNb-;qpg1EPAM(ME^ps)?4i@M~QXic5q&!wGA8~zyJ#}kr& z^`4JJ%2R4dCKVL9!V%6$c5)Gv^*q_xt7|K06))bGDUPP7^FtSfX;?h<0|XKb062A zIY|b0!pj0C)Y$7;i^P=d-~9Mh&zQKh^`h&1%>hsw!5hUsnpx4t z<}nU3;cAnu{B7X&Vn5^sgN95?k&<*Nw-dMSz$p_Pc^$xvIFk*X^*T}DEO_*uml7(B z&nEcAJ#m?Xu}#P#5u(vuOElFSM`G;J(?_?d0s0skGYz4+p=0BMwY@=f?C04B`6n16 z7Y+?9wH$J zAxS-==YiY@80*`{n1+s)KEk056AV77g?$%2H0xq(Q))9XS&VWbRL_G=l_J9>UJl0D zL}N3`NDj2QCw^L+J)AKpGPZ04N*&EdoH2o<_uVvg5ExqK?h8cD!pAn(v{$fP*#~QU zh>wrmGmlPAjvv4qPUcCCWLhX|Ka2&~1>W*WY1;yK(tBoXnGCEf#s(&kaR8=O7&`Rb z4)NokexjR!kF~8MOFmU5aQ$lW3aOlWOo#8pn)8ot^lQLVQZO5XoZ}x``u%x;$Cmjs zwt{}jE1RV@QuzczTVvNF(%{QMY#aX3$pievr_W(l1ZA{3C6z9Llh!WOKW`#3*AYhq z-tucRhL5MYjUq^yq;P4yz(j=;Uhu<*6tg}0;12PFp$~4~hxPm_+Zg8Ct>f7*BneZNsSb8?%&Jh@KlZTTrOg zc*d4a&)A=--&QSt^&=aCKtMfi2RM(tjY0_3lN)$zC%(pMOo(G{xaW#VQD)ml*8}*( zn%f398D{+~2NGYgRbLr0gOY-ta%{uQ8}bVGoMs=E!xb*`2zR1d+}H1qgGY~B`-@YJ z>*a;j$od&444i_t&M>U#WibY2>CmtI+6%Qc>JFq&fKMxFac!J|LFhSyp@oAfvh|$Q!ky#K zhS(4BtuuI=bE{5uez>A2b4!3M+hm`g$1$&w|CB6iS~rUj(~}eO8bJK3dJ?_67ebx{ zSHS|R%y8%`=YQMnAR>?_}JgGOix59Mum~lwBBOj7l{Dr%(^B9~CeuB#Ukb0`^qvuU*Y(62BICR)&Tg!A&&-M+!2eTcS zQp|kcb?_I5@TRuW`$zm0SeN?*o>tHfJx!tLIT3p}glz!EcCx$YvH;wLhF24aiOPLh zoyM4vMhXD7pn%KA%I|SJ3pjFVbc&HshPKa%R-zM#w$p3fhA+q*C$x=DN^`o8SMD%{ zlYy6XyKVf(AvWYbX0=U|B7A&%L$qy^lSpgCbq?mNVK#inCYah3&VIO?=1DXw=#`qC zbt3TAho;;JwjNhLV1kW_T;f+5&f5zw$zb{>8{!V`+%h~%KVy-DqlO+=H=VZ=FkY%TPJGOKbO-eUMZb@k`Qw5*kXQI4 zNn-VY-V}k{dvi=NgDj)aFv2b;9&Lhj62jH0Xgt5%4NV`a$nS9VFeZ8jwL3ZT-35mn zvUwAUQ9a=cgBJ%U^%9B`*>UXEt~NPJ9a#K=jILPgIq5_LF4);`bivL2J}%hVmz_pI z&(zfWn4ASNsVrtA?CTky6@SLgnCP>dnQ&s$k2bCduV@v=0M<$2v&?X_w&f?0 zdVL4q!ob4O|06wo;ixOrj>l#y;~Gg=-=WAx*pV-hTSqte=+)3!U&FCJJ(R7IGj_tH zSk_m_@)csRD}7KQl3@|As*N?`C_c!U@vo=O(oUUM9HYTXr$fev>%5uanu%NzjR zCb4pse%58Ff_FbT99ZTs=22SCWBp8Il>D>{j4u>gKeWxhWg0&$HJ{gkdPXCf61P@& ztiI#OvjYd~D)hvhL4pdPanYqKH?T(AS0xsJjcpoa4(T1TJw`VIoTCqRpI?P*;>dsN z5f0BOf=znyxkaZ2tJWn8N$N>lK}c;lWS?W5vOBR=JKko}KC|$3Z%PH$J5|jKJ-NqE z_ZknrZ7W~D$^f(y8P~onU3Oty2J4NY*@llDx%i|JpU9&wHDK(xtG@VU#^kYat*h>i zdSLC^jL7(-#cz$a=M=p%&kPDtW4)wR`B-^()-G4{E(m^LY+5LRq%6%7l<6vOPNhVCyvY=4yUI zIx&MxLE28(nmXlm7viLOLSs$b4|GCD7I{^>sJ)bo<7qB^r=YAS^^JFY6;xwEh zZpDM~;ZEeb0~BvkTQTEG0U3VZL5j9H_mXvxdHwoPMGk8H%GZ$DSUoG};o!Bp*+kXX z`qy7&0LlzDGC5UnIv&!hC5g%LKEG*AaEI$`J|`zF9*~_UC6v2ef%Yt=w?iGS=`x{m`*tc1v}Pz zf~slY{K=p-7He#u7L@_cNMwKhd*f^(-Vaneam*r{gTf>LelwEqaEL>^IXTI3UTi}^ zZkltHCYX)!fRgkGlZFWF0F?CZ*bebcbNh5(fov2_4=P{4lkUMPb=`l~2uhFxu>7&DseW}mFpI(L7m<98w3m<&s^gYwzKLS`@ ziH2UU5yjHI=Sa0E5;z6n)mm>R$Iaaa0HpF2H=cyKrST)6aY5j>Y2EFa4KyaOJpi`Y z0cR0NFVNX;eH&s&2RLs_Wk`!X1Ktl5EXMuVY^M5^Na4ay{PgzMr(hU*GqwVm<`|tx zHqpMHc}$IYj}CnPhO8RSa9ryZ-xY7p0CWe2u`wOua|f#J0CPySsjO015zUoj^|=$R z&P!8a>m2?Q`plg2TfXWox!mch;lqB)b!%4}(i&%-8hjt^C)?8v8krgXwGp&JSbXUmUuKNKj;seLQ@+i{*gD4%I@RALNg?5Nv zHQN3d?-dcg{ZuEQo!};N-E}JHlr|#Z=D+=Y^?ah~?(8cL)5{VsbD?G)a@Zyct*NHxP>~FNNVt39Nz-u{udkt;$vC~g<^Q~(o z@!$ErW946qkAsrqYR=YH5b{$F!kam>41*1>C($G?Qu;QuA8=!KcHIVdWNDr-8-7uK zNuNiULdrZEx{d!~v71dXW?a|C=vhDe#uyuYWb4hW)6k0ypF8ER{BAwTAx;YE-wb!) zU;16Was^(;$OUp5dXvkJY0hDAS|8fn=gyP6&xSuan8cZ0vW)z(=x@DiJPDG%HphC= z- zpYdSh-(EFF=R=BYI@>x#_%jYWdLEjhM|USaBzVpNLG3+y_(R$BD_RmMas$MWs~oG^0ClV~+&9ED$w?cD|Yz+=nu2k$xd2U}uu6PP0V zCo+iBf#`{lqWxs#{-;()(J&9)cV& z*MIxg+j{>(@hd`~jcXbH;1z zth?n%0u(-3tD58KJI#tQPuPp_{T#@NnLsv#(utmIWON>=r)G}FN{F5lNBD@6U;Bn9 z>MqnKn+0+&Jbe!0Sg#XY1|IL>WT_VXUT;oA+Kv6ir{@DlMjpC8`1rDX*N^ifn3Oa- zP>v=r{|3wSjsMrp<+?rvZ1#&IQ%o*?Q%fUy9{OfIvd7w82leqs-`IVe19y5!^8?p+ z%lE(O);9mymq@O`lr{MH-Gap%a!lvK(+9_5!wv_d}s`<0wzR2F;-6sG^f)1 zfAhBE<$Hhn)^a}|--)B-fGBwkg|A}DfUPxB;ADB-k7x(+!4Wu(Z^V|l+qB6&n>1q*9dcD_jHBlT z*vR|+hTp{?KmT(AyX9Nn__#hpI{B~9Yw%ik6(uW2wP}cuI}>`1H0k-6=fBTqX`C$v zyXpzH+GeRX%|8xjW>_S<&=S+Pnr``~H$Jia)W5&2PruNUE@20Cie;tIvIjt59r&b0 zjV=c|+__#ALk??qI+k=+1B_gv^QeSsUl&j? z;p|tZ|KgJ`FMscq_bfcG=0&dhz{tYj7c4!e`8Av9+C(?nNM0J_+A`~hL2+5Y%lGV- zcj`{^cVGXwo}+cX;<;dQvT7u2?0R+qYFq{XM198e*L=}E%d_>lL3~zo=0om&Voy%^ z%h9>f^lD0ytPpr zg~{1jZAiO~^T97J@yeh09w`1xwSh24F`NSEhCjRLSXJn`%mH@4#+$x@;up2ebwIl&_3snm%EJ(YEoj{-clclgY{Q#$UL- z{G^^VuQM1Gu)n(U2vif97a;}2J2D&cm4Ei0<mZtf?9#n|`tkjxXn6KX&EI1=R@*$+Kyw>;|^ zN6TfsKa#H^pu#R*_}$O*#n-X_6q!ggu8IzGT!q@a0d4&GoYsxW{s08 zxcb6`!zl91*VjDiv#}r4pKJ1goci!UFDRc`2%OJ$tT_0@2dCnL<$j-qr9L&M`lL5D z(Jg%h*(2AFmk(S^Onhux>cB?H;>YJE=cKZwR~3}pmJcYob}zo~KupBx=(Nh~M4*nz zFreXsw&7fy?>G)Rb7uLh_>fd0az4fHf;q3Jlg~yVw=Ucr;=5V{Uqw2b-#L3OowL9U z9j+Ix`1q<;8v}WtQ-xXig+I)9(3;nXc|pGNB1^pvR0~0A$kl-?YrweTR}h1GVi

c)ijgxDm}8EsRXFt3h@+Ufr7@DN z^55r2UpdZvo*$)c`MJ_3zXBARbH%T}ifygzYy6g*WBtspGU<*Ccb`wpyW!Ui$gZ}y zo>MwK`K>f-62KfvO2{S zXF|ni6T=gB=C>=mF~5ojWS?I%DBt!ouB^&}v*S8G>5&(6>bM<0W9)PIeSXbv;v2lq zgZx&0)nJZqzUPEz=3RZouldy~VSciFe9|fxrs_KoD#u$hYz3BTu8Twxs@yt>*lp{< zm_XbpVEfL5#v}%x;+@AY<0*cV$ZF-248A&7CXCUG-9e@z7Va=V8J*&{q4I$n{~M-~K{qUmg-Y{N~tC__Y!6wZ`uS zAN=8SKnb`wARia}P{>}4q*mFJ2rt$xz9z}40>2@prKgMpJ4y?1MK zsu;8LLY(s8tNKp-L`??i35r}^567PuI=u8S&*EdFoy9Nf;48%{S#m8d=h|q*N!*Hw zE&QzCc2jn4u4(uar*pTPKCQ7DC)&Cs49?>3$7+X~)XJA`!=HT>p7`~r%@S~FvIWT% zL)t28t$h|BY!xpHnSQNXihG*>p${(0U;hi2mrwZcOUrZh0ee^UiT1oYO{3$5Hop*u zLXEN0l1qM=vD`rN)XOLJdon_5oHz3`AzpsrE1f=|*Mk1={U^)6{EcJ3kodUYZmX=p z&l4~2a)h&L*mG4|<3d+3_?Prr)`vgu$Y1U7EWIl2?@iUEd5K>;n9zxxlFNU^0vTLl zH@o9AcfQkuuVr{d?>6N1tv`70$?|*eKGqA1!uC8^rS(s+P1LOQ9lYFac+7nk_^^=}_9|LQHrRm;gm z#jgtmwd-2xd;fSm;rGSZd-@wbDeXS|)%sP&lv@b1qs`Sf43!0V?3qvsHeeF4^Q(*h z^}o7zxuRcU@`@_U0N4FIMxo}rPTLvJc{K#}XhYWmowJJ2$Yjbl`u)zkPnNIv?#GvR zeQ>x@oZ)FOm|m&l>_ivC(ek;URCk@4f5BINBIPcJedSknv#$7sL09O4r%@qb_M zz2et2d?)PSD|vhJv?jf^coe^7;*5D_(i{GoNjc@GFgNZjMJ5=HK91L-#6s_k5ZsDS zGS%RQ&sF+5eNE*3{W~3);ByDsjH9O)4$S@$?yR>?gy?){V`EPI$n>{$7kZJt&E|jq z@9tl&>KhB0wjiX?fvux_ph<@^P`xU#l~@YcVmvoP|52 zFCDST=db-|m-UT`(xE24+%n&4gZ%FnLi&Yo)!)!<`8*?XqEn@~PlG4oI{hPQc|SBA-3UqQo@Ok7n} zIAZ21l@78Rn`X^sw|ukiJP&AnypS?sjm)BYgRrvd_2vm*-zj>cKd@`Ab&91Yp=>6{)F%4)7auKu@lUJhnvWozKNZb^uG+`E@Y3=U zeK~|@uUf1nf;jWRpXQgYuqA_|MTZQJmcB;TNR^GlS{T8}iC6rO{IH|tWqO{uY5h}C zK^05FmfvX7IMk$1hE*ehH{+tKyHIa1DdB;;rJvHi z@XysN8q8vy7k-&z&tLr~zqICPT-#vO+|kk)bI{UP%}!$rHS^6TDD1uXt~a|@W*~+c z8vo^wJW;Rw34f4ZJkG`2_D~Yj%WRNd2O^Mwn=s<$0*s{9@EYCPT5v)bA~e(n|~6M0EUxGtnrcN&$s(s zzN8S(XWAcol9+ za@NCPqQw`HsBTqo#8>DWj&U^~+CTP~&69^IHqX$ty#E|%_>m7|XO7~asM|V+|Xy_l(fh&fm#RNST>VcoN?=6S_DPi%0~BG=sQt4-78)-@|b)lahBHa~PL<9jHj zNE~dl9PG02qUPM@QPu+cEDu-Af8%z}zB%Ihfge*{9Wd$&G+)E(=&9+o!^CjO`cwNdjVRH+WU`h_MXAOitJp5x3ifW{$igPf9iBj$(b=HI#x==`-hy-E&gI#->XR(BW&pMdcoR19-nNcPkY4s2bR7uK27u z;T-wi{Jv$d3tg^Khr|3zu!D-f$3GV1rd-BjB{h8+psmB&uHFO}3e<>-KnIym}P_oSC zslstp61Dm&1NiV|^pEbaNt}ZX!rh1GA<@OoA~K`yhAgd{@foOROsg!`F}gM(u1!jB zP-&PeM7Vk8W1#d^)-p1e`o(13g|c~w?dj`;4_bZu^_E|g3d=E{cLES;rdxmDH283uG=7WUKG<2~ea{IxU4q0( zBCeM((XD0e;O571>R|^u&Ev*jpsQGwzvm-2(K$^ICifY)?_e`E(umG-isbY(H;sFS z_TV{-u;uIR9OWMt?$V=eCxZbQ9k$3lC>2^A@xz~@XvD&(_uWN31AO=Zpf(=jB!lHh zOT3|j8)NsuFr00(J`~5*Aa@-yCcZDeY#2MK^7+byjE?yuYo4B|14zoWZPTeh8BIOF zi#LZ9-0pPpQq1&2arSg`YF@vQoGhb26RLwnlb*1L_^M-Vlx>giHItHpV-y+pt6ZEK z556G7lZ4?GS?qbNp_S;OAM&IlDs9+mIL@;^vinA)D6z3H9OHAVWxzHP_n^luSJ#<< zbsIty2lS^g(Tp%sL>_Jx%DMrbLPR&IRuN*2au@Mv3b3wQaDyVnmOp4Ma3Q*l1@}l- z7!@6xqcC>X;&3#^WC@2>d~Pt-WCFI;DSS*he8-yHfN>hl!&k7gZRoJWX*}IU_<3Dv zFh%O=_d;$wPTu#$88_QzeaYlJH`gOD^~u}%0AtVi0{v!P<5awgzdH2uJ`V|wUL*2lawezA2~fq&{P;mfB?8T6HUC*4h6A&Uoa8O-j$RT~z$aZBVg6 zzF?cyl6N zdHw?sJ7Tp$XXHMr#>SS7hWS(q4Vv|F6FxR`qoAKa__u1W&%AQI4T^VKan^IyU>zfs zE|$R$NQPNwnbWKcmi{dLjG5%b9r@2i8f!K??SvY4H+*lPY@EblJRiC1P#E;CqroIW z@amJ2xy(A56v{9|GuaTpMMj+DK>H#%Xah4-!k=}#^ zneQH-ALI49-brtya+(0Rs?MoH;W4xa=7q~HKFb7Z1nBuy5&@vrkTKXDY=saRII;oP z3R%&P2^nF-NYearIVR*J3O2Ys934KH3%!qF8Ezacu`vg0S*Oab^yt!p+xLq-xy5gM z#Kw5jI=`XA!CkZ&zAqE&VEj1=NFmPhl*4MSO=PEas`~e2-T71-1sApc|fu*Q}= zsYFnC_DZcy+zSDb@&j)&>t^-n;oK7;%>Y=GI zf;q6^#lf=W>#ky4S#ll)lVVQT_DO*_|C(c%5cIB9nT$1w zdZdwu#x~{=-+@S!Al?*`YqRX_$W)w|mL<42l`iKk-%cwYqIN?eH8`i)kL=}d1?JZx ztLCs2KGwvGug#(X==ud4yo;s5T!B+uNNV9YMyc!;d~C+efEeaJa{IVw7aDzJFOkR6 zSlJt<<>?A3vyx@)YW!;#RD~3cJ<+yt$FWi*K*_8K6|i@y5t3Ja zJ+H|ads>I+vjj95MRGK=^x>=qv2joEMXBp_IFN4`AdHaye#ZCSN+T3ki zEEWhGJ-%>&Q^eAnKgqhuJba{|Jl+AxddOr{Cxi+(@50!IbHi4?hjyY5LQ=XVPTEpb zyqVjwx1@vOf~d3GC@cCi=V6PSGqd|Ua>`SZ|JP5mkUUL?=|EPi{@-nlH?JLkAw z*sMbLgtgvL+o_1?*wJfZjcXpC5>GR~M4yu?y`l7N54Pg1hB01ME2+8Z!14qfU-Yz@ zpP&@C_lf&Q^@(4j;1EbkPV$`KhCay2t@XoalE&DO(HG;)bGsV$(1$|8a365@r{WKw zNW$FkEp^Sm<|7b9uV3Ad{N#D~L@0goVuYqx6L^T_<{Zg#=0otZT7J0Sg93< zJ_mX2IquB#Bm6s#^rsweb>du#$y5q2icb}=oNpi;{UA7T{^iK)*yGw5d6=pq_?*D>mRC&iQRDaItw;A9 zUwyN}YMcO55)^&3H9%p>YklyFuHBgRqrZ5o{^}Fg-RyE2Q&BkPr4P7!;2dsBBY5kZ z6MOo=-HSke#!JD&S`O^!e_!8v^T8YV)+p1?{L!gB{K1puy1vT%sWe=-JBLXqC(&~o zh8QdS8g_rYT88wPo<6+$(H>5CKO8#&q^#c>*j4hprAvR9e{%Kyt8YGf`?u>?8Tz14 zS1k!Et{sV(!ehcu#U^0M9yMmukRS`=W<1D5*Xuj%0?f#3B#i1AuV%Dk0a#p(np`Z z@Ny<>{{ZDV5+@v)mOs>&&;9Vv>-)pHaOkS3YygE%;ePHnZ!h`bKx(H9HZuLnZ`piM z2ii=ClLN3rsu>=c{+jNjKd(=0rLpid^!u4*y(mWJPG6kjm0Yv8i=0jt@0q$c?3SO6 zo`T_+i0(Myt98b;JQvD(PJ8@c_^spR4R6xbATVp;gA^fWJoolt6Viy=aHkR(bL6>a z0*u#QIOR-CHs#1eI_@gp{LgMJH~1i?ZcMM{ufkCb2He+@V%l*Br$@ccN`(OGk)9u)8Cl^IS$70>cnNtJOD;^adIv1mfzOH@{j*A zpUGT+)Iu&-&YD8$81J|E-`Afpo?Sod(=~-f1KG?W4N<>A4H|trX(W)6k{Oa&+m(#9NV~FpO<-jgq5FpLo=R80h%`t-tc094&kfl2?<-(g>J|r?=r^r}OA> zmp&f(`pX~wSI3@L@|*kMoPV!t)up3lQ3afNHGkNJ?ukAA%&S+P!*d|=aQo0Nz5YfK zKR4s_UId|>uzYyqbjJt5=GTt(Ez-yS$U9G{Cqm(9+ajN> zgT~ide(a0*RMefm>R_qQXttNTKUJiWa#G(o>gibbxL(-&eO>l^>-4Yw{;}#f=Ndog zTpjgwLr5GKkp=Bm^VjU9%39U~*@|iCk3RCfSN<|`f4G7d?}tSDTy`AIwQL?;#$97+ ztSvnwvYK=4p}Io0?fv>@g@5oyeJpBc$rtZF^xS26hCWZ4#Yok->p2VeHu^YSPUGG2k^A|XtmgmW>+a9E=9)4OCk5TSW^(Rd;pI_JfySLre zQLOv*sbCN46V?6wuS}=FN|eBT_p(bFq*`MXpIA`Vg(EMp(umI{;a4t?=!xmyYV?&H2P7PMKv=d+vjRBWh(As6Lj0Qcn$#3?!%y6`&&<3aj!!;n$@xk0 z*`QFf2~yb7*ZgYBR84)J;s=KZ&x_vE!tWtII60`G5(@|IFyHPr=5zVG<@(X_<1hTc z_kGCwAo)o&!Uw+XL*A!{f;S*LxN;y5=0e-ZrK)pdNED2liw(!iVbw-%n7!XMpG8kA zGUJMmr0RBj5-MyJddQOpL{O*s7%s{`6u+WXrgQwlI?smCIg$&Q{AYgqCt0wKb7$_% zm%{TugWsEv_{Fa|uJO;}cZ_9uLpG0)>jq*Vhu`WPlbLjiH(IU~Fm-o{X+n|rIebs+ zBK*FBMohVN%r4@=_@qH>4)KXqe5CL#cK)Tu;+Dei@z-rsKEYOe;uO{W-~*^lGv{e} zg4af91r84J?WZul<4pXy&Q9bMAD7uEiayKu@j6WtFdw~+#;%<5b$dDfR;X#?4us;} z-~EhV6zs>~=Rof`?o~=VM~9%M_?8J+n!&AcCV)?AP=;fE71{~UeEA>#S{QucDki=r zzHybu$j{hvT>Nr&n2+r=zY;+&dlw*cHh$KbFJ$UN=-6jIG7AR2vDH_c$iN1FmhpRt z?{%2s!?BZglURd~-k|DP8~&9Flv)o?mLI$Jz3h>-Z8i{UeJRS<(K9vL#!-~$F*1Sp z9>4-|wb7EC2gB>kF9$2`EI#_O(HBeOdGZy+=Ze2BPH_+Mi?qgP47=j(>kB=mJ%oMS z9r<0iE@an9F`Z)KGra&4x%#2EIrCiSSMf=2pI?~4w>$UPbpC{gT;8zlrl=Bb2 zc!MuoiVfHWSDf^|NDlF(^ZW;&*`LSHX6X1EeyW$cIeN{P*pA<}=H;OUB#~>P2l%!Y z!u69#KlsSz*U2UJ{M*;+{q-Mwz4pdlJGFtZ-+TGiS1Ql<#B&y|xO2F8BP#-G95X!= zS3AtF&0v5*jT?Lk8~!j1%0_T}otooBko6is#Sgz&6@Aj7$ONp`$^7Ks*zOGN$=Vl+ z!3WfQyRB%BY(65Ff(S*v1=yWtyJ{I0gB$4W-~OP!g>&~BlI$ss{JeWJ0Y~lvE4La}LgwmJ{B^=-^LrxrR*K+!NY34Y z%M z<9FfUS32e(gAJbEtbl5ub8iasSIo+HYW6cI2(;PPCVrX9hj6>)HIID%gYPzH@6^%v zv^{*@-@5)2n!;y#NN$bBu|)+fn^0}89(_q=8AGE|lG!A3qm}-*G$sPd@g2 zSN`*ry_F8$fdaX8yu3>5_^=Mm3a>SxDq|(W496V3gthog+!l-+gI^0x3>K~U0B9_I z@g1v9#%%cbQY(J<)|7{e%NhR$c6@0R)3;{wt|Y5hT-qAn?23((Ie*Is_;P_4Gx3j1 z3^!RMCcZ=O#~*wM_}}BBm6H6+W|(D1K9`SA_)O&v{7zZehxLm7tBQH}eC`H%|3AL+ zwv$WC=ZSiwBbOHn*aasRMW->jDp-wcQfvqt$sDPv&GGOq`KuGkd^o;c>O`@?JJE_` zdU788%6;TNa;;()znFK!uf=i(n|UXb!}$}T5F5S&N6!Fu`(`Au^2Zij=Z|V?HNBZ# z{Jg_J&>P3Qlh3>HhAVHIXs5)?*?J{TB9TPPY-Gp32p`^F3!lv=`TY2MT!#Dn_EX5YDwXjm4@%zo zyA%j0dpPZ8aUi>rp!dHqyG~d+l6Q>+x9T-*oC&4dQmFv;TYcH~Spj>DJ0esIt zzWNO+#A`{>E5i(Xk;Z0`sjgNLsQM^ePYfMu`tZTDpWqGSgiZetwnduxeT7P8ynTsi zel~9SC}kpn5&t6m<~Z?*-@e9Xw_7%@1cxGiwOUv!*ZAgV{^YpI;WyoHSsAi`#H6j9 zt$aSe;%xY&tQ7Q@%CCLw|GfH*c7B0V=63;TLHuy07aBFXpK@e@kz6>#YSGcv3{ghz zzVXF3=^Q@()T&z5KP7&Q>i!XZTNu&$kfkNQnO!8-_aDL+?R~C8sjF4t! z6x@c9tB)3F@nK85F<=By?G&Gi4}X@LiXJ2XmM&tvDMDVeZJcH{s6W+y1bgFn`9~ZXTFjEjziZ(}(o3vn z`%X>ZGshK%2W48h%Jnqix>9=bSGbGC-{Va~Hp{r_k-l2)R5e=9GXJFTue#GuTPtHLO_kpoE;{;<|N8ou=yCIP zN<{A~WY5T@7mLhsKlK)EER*b9LF?v{dT-&+=Hpvd_~PVB{13->Hs|DD_AU++MKR^? zVbs#s_)ceV^X6!`7vaB08NBAP@4xarcZzYI{jMLv_MN@||G4r!x9+?3(b^}k&qm0m zIJo%3!Mf<)XVROminu6NX7e>E)#+h2O$}L)eu$)~=3}XaGUgyZ_V8KMnK#)7zjPHp z_Ts=j%wK(OAJ%4maf|Pa51wLAKZDR6(r+-k<@J}An;-pDHxE9y+0Rj)g#6$aUwirP zX!kYxQ0mVy-QN2yL-92;)+QS*i|kvrv|fAPK+-?Jmin%y1ZS6N0LGw(w2!|y(vgZ*y#F}>^b>-1db)Nj=f;xC|Ft8@YI zMIq1nn~#0+?)d1{!hey9e+8a5izk@{Oplez2GHqrSUlSN&@^wrvVyP!giSlmuO%9r zW`jOGD83?gYTjdlCEZT%G_f_YKb`yp!)N?Qcc8y6-5c~LFW-9YpKRX@b^v?Vs?#fW z*DlT`JnOH$|Jl3C_q|fP=kqnu&(d`7^YSrkS5(VraZMu&zIv_2t3qXyto_-1d=_pk z^vbJk!~$p|XLVszAW2V_Pv+Y=r{jaEb~--#@C&o@YkYyT{(x!uak=@SdyXFer}KN5 zFTlMk$hvZOMZ0@2f4q3@#*LTjFKs?eK|fUioJEMtmjUO-<02&yOE|p|V-%X=6Xv@X(oCxjr1jf2;npdQ$tQM<2QW z=azp~pZ|S`@O0`r&8O4l#eLPLy7n@?{`u15<>(>(HP?sj)ax^gp0C0^Q@=iWK*f2c zD)fL#sXs~F-K&MVM;neWi6M8@tERwteOT%%cv{JMqtu2a&-F?ld~arKwAH@y=LKKw z#h-2EA?L&VSjQ(K-_mq$Dl8u&b4}hKRXUGo8jtD{dqj15STlZy(C<7sI)2CQ_~fnE k9@EG3{4s5ok?kb>|H;3ubeVRY^#A|>07*qoM6N<$f~C=$asU7T literal 0 HcmV?d00001 diff --git a/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/profile/backup_config.json b/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/profile/backup_config.json new file mode 100644 index 000000000..78f40ae7c --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/profile/backup_config.json @@ -0,0 +1,3 @@ +{ + "allowToBackupRestore": true +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/profile/main_pages.json b/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/profile/main_pages.json new file mode 100644 index 000000000..1898d94f5 --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/src/main/resources/base/profile/main_pages.json @@ -0,0 +1,5 @@ +{ + "src": [ + "pages/Index" + ] +} diff --git a/harmony-os/SherpaOnnxTts/entry/src/main/resources/en_US/element/string.json b/harmony-os/SherpaOnnxTts/entry/src/main/resources/en_US/element/string.json new file mode 100644 index 000000000..29b5d21cd --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/src/main/resources/en_US/element/string.json @@ -0,0 +1,16 @@ +{ + "string": [ + { + "name": "module_desc", + "value": "On-device text-to-speech with Next-gen Kaldi" + }, + { + "name": "EntryAbility_desc", + "value": "On-device text-to-speech with Next-gen Kaldi" + }, + { + "name": "EntryAbility_label", + "value": "TTS" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxTts/entry/src/main/resources/rawfile/.gitkeep b/harmony-os/SherpaOnnxTts/entry/src/main/resources/rawfile/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/harmony-os/SherpaOnnxTts/entry/src/main/resources/zh_CN/element/string.json b/harmony-os/SherpaOnnxTts/entry/src/main/resources/zh_CN/element/string.json new file mode 100644 index 000000000..c545b1b46 --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/src/main/resources/zh_CN/element/string.json @@ -0,0 +1,16 @@ +{ + "string": [ + { + "name": "module_desc", + "value": "使用新一代Kaldi进行本地离线语音合成" + }, + { + "name": "EntryAbility_desc", + "value": "使用新一代Kaldi进行本地离线语音合成" + }, + { + "name": "EntryAbility_label", + "value": "本地语音合成" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxTts/entry/src/ohosTest/ets/test/Ability.test.ets b/harmony-os/SherpaOnnxTts/entry/src/ohosTest/ets/test/Ability.test.ets new file mode 100644 index 000000000..8aa374977 --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/src/ohosTest/ets/test/Ability.test.ets @@ -0,0 +1,35 @@ +import hilog from '@ohos.hilog'; +import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from '@ohos/hypium'; + +export default function abilityTest() { + describe('ActsAbilityTest', () => { + // Defines a test suite. Two parameters are supported: test suite name and test suite function. + beforeAll(() => { + // Presets an action, which is performed only once before all test cases of the test suite start. + // This API supports only one parameter: preset action function. + }) + beforeEach(() => { + // Presets an action, which is performed before each unit test case starts. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: preset action function. + }) + afterEach(() => { + // Presets a clear action, which is performed after each unit test case ends. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: clear action function. + }) + afterAll(() => { + // Presets a clear action, which is performed after all test cases of the test suite end. + // This API supports only one parameter: clear action function. + }) + it('assertContain', 0, () => { + // Defines a test case. This API supports three parameters: test case name, filter parameter, and test case function. + hilog.info(0x0000, 'testTag', '%{public}s', 'it begin'); + let a = 'abc'; + let b = 'b'; + // Defines a variety of assertion methods, which are used to declare expected boolean conditions. + expect(a).assertContain(b); + expect(a).assertEqual(a); + }) + }) +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxTts/entry/src/ohosTest/ets/test/List.test.ets b/harmony-os/SherpaOnnxTts/entry/src/ohosTest/ets/test/List.test.ets new file mode 100644 index 000000000..794c7dc4e --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/src/ohosTest/ets/test/List.test.ets @@ -0,0 +1,5 @@ +import abilityTest from './Ability.test'; + +export default function testsuite() { + abilityTest(); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxTts/entry/src/ohosTest/module.json5 b/harmony-os/SherpaOnnxTts/entry/src/ohosTest/module.json5 new file mode 100644 index 000000000..55725a929 --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/src/ohosTest/module.json5 @@ -0,0 +1,13 @@ +{ + "module": { + "name": "entry_test", + "type": "feature", + "deviceTypes": [ + "phone", + "tablet", + "2in1" + ], + "deliveryWithInstall": true, + "installationFree": false + } +} diff --git a/harmony-os/SherpaOnnxTts/entry/src/test/List.test.ets b/harmony-os/SherpaOnnxTts/entry/src/test/List.test.ets new file mode 100644 index 000000000..bb5b5c373 --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/src/test/List.test.ets @@ -0,0 +1,5 @@ +import localUnitTest from './LocalUnit.test'; + +export default function testsuite() { + localUnitTest(); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxTts/entry/src/test/LocalUnit.test.ets b/harmony-os/SherpaOnnxTts/entry/src/test/LocalUnit.test.ets new file mode 100644 index 000000000..165fc1615 --- /dev/null +++ b/harmony-os/SherpaOnnxTts/entry/src/test/LocalUnit.test.ets @@ -0,0 +1,33 @@ +import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from '@ohos/hypium'; + +export default function localUnitTest() { + describe('localUnitTest', () => { + // Defines a test suite. Two parameters are supported: test suite name and test suite function. + beforeAll(() => { + // Presets an action, which is performed only once before all test cases of the test suite start. + // This API supports only one parameter: preset action function. + }); + beforeEach(() => { + // Presets an action, which is performed before each unit test case starts. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: preset action function. + }); + afterEach(() => { + // Presets a clear action, which is performed after each unit test case ends. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: clear action function. + }); + afterAll(() => { + // Presets a clear action, which is performed after all test cases of the test suite end. + // This API supports only one parameter: clear action function. + }); + it('assertContain', 0, () => { + // Defines a test case. This API supports three parameters: test case name, filter parameter, and test case function. + let a = 'abc'; + let b = 'b'; + // Defines a variety of assertion methods, which are used to declare expected boolean conditions. + expect(a).assertContain(b); + expect(a).assertEqual(a); + }); + }); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxTts/hvigor/hvigor-config.json5 b/harmony-os/SherpaOnnxTts/hvigor/hvigor-config.json5 new file mode 100644 index 000000000..06b278367 --- /dev/null +++ b/harmony-os/SherpaOnnxTts/hvigor/hvigor-config.json5 @@ -0,0 +1,22 @@ +{ + "modelVersion": "5.0.0", + "dependencies": { + }, + "execution": { + // "analyze": "normal", /* Define the build analyze mode. Value: [ "normal" | "advanced" | false ]. Default: "normal" */ + // "daemon": true, /* Enable daemon compilation. Value: [ true | false ]. Default: true */ + // "incremental": true, /* Enable incremental compilation. Value: [ true | false ]. Default: true */ + // "parallel": true, /* Enable parallel compilation. Value: [ true | false ]. Default: true */ + // "typeCheck": false, /* Enable typeCheck. Value: [ true | false ]. Default: false */ + }, + "logging": { + // "level": "info" /* Define the log level. Value: [ "debug" | "info" | "warn" | "error" ]. Default: "info" */ + }, + "debugging": { + // "stacktrace": false /* Disable stacktrace compilation. Value: [ true | false ]. Default: false */ + }, + "nodeOptions": { + // "maxOldSpaceSize": 8192 /* Enable nodeOptions maxOldSpaceSize compilation. Unit M. Used for the daemon process. Default: 8192*/ + // "exposeGC": true /* Enable to trigger garbage collection explicitly. Default: true*/ + } +} diff --git a/harmony-os/SherpaOnnxTts/hvigorfile.ts b/harmony-os/SherpaOnnxTts/hvigorfile.ts new file mode 100644 index 000000000..f3cb9f1a8 --- /dev/null +++ b/harmony-os/SherpaOnnxTts/hvigorfile.ts @@ -0,0 +1,6 @@ +import { appTasks } from '@ohos/hvigor-ohos-plugin'; + +export default { + system: appTasks, /* Built-in plugin of Hvigor. It cannot be modified. */ + plugins:[] /* Custom plugin to extend the functionality of Hvigor. */ +} diff --git a/harmony-os/SherpaOnnxTts/oh-package-lock.json5 b/harmony-os/SherpaOnnxTts/oh-package-lock.json5 new file mode 100644 index 000000000..f538ae290 --- /dev/null +++ b/harmony-os/SherpaOnnxTts/oh-package-lock.json5 @@ -0,0 +1,19 @@ +{ + "meta": { + "stableOrder": true + }, + "lockfileVersion": 3, + "ATTENTION": "THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.", + "specifiers": { + "@ohos/hypium@1.0.19": "@ohos/hypium@1.0.19" + }, + "packages": { + "@ohos/hypium@1.0.19": { + "name": "@ohos/hypium", + "version": "1.0.19", + "integrity": "sha512-cEjDgLFCm3cWZDeRXk7agBUkPqjWxUo6AQeiu0gEkb3J8ESqlduQLSIXeo3cCsm8U/asL7iKjF85ZyOuufAGSQ==", + "resolved": "https://ohpm.openharmony.cn/ohpm/@ohos/hypium/-/hypium-1.0.19.har", + "registryType": "ohpm" + } + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxTts/oh-package.json5 b/harmony-os/SherpaOnnxTts/oh-package.json5 new file mode 100644 index 000000000..a79d5300e --- /dev/null +++ b/harmony-os/SherpaOnnxTts/oh-package.json5 @@ -0,0 +1,9 @@ +{ + "modelVersion": "5.0.0", + "description": "Please describe the basic information.", + "dependencies": { + }, + "devDependencies": { + "@ohos/hypium": "1.0.19" + } +} diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets index b0695f3a0..2675c7b77 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets @@ -11,6 +11,7 @@ import { audio } from '@kit.AudioKit'; @Entry @Component struct Index { + @State title: string = 'Next-gen Kaldi: VAD + ASR'; @State currentIndex: number = 0; @State resultForFile: string = ''; @State progressForFile: number = 0; @@ -73,13 +74,11 @@ struct Index { }; const audioCapturerInfo: audio.AudioCapturerInfo = { - source: audio.SourceType.SOURCE_TYPE_MIC, - capturerFlags: 0 + source: audio.SourceType.SOURCE_TYPE_MIC, capturerFlags: 0 }; const audioCapturerOptions: audio.AudioCapturerOptions = { - streamInfo: audioStreamInfo, - capturerInfo: audioCapturerInfo + streamInfo: audioStreamInfo, capturerInfo: audioCapturerInfo }; audio.createAudioCapturer(audioCapturerOptions, (err, data) => { @@ -162,15 +161,9 @@ struct Index { @Builder TabBuilder(title: string, targetIndex: number, selectedImg: Resource, normalImg: Resource) { Column() { - Image(this.currentIndex == targetIndex ? selectedImg : normalImg) - .size({ width: 25, height: 25 }) - Text(title) - .fontColor(this.currentIndex == targetIndex ? '#28bff1' : '#8a8a8a') - } - .width('100%') - .height(50) - .justifyContent(FlexAlign.Center) - .onClick(() => { + Image(this.currentIndex == targetIndex ? selectedImg : normalImg).size({ width: 25, height: 25 }) + Text(title).fontColor(this.currentIndex == targetIndex ? '#28bff1' : '#8a8a8a') + }.width('100%').height(50).justifyContent(FlexAlign.Center).onClick(() => { this.currentIndex = targetIndex; this.controller.changeIndex(this.currentIndex); }) @@ -181,11 +174,7 @@ struct Index { Tabs({ barPosition: BarPosition.End, controller: this.controller }) { TabContent() { Column({ space: 10 }) { - Text('Next-gen Kaldi: VAD + ASR') - .fontColor('#182431') - .fontSize(25) - .lineHeight(41) - .fontWeight(500) + Text(this.title).fontSize(20).fontWeight(FontWeight.Bold); Button('Select .wav file (16kHz) ') .enabled(this.selectFileBtnEnabled) @@ -211,8 +200,7 @@ struct Index { if (this.workerInstance) { this.workerInstance.postMessage({ - msgType: 'non-streaming-asr-vad-decode', - filename: result[0], + msgType: 'non-streaming-asr-vad-decode', filename: result[0], }); } else { console.log(`this worker instance is undefined ${this.workerInstance}`); @@ -236,80 +224,86 @@ struct Index { }.width('100%').justifyContent(FlexAlign.Center) } - TextArea({ text: this.resultForFile }).width('100%').lineSpacing({ value: 10, unit: LengthUnit.VP }); - - } - .alignItems(HorizontalAlign.Center) - .justifyContent(FlexAlign.Start) + TextArea({ text: this.resultForFile }) + .width('100%') + .lineSpacing({ value: 10, unit: LengthUnit.VP }) + .height('100%'); + }.alignItems(HorizontalAlign.Center).justifyContent(FlexAlign.Start) }.tabBar(this.TabBuilder('From file', 0, $r('app.media.icon_doc'), $r('app.media.icon_doc_default'))) TabContent() { - Column() { - Button(this.message) - .enabled(this.micInitDone) - .onClick(() => { - console.log('clicked mic button'); - this.resultForMic = ''; - if (this.mic) { - if (this.micStarted) { - this.mic.stop(); - this.message = "Start recording"; - this.micStarted = false; - console.log('mic stopped'); - - const samples = this.flatten(this.sampleList); - let s = 0; - for (let i = 0; i < samples.length; ++i) { - s += samples[i]; - } - console.log(`samples ${samples.length}, sum: ${s}`); - - if (this.workerInstance) { - console.log('decode mic'); - this.workerInstance.postMessage({ - msgType: 'non-streaming-asr-vad-mic', - samples, - }); - } else { - console.log(`this worker instance is undefined ${this.workerInstance}`); - } + Column({ space: 10 }) { + Text(this.title).fontSize(20).fontWeight(FontWeight.Bold); + Button(this.message).enabled(this.micInitDone).onClick(() => { + console.log('clicked mic button'); + this.resultForMic = ''; + if (this.mic) { + if (this.micStarted) { + this.mic.stop(); + this.message = "Start recording"; + this.micStarted = false; + console.log('mic stopped'); + + const samples = this.flatten(this.sampleList); + let s = 0; + for (let i = 0; i < samples.length; ++i) { + s += samples[i]; + } + console.log(`samples ${samples.length}, sum: ${s}`); + + if (this.workerInstance) { + console.log('decode mic'); + this.workerInstance.postMessage({ + msgType: 'non-streaming-asr-vad-mic', samples, + }); } else { - this.sampleList = []; - this.mic.start(); - this.message = "Stop recording"; - this.micStarted = true; - console.log('mic started'); + console.log(`this worker instance is undefined ${this.workerInstance}`); } + } else { + this.sampleList = []; + this.mic.start(); + this.message = "Stop recording"; + this.micStarted = true; + console.log('mic started'); } - }); + } + }); Text(`Supported languages: ${this.lang}`) - TextArea({ text: this.resultForMic }).width('100%').lineSpacing({ value: 10, unit: LengthUnit.VP }); - } - .alignItems(HorizontalAlign.Center) - .justifyContent(FlexAlign.Start) + TextArea({ text: this.resultForMic }) + .width('100%') + .lineSpacing({ value: 10, unit: LengthUnit.VP }) + .width('100%') + .height('100%'); + }.alignItems(HorizontalAlign.Center).justifyContent(FlexAlign.Start) } .tabBar(this.TabBuilder('From mic', 1, $r('app.media.ic_public_input_voice'), $r('app.media.ic_public_input_voice_default'))) TabContent() { - Column() { - Text("Everything is open-sourced"); - Divider(); - Text("It runs locally, without accessing the network"); - Divider(); - Text("See also https://github.com/k2-fsa/sherpa-onnx"); - Divider(); - Text("and https://k2-fsa.github.io/sherpa/social-groups.html"); + Column({ space: 10 }) { + Text(this.title).fontSize(20).fontWeight(FontWeight.Bold); + TextArea({ + text: ` +Everyting is open-sourced. + +It runs locally, without accessing the network + +See also https://github.com/k2-fsa/sherpa-onnx + +新一代 Kaldi QQ 和微信交流群: 请看 + +https://k2-fsa.github.io/sherpa/social-groups.html + +微信公众号: 新一代 Kaldi + ` + }).width('100%').height('100%').focusable(false) }.justifyContent(FlexAlign.Start) - }.tabBar(this.TabBuilder('Help', 2, $r('app.media.info_circle'), - $r('app.media.info_circle_default'))) + }.tabBar(this.TabBuilder('Help', 2, $r('app.media.info_circle'), $r('app.media.info_circle_default'))) }.scrollable(false) - } - .width('100%') - .justifyContent(FlexAlign.Start) + }.width('100%').justifyContent(FlexAlign.Start) } private micCallback = (buffer: ArrayBuffer) => { diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/element/string.json b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/element/string.json index 09e201b54..652fac4cc 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/element/string.json +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/element/string.json @@ -2,19 +2,19 @@ "string": [ { "name": "module_desc", - "value": "VAD+ASR with Next-gen Kaldi" + "value": "On-device VAD+ASR with Next-gen Kaldi" }, { "name": "EntryAbility_desc", - "value": "VAD+ASR" + "value": "On-device VAD+ASR with Next-gen Kaldi" }, { "name": "EntryAbility_label", - "value": "VAD_ASR" + "value": "On-device speech recognition" }, { "name": "mic_reason", - "value": "access the microhone for speech recognition" + "value": "access the microhone for on-device speech recognition with Next-gen Kaldi" } ] } \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/en_US/element/string.json b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/en_US/element/string.json index f94595515..652fac4cc 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/en_US/element/string.json +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/en_US/element/string.json @@ -2,15 +2,19 @@ "string": [ { "name": "module_desc", - "value": "module description" + "value": "On-device VAD+ASR with Next-gen Kaldi" }, { "name": "EntryAbility_desc", - "value": "description" + "value": "On-device VAD+ASR with Next-gen Kaldi" }, { "name": "EntryAbility_label", - "value": "label" + "value": "On-device speech recognition" + }, + { + "name": "mic_reason", + "value": "access the microhone for on-device speech recognition with Next-gen Kaldi" } ] } \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/zh_CN/element/string.json b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/zh_CN/element/string.json index 597ecf95e..00384ae7f 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/zh_CN/element/string.json +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/zh_CN/element/string.json @@ -2,15 +2,19 @@ "string": [ { "name": "module_desc", - "value": "模块描述" + "value": "基于新一代Kaldi的本地语音识别" }, { "name": "EntryAbility_desc", - "value": "description" + "value": "基于新一代Kaldi的本地语音识别" }, { "name": "EntryAbility_label", - "value": "label" + "value": "本地语音识别" + }, + { + "name": "mic_reason", + "value": "使用新一代Kaldi, 访问麦克风进行本地语音识别 (不需要联网)" } ] } \ No newline at end of file diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index 166430da4..e25097809 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -1169,6 +1169,17 @@ SherpaOnnxOfflineTtsGenerateWithProgressCallback( return SherpaOnnxOfflineTtsGenerateInternal(tts, text, sid, speed, wrapper); } +const SherpaOnnxGeneratedAudio * +SherpaOnnxOfflineTtsGenerateWithProgressCallbackWithArg( + const SherpaOnnxOfflineTts *tts, const char *text, int32_t sid, float speed, + SherpaOnnxGeneratedAudioProgressCallbackWithArg callback, void *arg) { + auto wrapper = [callback, arg](const float *samples, int32_t n, + float progress) { + return callback(samples, n, progress, arg); + }; + return SherpaOnnxOfflineTtsGenerateInternal(tts, text, sid, speed, wrapper); +} + const SherpaOnnxGeneratedAudio *SherpaOnnxOfflineTtsGenerateWithCallbackWithArg( const SherpaOnnxOfflineTts *tts, const char *text, int32_t sid, float speed, SherpaOnnxGeneratedAudioCallbackWithArg callback, void *arg) { diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index e9cd5be0a..fde626e99 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -930,6 +930,9 @@ typedef int32_t (*SherpaOnnxGeneratedAudioCallbackWithArg)(const float *samples, typedef int32_t (*SherpaOnnxGeneratedAudioProgressCallback)( const float *samples, int32_t n, float p); +typedef int32_t (*SherpaOnnxGeneratedAudioProgressCallbackWithArg)( + const float *samples, int32_t n, float p, void *arg); + SHERPA_ONNX_API typedef struct SherpaOnnxOfflineTts SherpaOnnxOfflineTts; // Create an instance of offline TTS. The user has to use DestroyOfflineTts() @@ -964,11 +967,19 @@ SherpaOnnxOfflineTtsGenerateWithCallback( const SherpaOnnxOfflineTts *tts, const char *text, int32_t sid, float speed, SherpaOnnxGeneratedAudioCallback callback); +SHERPA_ONNX_API const SherpaOnnxGeneratedAudio * SherpaOnnxOfflineTtsGenerateWithProgressCallback( const SherpaOnnxOfflineTts *tts, const char *text, int32_t sid, float speed, + SherpaOnnxGeneratedAudioProgressCallback callback); +SHERPA_ONNX_API +const SherpaOnnxGeneratedAudio * +SherpaOnnxOfflineTtsGenerateWithProgressCallbackWithArg( + const SherpaOnnxOfflineTts *tts, const char *text, int32_t sid, float speed, + SherpaOnnxGeneratedAudioProgressCallbackWithArg callback, void *arg); + // Same as SherpaOnnxGeneratedAudioCallback but you can pass an additional // `void* arg` to the callback. SHERPA_ONNX_API const SherpaOnnxGeneratedAudio * diff --git a/sherpa-onnx/csrc/circular-buffer.cc b/sherpa-onnx/csrc/circular-buffer.cc index 2fd19cdfa..2ba81807b 100644 --- a/sherpa-onnx/csrc/circular-buffer.cc +++ b/sherpa-onnx/csrc/circular-buffer.cc @@ -22,8 +22,14 @@ CircularBuffer::CircularBuffer(int32_t capacity) { void CircularBuffer::Resize(int32_t new_capacity) { int32_t capacity = static_cast(buffer_.size()); if (new_capacity <= capacity) { +#if __OHOS__ + SHERPA_ONNX_LOGE( + "new_capacity (%{public}d) <= original capacity (%{public}d). Skip it.", + new_capacity, capacity); +#else SHERPA_ONNX_LOGE("new_capacity (%d) <= original capacity (%d). Skip it.", new_capacity, capacity); +#endif return; } @@ -90,10 +96,18 @@ void CircularBuffer::Push(const float *p, int32_t n) { int32_t size = Size(); if (n + size > capacity) { int32_t new_capacity = std::max(capacity * 2, n + size); +#if __OHOS__ + SHERPA_ONNX_LOGE( + "Overflow! n: %{public}d, size: %{public}d, n+size: %{public}d, " + "capacity: %{public}d. Increase " + "capacity to: %{public}d. (Original data is copied. No data loss!)", + n, size, n + size, capacity, new_capacity); +#else SHERPA_ONNX_LOGE( "Overflow! n: %d, size: %d, n+size: %d, capacity: %d. Increase " - "capacity to: %d", + "capacity to: %d. (Original data is copied. No data loss!)", n, size, n + size, capacity, new_capacity); +#endif Resize(new_capacity); capacity = new_capacity; diff --git a/sherpa-onnx/csrc/lexicon.cc b/sherpa-onnx/csrc/lexicon.cc index fe5e595b9..505ea37a8 100644 --- a/sherpa-onnx/csrc/lexicon.cc +++ b/sherpa-onnx/csrc/lexicon.cc @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -159,17 +160,26 @@ std::vector Lexicon::ConvertTextToTokenIdsChinese( words = ProcessHeteronyms(words); if (debug_) { - fprintf(stderr, "Input text in string: %s\n", text.c_str()); - fprintf(stderr, "Input text in bytes:"); + std::ostringstream os; + + os << "Input text in string: " << text << "\n"; + os << "Input text in bytes:"; for (uint8_t c : text) { - fprintf(stderr, " %02x", c); + os << " 0x" << std::setfill('0') << std::setw(2) << std::right << std::hex + << c; } - fprintf(stderr, "\n"); - fprintf(stderr, "After splitting to words:"); + os << "\n"; + os << "After splitting to words:"; for (const auto &w : words) { - fprintf(stderr, " %s", w.c_str()); + os << " " << w; } - fprintf(stderr, "\n"); + os << "\n"; + +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else + SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif } std::vector ans; @@ -259,17 +269,26 @@ std::vector Lexicon::ConvertTextToTokenIdsNotChinese( std::vector words = SplitUtf8(text); if (debug_) { - fprintf(stderr, "Input text (lowercase) in string: %s\n", text.c_str()); - fprintf(stderr, "Input text in bytes:"); + std::ostringstream os; + + os << "Input text (lowercase) in string: " << text << "\n"; + os << "Input text in bytes:"; for (uint8_t c : text) { - fprintf(stderr, " %02x", c); + os << " 0x" << std::setfill('0') << std::setw(2) << std::right << std::hex + << c; } - fprintf(stderr, "\n"); - fprintf(stderr, "After splitting to words:"); + os << "\n"; + os << "After splitting to words:"; for (const auto &w : words) { - fprintf(stderr, " %s", w.c_str()); + os << " " << w; } - fprintf(stderr, "\n"); + os << "\n"; + +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else + SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif } int32_t blank = token2id_.at(" "); diff --git a/sherpa-onnx/csrc/melo-tts-lexicon.cc b/sherpa-onnx/csrc/melo-tts-lexicon.cc index 29857824f..ec729cdb5 100644 --- a/sherpa-onnx/csrc/melo-tts-lexicon.cc +++ b/sherpa-onnx/csrc/melo-tts-lexicon.cc @@ -6,11 +6,21 @@ #include #include // NOLINT +#include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif #include "cppjieba/Jieba.hpp" #include "sherpa-onnx/csrc/file-utils.h" #include "sherpa-onnx/csrc/macros.h" +#include "sherpa-onnx/csrc/onnx-utils.h" #include "sherpa-onnx/csrc/symbol-table.h" #include "sherpa-onnx/csrc/text-utils.h" @@ -62,6 +72,60 @@ class MeloTtsLexicon::Impl { } } + template + Impl(Manager *mgr, const std::string &lexicon, const std::string &tokens, + const std::string &dict_dir, + const OfflineTtsVitsModelMetaData &meta_data, bool debug) + : meta_data_(meta_data), debug_(debug) { + std::string dict = dict_dir + "/jieba.dict.utf8"; + std::string hmm = dict_dir + "/hmm_model.utf8"; + std::string user_dict = dict_dir + "/user.dict.utf8"; + std::string idf = dict_dir + "/idf.utf8"; + std::string stop_word = dict_dir + "/stop_words.utf8"; + + AssertFileExists(dict); + AssertFileExists(hmm); + AssertFileExists(user_dict); + AssertFileExists(idf); + AssertFileExists(stop_word); + + jieba_ = + std::make_unique(dict, hmm, user_dict, idf, stop_word); + + { + auto buf = ReadFile(mgr, tokens); + + std::istrstream is(buf.data(), buf.size()); + InitTokens(is); + } + + { + auto buf = ReadFile(mgr, lexicon); + + std::istrstream is(buf.data(), buf.size()); + InitLexicon(is); + } + } + + template + Impl(Manager *mgr, const std::string &lexicon, const std::string &tokens, + const OfflineTtsVitsModelMetaData &meta_data, bool debug) + : meta_data_(meta_data), debug_(debug) { + { + auto buf = ReadFile(mgr, tokens); + + std::istrstream is(buf.data(), buf.size()); + InitTokens(is); + } + + { + auto buf = ReadFile(mgr, lexicon); + + std::istrstream is(buf.data(), buf.size()); + InitLexicon(is); + } + } + std::vector ConvertTextToTokenIds(const std::string &_text) const { std::string text = ToLowerCase(_text); // see @@ -84,17 +148,24 @@ class MeloTtsLexicon::Impl { jieba_->Cut(text, words, is_hmm); if (debug_) { - SHERPA_ONNX_LOGE("input text: %s", text.c_str()); - SHERPA_ONNX_LOGE("after replacing punctuations: %s", s.c_str()); - std::ostringstream os; std::string sep = ""; for (const auto &w : words) { os << sep << w; sep = "_"; } +#if __OHOS__ + SHERPA_ONNX_LOGE("input text: %{public}s", text.c_str()); + SHERPA_ONNX_LOGE("after replacing punctuations: %{public}s", s.c_str()); + + SHERPA_ONNX_LOGE("after jieba processing: %{public}s", + os.str().c_str()); +#else + SHERPA_ONNX_LOGE("input text: %s", text.c_str()); + SHERPA_ONNX_LOGE("after replacing punctuations: %s", s.c_str()); SHERPA_ONNX_LOGE("after jieba processing: %s", os.str().c_str()); +#endif } } else { words = SplitUtf8(text); @@ -102,7 +173,7 @@ class MeloTtsLexicon::Impl { if (debug_) { fprintf(stderr, "Input text in string (lowercase): %s\n", text.c_str()); fprintf(stderr, "Input text in bytes (lowercase):"); - for (uint8_t c : text) { + for (int8_t c : text) { fprintf(stderr, " %02x", c); } fprintf(stderr, "\n"); @@ -307,9 +378,48 @@ MeloTtsLexicon::MeloTtsLexicon(const std::string &lexicon, bool debug) : impl_(std::make_unique(lexicon, tokens, meta_data, debug)) {} +template +MeloTtsLexicon::MeloTtsLexicon(Manager *mgr, const std::string &lexicon, + const std::string &tokens, + const std::string &dict_dir, + const OfflineTtsVitsModelMetaData &meta_data, + bool debug) + : impl_(std::make_unique(mgr, lexicon, tokens, dict_dir, meta_data, + debug)) {} + +template +MeloTtsLexicon::MeloTtsLexicon(Manager *mgr, const std::string &lexicon, + const std::string &tokens, + const OfflineTtsVitsModelMetaData &meta_data, + bool debug) + : impl_(std::make_unique(mgr, lexicon, tokens, meta_data, debug)) {} + std::vector MeloTtsLexicon::ConvertTextToTokenIds( const std::string &text, const std::string & /*unused_voice = ""*/) const { return impl_->ConvertTextToTokenIds(text); } +#if __ANDROID_API__ >= 9 +template MeloTtsLexicon::MeloTtsLexicon( + AAssetManager *mgr, const std::string &lexicon, const std::string &tokens, + const std::string &dict_dir, const OfflineTtsVitsModelMetaData &meta_data, + bool debug); + +template MeloTtsLexicon::MeloTtsLexicon( + AAssetManager *mgr, const std::string &lexicon, const std::string &tokens, + const OfflineTtsVitsModelMetaData &meta_data, bool debug); +#endif + +#if __OHOS__ +template MeloTtsLexicon::MeloTtsLexicon( + NativeResourceManager *mgr, const std::string &lexicon, + const std::string &tokens, const std::string &dict_dir, + const OfflineTtsVitsModelMetaData &meta_data, bool debug); + +template MeloTtsLexicon::MeloTtsLexicon( + NativeResourceManager *mgr, const std::string &lexicon, + const std::string &tokens, const OfflineTtsVitsModelMetaData &meta_data, + bool debug); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/melo-tts-lexicon.h b/sherpa-onnx/csrc/melo-tts-lexicon.h index da0644be2..e91cf33f2 100644 --- a/sherpa-onnx/csrc/melo-tts-lexicon.h +++ b/sherpa-onnx/csrc/melo-tts-lexicon.h @@ -25,6 +25,16 @@ class MeloTtsLexicon : public OfflineTtsFrontend { MeloTtsLexicon(const std::string &lexicon, const std::string &tokens, const OfflineTtsVitsModelMetaData &meta_data, bool debug); + template + MeloTtsLexicon(Manager *mgr, const std::string &lexicon, + const std::string &tokens, const std::string &dict_dir, + const OfflineTtsVitsModelMetaData &meta_data, bool debug); + + template + MeloTtsLexicon(Manager *mgr, const std::string &lexicon, + const std::string &tokens, + const OfflineTtsVitsModelMetaData &meta_data, bool debug); + std::vector ConvertTextToTokenIds( const std::string &text, const std::string &unused_voice = "") const override; diff --git a/sherpa-onnx/csrc/offline-tts-vits-impl.h b/sherpa-onnx/csrc/offline-tts-vits-impl.h index 972303dd4..5ef79f69b 100644 --- a/sherpa-onnx/csrc/offline-tts-vits-impl.h +++ b/sherpa-onnx/csrc/offline-tts-vits-impl.h @@ -40,7 +40,11 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { tn_list_.reserve(files.size()); for (const auto &f : files) { if (config.model.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("rule fst: %{public}s", f.c_str()); +#else SHERPA_ONNX_LOGE("rule fst: %s", f.c_str()); +#endif } tn_list_.push_back(std::make_unique(f)); } @@ -57,7 +61,11 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { for (const auto &f : files) { if (config.model.debug) { +#if __OHOS__ SHERPA_ONNX_LOGE("rule far: %s", f.c_str()); +#else + SHERPA_ONNX_LOGE("rule far: %{public}s", f.c_str()); +#endif } std::unique_ptr> reader( fst::FarReader::Open(f)); @@ -88,7 +96,11 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { tn_list_.reserve(files.size()); for (const auto &f : files) { if (config.model.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("rule fst: %{public}s", f.c_str()); +#else SHERPA_ONNX_LOGE("rule fst: %s", f.c_str()); +#endif } auto buf = ReadFile(mgr, f); std::istrstream is(buf.data(), buf.size()); @@ -103,7 +115,11 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { for (const auto &f : files) { if (config.model.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("rule far: %{public}s", f.c_str()); +#else SHERPA_ONNX_LOGE("rule far: %s", f.c_str()); +#endif } auto buf = ReadFile(mgr, f); @@ -156,14 +172,22 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { std::string text = _text; if (config_.model.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("Raw text: %{public}s", text.c_str()); +#else SHERPA_ONNX_LOGE("Raw text: %s", text.c_str()); +#endif } if (!tn_list_.empty()) { for (const auto &tn : tn_list_) { text = tn->Normalize(text); if (config_.model.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("After normalizing: %{public}s", text.c_str()); +#else SHERPA_ONNX_LOGE("After normalizing: %s", text.c_str()); +#endif } } } @@ -226,10 +250,17 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { int32_t num_batches = x_size / batch_size; if (config_.model.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE( + "Text is too long. Split it into %{public}d batches. batch size: " + "%{public}d. Number of sentences: %{public}d", + num_batches, batch_size, x_size); +#else SHERPA_ONNX_LOGE( "Text is too long. Split it into %d batches. batch size: %d. Number " "of sentences: %d", num_batches, batch_size, x_size); +#endif } GeneratedAudio ans; @@ -255,7 +286,7 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { audio.samples.end()); if (callback) { should_continue = callback(audio.samples.data(), audio.samples.size(), - b * 1.0 / num_batches); + (b + 1) * 1.0 / num_batches); // Caution(fangjun): audio is freed when the callback returns, so users // should copy the data if they want to access the data after // the callback returns to avoid segmentation fault. @@ -297,6 +328,16 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { if (meta_data.frontend == "characters") { frontend_ = std::make_unique( mgr, config_.model.vits.tokens, meta_data); + } else if (meta_data.jieba && !config_.model.vits.dict_dir.empty() && + meta_data.is_melo_tts) { + frontend_ = std::make_unique( + mgr, config_.model.vits.lexicon, config_.model.vits.tokens, + config_.model.vits.dict_dir, model_->GetMetaData(), + config_.model.debug); + } else if (meta_data.is_melo_tts && meta_data.language == "English") { + frontend_ = std::make_unique( + mgr, config_.model.vits.lexicon, config_.model.vits.tokens, + model_->GetMetaData(), config_.model.debug); } else if ((meta_data.is_piper || meta_data.is_coqui || meta_data.is_icefall) && !config_.model.vits.data_dir.empty()) { diff --git a/sherpa-onnx/csrc/offline-tts-vits-model.cc b/sherpa-onnx/csrc/offline-tts-vits-model.cc index 38efc6204..eb605a7bd 100644 --- a/sherpa-onnx/csrc/offline-tts-vits-model.cc +++ b/sherpa-onnx/csrc/offline-tts-vits-model.cc @@ -144,7 +144,11 @@ class OfflineTtsVitsModel::Impl { ++i; } +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s\n", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s\n", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; // used in the macro below From 9352ccf7d5597b69baa4942ed72f4e3563e77667 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 4 Dec 2024 14:51:46 +0800 Subject: [PATCH 092/183] Release v1.10.33 (#1591) --- CHANGELOG.md | 17 ++++++++++++++++- CMakeLists.txt | 2 +- build-ios-shared.sh | 2 +- dart-api-examples/add-punctuations/pubspec.yaml | 2 +- dart-api-examples/audio-tagging/pubspec.yaml | 2 +- dart-api-examples/keyword-spotter/pubspec.yaml | 2 +- .../non-streaming-asr/pubspec.yaml | 2 +- .../speaker-diarization/pubspec.yaml | 2 +- .../speaker-identification/pubspec.yaml | 2 +- dart-api-examples/streaming-asr/pubspec.yaml | 2 +- dart-api-examples/tts/pubspec.yaml | 2 +- .../vad-with-non-streaming-asr/pubspec.yaml | 2 +- dart-api-examples/vad/pubspec.yaml | 2 +- flutter-examples/streaming_asr/pubspec.yaml | 4 ++-- flutter-examples/tts/pubspec.yaml | 4 ++-- flutter/sherpa_onnx/pubspec.yaml | 12 ++++++------ .../sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec | 2 +- .../macos/sherpa_onnx_macos.podspec | 2 +- harmony-os/SherpaOnnxHar/README.md | 4 ++-- .../SherpaOnnxHar/sherpa_onnx/BuildProfile.ets | 2 +- harmony-os/SherpaOnnxHar/sherpa_onnx/README.md | 5 +++-- .../SherpaOnnxHar/sherpa_onnx/oh-package.json5 | 2 +- harmony-os/SherpaOnnxTts/entry/oh-package.json5 | 2 +- harmony-os/SherpaOnnxVadAsr/entry/README.md | 2 +- .../SherpaOnnxVadAsr/entry/oh-package.json5 | 2 +- new-release.sh | 16 ++++++++-------- nodejs-addon-examples/package.json | 2 +- 27 files changed, 59 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6493a95e4..1a1ce29af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,23 @@ +## 1.10.33 + +* Add non-streaming ASR support for HarmonyOS. (#1564) +* Add streaming ASR support for HarmonyOS. (#1565) +* Fix building for Android (#1568) +* Publish `sherpa_onnx.har` for HarmonyOS (#1572) +* Add VAD+ASR demo for HarmonyOS (#1573) +* Fix publishing har packages for HarmonyOS (#1576) +* Add CI to build HAPs for HarmonyOS (#1578) +* Add microphone demo about VAD+ASR for HarmonyOS (#1581) +* Fix getting microphone permission for HarmonyOS VAD+ASR example (#1582) +* Add HarmonyOS support for text-to-speech. (#1584) +* Fix: support both old and new websockets request headers format (#1588) +* Add on-device tex-to-speech (TTS) demo for HarmonyOS (#1590) + ## 1.10.32 * Support cross-compiling for HarmonyOS (#1553) * HarmonyOS support for VAD. (#1561) -* Fix publishing flutter iOS app to appstore. +* Fix publishing flutter iOS app to appstore (#1563). ## 1.10.31 diff --git a/CMakeLists.txt b/CMakeLists.txt index f5b626b8c..cdbdfbc09 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ project(sherpa-onnx) # Remember to update # ./CHANGELOG.md # ./new-release.sh -set(SHERPA_ONNX_VERSION "1.10.32") +set(SHERPA_ONNX_VERSION "1.10.33") # Disable warning about # diff --git a/build-ios-shared.sh b/build-ios-shared.sh index 9ac67848e..1dee7d76d 100755 --- a/build-ios-shared.sh +++ b/build-ios-shared.sh @@ -242,7 +242,7 @@ for d in ios-arm64_x86_64-simulator ios-arm64; do CFBundlePackageType FMWK CFBundleShortVersionString - 1.10.32 + 1.10.33 CFBundleSupportedPlatforms iPhoneOS diff --git a/dart-api-examples/add-punctuations/pubspec.yaml b/dart-api-examples/add-punctuations/pubspec.yaml index fef4dd729..ebf064dbb 100644 --- a/dart-api-examples/add-punctuations/pubspec.yaml +++ b/dart-api-examples/add-punctuations/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.32 + sherpa_onnx: ^1.10.33 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/audio-tagging/pubspec.yaml b/dart-api-examples/audio-tagging/pubspec.yaml index fa359d89d..20288d20a 100644 --- a/dart-api-examples/audio-tagging/pubspec.yaml +++ b/dart-api-examples/audio-tagging/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.32 + sherpa_onnx: ^1.10.33 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/keyword-spotter/pubspec.yaml b/dart-api-examples/keyword-spotter/pubspec.yaml index ffa074a23..7951ff1a0 100644 --- a/dart-api-examples/keyword-spotter/pubspec.yaml +++ b/dart-api-examples/keyword-spotter/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.32 + sherpa_onnx: ^1.10.33 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/non-streaming-asr/pubspec.yaml b/dart-api-examples/non-streaming-asr/pubspec.yaml index 52ee859e6..d747ba126 100644 --- a/dart-api-examples/non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.32 + sherpa_onnx: ^1.10.33 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/speaker-diarization/pubspec.yaml b/dart-api-examples/speaker-diarization/pubspec.yaml index 6706e28d1..05ed44f91 100644 --- a/dart-api-examples/speaker-diarization/pubspec.yaml +++ b/dart-api-examples/speaker-diarization/pubspec.yaml @@ -8,7 +8,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.32 + sherpa_onnx: ^1.10.33 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/speaker-identification/pubspec.yaml b/dart-api-examples/speaker-identification/pubspec.yaml index 4b47846b8..64b1383df 100644 --- a/dart-api-examples/speaker-identification/pubspec.yaml +++ b/dart-api-examples/speaker-identification/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.32 + sherpa_onnx: ^1.10.33 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/streaming-asr/pubspec.yaml b/dart-api-examples/streaming-asr/pubspec.yaml index cfc9037ab..538641a74 100644 --- a/dart-api-examples/streaming-asr/pubspec.yaml +++ b/dart-api-examples/streaming-asr/pubspec.yaml @@ -11,7 +11,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.32 + sherpa_onnx: ^1.10.33 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/tts/pubspec.yaml b/dart-api-examples/tts/pubspec.yaml index 9cf7049bc..ba4c4de0c 100644 --- a/dart-api-examples/tts/pubspec.yaml +++ b/dart-api-examples/tts/pubspec.yaml @@ -8,7 +8,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.32 + sherpa_onnx: ^1.10.33 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml index ca0d08cad..014daad04 100644 --- a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.32 + sherpa_onnx: ^1.10.33 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad/pubspec.yaml b/dart-api-examples/vad/pubspec.yaml index cd1acfbd2..230e62677 100644 --- a/dart-api-examples/vad/pubspec.yaml +++ b/dart-api-examples/vad/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.32 + sherpa_onnx: ^1.10.33 path: ^1.9.0 args: ^2.5.0 diff --git a/flutter-examples/streaming_asr/pubspec.yaml b/flutter-examples/streaming_asr/pubspec.yaml index 680d9d346..da2cd2d10 100644 --- a/flutter-examples/streaming_asr/pubspec.yaml +++ b/flutter-examples/streaming_asr/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' -version: 1.10.32 +version: 1.10.33 topics: - speech-recognition @@ -31,7 +31,7 @@ dependencies: record: ^5.1.0 url_launcher: ^6.2.6 - sherpa_onnx: ^1.10.32 + sherpa_onnx: ^1.10.33 # sherpa_onnx: # path: ../../flutter/sherpa_onnx diff --git a/flutter-examples/tts/pubspec.yaml b/flutter-examples/tts/pubspec.yaml index f8eca471a..646db8906 100644 --- a/flutter-examples/tts/pubspec.yaml +++ b/flutter-examples/tts/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' # Remove this line if you wish to publish to pub.dev -version: 1.10.32 +version: 1.10.33 environment: sdk: ">=2.17.0 <4.0.0" @@ -18,7 +18,7 @@ dependencies: cupertino_icons: ^1.0.6 path_provider: ^2.1.3 path: ^1.9.0 - sherpa_onnx: ^1.10.32 + sherpa_onnx: ^1.10.33 # sherpa_onnx: # path: ../../flutter/sherpa_onnx url_launcher: 6.2.6 diff --git a/flutter/sherpa_onnx/pubspec.yaml b/flutter/sherpa_onnx/pubspec.yaml index f5da2dbb6..8f7057a47 100644 --- a/flutter/sherpa_onnx/pubspec.yaml +++ b/flutter/sherpa_onnx/pubspec.yaml @@ -17,7 +17,7 @@ topics: - voice-activity-detection # remember to change the version in ../sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec -version: 1.10.32 +version: 1.10.33 homepage: https://github.com/k2-fsa/sherpa-onnx @@ -30,23 +30,23 @@ dependencies: flutter: sdk: flutter - sherpa_onnx_android: ^1.10.32 + sherpa_onnx_android: ^1.10.33 # sherpa_onnx_android: # path: ../sherpa_onnx_android - sherpa_onnx_macos: ^1.10.32 + sherpa_onnx_macos: ^1.10.33 # sherpa_onnx_macos: # path: ../sherpa_onnx_macos - sherpa_onnx_linux: ^1.10.32 + sherpa_onnx_linux: ^1.10.33 # sherpa_onnx_linux: # path: ../sherpa_onnx_linux - sherpa_onnx_windows: ^1.10.32 + sherpa_onnx_windows: ^1.10.33 # sherpa_onnx_windows: # path: ../sherpa_onnx_windows - sherpa_onnx_ios: ^1.10.32 + sherpa_onnx_ios: ^1.10.33 # sherpa_onnx_ios: # path: ../sherpa_onnx_ios diff --git a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec index e840a2225..2ff5a781c 100644 --- a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec +++ b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec @@ -7,7 +7,7 @@ # https://groups.google.com/g/dart-ffi/c/nUATMBy7r0c Pod::Spec.new do |s| s.name = 'sherpa_onnx_ios' - s.version = '1.10.32' + s.version = '1.10.33' s.summary = 'A new Flutter FFI plugin project.' s.description = <<-DESC A new Flutter FFI plugin project. diff --git a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec index 100966033..a3ec33d16 100644 --- a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec +++ b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec @@ -4,7 +4,7 @@ # Pod::Spec.new do |s| s.name = 'sherpa_onnx_macos' - s.version = '1.10.32' + s.version = '1.10.33' s.summary = 'sherpa-onnx Flutter FFI plugin project.' s.description = <<-DESC sherpa-onnx Flutter FFI plugin project. diff --git a/harmony-os/SherpaOnnxHar/README.md b/harmony-os/SherpaOnnxHar/README.md index e775cedee..5896e7acf 100644 --- a/harmony-os/SherpaOnnxHar/README.md +++ b/harmony-os/SherpaOnnxHar/README.md @@ -22,9 +22,9 @@ Pre-built `har` packages can be found at You can also download it using ``` -wget https://ohpm.openharmony.cn/ohpm/sherpa_onnx/-/sherpa_onnx-1.10.32.har +wget https://ohpm.openharmony.cn/ohpm/sherpa_onnx/-/sherpa_onnx-1.10.33.har -# Please replace the version 1.10.32 if needed. +# Please replace the version 1.10.33 if needed. ``` You can also use diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets index c6564edc7..2558f09c8 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets @@ -1,7 +1,7 @@ /** * Use these variables when you tailor your ArkTS code. They must be of the const type. */ -export const HAR_VERSION = '1.10.32'; +export const HAR_VERSION = '1.10.33'; export const BUILD_MODE_NAME = 'debug'; export const DEBUG = true; export const TARGET_NAME = 'default'; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md b/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md index 8f8243981..e94741a01 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md @@ -23,7 +23,7 @@ or update your `oh-package.json5` to include the following: ``` "dependencies": { - "sherpa_onnx": "1.10.32", + "sherpa_onnx": "1.10.33", }, ``` @@ -33,7 +33,8 @@ Note that we recommend always using the latest version. | Demo | URL | Description| |------|-----|------------| -|SherpaOnnxVadAsr|[Address](https://github.com/k2-fsa/sherpa-onnx/tree/master/harmony-os/SherpaOnnxVadAsr)|It shows how to use VAD with a non-streaming ASR model for speech recognition| +|SherpaOnnxVadAsr|[Address](https://github.com/k2-fsa/sherpa-onnx/tree/master/harmony-os/SherpaOnnxVadAsr)|It shows how to use VAD with a non-streaming ASR model for on-device speech recognition without accessing the network | +|SherpaOnnxTts|[Address](https://github.com/k2-fsa/sherpa-onnx/tree/master/harmony-os/SherpaOnnxTts)|It shows how to use Next-gen Kaldi for on-device text-to-speech (TTS, i.e., speech synthesis)| # Documentation diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 index 4d622d6e4..7e123d17e 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 @@ -1,6 +1,6 @@ { "name": "sherpa_onnx", - "version": "1.10.32", + "version": "1.10.33", "description": "Speech-to-text, text-to-speech, and speaker diarization using Next-gen Kaldi without internet connection", "main": "Index.ets", "author": "The next-gen Kaldi team", diff --git a/harmony-os/SherpaOnnxTts/entry/oh-package.json5 b/harmony-os/SherpaOnnxTts/entry/oh-package.json5 index daff21b30..e57a8e40e 100644 --- a/harmony-os/SherpaOnnxTts/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxTts/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.32", + "sherpa_onnx": "1.10.33", } } diff --git a/harmony-os/SherpaOnnxVadAsr/entry/README.md b/harmony-os/SherpaOnnxVadAsr/entry/README.md index fcdfe710b..4862dd945 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/README.md +++ b/harmony-os/SherpaOnnxVadAsr/entry/README.md @@ -1,6 +1,6 @@ # Introduction -Please download ./sherpa_onnx-v1.10.32.har +Please download ./sherpa_onnx-v1.10.33.har from Hint: For users who have no access to huggingface, please use diff --git a/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 b/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 index 78a4d6b5b..b871adad1 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 @@ -7,7 +7,7 @@ "license": "", "dependencies": { // please see https://ohpm.openharmony.cn/#/cn/detail/sherpa_onnx - "sherpa_onnx": "1.10.32", + "sherpa_onnx": "1.10.33", } } diff --git a/new-release.sh b/new-release.sh index 2fc30371f..052e9ebf3 100755 --- a/new-release.sh +++ b/new-release.sh @@ -2,12 +2,12 @@ set -ex -sed -i.bak 's/1\.10\.31/1\.10\.32/g' ./build-ios-shared.sh -find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.31/1\.10\.32/g' {} \; -find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.31/1\.10\.32/g' {} \; -find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.31/1\.10\.32/g' {} \; -find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.31/1\.10\.32/g' {} \; -find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.31/1\.10\.32/g' {} \; +sed -i.bak 's/1\.10\.32/1\.10\.33/g' ./build-ios-shared.sh +find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.32/1\.10\.33/g' {} \; +find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.32/1\.10\.33/g' {} \; +find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.32/1\.10\.33/g' {} \; +find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.32/1\.10\.33/g' {} \; +find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.32/1\.10\.33/g' {} \; -find harmony-os -name "README.md" -type f -exec sed -i.bak 's/1\.10\.31/1\.10\.32/g' {} \; -find harmony-os -name oh-package.json5 -type f -exec sed -i.bak 's/1\.10\.31/1\.10\.32/g' {} \; +find harmony-os -name "README.md" -type f -exec sed -i.bak 's/1\.10\.32/1\.10\.33/g' {} \; +find harmony-os -name oh-package.json5 -type f -exec sed -i.bak 's/1\.10\.32/1\.10\.33/g' {} \; diff --git a/nodejs-addon-examples/package.json b/nodejs-addon-examples/package.json index 47bed5b88..12a5d80ca 100644 --- a/nodejs-addon-examples/package.json +++ b/nodejs-addon-examples/package.json @@ -1,5 +1,5 @@ { "dependencies": { - "sherpa-onnx-node": "^1.10.32" + "sherpa-onnx-node": "^1.10.33" } } From 84821b1f9929d39118725199284f6b77c73c0984 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 6 Dec 2024 10:11:18 +0800 Subject: [PATCH 093/183] Fix building node-addon package (#1598) --- .../sherpa_onnx/src/main/cpp/non-streaming-tts.cc | 2 ++ scripts/node-addon-api/CMakeLists.txt | 4 +--- scripts/node-addon-api/package.json | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc index 67f348e9b..ed1f3afb3 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc @@ -406,7 +406,9 @@ class TtsGenerateWorker : public Napi::AsyncWorker { for (auto d : _this->data_list_) { if (d->cancelled) { +#if __OHOS__ OH_LOG_INFO(LOG_APP, "TtsGenerate is cancelled"); +#endif return 0; } } diff --git a/scripts/node-addon-api/CMakeLists.txt b/scripts/node-addon-api/CMakeLists.txt index b71f7a0d4..4fdffbfc7 100644 --- a/scripts/node-addon-api/CMakeLists.txt +++ b/scripts/node-addon-api/CMakeLists.txt @@ -11,9 +11,7 @@ cmake_policy(SET CMP0042 NEW) project(sherpa-onnx) -set(CMAKE_CXX_STANDARD 14) - -add_definitions(-DNAPI_VERSION=3) +set(CMAKE_CXX_STANDARD 17) include_directories(${CMAKE_JS_INC}) diff --git a/scripts/node-addon-api/package.json b/scripts/node-addon-api/package.json index e7e1727e1..f0bb57d0d 100644 --- a/scripts/node-addon-api/package.json +++ b/scripts/node-addon-api/package.json @@ -3,8 +3,8 @@ "version": "1.0.0", "description": "Speech-to-text, text-to-speech, and speaker diarization using Next-gen Kaldi without internet connection", "dependencies": { - "cmake-js": "^6.0.0", - "node-addon-api": "^1.1.0", + "cmake-js": "^7.0.0", + "node-addon-api": "^8.3.0", "perf_hooks": "*" }, "scripts": { From 91a43cccffd0179ca8b59f2c236954d25480d717 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 6 Dec 2024 17:38:40 +0800 Subject: [PATCH 094/183] Update doc links for HarmonyOS (#1601) --- harmony-os/README.md | 9 +++++-- harmony-os/SherpaOnnxHar/README.md | 37 +++------------------------ harmony-os/SherpaOnnxTts/README.md | 5 ++++ harmony-os/SherpaOnnxVadAsr/README.md | 5 ++++ 4 files changed, 20 insertions(+), 36 deletions(-) create mode 100644 harmony-os/SherpaOnnxTts/README.md create mode 100644 harmony-os/SherpaOnnxVadAsr/README.md diff --git a/harmony-os/README.md b/harmony-os/README.md index 280258bd8..5b14bb7bb 100644 --- a/harmony-os/README.md +++ b/harmony-os/README.md @@ -2,8 +2,13 @@ - [./SherpaOnnxHar](./SherpaOnnxHar) It is for building `sherpa_onnx.har`. If you don't need to change the C++ or Typescript code of sherpa-onnx, then - you can download pre-built `sherpa_onnx.har` from us. Please refer to - our [doc](https://k2-fsa.github.io/sherpa/onnx) for how to download it. + you can download pre-built `sherpa_onnx.har` from us. Just run `ohpm install sherpa_onnx`. + Please refer to our [doc](https://k2-fsa.github.io/sherpa/onnx/harmony-os/how-to-build-har.html) + if you want to build `sherpa-onnx` from source. - [./SherpaOnnxVadAsr](./SherpaOnnxVadAsr) It shows how to use VAD + Non-streaming ASR for speech recognition. + Please see the doc at + +- [./SherpaOnnxTts](./SherpaOnnxTts) It shows how to run on-device text-to-speech. + Please see the doc at diff --git a/harmony-os/SherpaOnnxHar/README.md b/harmony-os/SherpaOnnxHar/README.md index 5896e7acf..a378f73cc 100644 --- a/harmony-os/SherpaOnnxHar/README.md +++ b/harmony-os/SherpaOnnxHar/README.md @@ -1,37 +1,6 @@ # Introduction -How to build `sherpa_onnx.har` from the command line: +How to build `sherpa_onnx.har` from the command line +---------------------------------------------------- -```bash -git clone https://github.com/k2-fsa/sherpa-onnx -cd sherpa-onnx -./build-ohos-arm64-v8a.sh -./build-ohos-x86-64.sh - -cd harmony-os/SherpaOnnxHar - -hvigorw clean --no-daemon - -hvigorw --mode module -p product=default -p module=sherpa_onnx@default assembleHar --analyze=normal --parallel --incremental --no-daemon - -ls -lh ./sherpa_onnx/build/default/outputs/default/sherpa_onnx.har -``` - -Pre-built `har` packages can be found at - - -You can also download it using -``` -wget https://ohpm.openharmony.cn/ohpm/sherpa_onnx/-/sherpa_onnx-1.10.33.har - -# Please replace the version 1.10.33 if needed. -``` - -You can also use -``` -ohpm install sherpa_onnx -``` -to install it. - -See also - +Please see https://k2-fsa.github.io/sherpa/onnx/harmony-os/how-to-build-har.html diff --git a/harmony-os/SherpaOnnxTts/README.md b/harmony-os/SherpaOnnxTts/README.md new file mode 100644 index 000000000..03fb76313 --- /dev/null +++ b/harmony-os/SherpaOnnxTts/README.md @@ -0,0 +1,5 @@ +# Introduction + +Please see +https://k2-fsa.github.io/sherpa/onnx/harmony-os/tts.html +for how to run code in this folder. diff --git a/harmony-os/SherpaOnnxVadAsr/README.md b/harmony-os/SherpaOnnxVadAsr/README.md new file mode 100644 index 000000000..9d9338e8a --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/README.md @@ -0,0 +1,5 @@ +# Introduction + +Please see +https://k2-fsa.github.io/sherpa/onnx/harmony-os/vad-asr.html +for how to run code in this folder. From a743a4400fa05d6a704377bde0e18f7c5d2f4dd3 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 9 Dec 2024 16:40:15 +0800 Subject: [PATCH 095/183] Add on-device real-time ASR demo for HarmonyOS (#1606) --- harmony-os/README.md | 3 + harmony-os/SherpaOnnxStreamingAsr/.gitignore | 12 + .../SherpaOnnxStreamingAsr/AppScope/app.json5 | 10 + .../resources/base/element/string.json | 8 + .../resources/base/media/app_icon.png | Bin 0 -> 2777 bytes .../build-profile.json5 | 40 ++ .../SherpaOnnxStreamingAsr/code-linter.json5 | 20 + .../SherpaOnnxStreamingAsr/entry/.gitignore | 6 + .../entry/build-profile.json5 | 33 ++ .../entry/hvigorfile.ts | 6 + .../entry/obfuscation-rules.txt | 23 + .../entry/oh-package-lock.json5 | 29 ++ .../entry/oh-package.json5 | 12 + .../main/ets/entryability/EntryAbility.ets | 43 ++ .../entrybackupability/EntryBackupAbility.ets | 12 + .../entry/src/main/ets/pages/Index.ets | 428 ++++++++++++++++++ .../entry/src/main/ets/pages/Permission.ets | 26 ++ .../main/ets/workers/StreamingAsrWorker.ets | 294 ++++++++++++ .../entry/src/main/module.json5 | 64 +++ .../main/resources/base/element/color.json | 8 + .../main/resources/base/element/string.json | 20 + .../main/resources/base/media/background.png | Bin 0 -> 57364 bytes .../main/resources/base/media/foreground.png | Bin 0 -> 12430 bytes .../src/main/resources/base/media/home.svg | 1 + .../main/resources/base/media/icon_doc.svg | 2 + .../main/resources/base/media/icon_mic.svg | 2 + .../src/main/resources/base/media/info.svg | 1 + .../resources/base/media/layered_image.json | 7 + .../main/resources/base/media/startIcon.png | Bin 0 -> 20093 bytes .../resources/base/profile/backup_config.json | 3 + .../resources/base/profile/main_pages.json | 5 + .../main/resources/en_US/element/string.json | 20 + .../entry/src/main/resources/rawfile/.gitkeep | 0 .../main/resources/zh_CN/element/string.json | 20 + .../src/ohosTest/ets/test/Ability.test.ets | 35 ++ .../entry/src/ohosTest/ets/test/List.test.ets | 5 + .../entry/src/ohosTest/module.json5 | 13 + .../entry/src/test/List.test.ets | 5 + .../entry/src/test/LocalUnit.test.ets | 33 ++ .../hvigor/hvigor-config.json5 | 22 + .../SherpaOnnxStreamingAsr/hvigorfile.ts | 6 + .../oh-package-lock.json5 | 19 + .../SherpaOnnxStreamingAsr/oh-package.json5 | 9 + .../entry/src/main/ets/pages/Index.ets | 8 +- .../base/media/ic_public_input_voice.png | Bin 474 -> 0 bytes .../media/ic_public_input_voice_default.png | Bin 438 -> 0 bytes .../main/resources/base/media/icon_doc.png | Bin 295 -> 0 bytes .../main/resources/base/media/icon_doc.svg | 2 + .../resources/base/media/icon_doc_default.png | Bin 295 -> 0 bytes .../main/resources/base/media/icon_mic.svg | 2 + .../src/main/resources/base/media/info.svg | 1 + .../main/resources/base/media/info_circle.png | Bin 636 -> 0 bytes .../base/media/info_circle_default.png | Bin 636 -> 0 bytes .../csrc/online-zipformer-transducer-model.cc | 17 + .../online-zipformer2-transducer-model.cc | 10 + 55 files changed, 1341 insertions(+), 4 deletions(-) create mode 100644 harmony-os/SherpaOnnxStreamingAsr/.gitignore create mode 100644 harmony-os/SherpaOnnxStreamingAsr/AppScope/app.json5 create mode 100644 harmony-os/SherpaOnnxStreamingAsr/AppScope/resources/base/element/string.json create mode 100644 harmony-os/SherpaOnnxStreamingAsr/AppScope/resources/base/media/app_icon.png create mode 100644 harmony-os/SherpaOnnxStreamingAsr/build-profile.json5 create mode 100644 harmony-os/SherpaOnnxStreamingAsr/code-linter.json5 create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/.gitignore create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/build-profile.json5 create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/hvigorfile.ts create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/obfuscation-rules.txt create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/oh-package-lock.json5 create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/entryability/EntryAbility.ets create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/pages/Index.ets create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/pages/Permission.ets create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/workers/StreamingAsrWorker.ets create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/main/module.json5 create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/element/color.json create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/element/string.json create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/background.png create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/foreground.png create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/home.svg create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/icon_doc.svg create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/icon_mic.svg create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/info.svg create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/layered_image.json create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/startIcon.png create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/profile/backup_config.json create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/profile/main_pages.json create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/en_US/element/string.json create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/rawfile/.gitkeep create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/zh_CN/element/string.json create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/ohosTest/ets/test/Ability.test.ets create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/ohosTest/ets/test/List.test.ets create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/ohosTest/module.json5 create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/test/List.test.ets create mode 100644 harmony-os/SherpaOnnxStreamingAsr/entry/src/test/LocalUnit.test.ets create mode 100644 harmony-os/SherpaOnnxStreamingAsr/hvigor/hvigor-config.json5 create mode 100644 harmony-os/SherpaOnnxStreamingAsr/hvigorfile.ts create mode 100644 harmony-os/SherpaOnnxStreamingAsr/oh-package-lock.json5 create mode 100644 harmony-os/SherpaOnnxStreamingAsr/oh-package.json5 delete mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/ic_public_input_voice.png delete mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/ic_public_input_voice_default.png delete mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_doc.png create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_doc.svg delete mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_doc_default.png create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_mic.svg create mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/info.svg delete mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/info_circle.png delete mode 100644 harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/info_circle_default.png diff --git a/harmony-os/README.md b/harmony-os/README.md index 5b14bb7bb..566e1a74c 100644 --- a/harmony-os/README.md +++ b/harmony-os/README.md @@ -10,5 +10,8 @@ VAD + Non-streaming ASR for speech recognition. Please see the doc at +- [./SherpaOnnxStreamingAsr](./SherpaOnnxStreamingAsr) It shows how to use + streaming ASR models for real-time on-device speech recognition. + - [./SherpaOnnxTts](./SherpaOnnxTts) It shows how to run on-device text-to-speech. Please see the doc at diff --git a/harmony-os/SherpaOnnxStreamingAsr/.gitignore b/harmony-os/SherpaOnnxStreamingAsr/.gitignore new file mode 100644 index 000000000..d2ff20141 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/.gitignore @@ -0,0 +1,12 @@ +/node_modules +/oh_modules +/local.properties +/.idea +**/build +/.hvigor +.cxx +/.clangd +/.clang-format +/.clang-tidy +**/.test +/.appanalyzer \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/AppScope/app.json5 b/harmony-os/SherpaOnnxStreamingAsr/AppScope/app.json5 new file mode 100644 index 000000000..88d26d63f --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/AppScope/app.json5 @@ -0,0 +1,10 @@ +{ + "app": { + "bundleName": "com.k2fsa.sherpa.onnx.streaming.asr", + "vendor": "example", + "versionCode": 1000000, + "versionName": "1.0.0", + "icon": "$media:app_icon", + "label": "$string:app_name" + } +} diff --git a/harmony-os/SherpaOnnxStreamingAsr/AppScope/resources/base/element/string.json b/harmony-os/SherpaOnnxStreamingAsr/AppScope/resources/base/element/string.json new file mode 100644 index 000000000..1bd22c6e6 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/AppScope/resources/base/element/string.json @@ -0,0 +1,8 @@ +{ + "string": [ + { + "name": "app_name", + "value": "SherpaOnnxStreamingAsr" + } + ] +} diff --git a/harmony-os/SherpaOnnxStreamingAsr/AppScope/resources/base/media/app_icon.png b/harmony-os/SherpaOnnxStreamingAsr/AppScope/resources/base/media/app_icon.png new file mode 100644 index 0000000000000000000000000000000000000000..a39445dc87828b76fed6d2ec470dd455c45319e3 GIT binary patch literal 2777 zcmV;~3MTc5P)9*YHQQH znh@I(s7WDIN`nJ+5@|<)iZcg=qN74U#DNnD1Se7u4fs(|1ivr?9ayP|B3iYCD$mfQ zCQ{S1n2)}^yxe#1J=_0pt-a1UPwQ^Z*?X_`Uu*sM+8<}X+baE^a`3seUF}?bEaiMO zrD`Qrd5@qw^epHZ>Df|p-qKBUEB%*?!m0{PHC6j|RplEgR~PkM5a^}N)Sfwi>W;Uz zdhwo_4HXBU%kRl^w@&7iKPx$e-n9%#IU!&oMI~iNsw0n19qSX;dS>I`G_G=WdcN9r z;_Rtv9XC<7kbL+HHxJ782T~pg05t)tf^>2vNJqfYt{YmqQDoBxkv+ra*BxxhcuK2v zm5%@Y)biQz)R8O%e=o%n${;ojY;EUP>`Qj6Cq)7GHm)C%2%^+hI;Z4T#a|oKIvshv z5H%!I+|I4PEXaXj04%ybsVolr%vhKnW7AEhC?eP!o1{y;8m2R#;}{6VZPc!+)ou0C zVWz$|1#2(|L5z%EYRxOzP+uLB>qYGuajX-<#^u;Kw&2uh&93)h>nHaFA%{&2PW=Nn zr?*a;gk3xvRhQIRa1de-!r(ss&?tRmZ=L2FMkhxI3lK6Jn<>5c*ID|@KU#^MCIo6> zpFA{|R(4fsBwHIW z9v!7G|7enadv4}~*8q_h%tD^j$7=PCnn0=dR0GKA(fgb9`2IRg6ksBIo+Gdw#|-3eSe=3tmDe zIqVN)tScM`0W#Z>2wc>~2Uv=3L)~D4gXqZtPQ8rifbYJqwkG>bv}95G7+};9Br?hF zWSa3b)X}z#79W9kukM%6-b_54WDJm~Ub=gsrJ0lz-8&lrQ7zfK1qzuZQkZvcE3|~S zZWmk0ETaNIHnMALn>akuvHLf5c4`y%!f+u>ZGp%@q_;T!`76_snc_?K;Wx%YpF;5K zw^F+BCYUPy`fpRif@5O@Im5cf?evD$>KlAgX;D0*HiO0`Yg3j;R4jT(9h(L_TsY6yxk*@ZBe%+dMqY=cB5oGs{D$QwOFbH)G$iVf<3Olcd7^#fr- zM{!ILWt#coT)s9ySkwDCPHv0oww8g8K%Yr{aR}msELVX(}JQr%F4Q8=KKn*OjSO*uSp;JK%GwhRF_K??vGC$ZqmJX z@+}8sQ)9Z}3*DiWl+L_7OXn_^{SW~2&C*b^;%IP!j$lkre7H&bMR1}7aTT*G8P}|G zHM1)hZDe{r_E3{{Y=d}}_PxJO_w4MaE4)$<<3JwzPdwPzfNemK(-X;{UCzmVr0zu5 zEnT}fzx)oVd!*W77`1Ig`DFcZ6TkPaI$hO1+`cGb$({ukz&{p4Ic-Xnwrg-KEkDqW zW3l$7Q`V$!1T(=QL1jgjIachdr75>-8>1A^h+;rTrD^nnwf?bw(Rang!*16Odj$Pn z@)JN5&5w~}ae6d};oa|&G>sT!)ixE#5;QW(u(=bqYHXcOflE%@t4A?n5fTUm0F~8_ zwpoz9rrU`@G=vsNjDRY(CrF(jIjqg8bd|CP02>eFag7T?u;C^ir+Z7YKmBYw;%%XdT2T}a$X4yR7EI;zaof3a)5Z;`OwVi%D?gbkBj!{;z2tOBSFk&E1DeiZXD**uvNqL}+|pO{ ztO$}2NMRit2ddU?)7Prq&*&H3X>&=E{-+j4iUz zrvL;?0$^@lyl=LHz9G^$SJV6ID__@7z->Bh>Vm=6AK&5bP%@heveHja5F@agGgUsY z@L@W2+^*NVoId0!kS~4XkWb%y;f}XBf>S+NIw9aHK;vN+4mJ|em)_QjIVfb2$;bwv zDKmoq6AThgKydS6Hs+UpKPWq|UA}s=UOEBZNM3oNT5qTAabY)X>L6jxfGDuu7&GD_ z=@@m?sJ-o2GS}&hNRW}-zHkr>o4&138@a8IC-FjSBxzjx?(*3@YmdmWGAd%0QvXzS zJ53JpX%Fp!=>v&`Hd7F@+Atw2vx9%^2M-APg0Jd|ePsRn3*B$#9Z5hCou4fo7W#SN z#}-@-N=##yQDh26pNzr9f*Q88krhI5@DHcf{dU-~PLSs}MvI4s1i|<=qxD~9`7>*~ znlw5lr$_6mTG4XbBNF_79BzvZ!TeIP)exdk3)kSHjYdW1P10ZJ_NCJSlrCuIU#gqw f88(SSw!Z%ZUzhC#9QlKF00000NkvXXu0mjfG$}gK literal 0 HcmV?d00001 diff --git a/harmony-os/SherpaOnnxStreamingAsr/build-profile.json5 b/harmony-os/SherpaOnnxStreamingAsr/build-profile.json5 new file mode 100644 index 000000000..8e63d9768 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/build-profile.json5 @@ -0,0 +1,40 @@ +{ + "app": { + "signingConfigs": [], + "products": [ + { + "name": "default", + "signingConfig": "default", + "compatibleSdkVersion": "4.0.0(10)", + "runtimeOS": "HarmonyOS", + "buildOption": { + "strictMode": { + "caseSensitiveCheck": true, + } + } + } + ], + "buildModeSet": [ + { + "name": "debug", + }, + { + "name": "release" + } + ] + }, + "modules": [ + { + "name": "entry", + "srcPath": "./entry", + "targets": [ + { + "name": "default", + "applyToProducts": [ + "default" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/code-linter.json5 b/harmony-os/SherpaOnnxStreamingAsr/code-linter.json5 new file mode 100644 index 000000000..77b31b517 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/code-linter.json5 @@ -0,0 +1,20 @@ +{ + "files": [ + "**/*.ets" + ], + "ignore": [ + "**/src/ohosTest/**/*", + "**/src/test/**/*", + "**/src/mock/**/*", + "**/node_modules/**/*", + "**/oh_modules/**/*", + "**/build/**/*", + "**/.preview/**/*" + ], + "ruleSet": [ + "plugin:@performance/recommended", + "plugin:@typescript-eslint/recommended" + ], + "rules": { + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/.gitignore b/harmony-os/SherpaOnnxStreamingAsr/entry/.gitignore new file mode 100644 index 000000000..e2713a277 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/.gitignore @@ -0,0 +1,6 @@ +/node_modules +/oh_modules +/.preview +/build +/.cxx +/.test \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/build-profile.json5 b/harmony-os/SherpaOnnxStreamingAsr/entry/build-profile.json5 new file mode 100644 index 000000000..4e4aa2bc1 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/build-profile.json5 @@ -0,0 +1,33 @@ +{ + "apiType": "stageMode", + "buildOption": { + "sourceOption": { + "workers": [ + './src/main/ets/workers/StreamingAsrWorker.ets' + ] + } + }, + "buildOptionSet": [ + { + "name": "release", + "arkOptions": { + "obfuscation": { + "ruleOptions": { + "enable": false, + "files": [ + "./obfuscation-rules.txt" + ] + } + } + } + }, + ], + "targets": [ + { + "name": "default" + }, + { + "name": "ohosTest", + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/hvigorfile.ts b/harmony-os/SherpaOnnxStreamingAsr/entry/hvigorfile.ts new file mode 100644 index 000000000..c6edcd904 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/hvigorfile.ts @@ -0,0 +1,6 @@ +import { hapTasks } from '@ohos/hvigor-ohos-plugin'; + +export default { + system: hapTasks, /* Built-in plugin of Hvigor. It cannot be modified. */ + plugins:[] /* Custom plugin to extend the functionality of Hvigor. */ +} diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/obfuscation-rules.txt b/harmony-os/SherpaOnnxStreamingAsr/entry/obfuscation-rules.txt new file mode 100644 index 000000000..272efb6ca --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/obfuscation-rules.txt @@ -0,0 +1,23 @@ +# Define project specific obfuscation rules here. +# You can include the obfuscation configuration files in the current module's build-profile.json5. +# +# For more details, see +# https://developer.huawei.com/consumer/cn/doc/harmonyos-guides-V5/source-obfuscation-V5 + +# Obfuscation options: +# -disable-obfuscation: disable all obfuscations +# -enable-property-obfuscation: obfuscate the property names +# -enable-toplevel-obfuscation: obfuscate the names in the global scope +# -compact: remove unnecessary blank spaces and all line feeds +# -remove-log: remove all console.* statements +# -print-namecache: print the name cache that contains the mapping from the old names to new names +# -apply-namecache: reuse the given cache file + +# Keep options: +# -keep-property-name: specifies property names that you want to keep +# -keep-global-name: specifies names that you want to keep in the global scope + +-enable-property-obfuscation +-enable-toplevel-obfuscation +-enable-filename-obfuscation +-enable-export-obfuscation \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package-lock.json5 b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package-lock.json5 new file mode 100644 index 000000000..ce848c049 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package-lock.json5 @@ -0,0 +1,29 @@ +{ + "meta": { + "stableOrder": true + }, + "lockfileVersion": 3, + "ATTENTION": "THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.", + "specifiers": { + "libsherpa_onnx.so@../oh_modules/.ohpm/sherpa_onnx@1.10.33/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx": "libsherpa_onnx.so@../oh_modules/.ohpm/sherpa_onnx@1.10.33/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx", + "sherpa_onnx@1.10.33": "sherpa_onnx@1.10.33" + }, + "packages": { + "libsherpa_onnx.so@../oh_modules/.ohpm/sherpa_onnx@1.10.33/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx": { + "name": "libsherpa_onnx.so", + "version": "1.0.0", + "resolved": "../oh_modules/.ohpm/sherpa_onnx@1.10.33/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx", + "registryType": "local" + }, + "sherpa_onnx@1.10.33": { + "name": "sherpa_onnx", + "version": "1.10.33", + "integrity": "sha512-cmZ8zwOMx4qmDvOjF1/PL6/suBgReanSf5XdQTuMWWZ6qN74rynODHrt4C+Qz754MTXg0q/phAKeVjGA4rHHSA==", + "resolved": "https://ohpm.openharmony.cn/ohpm/sherpa_onnx/-/sherpa_onnx-1.10.33.har", + "registryType": "ohpm", + "dependencies": { + "libsherpa_onnx.so": "file:./src/main/cpp/types/libsherpa_onnx" + } + } + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 new file mode 100644 index 000000000..e57a8e40e --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 @@ -0,0 +1,12 @@ +{ + "name": "entry", + "version": "1.0.0", + "description": "Please describe the basic information.", + "main": "", + "author": "", + "license": "", + "dependencies": { + "sherpa_onnx": "1.10.33", + } +} + diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/entryability/EntryAbility.ets b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/entryability/EntryAbility.ets new file mode 100644 index 000000000..679d91453 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/entryability/EntryAbility.ets @@ -0,0 +1,43 @@ +import AbilityConstant from '@ohos.app.ability.AbilityConstant'; +import hilog from '@ohos.hilog'; +import UIAbility from '@ohos.app.ability.UIAbility'; +import Want from '@ohos.app.ability.Want'; +import window from '@ohos.window'; + +export default class EntryAbility extends UIAbility { + onCreate(want: Want, launchParam: AbilityConstant.LaunchParam): void { + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onCreate'); + } + + onDestroy(): void { + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onDestroy'); + } + + onWindowStageCreate(windowStage: window.WindowStage): void { + // Main window is created, set main page for this ability + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onWindowStageCreate'); + + windowStage.loadContent('pages/Index', (err) => { + if (err.code) { + hilog.error(0x0000, 'testTag', 'Failed to load the content. Cause: %{public}s', JSON.stringify(err) ?? ''); + return; + } + hilog.info(0x0000, 'testTag', 'Succeeded in loading the content.'); + }); + } + + onWindowStageDestroy(): void { + // Main window is destroyed, release UI related resources + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onWindowStageDestroy'); + } + + onForeground(): void { + // Ability has brought to foreground + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onForeground'); + } + + onBackground(): void { + // Ability has back to background + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onBackground'); + } +} diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets new file mode 100644 index 000000000..d2c48b421 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets @@ -0,0 +1,12 @@ +import hilog from '@ohos.hilog'; +import BackupExtensionAbility, { BundleVersion } from '@ohos.application.BackupExtensionAbility'; + +export default class EntryBackupAbility extends BackupExtensionAbility { + async onBackup() { + hilog.info(0x0000, 'testTag', 'onBackup ok'); + } + + async onRestore(bundleVersion: BundleVersion) { + hilog.info(0x0000, 'testTag', 'onRestore ok %{public}s', JSON.stringify(bundleVersion)); + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/pages/Index.ets b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/pages/Index.ets new file mode 100644 index 000000000..3a08b9213 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/pages/Index.ets @@ -0,0 +1,428 @@ +import { LengthUnit } from '@kit.ArkUI'; +import worker, { MessageEvents } from '@ohos.worker'; +import { BusinessError } from '@kit.BasicServicesKit'; +import { picker } from '@kit.CoreFileKit'; +import systemTime from '@ohos.systemTime'; +import { Permissions } from '@kit.AbilityKit'; +import { allAllowed, requestPermissions } from './Permission'; +import { audio } from '@kit.AudioKit'; +import fs from '@ohos.file.fs'; + + +function savePcmToWav(filename: string, samples: Int16Array, sampleRate: number) { + const fp = fs.openSync(filename, fs.OpenMode.READ_WRITE | fs.OpenMode.CREATE); + + const header = new ArrayBuffer(44); + const view = new DataView(header); + + // http://soundfile.sapp.org/doc/WaveFormat/ + // F F I R + view.setUint32(0, 0x46464952, true); // chunkID + view.setUint32(4, 36 + samples.length * 2, true); // chunkSize // E V A W + view.setUint32(8, 0x45564157, true); // format // // t m f + view.setUint32(12, 0x20746d66, true); // subchunk1ID + view.setUint32(16, 16, true); // subchunk1Size, 16 for PCM + view.setUint32(20, 1, true); // audioFormat, 1 for PCM + view.setUint16(22, 1, true); // numChannels: 1 channel + view.setUint32(24, sampleRate, true); // sampleRate + view.setUint32(28, sampleRate * 2, true); // byteRate + view.setUint16(32, 2, true); // blockAlign + view.setUint16(34, 16, true); // bitsPerSample + view.setUint32(36, 0x61746164, true); // Subchunk2ID + view.setUint32(40, samples.length * 2, true); // subchunk2Size + + fs.writeSync(fp.fd, new Uint8Array(header).buffer, { length: header.byteLength }); + fs.writeSync(fp.fd, samples.buffer, { length: samples.buffer.byteLength }); + + fs.closeSync(fp.fd); +} + +function toInt16Samples(samples: Float32Array): Int16Array { + const int16Samples = new Int16Array(samples.length); + for (let i = 0; i < samples.length; ++i) { + let s = samples[i] * 32767; + s = s > 32767 ? 32767 : s; + s = s < -32768 ? -32768 : s; + int16Samples[i] = s; + } + + return int16Samples; +} + + +@Entry +@Component +struct Index { + @State title: string = 'Next-gen Kaldi: Real-time speech recognition'; + @State titleFontSize: number = 15; + @State currentIndex: number = 0; + @State lang: string = 'English'; + @State resultForFile: string = '' + @State resultForMic: string = '' + @State selectFileBtnEnabled: boolean = false; + @State micBtnCaption: string = 'Start'; + @State micStarted: boolean = false; + @State micAllowed: boolean = false; + @State micBtnEnabled: boolean = false; + @State micSaveBtnCaption: string = 'Save recorded audio'; + @State micSaveBtnEnabled: boolean = false; + @State info: string = ''; + @State micInfo: string = ''; + @State micInitDone: boolean = false; + private resultListForMic: string[] = []; + private controller: TabsController = new TabsController(); + private workerInstance?: worker.ThreadWorker + private readonly scriptURL: string = 'entry/ets/workers/StreamingAsrWorker.ets' + private startTime: number = 0; + private stopTime: number = 0; + private sampleRate: number = 48000; + private sampleList: Float32Array[] = [] + private mic?: audio.AudioCapturer; + + flatten(samples: Float32Array[]): Float32Array { + let n = 0; + for (let i = 0; i < samples.length; ++i) { + n += samples[i].length; + } + + const ans: Float32Array = new Float32Array(n); + let offset: number = 0; + for (let i = 0; i < samples.length; ++i) { + ans.set(samples[i], offset); + offset += samples[i].length; + } + + return ans; + } + + async initMic() { + const permissions: Permissions[] = ["ohos.permission.MICROPHONE"]; + let allowed: boolean = await allAllowed(permissions); + if (!allowed) { + console.log("request to access the microphone"); + const status: boolean = await requestPermissions(permissions); + + if (!status) { + console.error('access to microphone is denied') + this.resultForMic = "Failed to get microphone permission. Please retry"; + return; + } + + allowed = await allAllowed(permissions); + if (!allowed) { + console.error('failed to get microphone permission'); + this.resultForMic = "Failed to get microphone permission. Please retry"; + return; + } + this.micAllowed = true; + } else { + console.log("allowed to access microphone"); + this.micAllowed = true; + } + + const audioStreamInfo: audio.AudioStreamInfo = { + samplingRate: this.sampleRate, + channels: audio.AudioChannel.CHANNEL_1, + sampleFormat: audio.AudioSampleFormat.SAMPLE_FORMAT_S16LE, + encodingType: audio.AudioEncodingType.ENCODING_TYPE_RAW, + }; + + const audioCapturerInfo: audio.AudioCapturerInfo = { + source: audio.SourceType.SOURCE_TYPE_MIC, capturerFlags: 0 + }; + + const audioCapturerOptions: audio.AudioCapturerOptions = { + streamInfo: audioStreamInfo, capturerInfo: audioCapturerInfo + + }; + audio.createAudioCapturer(audioCapturerOptions, (err, data) => { + if (err) { + console.error(`error code is ${err.code}, error message is ${err.message}`); + this.resultForMic = 'Failed to init microphone'; + } else { + console.info(`init mic successfully`); + this.mic = data; + this.mic.on('readData', this.micCallback); + } + }); + } + + async aboutToAppear() { + this.workerInstance = new worker.ThreadWorker(this.scriptURL, { + name: 'Streaming ASR worker' + }); + + this.workerInstance.onmessage = (e: MessageEvents) => { + const msgType = e.data['msgType'] as string; + console.log(`received msg from worker: ${msgType}`); + + if (msgType == 'init-streaming-asr-done') { + this.selectFileBtnEnabled = true; + this.micBtnEnabled = true; + this.info = `Initializing done.\n\nPlease select a wave file of 16kHz in language ${this.lang}`; + this.micInfo = `Initializing done.\n\nPlease click Start and speak`; + } + + if (msgType == 'streaming-asr-decode-file-done') { + const text = e.data['text'] as string; + this.resultForFile = text; + this.selectFileBtnEnabled = true; + + systemTime.getRealTime((err, data) => { + if (err) { + console.log('Failed to get stop time'); + } else { + this.stopTime = data; + + const audioDuration = e.data['duration'] as number; + const elapsedSeconds = (this.stopTime - this.startTime) / 1000; + const RTF = elapsedSeconds / audioDuration; + this.info = `Audio duration: ${audioDuration.toFixed(2)} s +Elapsed: ${elapsedSeconds.toFixed(2)} s +RTF = ${elapsedSeconds.toFixed(2)}/${audioDuration.toFixed(2)} = ${RTF.toFixed(3)} +`; + } + }); + } + + if (msgType == 'streaming-asr-decode-mic-result') { + const text = e.data['text'] as string; + if (text.trim() == '') { + return; + } + + const isEndpoint = e.data['isEndpoint'] as boolean; + + let s = ''; + let i = 0; + for (; i < this.resultListForMic.length; ++i) { + s += `${i}: ${this.resultListForMic[i]}\n` + } + + s += `${i}: ${text}`; + this.resultForMic = s; + + if (isEndpoint) { + this.resultListForMic.push(text); + } + } + }; + + const context = getContext(); + this.workerInstance.postMessage({ msgType: 'init-streaming-asr', context }); + this.info = 'Initializing ASR model.\nPlease wait'; + this.micInfo = 'Initializing ASR model.\nPlease wait'; + + await this.initMic(); + } + + @Builder + TabBuilder(title: string, targetIndex: number, selectedImg: Resource, normalImg: Resource) { + Column() { + Image(this.currentIndex == targetIndex ? selectedImg : normalImg).size({ width: 25, height: 25 }) + Text(title).fontColor(this.currentIndex == targetIndex ? '#28bff1' : '#8a8a8a') + }.width('100%').height(50).justifyContent(FlexAlign.Center).onClick(() => { + this.currentIndex = targetIndex; + this.controller.changeIndex(this.currentIndex); + }) + } + + build() { + Column() { + Tabs({ barPosition: BarPosition.End, controller: this.controller }) { + TabContent() { + Column({ space: 10 }) { + Text(this.title).fontSize(this.titleFontSize).fontWeight(FontWeight.Bold); + Button('Select .wav file (16kHz) ') + .enabled(this.selectFileBtnEnabled) + .fontSize(13) + .width(296) + .height(60) + .onClick(() => { + this.resultForFile = ''; + this.info = ''; + this.selectFileBtnEnabled = false; + + const documentSelectOptions = new picker.DocumentSelectOptions(); + documentSelectOptions.maxSelectNumber = 1; + documentSelectOptions.fileSuffixFilters = ['.wav']; + const documentViewPicker = new picker.DocumentViewPicker(); + + documentViewPicker.select(documentSelectOptions).then((result: Array) => { + console.log(`select file result: ${result}`); + + if (!result[0]) { + this.resultForFile = 'Please select a file to decode'; + this.selectFileBtnEnabled = true; + return; + } + + if (this.workerInstance) { + systemTime.getRealTime((err, data) => { + if (err) { + console.log('Failed to get start time'); + } else { + this.startTime = data; + } + }); + + this.workerInstance.postMessage({ + msgType: 'streaming-asr-decode-file', filename: result[0], + }); + this.info = `Decoding ${result[0]} ... ...`; + } else { + console.log(`this worker instance is undefined ${this.workerInstance}`); + } + + }).catch((err: BusinessError) => { + console.error(`Failed to select file, code is ${err.code}, message is ${err.message}`); + this.selectFileBtnEnabled = true; + }) + }) + + Text(`Supported languages: ${this.lang}`); + if (this.info != '') { + TextArea({ text: this.info }).focusable(false); + } + TextArea({ text: this.resultForFile }) + .width('100%') + .lineSpacing({ value: 10, unit: LengthUnit.VP }) + .height('100%'); + } + }.tabBar(this.TabBuilder('From file', 0, $r('app.media.icon_doc'), $r('app.media.icon_doc'))) + + TabContent() { + Column({ space: 10 }) { + Text(this.title).fontSize(this.titleFontSize).fontWeight(FontWeight.Bold); + Button(this.micBtnCaption) + .enabled(this.micBtnEnabled) + .fontSize(13) + .width(296) + .height(60) + .onClick(() => { + this.micInfo = ''; + if (this.mic) { + if (this.micStarted) { + this.micStarted = false; + this.micBtnCaption = 'Start'; + this.mic.stop(); + this.micSaveBtnEnabled = true; + + if (this.workerInstance) { + this.workerInstance.postMessage({ + msgType: 'streaming-asr-decode-mic-stop' + }); + } + } else { + this.micStarted = true; + this.micSaveBtnEnabled = false; + this.micBtnCaption = 'Stop'; + this.resultForMic = ''; + this.resultListForMic = []; + + if (this.workerInstance) { + this.workerInstance.postMessage({ + msgType: 'streaming-asr-decode-mic-start' + }); + } + + this.sampleList = []; + this.mic.start(); + } + } + }); + Button(this.micSaveBtnCaption) + .enabled(this.micSaveBtnEnabled) + .fontSize(13) + .width(296) + .height(60) + .onClick(() => { + if (this.sampleList.length == 0) { + this.micSaveBtnEnabled = false; + return; + } + + const samples = this.flatten(this.sampleList); + + if (samples.length == 0) { + this.micSaveBtnEnabled = false; + return; + } + + + let uri: string = ''; + + + const audioOptions = new picker.AudioSaveOptions(); // audioOptions.newFileNames = ['o.wav']; + + const audioViewPicker = new picker.AudioViewPicker(); + + audioViewPicker.save(audioOptions).then((audioSelectResult: Array) => { + uri = audioSelectResult[0]; + savePcmToWav(uri, toInt16Samples(samples), this.sampleRate); + console.log(`Saved to ${uri}`); + this.micInfo += `\nSaved to ${uri}`; + }); + + }) + + + Text(`Supported languages: ${this.lang}`) + + if (this.micInfo != '') { + TextArea({ text: this.micInfo }) + .focusable(false); + } + + TextArea({ text: this.resultForMic }) + .width('100%') + .lineSpacing({ value: 10, unit: LengthUnit.VP }) + .width('100%') + .height('100%'); + } + }.tabBar(this.TabBuilder('From mic', 1, $r('app.media.icon_mic'), $r('app.media.icon_mic'))) + + + TabContent() { + Column({ space: 10 }) { + Text(this.title).fontSize(this.titleFontSize).fontWeight(FontWeight.Bold); + TextArea({ + text: ` +Everyting is open-sourced. + +It runs locally, without accessing the network + +See also https://github.com/k2-fsa/sherpa-onnx + +新一代 Kaldi QQ 和微信交流群: 请看 + +https://k2-fsa.github.io/sherpa/social-groups.html + +微信公众号: 新一代 Kaldi + ` + }).width('100%').height('100%').focusable(false) + }.justifyContent(FlexAlign.Start) + }.tabBar(this.TabBuilder('Help', 2, $r('app.media.info'), $r('app.media.info'))) + }.scrollable(false) + }.width('100%') + } + + private micCallback = (buffer: ArrayBuffer) => { + const view: Int16Array = new Int16Array(buffer); + + const samplesFloat: Float32Array = new Float32Array(view.length); + for (let i = 0; i < view.length; ++i) { + samplesFloat[i] = view[i] / 32768.0; + } + + this.sampleList.push(samplesFloat); + + if (this.workerInstance) { + this.workerInstance.postMessage({ + msgType: 'streaming-asr-decode-mic-samples', + samples: samplesFloat, + sampleRate: this.sampleRate, + }) + } + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/pages/Permission.ets b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/pages/Permission.ets new file mode 100644 index 000000000..40ef391ad --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/pages/Permission.ets @@ -0,0 +1,26 @@ +// This file is modified from +// https://gitee.com/ukSir/hmchat2/blob/master/entry/src/main/ets/utils/permissionMananger.ets +import { abilityAccessCtrl, bundleManager, common, Permissions } from '@kit.AbilityKit'; + +export function allAllowed(permissions: Permissions[]): boolean { + if (permissions.length == 0) { + return false; + } + + const mgr: abilityAccessCtrl.AtManager = abilityAccessCtrl.createAtManager(); + + const bundleInfo = bundleManager.getBundleInfoForSelfSync(bundleManager.BundleFlag.GET_BUNDLE_INFO_WITH_APPLICATION); + + let tokenID: number = bundleInfo.appInfo.accessTokenId; + + return permissions.every(permission => abilityAccessCtrl.GrantStatus.PERMISSION_GRANTED == + mgr.checkAccessTokenSync(tokenID, permission)); +} + +export async function requestPermissions(permissions: Permissions[]): Promise { + const mgr: abilityAccessCtrl.AtManager = abilityAccessCtrl.createAtManager(); + const context: Context = getContext() as common.UIAbilityContext; + + const result = await mgr.requestPermissionsFromUser(context, permissions); + return result.authResults.length > 0 && result.authResults.every(authResults => authResults == 0); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/workers/StreamingAsrWorker.ets b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/workers/StreamingAsrWorker.ets new file mode 100644 index 000000000..d521cc3e4 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/workers/StreamingAsrWorker.ets @@ -0,0 +1,294 @@ +import worker, { ErrorEvent, MessageEvents, ThreadWorkerGlobalScope } from '@ohos.worker'; +import { + OnlineModelConfig, + OnlineRecognizer, + OnlineRecognizerConfig, + OnlineStream, + readWaveFromBinary, + Samples +} from 'sherpa_onnx'; +import { fileIo } from '@kit.CoreFileKit'; + +const workerPort: ThreadWorkerGlobalScope = worker.workerPort; + + +let recognizer: OnlineRecognizer; +let micStream: OnlineStream; + +function getModelConfig(type: number): OnlineModelConfig { + const modelConfig = new OnlineModelConfig(); + switch (type) { + case 0: { + const modelDir = 'sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20'; + modelConfig.transducer.encoder = `${modelDir}/encoder-epoch-99-avg-1.onnx`; + modelConfig.transducer.decoder = `${modelDir}/decoder-epoch-99-avg-1.onnx`; + modelConfig.transducer.joiner = `${modelDir}/joiner-epoch-99-avg-1.onnx`; + modelConfig.tokens = `${modelDir}/tokens.txt`; + modelConfig.modelType = 'zipformer'; + break; + } + + case 1: { + const modelDir = 'sherpa-onnx-lstm-zh-2023-02-20'; + modelConfig.transducer.encoder = `${modelDir}/encoder-epoch-11-avg-1.onnx`; + modelConfig.transducer.decoder = `${modelDir}/decoder-epoch-11-avg-1.onnx`; + modelConfig.transducer.joiner = `${modelDir}/joiner-epoch-11-avg-1.onnx`; + modelConfig.tokens = `${modelDir}/tokens.txt`; + modelConfig.modelType = 'lstm'; + break; + } + + case 2: { + const modelDir = 'sherpa-onnx-lstm-en-2023-02-17'; + modelConfig.transducer.encoder = `${modelDir}/encoder-epoch-99-avg-1.onnx`; + modelConfig.transducer.decoder = `${modelDir}/decoder-epoch-99-avg-1.onnx`; + modelConfig.transducer.joiner = `${modelDir}/joiner-epoch-99-avg-1.onnx`; + modelConfig.tokens = `${modelDir}/tokens.txt`; + modelConfig.modelType = 'lstm'; + break; + } + + case 3: { + const modelDir = 'icefall-asr-zipformer-streaming-wenetspeech-20230615'; + modelConfig.transducer.encoder = `${modelDir}/exp/encoder-epoch-12-avg-4-chunk-16-left-128.int8.onnx`; + modelConfig.transducer.decoder = `${modelDir}/exp/decoder-epoch-12-avg-4-chunk-16-left-128.onnx`; + modelConfig.transducer.joiner = `${modelDir}/exp/joiner-epoch-12-avg-4-chunk-16-left-128.onnx`; + modelConfig.tokens = `${modelDir}/data/lang_char/tokens.txt`; + modelConfig.modelType = 'zipformer2'; + break; + } + + case 4: { + const modelDir = 'icefall-asr-zipformer-streaming-wenetspeech-20230615'; + modelConfig.transducer.encoder = `${modelDir}/exp/encoder-epoch-12-avg-4-chunk-16-left-128.onnx`; + modelConfig.transducer.decoder = `${modelDir}/exp/decoder-epoch-12-avg-4-chunk-16-left-128.onnx`; + modelConfig.transducer.joiner = `${modelDir}/exp/joiner-epoch-12-avg-4-chunk-16-left-128.onnx`; + modelConfig.tokens = `${modelDir}/data/lang_char/tokens.txt`; + modelConfig.modelType = 'zipformer2'; + break; + } + + case 5: { + const modelDir = 'sherpa-onnx-streaming-paraformer-bilingual-zh-en'; + modelConfig.paraformer.encoder = `${modelDir}/encoder.int8.onnx`; + modelConfig.paraformer.decoder = `${modelDir}/decoder.int8.onnx`; + modelConfig.tokens = `${modelDir}/tokens.txt`; + modelConfig.modelType = 'paraformer'; + break; + } + + case 6: { + const modelDir = 'sherpa-onnx-streaming-zipformer-en-2023-06-26'; + modelConfig.transducer.encoder = `${modelDir}/encoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx`; + modelConfig.transducer.decoder = `${modelDir}/decoder-epoch-99-avg-1-chunk-16-left-128.onnx`; + modelConfig.transducer.joiner = `${modelDir}/joiner-epoch-99-avg-1-chunk-16-left-128.onnx`; + modelConfig.tokens = `${modelDir}/tokens.txt`; + modelConfig.modelType = 'zipformer2'; + break; + } + + case 7: { + const modelDir = 'sherpa-onnx-streaming-zipformer-fr-2023-04-14'; + modelConfig.transducer.encoder = `${modelDir}/encoder-epoch-29-avg-9-with-averaged-model.int8.onnx`; + modelConfig.transducer.decoder = `${modelDir}/decoder-epoch-29-avg-9-with-averaged-model.onnx`; + modelConfig.transducer.joiner = `${modelDir}/joiner-epoch-29-avg-9-with-averaged-model.onnx`; + modelConfig.tokens = `${modelDir}/tokens.txt`; + modelConfig.modelType = 'zipformer'; + break; + } + + case 8: { + const modelDir = 'sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20'; + modelConfig.transducer.encoder = `${modelDir}/encoder-epoch-99-avg-1.int8.onnx`; + modelConfig.transducer.decoder = `${modelDir}/decoder-epoch-99-avg-1.onnx`; + modelConfig.transducer.joiner = `${modelDir}/joiner-epoch-99-avg-1.int8.onnx`; + modelConfig.tokens = `${modelDir}/tokens.txt`; + modelConfig.modelType = 'zipformer'; + break; + } + + case 9: { + const modelDir = 'sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23' + modelConfig.transducer.encoder = `${modelDir}/encoder-epoch-99-avg-1.int8.onnx`; + modelConfig.transducer.decoder = `${modelDir}/decoder-epoch-99-avg-1.onnx`; + modelConfig.transducer.joiner = `${modelDir}/joiner-epoch-99-avg-1.int8.onnx`; + modelConfig.tokens = `${modelDir}/tokens.txt`; + modelConfig.modelType = 'zipformer'; + break; + } + + case 10: { + const modelDir = 'sherpa-onnx-streaming-zipformer-en-20M-2023-02-17'; + modelConfig.transducer.encoder = `${modelDir}/encoder-epoch-99-avg-1.int8.onnx`; + modelConfig.transducer.decoder = `${modelDir}/decoder-epoch-99-avg-1.onnx`; + modelConfig.transducer.joiner = `${modelDir}/joiner-epoch-99-avg-1.int8.onnx`; + modelConfig.tokens = `${modelDir}/tokens.txt`; + modelConfig.modelType = 'zipformer'; + break; + } + + case 14: { + const modelDir = 'sherpa-onnx-streaming-zipformer-korean-2024-06-16'; + modelConfig.transducer.encoder = `${modelDir}/encoder-epoch-99-avg-1.int8.onnx`; + modelConfig.transducer.decoder = `${modelDir}/decoder-epoch-99-avg-1.onnx`; + modelConfig.transducer.joiner = `${modelDir}/joiner-epoch-99-avg-1.int8.onnx`; + modelConfig.tokens = `${modelDir}/tokens.txt`; + modelConfig.modelType = 'zipformer'; + break; + } + default: { + console.log(`Please specify a supported type. Given type ${type}`); + } + } + return modelConfig; +} + +function initStreamingAsr(context: Context): OnlineRecognizer { + let type: number; + + /* + +If you use type = 8, then you should have the following directory structure in the rawfile directory + +(py38) fangjuns-MacBook-Pro:rawfile fangjun$ pwd +/Users/fangjun/open-source/sherpa-onnx/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/rawfile +(py38) fangjuns-MacBook-Pro:rawfile fangjun$ ls +sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 +(py38) fangjuns-MacBook-Pro:rawfile fangjun$ tree . +. +└── sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 + ├── decoder-epoch-99-avg-1.onnx + ├── encoder-epoch-99-avg-1.int8.onnx + ├── joiner-epoch-99-avg-1.int8.onnx + └── tokens.txt + +1 directory, 4 files + +You can download model files from +https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models + +Note that please delete files that are not used. Otherwise, you APP will be very large +due to containing unused large files. + + */ + type = 8; + + const config: OnlineRecognizerConfig = new OnlineRecognizerConfig(); + config.modelConfig = getModelConfig(type); + config.modelConfig.debug = true; + config.modelConfig.numThreads = 2; + config.enableEndpoint = true; + + return new OnlineRecognizer(config, context.resourceManager); +} + +interface DecodeFileResult { + text: string; + duration: number; +} + +function decodeFile(filename: string): DecodeFileResult { + const fp = fileIo.openSync(filename); + const stat = fileIo.statSync(fp.fd); + const arrayBuffer = new ArrayBuffer(stat.size); + fileIo.readSync(fp.fd, arrayBuffer); + const data: Uint8Array = new Uint8Array(arrayBuffer); + const wave: Samples = readWaveFromBinary(data) as Samples; + console.log(`Sample rate: ${wave.sampleRate}`); + + const stream = recognizer.createStream(); + stream.acceptWaveform(wave); + const tailPadding = new Float32Array(0.5 * wave.sampleRate); + tailPadding.fill(0); + + stream.acceptWaveform({ samples: tailPadding, sampleRate: wave.sampleRate }); + + while (recognizer.isReady(stream)) { + recognizer.decode(stream); + } + + const audioDuration = wave.samples.length / wave.sampleRate; + + return { text: recognizer.getResult(stream).text, duration: audioDuration }; +} + +/** + * Defines the event handler to be called when the worker thread receives a message sent by the host thread. + * The event handler is executed in the worker thread. + * + * @param e message data + */ +workerPort.onmessage = (e: MessageEvents) => { + const msgType = e.data['msgType'] as string; + + if (msgType != 'streaming-asr-decode-mic-samples') { + console.log(`from the main thread, msg-type: ${msgType}`); + } + + if (msgType == 'init-streaming-asr' && !recognizer) { + console.log('initializing streaming ASR...'); + const context = e.data['context'] as Context; + recognizer = initStreamingAsr(context); + console.log('streaming ASR is initialized. '); + workerPort.postMessage({ 'msgType': 'init-streaming-asr-done' }); + } + + if (msgType == 'streaming-asr-decode-file') { + const filename = e.data['filename'] as string; + console.log(`decoding ${filename}`); + const result = decodeFile(filename); + workerPort.postMessage({ + 'msgType': 'streaming-asr-decode-file-done', text: result.text, duration: result.duration + }); + } + + if (msgType == 'streaming-asr-decode-mic-start') { + micStream = recognizer.createStream(); + } + + if (msgType == 'streaming-asr-decode-mic-stop') { // nothing to do + } + + if (msgType == 'streaming-asr-decode-mic-samples') { + const samples = e.data['samples'] as Float32Array; + const sampleRate = e.data['sampleRate'] as number; + + micStream.acceptWaveform({ samples, sampleRate }); + while (recognizer.isReady(micStream)) { + recognizer.decode(micStream); + + let isEndpoint = false; + let text = recognizer.getResult(micStream).text; + + if (recognizer.isEndpoint(micStream)) { + isEndpoint = true; + recognizer.reset(micStream); + } + + if (text.trim() != '') { + workerPort.postMessage({ + 'msgType': 'streaming-asr-decode-mic-result', text: text, isEndpoint: isEndpoint, + }); + } + } + } + +} + +/** + * Defines the event handler to be called when the worker receives a message that cannot be deserialized. + * The event handler is executed in the worker thread. + * + * @param e message data + */ +workerPort.onmessageerror = (e: MessageEvents) => { +} + +/** + * Defines the event handler to be called when an exception occurs during worker execution. + * The event handler is executed in the worker thread. + * + * @param e error message + */ +workerPort.onerror = (e: ErrorEvent) => { +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/module.json5 b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/module.json5 new file mode 100644 index 000000000..80e93ca6a --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/module.json5 @@ -0,0 +1,64 @@ +{ + "module": { + "name": "entry", + "type": "entry", + "description": "$string:module_desc", + "mainElement": "EntryAbility", + "deviceTypes": [ + "phone", + "tablet", + "2in1" + ], + "deliveryWithInstall": true, + "installationFree": false, + "pages": "$profile:main_pages", + "abilities": [ + { + "name": "EntryAbility", + "srcEntry": "./ets/entryability/EntryAbility.ets", + "description": "$string:EntryAbility_desc", + "icon": "$media:layered_image", + "label": "$string:EntryAbility_label", + "startWindowIcon": "$media:startIcon", + "startWindowBackground": "$color:start_window_background", + "exported": true, + "skills": [ + { + "entities": [ + "entity.system.home" + ], + "actions": [ + "action.system.home" + ] + } + ] + } + ], + "extensionAbilities": [ + { + "name": "EntryBackupAbility", + "srcEntry": "./ets/entrybackupability/EntryBackupAbility.ets", + "type": "backup", + "exported": false, + "metadata": [ + { + "name": "ohos.extension.backup", + "resource": "$profile:backup_config" + } + ], + } + ], + "requestPermissions": [ + { + "name": "ohos.permission.MICROPHONE", + "reason": "$string:mic_reason", + "usedScene": { + "abilities": [ + "EntryAbility", + ], + "when": "inuse", + } + } + ] + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/element/color.json b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/element/color.json new file mode 100644 index 000000000..3c712962d --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/element/color.json @@ -0,0 +1,8 @@ +{ + "color": [ + { + "name": "start_window_background", + "value": "#FFFFFF" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/element/string.json b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/element/string.json new file mode 100644 index 000000000..8679134da --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/element/string.json @@ -0,0 +1,20 @@ +{ + "string": [ + { + "name": "module_desc", + "value": "On-device real-time speech recognition with Next-gen Kaldi" + }, + { + "name": "EntryAbility_desc", + "value": "On-device real-time speech recognition with Next-gen Kaldi" + }, + { + "name": "EntryAbility_label", + "value": "Real-time ASR" + }, + { + "name": "mic_reason", + "value": "access the microhone for on-device real-time speech recognition with Next-gen Kaldi" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/background.png b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/background.png new file mode 100644 index 0000000000000000000000000000000000000000..f939c9fa8cc8914832e602198745f592a0dfa34d GIT binary patch literal 57364 zcmYIuc|6qL_rIk#Su&MMQlYU)cz{|$Qc0x~A^BEf( z`{n=HaSk>%wsfNM*uUkN^8dI{qxxW z*@b_`#>VlLWSG9 z0>QdPQ-&i_RCVdp2s$-u%S362^SHV0`EO6;@n(xK));G>#qwhPWrDXGk@OBMV}H!J za!48&`xhWJKj{_+f3ir<>Jg6Ax<&Xgn;)U7UJyAw{(u?zlf{oLsJTS-_o1?+lSg-j z8fcZj1*Ad(!X>WuuxM!H5t@V3*8vLL6`QnC!q!BwQjI{yk*;~@|3;B)`p$WYcDmnZ zt`R zr=oS6o-D$WZsYKh1PiOdhhX&YWGOzpc<6ITKzr^zi-#>z){t;yz3tu_a!>)(tTU9d zd}COuy~Tb}UIRNX@aVGJqEKUa)1#E-u}pl!sY)z4cu+Hu9==`6=0Ob#x-%q}t@jBp zmoiZDcfF1WL{PB0ZO**8yZ+%;LF6K*JDUoHrJkl0Wzak+Y%E( znUmuA^p@Jv6{%Y;MsiZ4O?#ID2b2ssEq6_KGL z8T%zdA3YhMnkBu19bNsa_$$_1^16jadx`0ZzPx`M%T>qZpYyNYOeDdmqLTNWpR5T% zOlRrW_xNCD+*3_WSxvt4P-@qQ9g_$aedDk-hcV~t>Oxw;UaAk1V?9m5<2k4%VrM$- z?{KH{)m_U~yJcBbX+vqVfq&4)Vf+FvAHd|s{V34=f#uJM!Tp?b32THmfzNn1unwY& zPNtaE{ZZ=OkZFh*xW2FT&fDF?64Q%l>dwdZ#Bg;^v;dAbU*QLEQG@_|ucNXFyx~H( z#h?kJKeI3jD^U~`e`*^zcm?PlIWj|tL_a8NC?HVl*gX%;5PW5Y%ZZ*G=jPn5#o+Sh zhnE>D@Wb!f*O>cZ0}ZT=HlEdoWVWk}5H1S;$vxe#Rv~;l5rJ=w--wPl621jCW}B|gxECKzT9 z3FKlD{=OfN5$J3?Ag0g4F5t8_D(RvO8W!*~?#q{Dhx(Sj=)^9ZlE|LyI?p1NXMWr| zGGbzFN^3)5?P^vfnD7XZo*8yf&O&>7XULUUvhJT@rHcF>PmjodH~u4RSmX4TH?v`IKg2cy7 z(T@e4&pPRHRczikEvwvO?jbblSVp z2qpyT+LHUFhHwcunP(^h)G#uA95vF`Gd&1k%F@wuCk3DnjNjw;b}*;dY{F5{7tNsg zLf4y|)RTV`PjQ^!NoWB3YA@S@Cw zUAr?mUcn7g)n!3J`D7*H`y{%TuT$wNY;))rP(y@kdFdPH#h|rjcW2#oRybxTchXlQ zwMW{bVcqRRc_2r^tI)Zav_+qLwdd|Bw=*pM!|pflbT%K&Eof^{6+|k{2_;HcrWd3? z#z;>@Y3dp#B^R5c9RhH8lT5MRr*;>xd<%C3sV2Y}>{On@a*oump`g#H<6V&DKeZ-?Zic$S$>ulEiZvJG8kHMeSzVE(R|E-<}cEG^n2E*Cp z-25-DQv_Mf+&WhT3r?23Phid$q`Z3HE($RgC{EJA0Yc1SP6(a(oZ4RU2L1~H6k0Q< zHY1Mj{)b(ll3Wr=HakbiEk13zYKN&f#9*}tMZiQ7h@Us+N(Jk`aWQHf)r!ObZAT>_STJuzjuO{qHMlTjN9^hPZ8sZBMl zl&MX}xk{d5VUEInRK9r^Tnx#HE2;hFoa7?NDufAxZV6Mj9B^NaAt4;oStAtWfVg8< zjQAfLPj#u>Xp*sALAi;M(f1>la|_-k(E*-1Sa_Vdt$KsCNAwAbm8CmvpDbwL$`Cx8 zkBC0&3#@q@7E3LVtGQcrGS=s-uh6FHuC)WTtU_@t5c_T~`Wv+F0Jd$a9s(?ucd&l{ zWThjQ*u4YqU6Wq{+^0sC%S;vXx~qO|+s%Am1m-N}zkd84>GT;5u}a1*p9&!g%3wk2 zl=rj+H9g>!z4_zdU1iItL}Zox?lwK^ykQ+_#Ym~p>s8CgcLQYV4wezL^K-_HzM$r! z1m$U&G13HqDckgHschNcoe73o=)$P$j46Y)SnaZK(U|F7d#{AGb%>@b+jX#5*Rf5x zq}@ejPTyyn&&@n|dDGl-o-=XF%6dndW+}@7JDd?6b}Mt-SX_GV^3{!3Yz5a~X@$Fw zyDIkaWq*rtn{8knumG6=yF(6lzQnq)&M@%8RzdC%{%-0Ey{v&0pp-aIPP$bTrF|=~!MvLftx2pd=0-86i#@A;;(b^r-TzBJn~W4d42|-V*)} zt}h95!TwDQ%xWD9TFS{BwGO@d9P>kia=+LQ@r>0>5VvEV8(&tEuw%+YP*Qm6KzZs9 z#qL6SPwl9DtPZ{0`)Vph`^ryNV|=I7r2Vf@LrX3<=$f6zv1^z*!<6j{f$|6Jw=%s2 zb)|d{?()1m_Xoab$B5r9#&taTI^E@0yTQ$UB1_f0nc<oQhFOi;b@!o=y6w&Tsrw|K5XXEJEA>@Eb?8hi( zlT-*bXZd6g*C+W9V6B5iF$2f(_-ek(ko^JW%$@}`#GJVV0S8A~FwzM(JdY)c1B&ls(qJ=bvy&S10cqD8@1Clbooq|3kmbD_she z@O#tu^ibgYfM#HD%WIF%%uf7+)sc&Dejs@WRQE+Q1jXlN2z>9dB;X9e>Y3a-&-A;T z>||D+o$j^$E>F`4y02DTELRMYH*biv(5+ed(cQq&82Gu z2~UNnOcNc&MwT3lD@S}nPJMsvOT%0L{`dN}DU&^Z#6?2^aE!5ulUV_Zct}2~K6R!_ z4ReuaX$@AP?M!XMpi&ZJwsY2up5F-xe0{ym`9#@pr%63v->d&@UoFthcC1`k$L=ze zYX1{xl49Q=z953h>NzyMc3UuH96t7)-k|lRw-P=T%Q`;dC7@r`uCOq8Eqi7gKC)~7 zb(*Q>H|T2(e>5DVf9nswM~C%V2G2 z#B|VOitZm{FlV>EydvsFF|Ue~ium0%0KOaFiMOLk(X}jHq@dI@*AM2G6XzCU zSpFR?#U4MPz~VZR>RA@a!CZu45#f<)^f#kJ+ULtRLJKzSj=cf+NxQ}Kw)Yme6wJz; zu3W=Jz<}rEm$g7sNy>yr-Z|OiI>qQ4m37~);`_~Xgr~N4wOAssk(HTh5er1XtFm+! zb`5FT&FoKA{ADaUP!Y#o^sGPb?mT2wBY9ZfQ}ujLk`C_dyTvT&)34sj!RXJcZ%lCzF?kE~i-xCSGh{ zy%iUR0+S_JP(#%W9!Npk=RL(8tFB7(up1ms-Q#8 z$-{dva97!EQB<5#@0KgW&2S|ddKN*<(?}37-=O@1bF668sG)3(D61=Ech&sJ;j|An zqu1a;`}bcMj;#tF3l~&Ue9ES7GRw~kIPKK&q&^No_3M#yjp?ygI;To&wcXbe%ju*z zpMI!gbi8@{AJVkgXR+py{dMSfko}H`^q^elZQ-5<2bG-K8tYq8Q@*4t)`Blvz!#v> zE;3kk_e^|Kew4?}eU;3n)q48yWgAm)d+F(;W(>jPB_glQLiH|IE=EDVFI*j_FBebS0vXyh5@x9LS?RNi7vXf?RckfXjvy^Pifki$9G zzwp&k7S+aNOI8%DUON~#xxv+a5rJDE+^6;@RcjnwKZ|%#%Ukq~@&vL#Pts;`f?jwYL)Y zDOROB^T8hlFfA@(=$bFYKWy{F^5$#{h*A1FG5GZZ1?>Y+!}UULap(oEekfHZCJkpC zppRS@+Uvrs>_Df!YT#HWpuaEwRq)V49)TgZ7Jf{A6@tpv&>tG)c9F&eZWo)(tDPDB z4Fkl6@ov*S4!gboeokhZ>My7@q%!Z93-zy>Y(_9axnH2W2Ie&#X2Z->o1A6ZoV(OgY z@PpdL`E%U}QN-vzdLCdkVX)Vp-z|CGg)^e06LvMfbj%1)ZdXNB>r>{Jk&ApwTkkLr z-2C5e31{3c{*xsm?)EItQ%pSW(%723B}AHgke#M{7KJW6TT*>9^+`FIe4;VHRwSF$ z9rBta7_>vwCuV;vFY=|NZ2KlX$A`EUk*phH=Pd~I8Kkr|v!j3sBAD^fPD!FoPpnHf zqP&jc&^s{jm0M&oBNXjUol2${7|G^u7UtOd2kxA0b?japS#xlwo_TaY+jh-`+$sfO zFLgfqb~kaemX{ErUn7}?_tb>g?G@UyT99HoY^;BG(5|gh>F3J!9J* zvrz6TP+;XdE$<41%Vony^Y}i*aCz@+4v^38p)5?Nhw`m%Cbg5Lpz%VOxaBnlA9P;N z9D=#{(>`$N_!?&CKf9eJGzIc>dhWes8XtpX`{gOhP;HMklZ8~@Yu~YT1bZZ{VwrAffDNiZ6Mh5vEzpq z=5A;0ff@>1MG@vbwRU!?7ZFD-SYng>JN(=>uwrkrl@4u6M^n6jl1shsk;DM`t#|F? z(H9W(@&~b(mmUR)30H=vAZdIrX%9iR7rMruZ_I4$Eq7YnBI4Z8T zj5;RTUu8?(ZsW>30%Hk#$^zfAtgZ&y!|p@5%e_4oe7)3{Y6c^x>zv=o_XPiF*wI1y zNe5L3p=L;8_D7-+5I+LfNgDYrOIUD_Iu_VJQD^=4v=Gd z_u%h$8{Lobyu6%VkeZI%T_vssgc#J4yD+&6pVkdLYl@3@NdcQbwl!J%4{RC80oF1z z`ksIXyrZT=Apq3kOR#m795+y}-8NizKBNESZCmBS#jqG`n4kCydp-4DZ^BF-zWD2# z1@F?p*^9m)EPrkd^E&cimk<1mN+iwSCVTHpqz^#`_Dj;-5xURqxK*!kp5asE##*=< zc{bFC-`m;q4VL3=| zKN6@)%XIu=yS*-K-9Bw`jN+-lWBttd77x>|g)~$UgPB_qH0h&bm}j3#sdLfV&xcR^ zQFk=d3;U8~YLqm@^61C zmaLbHw=dJ0oLP?>eyJ&=wgtZm!2mS9V!i~62x+n`%jyesf0bKruxRDH-)c2uF;&qT z4Z0drBbHg-G#ueH1vVaEJFTw$U))8mlUjFz?!PDqNpcIqZ%B6$Ju$CzrK@_na@?na5LpJODS}`)`8j7i#>C z0RNEb>nnQ8v$qXrgh)-(=VVRFwj4 zZKH}5T4rlZ$PiI2z3_*{`av5A0jPJY!Y*RQ?XbKPZmNdwp6ufAH4m~1%r{gYeOJBR zai+gl7I{Z35P0Q7EoGmkkLGHe5rR^{bdxWyMiC1~&kI@I-bYJrdGv{=O7!p&kKxN3 ztOoyzWj_asX!zA>`fa~&>#$n@3{c@VVcl3(1m5=dCI-~1uR+4s;@87ozKCU|Z(EhE z7~Csbr}e|&-zPK~*W}WcKqB+rv-rNRzvqfY299AvP zA5u^Rs->xN6b@MzP_f(M+}|~RxUHs#zO%D772V@B$F;5<%Jx|0#Oh_?#%yrHfV>}I z!Lfe59_VCjJ!pEQOWyUr;CdyL z-RzERMQjU_j%}N!Av?++44uVMc#r_KCTZxxSZL>4`xbm)#)*?4I#nFDOZLv10s^{6 zAyo6zfA)w8n^jk|KBb4J;|Gbx9)grFflY-Nyl_v8_@}gizDNn(Y2l6TqM&aN(+9Qg zTBo#J4N$h%f!;K&2NqBlT~J6aqHGy6HI`Xn*)UV$w2>iLk~P=l)VTdah9Ab`z%}dg zxIvG$xPG=H0NRw|6_-~Bzh+BPv9&C;z)58?`7t~$HupdHcF!F5dirrGrn3d}wAHr! z^@&!aoW@3sENjl#i@LzRYOZ4b#v|Jk_Mo$-VYlgbE3LQVKniS1mH)uO`90X{bc~{1 z*%Wm4$E_2-W__`4`mDu;Ld(wv8e147=mMu!AKSC=mw*4n^8S>~fm9mJgf4~8t(bb> z^_3WSK>aAZ6lK3OZ#_7g@)?z1#pZ zoR2>rm%_enbG!+Y34#Jmal)V9@-s8li+_Le^~z8cxHeF5vR%p~{93TJv%YmeTB|@^ zc=}q4Gofbju_Z#%Iv9|44|pawNvh^mFGBA_KZ5C^rx-l~Ytqf4;%SxezE8%O)aJh& z>2it7b`epB=P&=s^y`mJMjMq&9Jvpdhn}6sFHk)q%d zE_RV6%-}?H)w7yAW9TA)&7XbMyu=N}tRA-JTl2iG6u8;@?;!BW;ykyof{i+alo zJu1v~ITow6y^)5crWdi)&;yNs0d)3*vN+aSszJ%`1`(%9X-Hi}3gH#iRg@{Svm?cP zM}T*)U{A8FTQ7b@oc$7vr_EeTIj6N%Cr}VI5VcfZk+@1UFc>zpJkm3S%cb<~=~`BV ztbyjzOPJuDkTJJ!hL^nLk}*=2EXd?->%+3NWrq&5a$%1G{r2~cLQT2W>8!pd$9G;K ziQIDUErsVk$XQPRm)pDFYVuLFlx&eiFlnoixT|jvAoB)ryM_}euaYFXrdKLqi|4AL zG`rnvWi4Qa>Wvo=;Y+t@ecMjl{#37K;?VkYdoSbT(2m}8!k~RT{yv0l8cPp{jtiXr z$7KAJAvM_g4ak}0Yo*q!sO%PN_CK)Pv>lC7xoB~vG1hs?Wv>^kpOBU0WV@$|oL!cE z1FV3%^4Pjr5Fqc)|Sv+upxx8BCM z9*cYQYi3jY(^pUL8`I|3rHf+5>sq98e!hkPsfNMQ1@y7Tnf4{F2p zx9AO&@zYO;WpCQUf4G@!d<{t43@&RHh2Ukg^D-8_;De`dc{hz?yPS_7BzU!x^P-tj zBWt_uk{g94M1uo_&0l?m1qh!Q>=dKy5cx zRa7mv(}`xYKJOm)h3s8goQ*XK1OT<#&Ozf35uTB^VD8m)Z6Bnlal5r-bkso}J^TcM zo)ZSc#2@`h0Si}lrnCFt67JFa*e&}2avKCL|IIk<$R2*5sILkv4P( zesTX_tP#NqXN#>Q{4oe!N=G{SZ_I#~%^kq5ilGc=Q63_5uRt!D^j$k=&$`Ha&bGlAjZ2&hWa=M};Cw|5onME2e;8le z)-hK+mgNbGw-4puLN6g_q5p6T?0XM^dMo810rSBSw7Rrl(jt2JNVBwhB0o3``lZ1y zBr`Dy8LdVilxv`X5b0N8#{#(y<2vQrLj;qv`XA#RZ+@Q~*aYa^UY~;#F>6BL>75+E zeH2(L#HhLeI=Mz1#%^96zY$Se;@N)biYOvM6H1p6-4LcvA=&GP()#?u=_WXgAoZl* z+bR{6BA52?12Rex)v?(LMRsKvf9{KzP<^4&NISV{2!a;wEhr&E)EloHqSR9%ezb)? zl9X;qQSTg@es%UevGs9-KQk6RqJ;Ui(v@S0=JpkXQVYgXlRKQcfFLT2A%*#c?7(b} zjki==Q^Y#Qf}ZVpFtF6<4SbGKkkU>I6wY*Ps*EAzemS5Z0r!-oD>~r!<<+c~fHK+{ z`u4nWcW&4!()0%2>r>@zr$F6$;5*IAuq5bc>cn-IEZ+B|hkO&NPeBi&47YiU-<$w0 zq-j9aGH~K;Y%0{D&e90RZ(J_@o*`(e0TgqWM zz>V1_2|7MMg_6zbeK`A2oW6>`dUuDIll*?4hKaK{^>2t!B*N9o7_!iC51?A=hss#S zTOD48mGM}}JkMLeB>f0zNw|zPj8Efyx1Qh?QyT7Bp*PsC1%+$kgboSqDR=rTEs%8X z-t2|68n3XC`A-sBYO9tXuQqE7{}pE3mRASQTvScN7(%JH0{M|k4t%rE7xh`qUf4A- zgEE3f#zcuMyMYyiu;w=#PFC-_W0rb;u#{l@E}K0uMy~Ec1MBz-KglT}I_AG%m9nb!XAkpoW-`_85Umy)5g0j(3(>`;o1;w;CKp zLKdGc@@LrE*Y6B#H>jMeTcD6nZx;FZw zZ?8nd;T;sv#~t>9Stu`V2=$pLBHrDq3VNw9{KZU-50LlNLK@?o*hLF?1Kjl3op`;u z=nFLXc(CuUKp%gcxwwBm08`iDki>51cyobB9Eypc5@0Uv%$x+m$P}vtzJ@yXv2Y(6 z%G|Dfw#*GyPhoZ)9Obc;u$h*k0~W zv)EW8ChYvHNP~Ws5(MQk4JSGnG!l*4I-odrw$8E;u9uTN)1sDTSK-9%H|jqRi1XpO z_RLbdR5?V7FZiM9a@_RLzrIa?o8u(&ct}&dJFEmRO#py=1J(LW)$S@B$xLi6T)SOw|;fa7Myzv z?MOZ*b$o!rCg?J9&v6SsP#m&goHWvlC%0`IUKT~X&=s1cU$O`0Ea`_f|aU@(<=bXW{`6+7W#cu@H9t zagx-Usc&&vez&!Mjqpdk+Ol(}Uo_B;A&JhUaOe-iG9|*Z<)SYRZ;!m{$5X=V;9Cl+ zs(#H}WR`823f+9`wmRKF;(;wyt*?b3@Y`H^;&@1GipUF_{Gb_RzIV!3$qMq++{iyr8Th+msVi*eA69cY1K|TmaXNA-rCXT%k z%$21aDiQY_-+BI`52BI$rv}FI)tg7-CaaD7_O`l9ngVYH9#Xu44ly2flHy-xuzEyCWC^6c-^K*QrZW zNG1PL`B#xfh_CD57q**Q+=Ty9EEolHUwT`)Z`SWJPvsxa-f8_iHO;AQOj^^?v$Pd6 zy~3pjahT&?UwB@2zW1)s8+UfK$SFAL~tHHx3whuvPyW4mh3w z`_Q5~nHOsoDT0sx@+N~J<-Y&TvqV4MCkgXgo^ntecjdoSopR%@?wkEfAuHDOIVHQe z|K0}d$IAWT3jC{=QJCD$*L3=%k#f)T)tT7R=nTHqn)i5$Q)sm)53ZV1w&{swK_X#n zpD3;2Eb$r)$CDg__L8tv=0-5U5hB))B~SI2(6`QM95phAkktAVs0hU305vOGT{|^t zH`?>)3!24y5TBnjRfAJG|J9jjj_JYwB?gujfD3QwPf@~K(A2Z4KynC|m! zMt!}`yx4=~u?@-#ab5-T?In;dGAUlGajcN(yFF%ypy(av6(B6-=d(A}}k7wcgUJ%c_TA&p~<@ZA~EU-mvx5S_ykM?O8{R|mH|RE75BR5QQ#CTpy{;f{(N zFpFjUOJ}EEwov(%eX6wm&~H5dD|PO&*VQvG&6Br6eo1I>i7L)sk`T?{8}`lQfCB2R z@nDF(51Rl?^;uv9K%Wz-qpmyIoZjoO+tGhY)P>lU7U1Rpv;b{^)mu_I7=1e%POI7M zneWYe`!E(sG!D4Pm@9XD2!jhItDw15w=Vl)ioN}tjFK(3~fxy=!h!`6@!cQuCP6#aH;{{dyV2@O1#ZX{Zl4pLmD z7*{Ip)`V*gV-QVaE+>|4R`><5Z1*;n%pfkb3AiZ1s39)5f5khONJ{XZ5dEj{AwE^i zj6G1{WVlyMNlC3!_Nyk^Z0DjKo$ha)xbx}7UO*rnNj8he_fyO?v!so#$d4^uhxAXf zZNG(a)^5wM7^{-xB|`JITdre*!q^0$>^GMLKm@oauH?5G^;l>0Hp)xxzomAmYTE02 z+c%CPd*0$Be%v~(u%mMywt>EgIlKPOZH{Q%Y5c6=;F0usNLUPph9Xez1H1>s1YOPG zz|s4D9}W5qUuupaM_2#&;|@Jl=mK~Bc0i~OYb643=Gzqz>i%czm6IJ}e-nj~`8ZFe zGWf#c?5=VP0hlqMCIlRJj0p>6ob8O5e(*AYuP~QI>C$d^Yi`)_a|r1LwH(~NZ9P?Y ze?ts^N2upq=Br??YX8%HZ%xopU$9Z$(sjX zPFNIynnhW{IRi^L#G9#+Ew!gHJ%T1dibisJk2~6dM4r$&WR1@Yh3+PZbrp7G519Z>UKXw(mZMT+M-ozzkggshV_x`b zthj%~?f*E&m2-P{17aTUsk&fyuduoa3w}G`Ii-fByRE*XlORaY&Ax;2q^Y}9DeUiq zyMK?>G}eX;GkTjbS%GZr z5T&~;Y#yW|>Ep#W|B^P_r=X1$4uFNPGyw?zjr2lT?F6>ZQaaY;=%~?w4R^35Z=yWu z?(pW}`Hbg{7^L5u3abb48R>Wz-8&e~ld& zG34mkg*Nsz8LkANRe$e1~y0OAYcFkLVXfFw#0X3 z=EB)RkCjS-zhk?;_Eww$ZWCeYf2AIt@_v0+O&5H%+nUcKQQZ*-D#Mj9~nh zx&c!=`cApy)!}O~mTV6{@dbum`*7{`e8wKXQ$qf(L_&%pEl%&9Hz4Ua`%w=5(|{Fe zG=KtAxRHvVR%isJiC+qS)RMDX`xiqORyFg!x&NkABWs5}rYfi3W6r|&5P*I>{#$0n zSspPdl-FAPCWDVqU+`hp5SJ)}U4;QxQ*A|gM$`7-D_HnBBw1Px+%y8Fr*ZBkK&P(5 zLO)g}sM)3#vqJr|zOLiUYMzC)Ip0^+BMHE(YMU_d9|WolPeKCgmx*JYTE6;S>Wa~2 z4x7~9yMFQiL85QHvJtCUi;sWX->6#j?bP;4-B$$B=t*-7v~dwa7d_l5=?cxUgm6Cd zaZr_|B^X5;{k6{%BEZY5G{tgIXaw~PMvhi$_PDnHbyno3v;_gj5-=Qm12)lz+O@kglm5{q;M_RZxMCq-* znMrLfk)rYkS^lo@-6`Sd+^FUeRw9NYH^+}naYE(H+Zh38KI`SA9vUIYM`w7n(({Fc z<0<5oW06nE*}@UB$5AV7a^dI2srSJRcWrClmn7EQdBmJ6?_NrBl@wo_%pe-;K3ph= zN1j@y%^Bw-|7I#-OsQL<1zRV2i1N8h%Jz zJwR0GxN$z5cL7T2`h@=Nn-d!(GsG9!?+6zh=pQ$E{l5S3TiBHQ1&Bvy(*8{} z3j>EOJw+p*2|#VfcRh@u)N+@NMx-@QrQhRg>Tr5cY}aHl3CA*moGLkK0}rdRVR=E^ z{#;gyR7l*RccCrEo*H}%3X|@5YPQ+FM>u|=k#sp-M{J+EGRGl7LH4Z8UIUZqJ%O1S$-a-TXZC__K^ zV}HQ($I)a#fHDGwtEVN4+}*Rszq5|ewZGZEuA5Iw2OpA6%g^thr!`g2lSe?v{V!Zs zZR|Oezz_e)(WIs7nejBn3Q;m~{el(T15QaA3slu+pDiHa->pWfN1C6rVtf%}cuYmO zgKLKj2iNqdxC5nzUkN5bWkY7QyW{~Jm`(yqq=456x~COUo&to>DhmwrE0T1u8eLBX zmGKaO;crc6pm6&VjM@?bZCAXTbba*pRUvkbglVZYwEkF8YfO`T(Y8Hj5McaI z|C{H>yx3qKlRMuy-lc?Sc1!2)CVr8jr{HCfqbxH-_?m>w0h)fl`U3oh{a{=<4u=GG zzB1dSG{rJNtgG}nPU<2q1UPrW{mUkc8)_`L7OAnol7dZB_a(SX@-|yK8Wwm(0F1NEm_aN1wVsURw>% zPcJ-K`1h9E5@B%#7Tn`q0}2)m8v1N;72R}2#~JeoV=z!u6nMx5Hh%7WcQf@>B}s}j zpX2a$CtQcsC3W?=6QyG8m#bS^7MwKolNJR0blaxwZnvS?S;Zd`$Td4sdlY4B=DpVj z;GB--4WcwwL>bZgwia+-FoH)nTd?9m$)`kWfURntsPevI9OkDUq}At_Fhr2*m>J<7 z|K^#22*1UDq{{(|XIx*ulqtAAdQ3OrRygED^IBKe*@i}bZ9_@AZve0qu;T?J2LZ}j zw%cP}y=TD%H^Z>GUW2*063o&E!US9==;FnvZpXFNHRbelmmD_~T)}7{w z&e;xBEsak%$=pypJ3t9=dtnbS!6w40@X`hEdjEiR%*$gfB`8X5t54B?{Y@k+{O-C( zyWn|kD&H^1e6{Z}+mjH!-{_d1n-62-&sj0eAIe`j`?O4m+Khn*F7;(ko`grc}wJs-Gcu{X=-q9>JtlE}duQ+wL-kpryH@ zy?9QcUQwlU%a{$3@vO{6uEg-;vQ6$i3UQK;nO(8qR*T1<;wvvr-5aev6Kzq@WY?yI z8CkJ-_v2o5#Cy<>1tkp7W+umyd18ce*OX=Fs(i}ooB^lb_(Z+B(#0c+peWSQ7vamb z`z_V8WZ6ITb0VsHVCX3uI!$aMYq+2H_VJv|H+xOae}8%g0Ho5T!|3N(fPIQlqqpY} zehINqo%!U~bwZHmWWWQHbG6yOu;gWGMqLHRHz7_bwPG8clq4AvuJY+yO|fZb!!O?8 zu}-gsTJ7>_YGOwb9ZP{7Y~E_-54t0uZ3t;;kkys%#n||9@a5P2V=teS?-R*n9l4LU zX`b4bjK#bVZd&U8y01tpmu%od$DMxAMMv9l&MoL=#mqz@UrVGR_l0_DR1(?*60e1Gde-2*c+IsqkdsUBQplCu zbAh}kVEU~E+wWc#ljwacB1;-}=6;qO#+T9U6+R*7gTqwax52TW8BT?9baXZbe&3!{KI_6)y4?e%W{LkWI2jCl?{Trz8L**uH#O^Q>E0F; zvZVDQPmj+y3P_#pP5&8F;btP7L{R3-N@^b&z}P6C*IselB-bHG;@9&O))tmx7<0R@ zq~8V%kqZ)eZcoE~O~sQ8B8+i&1Ue*r4H|9dY8S&zqWooS;5LT2)V0emG9SEr9t7AM z08Kh_ER&MkZz||l>!~yU@mi`?QQ4AitwkZp6F1DCU$U*G8x922-bf6%3pYrD#i2*< zwpz(G$kV;(&?c|bI?kVkWtK(xu`&B#;UTMoJn+{-FXYMJH&~sfC%3D^A2%%pYB~Fx zYFb@KR!L)a;xpqnrzd^@O_;-5c!|es9)R%NkQ;Y{;h&+Ck8^jTn&jZ}P=M)n>!7A9 zbI=`ms%#Cn4 zcD|SP<@REH*!8~greM*drUAx|97aK~i?nk84xe+fW zZ{VZUt^WcR{^_IyCA?BgZ6gdxVu5?G1|-aEz1&EUsaWP+cJ~=7?fk17Km5W&X3{&= zr6*juZl+Xa>izM!qk7^<2X1*30KepqIdjyV2i+e+zNXSxbK7Tpa}Fm~tK0+5Cmz|g zd=qVePKdNVx^>DVw^plZ?2M6Lxb`!8Ti#RkyDG;^w5l=4mTJ7GuF?>G>j?|lQi82< zNSi&Ar21!4wJGm%haIm3(&qHRaalgKQ+Zo*VUmdvO3d*r$tQiZdevGg?sUI{@hBMB z#c4dG%$ziRt^bWNf~3wy9fsIN_Xz#^hwnqZ)3n%{%nU9mIShVGJbF@_aV%R@{2`Bd zRRV1z;iLf8vnhQhV!*)}h_XFiU+=HG5zruPk-I(^EEW2+SP43iUg88Ktt+fn{a3`C zxH5^rzt^)}NibifBptLnWW>O$q<;o81Ytp^|JHO2c^)R9nQizz@=pua-L?WcDwzsk zqLYg1NS}l0EoS1SEwfU_n>3wtIkq4r(>>1vzP9Z)u* z7!cFZk(y94Ta9;@KGI}VuVTz%OclFRP84+NBUYBAN9)j18h-Dk(N_YxRc+#$@;E!G zk3>;{dx`$+A4-y+OCDz=U?O~&oq10pF2=@SEP`h*hn*uC*BdqRBV;NUWL%7GQwvf+ zy^@Jg8oV=aF&&>FIZfBUhPx!`mVdKBuW_kcOjuX6o{4h~GUS(Oc#=*IhjnUUK6V>q z3|r^NJ1i%lyLPs-RMaW{5i$=F!>FC4M0Pj0<<@G%muXC?eGi&&ai*KS|^#9Ba>V z1r&49PJmi&clkkAhrws5!q)&@Ng2>63rG~VPQPfM6P3_7JQhw!k2;x7`97!rb;o&f zj*N+5e^fk>D^vzYxcBT!!vc`_!+5f!_>XV3z@oz}r2l;7v?ybOOoLg1yQEm1p==et z8!M{V&DaVz@Xg1^2sOzN<|B~4p!Qqom;IvMJuhY^iq(pcg1vcJBD)9j$F|MVwyRM%Pf=l_jD+NyPHL%YE6 z$(-O5y>IX=Oj2(?JA*YBgFzC#Ok z9`8k0Tqim&9(eUu$uOl3X@wSOFmmcm0q`1mIA64Ve_<%3$nNID@10j(FXICMN0-)z_1h!Y(wFt@%rzn&KWkzAN|(aV{DA=J;-G z#?ZdfVo{uhmv0)tmnXPt7NlYVPN%)+Ps(HATs zB#a;EeCAVi=f9W$o`(OvXpJzf;CLh}-04ibR;6BeF3%HSpb7P|@BS;Ns&;?bSOo4F z4DlH!B~h1(AX80$!u6fC-}OET`Dlw`(|?}OMDd~ z>qFr8tnPYIjcmoZtVUn^-ei%&OQA5Tc=Z`Iz9m6b8v)SNDYgGI z&ufpuaQSeQ_2BtH5K)eKXd4pr>O-P(?zf3-LUZVNwLsusL-~7SqM_*WS%%V#M4_TG z{P&M5x)q1sQS4zgx}C=u@Q?t@YU*P&n!}ih@#Hx{2kRN*I*QhP*keYtJ=k?c?y9!B$5bcgrQql3d(MDOE& z$&4)a62X+@f)63w)4wmU=x5`h3F6ai?c0HhJ~iZLYXK!aa#)hyA>2 z|mZaulq=2%a+*w}~-#`f<0;rmBC$8kUReVyk83I8Vz z9h*!SORnHE+X=(t1767g6#NDfz8iGC>whkQKj)G}l@4r;Kv22N_b&h+TX2u|j7#Oj z(K3uiNL1XY*yk@SMq0V^nF^C4tY7F%Xkl1!XVbIhi9k&fR@zT?lM-aSH@RdqE*fzT z0x=nU5YhN`oe2_Me7X&Slwrh-emZTam}o^KV=~utowP0%qBQVdeF^BBD(JrsnqT=g z0Kw~8J^_6p*PaLgV@w0$mjgf4%j*&bCxW;?u04g`wLQC{3<iiFFlUUNQ@-0`3U0PTr^* zMu`6+{ji*^jscj}HzT-Ix^mFBSE+}Zet434IpXr-z;GbHM|<6Z$ud>QLOHm$q>Yj? zi=X^?XVKh5dmh63E6q?c-(MkM>f(9y>kJ)X*W=($$*zh%V_IowxHcM_Px=q^tBS~D z^CNokYN*qIzqTFLw@*J|W1E6Y93dEjFM7bVH;omm!&C=Z%kF zDZ!5^rmEV)HlD6O6Tr*st_e4;^fb1cMxb2+e*K7{dMXd+lY~LT*&%qoG(^LQ;xu2U zlX&3i8OG86!Vntf_USh9iF4*U|J`}Z=mVM)PeAs{D4WZ*4$7P zB%t)P&$2Kr04o8Xy;J`g@KPzWe`1T}m6IZ9MOy`GPfato?=$ik(>JsouPv<{^B1k$GpotiH# zAFc}^jX-(p!24l8(M&7@pUe|Pfm=;J8d^`&7M`y}lC2ikiklLO3&7s(v`TZM_wLvp z)BGvu*V#(5myOg0-#f?hZM~gOm)pbI4r6l2`c;x+BoKN zlf8pTUa5LIE_z>y*IP(5Wwu|3hR`D}LJe2Z{OO%LwF75itx_bm2;*V*L_d!<^U`113iZ?AUR2fo{~@G!O7S z8ry*a+L@ya1s~1tUwKIw=9Y$~W4(^vWXYd@p8Pzd41rg5Et!ZFn)0i|BZzsFQS{Ma z45FpX$A2OpdxJDya+vhWuRX!EMr)~=G60EB#(9=Cm{yUH#1~9tH^>Jf<0R6m#c}G< zi(K*ezx7%l*|KrLE}7Nbi?ghND_o~9`pZ1q-*}Q*Q`{_{6rWZ;i3So3-$FI8e&&NC zWaY{pZS>)b>-mE2`c_1^jB#|!C|63e+q*hQFKyk1RQ#UTkJI!M6}>*G=VmpY(8bq{tn;^1f`?7^Zc-BLmxn4n zI7ms3JW&2@wCq%Iun#b{=0FF4fUU|6)~D`fAdrMDf-%qb7}(_}O-Q%nk`;V~i0&E` znTDr*@a5IOZ9_&vz`~lLmNpX8``JG1kxEJD;}0!4K|3<0TVqBa%r23*zlrBZWH4U0 z5PQ(DoTHN$fb7YEFYgjdU<)3`W~2TCFZR=#A)q&Z+nJ$iP35--s`>pS@B(Z1_+$t{8(iqnGXFSA(Eez$U z(rAcMIv(%#M&j7W?q4q*k#Rn$E zuip+NtT*wwH#{;4u5GD8u}hZ<6@&20Q`j4GxWAW}!MyTY;KIYKaj~9lLj|ADb-{w> zXQV5^!qH%Z=(nxMKm85L9tLs3cFQNel6fR6KmK|2x@yy>gzqqyx%l2?3(eDsLCocG zdslQ2dcLqbO%Nc`$|v^)KCTKql8YQ&?l90WQGtlNjj$*dWc`kau){M=;cMhq|fFjQ_6$TE)+((=L zN}9jU#9gO~MwryIRsj`Atd^e}?`()lD^;B%s>2xr9u$3Ux0maqBQ-M>|74?_%Xg7K z!Rj9hvpde``3walaYgh+!5Q07qw5!{qQ@py4<7ToKiaHbesEVf#mwc)!Ha{sUwaYR zYil{4w$X?jszTm52%aZddax+>6ZVji-I*L2fukc8YS$2F;Fp7qW|#QMx9#UKh&WC@ z@b|j|WKkGzxI%6W_|)$N(vBy^<2S&=M}T&+nZ~}8nxXRO<)lH7nb=UnCA)@o7GYXG zo3mta!~WY5Dh@By(QrLSG!7x6di% zS9=>}2G(da?F-j0X5}QM<)9<2P^&l*D$0iYCMgnRBFhgP;FHiQ{{xc#7njIn&F46G z?iOCDCSZ+j2-Bt2p^J`aBdnQ2?1U{L4m?WeF)8Z<2czjUtR`T$m;{Z_29g z>0R-hEnP?RcHD}C;UCvlJW`!Q#=eH%5m;&(#~y)~Xxx)!XmTP*e;VXL8x+aO(;`p| z^Y7W=lRA)%A&Qg4Ci82P=5l54I9(e#7KD~f&prgcc-_0=Y$*(6kGR#%a+Hj=nMsHH z{nStbI?Mq~mcO0m3g4GMOW%!sg=~(F zHo*;$bSAPDVg*dJd-V~f&<4;QrUGPQ6G10(WzW(3hbT`A_0#Y>R2$q%MZMcYywII% z>aI2%Lsu?S5d6~Z&+thwjJ}cHCua1T#4KIVsE)J)J~nf3t4Di|CU2=n)FGexBvJ*U zcqjy-l@EC24Xf1KX1_uW^(#D5hrp2oIs)xY*_=Xl}7sic0DaxuVQ;Vj(H8jl6{ ztl@;=7&sO8d1Gy79NJS|g5yuZoY}H4{hxfL0oDiPGb?VB&s?rXwe~sbb+Sdvx96Mi zf7XvCdY<~>#8qEs6=adRIh)T#cly&iVqloGZYgq2DE$sBY(0R;w#HyO5m{Xi|j`ryzeJhFvObXi}zQ$^dkUa z8-=*j7t{_XJ~$Hv+WXY=obm2O&HfejylNDi~KEqaO>WLW#z~4D&S_4?L?|I7O zd9bOA>y97h8sWz}k$zJxC8agx00PU z=&q>}m9ckFl0H+8hHU7@QXQTDL?Q5QW~dH6U!?M-P2yvDhHyR=*S$jlFb&0tEg}In&YcQjdt18>ST2pa1*s+G_eQ z$i_(cvP~<#>q^Bp?-6%4Xz=QHw?E&1dQfBsGqE1{N7)PW@SLg91&af=IdJ<2o23%I z=B3MHDwg?zEY+b7?2pWuog5RCD;Ts$p6L=wk|sWaAE$aA+6Z*uB?%5v$opCbw9)s| zLe|cu36WL79#gea+kAOY86xuP@wbA8`P>mQkI<_463)vU;mhz}ev%wYe9GJV8DG zsI*WsdD7gNyjS4W75N&vocg7{z5xOXo$IkwyV2@+8uJ0z_5FJ|yr3t0HolQ8DNX*! z@UtBrYSwpRoJm))>Ui-&I|GfHtg}9}+AglmSHBzP+5p0(>?gKNG`pAQ!o9wA#@CUV?kk=n|xk;NAC7^On%cCA6GUg(8h74Mx zmW0D{fTc@BUs1k3M=8z#svN%Ei)~)D$!SRh)g|_VkdkQiW;lkt?N}oDiND=P-Idjx zkXC>GUNXXJwB{;*6!`ng08u+T37|1I=G#2R0wvra0A!Sc!<9r=?}l{$d_EW{5PB5< zwUrHoXWjP(om^Xc&*V*LNj~HwO;dHpPQq`eu13BY+nHVMI=pjOlsk;VH~8AK#p3E# z1Ayw~&8+%!P<)FVQz)NqdGfTyNTcPU!_)~5lQhDRYkp zC_%1KG3Srg*YlBCiN@6Rz58(IAeQR&A_FooBDOZM83P*b{nB%0neKaT#g$Y7rGmbH zHMCz_Yq+w?u72_rRDz6F4}2GfvaFfx80_zu;fIdvk1$FYLSXCbPQ#V%gzb)_Nq(}y zU3ZOC)Aq>!)bT44i|W`IwFgrG;@_%k*I%D4G6?l|eYRk%UGdM|8h^+cnFz~LymyV5 z5h^5j|4ieG`CvT0^v)hdx>x$4e6v^czfVQlAfgj#Fy_(pxneG?yXsOU8$@^>PX-We zw`wab$am3g+C&Uz4)|>7a*fvwKsEZ&?Ybqt9)qDXf}-cC5E22Loax}F)rj@7O7$(2 z?!By3nfztcBnGSUa1VZ)041(8iYs;m!`C^1Tiyg?|0l^IwgFc*BSY;i+Ru*Uh}%B( zpGlO&;XTgsH^=xdf>7^jmsz*4(_pfM?Wj~cXnBx z$yXh{O^XBq{@qVmy!3{Fe;!W@={=aK2j2UzP5%pMBJj0CeFX*AMz0*|e5> z0wrQ0n97T;j_W9N+s3LX;fTC8`{qy)IZ0K9riL!D!5uE5b9WPVf&!-Q=RVOjTSwBi z;k8~2s=sRnuy~C3mJ|d`StNjPSpD|gN1T; zzn|xTg~NK#smNy7NR@gBtcTMt3~%0kdbzV9%NPq6P)tbZzz0`C{C#mdv%>;Ao>|XF z9T!uW%f{;V^q70#wi`Y&^GyCG4UkW@$`FG>2r$|+R>cng%Ay@aip@1NWmZ1+gcN$V zGh=iq+^Iy7a|>y}@#KfqSDsgM>yr($WF&@~n1*KGhMF{vmm|Fakd5mo!~zM$Gew zn{T}s^aD5dq_;fJQ%))f`$5s3r1`G7tNu9Cv_YzL=G)n86=SkQN(esj_>Q{^f$Q0l zj$sILcM@Rv$kp*t$s4ktEp{iiV&b;eWR+O7^3?$9y^dc_N(V^%wbpl*ZmZW}s~61t zC)3`KlBcpmunVa)|J8NwWr3e`izfB^AQkzeKpWXQY){k@)2p5_!R@8GcPFT#3p_sS zU2P7<-pWbsgYLk%M&LUO#ycYKV59bKe8nkHyyH-9+I^Gtsekp|x9$Vh6x$K2JW4MH z?B97keW}HJL>CBgaJvcIuqZwH&v0t{zp6rmOjcJdt=5#U0gz%O;r5BPbli`~bn-B~x)jPcuX;Qa4p=fVKCY!AcXB)_9R@svcMQ3a+3Qf#anpAW6c zy`hp8b*Np5O#tA*6rhnIK0?8wYULw21)NewAS@DQyw=aryfmQb0zC~6F(8jHAmH%yD&YeYF3g2R$mBpYO8RPkdMs{f+{XJILUCPEi(lE9^uM}al?6z}`_pj_)mbUDDEc^i26 z^#|94ClCxrF#PNB6U=hBSP%DQzhg!rc^sg`bNY4$x@IgCJ_Sk>1Ce0sp47kZzXIY9 z|7!cT`@e6#M>bl%n(^E0X@sPdj`Wk)&2m9A|eG&Uv*S&;NUT2*W&tD|}H=7Wpy5$Op4C z;lrxxFPj050yU58a@~5snJrO;gF|XTcxBFwrycmk?zoNvu6Cu}Gr@DrqBwXLlharC zl1vBO)RIe=mBUAV+QtI_*stF9v3zwjExdyrp!b|Em z^Qi{xZ+SxKi*%CxJR`=belBN2@N*NRaj@ydsNK{UIK2gkP!gwG=z;sfD^oQzTA#La zO5vBp_e3}q=cE4-Kbqa{n-PV-zF=n@csZ2&dJ< zfPr0T)65}Y8PR7?#2yb`jv;P)6TsvSoOqenNdzgKy#1i7h!>dojt|V;PIc}Z;55sXdP=l9(^p|759HpLCBthH#}Aa`oZ`9GAO=*n{lX#bRAm^gh`ld{8~~gycM6iYEUB7zn&$9I}i%`)4W;V0V(Jht>^f zV!k8yO{{Cv1jw`yBk8d85UqHM5mK#FpJ3fnn2WQtrDy9`CEQO68Kxw??(_}4`m&iQ zn>(Hh5S=F6y#FT24V9j|Trq(4`!-UVkr>`Hu!LD=3vz0ks3PQsHSoStgeYXiK=vGzZpKaR8a6rQN!4etGo|kBLTOdJzt8YADqF*68=L zY+4i#i9+9$xs`EF*s$V5G6!#;J-EZDvfDh2F4xfkUa^ny{IpzpCqRC?vPY5~C+HEo zw2A<6CfR4qiAr<&J`>#S`=sNLi@g%rg=i@z|;p+JN}{J+d~3!bwR|1_p_WZ*zFg8JdY2H&$(=>qm|h~`0d88 zWfyZh%%J_j4Dq6hl=rxTCAnU4frH$_ytGsCU*D1mn`Z+sw9>F*#!002LkOF@J|RgG z&VYXmonzYG{uD{CvS4 z2zvgHZG^kGrEZme_YMX^>Jp5Ekly?SG)UqM2$JF;2kQZuO3HlZJBAWt5XB?QAtk6p z;PZBUYmLv}O4#vA`t8Ta9W!j|LYfuO*R{kX~Gkj&k=x{OR zgyuxc7eyW4QKwM~Y;XaJ4k9|Rj;;=@E%@FF)P+@9Wx#6|HcbPs9Er>v%et4vJrx)Y z3O+mlAgaHtAg>Nf|0Z2za?+B6+hfpony5lDAE$d(o?L1}N0%V|tJR#e1J<;%&1W}W z4sdoDCj#!=VGrjHHMfK~!Aastb2s_g)o|qjTPwpxh%bS!912Ze_R1@tsT?0hUX>l= z0g~f3qq>IyyT|fEsc3UU%%e9f@6tYuSbu!PUgly3^o}%#>ptxjwWfP1pM1AwR0`_Q z%ul*q5UsD$nLPe0@(4Nfp56?GD!KCH8Cq7Ut-*bUr}KB^_liJCg=aP&2w@$IA|4wz z09gyWU?8N!5TMlMU;(rK)zk;6jObF@{cH>4aH;$*7AvDf@#!;Um?R*(8&!b z5TAj!VC4&7_>dCm<;$(+T{TeoPk0>2{Bi?uVfbTXN!yb(S#~8f2){1p713Ty*{jc_ zRf2HseOZT8+!fPXa&@%N3i994vCh!EtP(;}!4)kKE%-$Ir&(6wqjxugE|6~v?;rNi z^h=ZRn^;Nzm0U~}M7eO*=BYA-tWFv8ZnP1qe?Ete!mwVw)ZOGc|2qNyR1{vBFqdt9 zt8xG7xKiWPD||`~g42zB1A?)^}Kb zHZN&k&5<=QopZ~J#!ma`OZ1?J|EfUB-SQyjl4>N4fd(x7L!Tv?k{Xl|Zi zj!2NPdK#Lr$aN7wpAeRyx5Er=tJ$^W!M|(Z|tTlIzdC>lf3BIlUt5Nq<^Tm~-|%FF_W;5qeHfl!yrS z9V6$z>|&Do^kuvZw?FH)k}b0zXk(QJeS<=)fX#LP&{-( zR1mXZ<8?!2fYl{@0Ezi8RS2-g=bTa3d*Q&5p}B_RA`OEM>K{D%u@0Na==gQGyV{eE z-kFU(OR^Kv7pt2ORs?Lq@qv7IXi2vKqKf33 zR~4e`{tcY0mG_o&UQI&*yPiUi5dRcXr0|&)XZQi&;?5gVlgjsGONiCF!slVgk!>pJ ztZJM|yhmK~(d5AOK36q1cB9m~^hW}b?T;y(@{Wy2Pli96zt0DS-1xLeo%g87+w+(p z>nEs|=n}0MPb;Eh_?gkGvf)rv3^I(x!*_Q~yK^$LoJi7p0jnH_?F3AMe?u6qKfACz zxBXJe>2EQe*q$tu`?_BD9)1(HV@WigmKpH)8qa8vN?apP0c^wh78>C_RjVEiq^C_M ziLc~F=qyRnDrNWFk00VNCHidqC;&lO-YJo^ilZH&&-2-nnG7s%+mw0h_s~!K*O8R3 zdXceMp|+2$u<*a4dybOy{rsWgc1HcLhxIs2qQ3&MoFc#~p7=ka}> zSXC^xPkO?8?qUqhJM_C!S!&(m8G3Jwc`Rc0Lv(=16$e0NUMq zg&0AcMq)4ca){?MH15c7r++038WzbRm^di@BInT7Q-|RVTyl#F$ zN#cH-@iNC$)^ouQ!q6}$)J3U?09q+e;jv%7R-)S-Tg~Fv-s)g$Za{wkkBTK+0U;hs zJXGJte6PM&iTX!8$oZr`sB{db{2cefDoJ1AZ*D#m-oYZdmG{q?_rL4IK4v0^_kBK= z-j#xDpZt3e8`$7C&CK}3T!m8lU>~eN6kQ*41SgS%V5hKZw=j)Y0#FP)dY2(Th|uUH z*sKv>v8vZVEx?Sto1+TzzFaFnv5g#17WrL9fQ9+6OXt`vpdPYF5qWs`#godJitEns zqdqueW_c6LUNyQ!6e)bV(zIh${I@c-qB98Qqq!2VR${EvJCyR!=6RF<@y{hl_Qyl2 zRdh>gWyr&rj-TmBVa~l0g-EWuk#WqPgx0ure2V|klh;4=KQV%yBZ<&=`Hd`3vbOwb zM`EK7C~{MW#PqMwf&TJ@9#J1^mA=^L?)=LLp?z4} zz^fRs$dnB19)LxSBwkz09b)2&L~W|Jf5_!{@4+(syl>;jtxMRO)@!;>_C* zf|Li*srkh>E${4jGP6<;xw<_rokHRO<7G2pVd?P#keF5p9sPK4xZ#+U7-rMwnLkG= zQp}}lGrZ!*cZq-z186@_t{%;RgXMksAD(?aQ)6-CqZ=`L_M!Oh1Io|y@hP=8=Z;nE6WMYM!8hA-?f{1$b8cd%+$!rUIY(C?#tyd?@}8%cbPu%fuV zHmJ?qK(RGCn^1^sz0*lppm$UUzNT_2bypgib!{*TbgoE-8kMliGrE|*OR;L`nD~#8B-YU(wWNs_(+5Un**Ep zff5*To$NlVS%x59R8Luue(S12jXGt_L*fDL?dgaseG8>+IdO-~L@F|zkWY>U^Dh1x z0rk7Qi)kd!8?2c~1Fy)kWslqI^)fQSdt)j@1z`Z2M)M41OCzTRx}ZKg!ot(XDZH5;arI>LD3nB^1q++cv|OT~`i z8ZoAX%GydeBvt!>ee56IT-VRx%(otrPQUJ(00XuH?IE}$Y?tClldCSub+=SuqEB+D zkt!~vrgb*u#_nbS1i$a3D{OkQhQ9C*_ovEATl&}ISmP<2KAlQ_-Grxw;okhm`w5qK z$_!LEkAFQ2I`dNsF(z*}iya2}T2Gyy!JHg6a?(VNYQ-;G6|4Wf_7F}vyw!Qmqj_bZ z4>QdG;vN z=^|&NU-I7b*sajdJc@(!q=!6FXSTadlX49Q)nc-2%~l9^p=1bvHRosomH4qXkdb@k zwK%z;z?zgB&4?-P8#|sLzsT z%{Y;tU%0KwHCb3~$ktLakPPO$8i3d~dkjW@-}c&{roA_Xy008E#BLYgH~|6E5d|T5 z1-=~Mav%F2rjId+NmKW#&3}4tNTnvK&2WU!&Nh^Zcj&P(k)yJceJO~@ zoS%KO6uItbmOcCzhD!{lYhWV4@#fZO*oy7o-8*q#kz1lxvw;y#OF@^7UpH9N5Gr9D zYX;BMkr2>|+2vZuzwSUhgC&IIbE^sZG9UEj@$y~S&z<4_c`&!!@pbI=$YmMMAVTzP z!hhUsnCf~c_FROUC;_J{ehp==1oXfm^pPqb?6%TBxJWN{YB}-$xNgnc47!yy?)4~9 zW6^M%8DbP(-}y*_8Fcpo(^}Ga9~-mB)pA8)~?JOV4olI{h0(@B+Q$xC5d~le-8b& zY#`>{j%RNi=Y+3Q8JeK8lqc~AWDpn6ABE0bo)xBW^l5+iByDp*_AG z{a+ch7yxnh2-*Dy0ou!wH}(i)Tdy_C+LlrjNC}H6oR&W~t|{>)!iqZ@y6F z{Z9uEMXfon-58Px??G!D5oo{xn_qE58U8r<{UL@3iFJ7md=6aaM45`lyZE<6eG8P0 zM+Mung>esC$yKLmsfO4+x7~jV3cjMTb@*iwBQd_KiT~bVMD7G_Fp-i#3Ag3VvwvgJ zeDa^SDwA}O33bLZdDOqk{PT2>}^ZuiwC z;D=h{g{AxG60UoTEx_=y8X}RY`67bD=rAHwZ~`vs`Cl9+)W^D#c=^|MK^l0IzPS41 z>RH|V-K#!>g^OjYfWDh6G?-KFP~=n8*#jfad4nU}&x-_VP)ifu|NZ2NXLv%`xe)Rm zaN2*^Is&#*_a^vh`05^UOnY*g&NH5O**!7oW}4H9xfyUZnHgZ~0K+~v_b!(td%2#s zA|rICEg_#ru(Op_*H7m-p+vt=$fN zl0Qxne}1|j#4)x@(su-^ZXsUZ&0`U>#&wsB4sdxCkP>pfg9q8I)PzY^z-%`J?NJ5B#wAUF*E2Sh8%o4VuZNg zhn+rNdZLtMTj=$|uiVd*tJpT=#8*~vliD`09q3=`vI~SPiE2whwhMl##D7H+MK?>c z9qx91xPZQD#cTSpLwZk5pbp&Wau1%yZ&}IM+_TuhJ}t1BDZ>aUr;y5D*_dLM_>Nhu zW{83uG!i$muzqsesr7=fVVV|SlyYf&jCFxqiSH+5-I=A@KglOh93TnIQ06WWwkHLi z`0(;_E#OI;>y-BS` zRm|I);;aH=hTh%rn;-wey*2XFe+YF-UJX&cX5d(H!3o{=vw*t1xcbYe_}x`48RXm( z2qznisI9=Rd#nlMm0S%6sVZoNE5d{J7WmoU2tT+%aICh?!;F{08 zghazF>D0pG24#JQ)Ma6K)cNP>Qr8}e3zM4XO&dkAwC6^+Tqz0GK((Yks9PR52Y)ee zaK?{9Fh z1OzF{6Z6zi=_B4F_4tM&(p6ufcX59*0K|pS-EFRos`0#BxB7L5LxZ5_UPTdAX^u+4 zk$9hZ+`{9j{Wzi@62z>L9lE~Nu3YmmKinE@mFXWlux76q1Ml#$2J zy~IT%@vm!(DmvUe<1z?0uks9UEt46=ExfsnMMi5nUL=8;h@pbhLh_fZRqa!_-VAAd zZ4kcH@p+K$r|y5suWeCLiF|VN$gz@cGdn9NDaOHVBs;=*wIW}drsdk;6KY3lo`2{AI5+U$BDWJUFm)aqj6;(x(Lbi7|Yf6yphgBoS@~ z@&3jP+jYo3-s7Jh6Ll86nw__T=~6!L{6`!G;#on#%J<>gaa>pc!8nirBEEOvD83b2DkFGe}n&vL_Vt7~BYWb7J?oTY5-bIK) zp$Wj)JV^Tv$30cGG-B}zio@Xc`g9iODv@tv5F<*T9f*EXNsILj(&5p#`)vj&LmKE@ zJYK=(vAM@6xoIfSeNoq*%i(xKmjsrk_OgAueO~k`*L~Z7e zG3nQs*XWS(`E4m7!$u$_u$@tYTjlC(IjL@S==w_alVmiyuJ(^(Bk{5D*_u!pd?>(} z^uz1f=n5YEtRF!919q7GvVTZ946bY&zn`pou#&sWCoFn+UqEnf?{`r&uIVIm^~=t0jOnZog6W`^$>?)m1L z2WWq_QHkKRuh>q}4<3bzfY;F?HpDLG%OYwa7>9-nN+Ul$mb z)}d>ObXR{(Il?cG)(n0iFAyZ)9h^xvS4GnJ9BiMuw#9}|PnZ4``H#`sEItn+NY_H$ zMv-g$J)?uqt%56~B=5pwGp^d|uO2)V^?gePPWIHo$*p{ z6+>TaHo3+CrpMqvE_U%n%+Vyhm-mR_ATK2a?1MwQ%*mg=@YteVRT%l&W=yGK4z;hMYLiI-d7jH45`uo~Q7q7}y zfK7gF5dWbfX3pw)gOG;zXTO37mt-de`NkO^)!O{6<{4L)>i%1|53+~T9A(i`akJ^c zVFDALp43U8v>D_o9SpxwQi_`DP?%B&Ku-1){GRrlX=HAikQD)Me2ovR&?D%ca(EBy zc=&6#_LtuIsY!%%sA6fY@p~ziWhoQ=OCt;>AmG}gWuKyRHw+T%Zbbhx{2bgE2x;5! zB)Z951iOh|T-)vNQ3|j7e*I<$-p-u(XT(}{B8#*cX%1cNXeg+HS=?>T`tI0~hTw>N zhzHIt z-wJuuWFu!DV+jd3l5|wjKaQ|98RQ;JOz;H4ncj#z+^U` zrh{^b3RJ;17r6k%*gQr2UScJ8CD{Z1z(^5DtkdW}FR`S0=iBIWdp-)hfq8OYqaLfU z1j)d>Q8r|9uSww}e2xa&1zfFBm|-k`-&=jWhFe5At#mxI%{ zxjnzZQw#Kz8CyxCor{W>(GN?%*p)0Xv_PMTs$O2ZtL9|Ug4sOdsva*IZz%yyz6G$* z;-;YwJo=@9yjDSv?qfC`PdR~rF{7Wd);QPDwHYZ!7!Y7Gm~U! zPTv^s34I*{I?#&xv?sFNk?XNy@n%dg#LZ~za)Xn18G{%qTRd_Op)?D{3rivId@I6w zWO>o~SO{H*=eR5;{Z(3$xo3UK!SZcP9P99=JicQ3&^^Dw^?L%;Fj+G>Xe>|_dx)<~~ZxS{*H1P97@Za9mlfgC*wjU)~yV?`)M#>TrI1Q(tWCw*OwNV6^i5qdA5vX?j-LrqYfo7yX$8s?i zB&WcgzHzMi`pM*atDU{M*6tg4=^GUi0(f9>GJ;sxPN-fqYe^WAM3x@MzT=A*ViVp~YzR!-_9svJmMlBU;YuI& zB7T*I{Ix8mee5wL*+JO8dUtdMBbwX!t(~x2fO~qFx(8f*9Neeg4#bHB=YUKSmdzEziS6~iVSC^u(*farDs5R(tY^Xw6_y%; z^E>>!^z6x7;=2R?S(xHg#>*bjZ>y12AMNW>=vUWb> z{bfD^cEU>vj`kl$t;6MidWc4%E?U$wc+7wgbwC7g>^gFH1o2o@d(9PE>al6T6J;pAt)TKLm zG5w}$NZ@v)%JyIY?_6iiObOg2t$}0#g|R3~p0~x^h4LjU-918XT5Vz;XmRa@&Ycu3 z)(0M;zK)$F*|@oUcs1eSgQp#Fq&9Ykc^C_x)1XTA82F*U+S-Oo?Gl)RDsMpc70trd zg3{VgqdG=0Xlem!%O1q5_Fj|y<8stHbqkYdB(dUj%{tB8qLLJj^v^mPDp^~H?Yw_~ zkM}I-*RTA&g+nbnt+uww4yo;%)&wz0L)F6@1q$e>4xDKg-+Bjx9RRI7H`SOGIGhxG zD$V_3JanT!yi%WTyM-NfD8m|uru{+MME}-aT@wny`_(~~bd+yN1DR4@833DS?Yqm-|<5+gF7u)C>4f?f}&Xc{@vbRpcB?YG2!*^m1M)UieMh zw~N)&APr53HF6MxBukt?E$KQC zB6A}^=jseIY#R|bC#fB9q)U-tfj;U+X^&&GiiY3hT${ym`!k$>pSFA(8+*`kFHK2q zAzFTtdV4^C+7<0JROnyM>u0C_Dqx*`=y-KKDM-PGzwiTFX!XdJu=tEBfkT!=(Tl@2 zz!_e0q8m8?nYo!t_k9D{N*svv7bn9Y-9Y^K|9x=S6m#G$rc(wM0aXw+(%A(J6C`6S z+jY@&Q3v8v$9>(}aL&d)Mz+jc8?^qi8FJ|+3TS_^d-=vx zKFR8FKAp!#ex_PL&W?_3Fw~_S;9jSiqaVR=65uVF2ImC3+dre!&uGe7NGn>-_jI%g zj1)1_#*OVA*!_CK(Ido zaR)cL>XJ5VK%w3MpW!cuVY9{^!l)JzJDwr6Wt#I@(nF-1rw-P0a_b2_`=<8rYuS%R zn@fUwb*pJhgylPNKPBuoI=lT3=wNYD@S8PXU>Ng(7z5dny=~6v-k$-tPIftYNyJ>U z?xgCCsQddaz=^zurlg+=_-(qqp4(*B$J19*IALzYuZaQ`@11i_r(kQ$$XLPN?V5ul ztIh)9K-#Qb2YiJJQQ=e?GR;ixB86K%-GlKjt=0`kRqn(XMeM=VLhc}^&#Nrh!uS!Z z%=x8p;9w~NqLaz$`v-5wrJWwMoZfd%!M#ExN&m;a5sYxy|6BkR&5lBpR{mTh@@O&V_ar;XKeAZ*~?F4PEGzjal z(F_R1QT?90Le7%LUCR^%S*B;lk?&Xf}{r(5{mwO-Y zdtT=}pA~+SSKH!J@e;dPI{T-7&!;Mo) zhWCtZ*wr{k8#RuE|LSgxnf`TL;vhKSL}Fe|-fQT_#Hv^@r}wor1OAm;t{17?V|QkK!+JqCehFni7@_sOh_S3HiwgNHRV6>J%EwIQdXB>rIBo^_yCT zUx(?^>NTtUQtkCi*6#=vlTx4KDH0{p%lDMb9ehT3K$6PS-39q>{<>NR zm;Q?W6vAX|ck2|BQDgYMp<*klK(QoAYGrbq4=m$~a^5f-DqP;d0LZwv)>vdBEqUwF z?B35U0^_!80O1I<#q$a!MkU*&>y`J=Xe70qdF45 zLGzB#Blk3N57~M-L{F*;N60obdO(5`~06DL?qHL$^kx= zZ&>@B(*8Qimsl>B)(;P+#*q84%;u=Ek}`aI!aucI3mFLhzspI#YoT0@i0}~-nO3_E zDiu&ZT^j5Nw_7~R0Uc8X{;+!2{NSTvIC|ETwaxem?A9u;`||VXmc*7E#)F&*ATbHv zj?(kR-LL>|!!}D=?QFPEMFY&xYl<>o-kl9bfhoN-f55_9j3*M>KMa%&U+A6Q==?T8*J;%dbIRf-;pYA&M@X;-D*1i z7wouNogBnKFJa&IvY1vA|Np5K0%Y}@FW<8GM&%{p(haA776W?f?_Mv${1}+&Q zwqiY{_>6{XZd(sSnX*69BnIb?zu+cD?|-WnbeUiUiP=Cb7RpQ7%e7+5?s6eMIPGjU zMc(O&B1N##BW-b~)1~Ec+1X2sfFAAk)10mHJw|})SYZD6SK$eyt{$9OJ5RosaMzLJ z@qN0pgrW5!b4zH;U{o#0Oxkph2JD)ao%=C$+BD)s}q-aJI zRv_?_7i8^a!G8}&9D*%hrhKzbbt~5$gZ}tty!?XPp?@Ohg+sdgud6Z$evIBSgEkXT zFr1qTb2_M+kCX*=cE4qSxQO0Am%3QRI=FZmSq1WSmxnWwXg9UZ0pewPh_EQq!vT$B zr>S6+p;SF961n^rFJk%>Kj-21{K4c)iIG$o^~lR*fyyIkfmj4G*VJ3y?UlA;T)-*a zp=(PXBLDCBos+S9)o-U49|Q;`3cK>Etz7xJ!nSU!y1itzR) zcpaG+%B%9lU;Vz;WQ^FyHr(GW*FsyJg463D9G~_TC+so+tAqkWkS-!KHj40C#{`l* z@5g&wi85gFTWcxhtDn3UdjRJ}c5X`dE&Yc1j-vS8=yex>-1SUo&?YGzuD55o#H zqu;vsdRpMw`G`-_89A+FfdAZcJ#8dhXy?z`q?WOEW2f^zGR>T^p?i$2tA|TIzp;O|ZwINSoEoHpO z^E$(+rz@ycjUiyXPQaOd?C_wNPj;M@oP$EzWCn~|6`|sxu74>Hp}A~W7KefshCT8b zZY3YJ-}z8ieFhH&N5sk1=sqV?ZB@rFo&V9j>vNdAyGs^Q74Y-L^v3&7USa)(Vqo1c z*5zUw$Za=yStsg^)izn$fK4x%YT71W=E>mxKY;sf4vwrkY(SY|Fjp_e{IVOMcoOc4 zBYBhHpj_^?LjFoa*>utBiIsMyQ@V}ACt~Wz&p*Z=u2;$4=%K9uhU=K}T6fqD3qnt6 z_Ex4S8z@F5T&vv?+}y$Pn2+97bMc2P!)8rU9w8Cxm-=O^ca2HiO^SPZ^kHQ^N3RZ3 zn+W1i7W+E(TVr>>r?uQoQ+&+)4>A`&%0+8##oi0TZ_aEC^L|Y{j6LF*@&GQ_?5jab zrX%chQIWK&3O!ckoBz6*12;xW2*!MMe)utN14?lyz_flV^mn2PeyuvTZ{Pz~mkkIT zr1h;iH3P;wql4n|Ul-NJdh5LF(CquRW$szN&1zH7&!q73bRHo4>4p z_O*+feaIKIZv$l?2Gf&nBNkyB^&~l@1^Q3dG@yj|SgBE~sQi*olYapT+1;qP(E>bwc?=sSAhQrrN8%ey; zNyxa1bNH2;zzrQCM0=>y?ZDv?KUsMKm%@$IezQbo_@!-LrzN8t3G=a3T@0a zB$-^g`m+gnEBCoI_3mL7Ge;chmf}$BJqKzRDc}&e3`-1tvp#zpbex7`E>-kQ&?V5D zkWlr)w}l|sG0r8O`?1v#OT6>NiuRwlNoE}v9m?EtsD539S1<-JyAHOvGW(MOqtivR zUB4Q;sFYMLIFAKT=UC1#c(OsEMdN4}N(^Zq&Z8jZFUuikG9>Ico@N`*let@10Tl(Y zbC$~O7v0(M5vm4Z+oCkt{#_J(M)qFM`u(zL!U213*Zz$$hVRCbb0cVg#W#mI6)wKqz$W>3pn>%45liDw^ETFqD7 z546xl)PqV8>K3nyXIzRANr|LDRv#!*t^i_!J?iea6g7O!@%edv&-;)sX=PAuebbj` zqEpWYQty;ciJrz*|Kr#seFjl)C~TS#4Ih^8k$!_A#CeVY@@!>jZ)W&*(%Tsr zj}x5JkSy%X3G|Zv3HdEXj6+p>{_qyd{MmjZ&}@cJp*ncyy`D~b>q7W5c~WvGCw9fM zNaFDRu#5~pGjbzF*2{1>A|n}^zn6s)%u+y$fIS8t{yUziuPEmB=+Wsbg3aB z7EG(0D^^&jBrb;}6|ftWg^pzVYVDc%nzm8BlQE}zQ|mCG>KU!47Otu}X*KH-1R`I= z)4z;tRejDuKHRN1*B1fL1VwgZ1>nmmpSO?Uj~`49|M#bIj)$#W9C*c>`Gehk?07k3 z(78ie-MDA#y(o2*M|;+BX}7$By<(i*_Xa##+seuG+HG=eH~@&fcYSN5-FIlu17Y*E z2_$t8*(BR_X4rhuvp+MTs9+YP{dyvo@iNGa-Mj0JtCoB-U%~-nIqt-xB?*}=> z!Q#P-xyS<}D9beLe4L>Zi=$P4<WAFo; z1Ik5R)Fjxf^$CpT&ueiU_YIUm`pf}vDZx(8A?rVxK4=Z%cKEL`0Jb!>PqtJYjIaDU zKhpWjZNCpjXWg}=86)5t8vLDqA>N$7%Sv93V{7^s47ba;MVFoI!dtYzOY4lLLHraP z{Y=_C2O5OG>}6~fQ);n(y!*!8gOq}HM&!ixtpb$Ui+17W2$zX+P@)YbqD7#Z7Uli@ zrBaXv_3QPT8-_iLxvgY&SSEYQfAa%5S=n{6$~%?4+)tzrzwZw zT9oli5B}_tx8nw}EAYME$%7l6^~*guhP7_*+|&J@9zd?Oovw*1$7qxG=RtGV6y%}b6qBb!V$-MA|P^@|a`8a$7bdCBCyi!vY_bmgYLMRl- zC%-38_HuR~B;;GTrED8rcYHy6*lTVa5=s}rBqW=k4$G%54}G`g`D$(!UGVeLts>`b zX&YhX&u!-8X@r_$1o}hKG^WKrW+{s6UTu_zk{_)}+9&ZZBNJcpnF>HJ+NF+zPVTLe zC`gtFHJvxE2sR`!ej2t$xyiSg@JRH|BE{jX_t8Q(xkFmFyo|;i9QMH#1m1AM)~i*d zTIk_OMO#hM`sjLjqTltyON}R#ZZvArA>`cua+RDPrn%e+5=P(<;Ah-3Vz4Lp4N&LH zxFthC3Pd#R>3@5}O64(uVZdIEBcGWk?Am*;&Z*F>usHRkvBd0*jQpX1?*)E^vjYY= zYkft|Zv{4_FmNj5&HkCEYsu$5J_r{A>k~PO_(1dJ=7$%DC%FOgM1$sU>8Zo<+Fu~p z*Q=UeemyYo&W}*W8z@1xM?C8KxauaW<-h`Pe60YT8g1atirF9wY4CVa97`{%{wv=; z+1u@n&6OWdOYmOgoto`9nd0RuKd&>1RD4LX^hNVT`OKcfM`ZyXMh-4fLu=X}QIxi>8fhws)z>zwT2V&}Dp=ov zjwy#+!j2DK(OvKeb9YW=MOyD` zHn>&8`!8^(u#|n@{FCd6DQuAQf@-&t->L#BaUzQUxV@5`cr*+w1yMhf)*=x zoV}dHfw3C!V@7Bp$F7vZWsJ)HjZfH!C*S(Kb*aS}>Lp!YXOK!kJ0i_y`faDq(0{xD z2nKPgCy!f>tS;~fHvM>m#5OGT3{UYbx{Fk>IQ7+)$Du0qsu}JQUG(tfXy{piOu5-Z zkz?7d-zLm-Kx4tYk?-DXIZ15C5PGD`+vJw90ZrWZxLXgDeIEVWy`@oi_L45W?ta$< zBh=UUHB$jU0?W}v{okg+(3ZlKg*x%X zHC`?fE9u5v?B)a`JCmh5_IysX;t>_gig{wKP81wYO9{SBx$nUv9T}2xaDa9k!ka?4 z&DbUi4gv@;bRiJWVL>8jdxUYU;8Pfn1~cVN`R_?Xi*sJGfqsoCbiK(uHypUK1>z!A zzcac|az+3kG3G|YIh~iHUwuMQs#il7Q@XDR(`(c~9Ou#QwU7A)c>#D{mj$BI^UsQB z7xL;e-g|u2fw^<$3=5!k}S?Xg7AhdpF^JUM^F zOR=@eQ?P3G^fD@hAATp$c>}y|;(kFo=|N_TZQM!K*wUvt|5;ABU))UOa{#8T8=p!D_~U8%ME>V2Irm^m$HnxvYMmNC$e1*MOmbXBYvJt*bW`1 zZl%R~Z_QFf%3Y7re)wrsQgiulGeY6N<00;VjPvB;e+PpC|KLiUb1}b z`5L?bC0VV^IW?ALoblV0#V?F57jW(KJ=;y%-;bb&k6> z!0N^Gqu>83e#7WZ`$k6l-^*%8ft&a@uz!c;G_D;OsdUPuZW_44LXBQ__Q(5^QL|z` zWp=nMwRRArI5a*G1PRzqnKU?jGy=MOA_knp2fEImd2qC8-M1(B+qU9O?5FO@g~`q@ ziUEPRl!rvLu5hd`=J|ojU?xJ=48cAEcC|Hf09TKV^Gf?R((Vw{{i)&#Swe1@dF_ z8bF7y|FPH!Ep$bKrghtD#m02`dBkvBzdsx(W*XooPL!RJ!_^jDZTs&a*I7Gb9M)hs z+C!(PgGdydXSb=V;dd#1YTSeYb~XavtesuF`G()j_UAli_Q-qbh5glUxc|&{6hQ3r ziu39m5)Z6t@7`?stYxs<7WY~pqtLi#@IPZcv(q0}=kfO9b4hyKeyJRERpi3jWuj3Nkcbl$TzOQTl|+a_wH&*%phVtk^V1ad--#iLN77V8e-0e?YT^! zf-HP+q75i=@h@uR7aS)VE_}KBaxahk+X!O%uYwB^P94otejug)@7Z3Smk0BMn*B6v zpMV354hSh?c~e8_r?@Ejo{6}9f-5|!J>mlv-R*u)`J4n;0UmEd++l+HQ;B>mZ~mNFY%`>JuCWKvbnPFLrOAxRE)+Xt}yt4YA&DG`lK z`7y57u`AO?yx_);#vn&)v1!MO&1;9o=l0aOqYy5ZZ z1?$>YqV;%#ds``o!_hVxyXpE4JEWHC@kz#hhZ=;tt3%0+z@_d?|A=NJD&79wGWo%P z(%wYTgS3r(0p#bZS{*x`8XR_0`thirMoGNqs4H`L`5)xT!q;>7s9dL4xF;iAC0TT1 zfP|s#-gv}OAEIj?N;S^BZe_oQ_h$_6gddG{ndaFJ z{3p4o5Z?DIu-fPK8|mU4dE{&pq&$9x}{~okfwzMlJ+Tjnua5nC<(Ge85&_ z`64SI==z}c8cueu@#f|oSyG^N3$Z*1>-~;V3o7|LKNe0MKe6>STsPbFOuZRb!R}zz zcFz@_i*lB(^B|J6rrT@Ya8V-vq)2Z8opKVK%SxV@4qOB$aU7e~1|>Mrq)Wa2dn^4Y zm8tFab)!=tG_x3jYhEmbe+(G`QT}dF#Ib_W=%M`wM5y2}$XWzOR+r=3xSscSDy1VS zDMimsiD~n%qigf;X+yE6@gt_V4=(f55_A4Rmnnmf8;gu<3acYF1ky+6-Zngk4|cA2 zgyChD{@&=f@4)6atG(O8+w0Nk_yQW>Y0+t2cJu`UT%6RxzSLN`UK+No{D8}$MLe%5Z7xd$z7+H zq_va|EGiLjYcUH9xi5511H5|1&kfa(>s0t#1^eMm5GKyaD+bCw4xax^0m9a%1R|Dx zEd1+sv_CkVrIy+^Txtd5L(1wNn=$)c>tu4w8r|#J3dQK0&F{aK#t1+sat2(mH(;1Q z=zOg*e?=Bf-e6@4YPMFKD-$^Q3b89UL9_R&L9YmcuLzdv53gQJm9)qglViHSw&l#z+UO)(6kwwhneyUv$=c z4&H zwY{VMxu?@_;7*V#@Hh=vZCQaooPCl(v||t{?w>40S2k&S{SArw1YqczbymV#lKXp8 zO;TC^Am-wvjQs0`V5sUl1pWa6(N9_h5cXaCl0X|bH7VOGLpBu|aOXcb^mQZ7+-+O+ zWwZi4gZ&cX_w_olH|F?d*Hb|E#Gy?T0);5%b}ajZwBJS>ncnpO_Q~0L=a0qLSy%}6 zKkc>Y?byWMqTL(ATr`x@r>T2un1M1cX%EEnEFjYmBdkmmS(^Cx>j7!31XiitqVsOB znK0ILnxm(VD?VS(^6KJ7L{&UuPOlF8B2Xc6>l@8>FfMw~Uvb2lCe{AqC!Ooh5t5rw z?6#CBZdJhUx)B7p}ImJCvuH2<%YgQ3N zo3;Os4HJxYYtnS|nqq`9$%vK@+m|f!u`nE@_!nRDk6{iE<4Lln_nH_&dUJLNe^ zL;DS3P(xnN@w+W))Rb{=^V2_Wgn*P`Oc{ynf1NPseSdg(lk&Cq$u16Z{C6B}4U>3=a)uaH0tg_D4~#r!ql5;4_VtN_)sb_o6B0(t)Ip)X7Ov6~Dq6e|Fw zpYm&PP(C)k9UHm7pwz`QsMse}gOYyTPDS!=-)-zNft-h!2S@euiZm86!15SCeRqgi zAkLdX*>8Wb!fFq$uU!IE!FYLRwmBJy)UGoQI=ueX`R!K!#1H?To*UY^Ik_oELCR`bWUXv9zn_v)e@D^=;u0Ms9Y|P7MD&>*TsBrGq4f5OL)4i# za<~Qos`b*53M0X?HI$NQ_)#qByNegESw(?*Z%Redvh~ZU7g0#cDI!|kO^U&R=LX*= zTG+}T_B%aW@NOrL+x2`Bh@`rX5OjKM>X*evOD7%q`z6eZQ`95xMZO+mvc%^?7s2=+ z!->Ust<%q(IyNmoj7YCjk~I&ry+cA|ZVL@7r9>(`^UeL`qbxT7^y2LSD}RQfMNO`c z#C=y1FC}eK%I}%m?JBhm3KObP#m0}uF*F}I1WFWN=XPH!e-FF!W+ep-7Dv!#0PjVC zT><#uJsSup`*_0S$2BCogeM{au9gl!9Zx)o1ml%hpa0lQN{4Ix+Vz0K0`Mz6?3avC z>ly^H6DRA1-NqUA$~IB@9Y~D1zN!^nS|QBkxz*K$P5IuM>yqotF(dxh8LY3k$P~GC zJNQa~_+Jv;ALsBCMv{41_o~bJr1kzKu<+UsY#7$3PuDaIX$ljg1TP?&c8dun`b6f+fPmOfc3*voorAuD8!)ALz z9zmE=$M(#ucTl0&f)2S$r7i%;8K-AK7e{pAhX6C}_7JKR!Q>=*E zI>zmtr1{dOf&z64lKZJ(FOABJ;)6a+3FP~I1>%;DVV~|x*b@YHBXHT8xY8#0=_2|4#`FMq=gy>8??~k+8Sri<=(^<)lp~ z(x7CwP&6=LW~EkW(uA;#Ip)W4GFVCdNL+Q3??o6xP~>Ize#cgUbMRg&d~VEgZ>@8D zV(L#8Bhc`&8jhMSpM1rQNcvVm<^fNn(c$ZFC-Z^v6>d@A48ne63-!K&@ezQI0NjcM zIm4fR4GVL52{XdHDj*+Mi0hq&PoJWMUGxj7HFZVAh2mzd*24onvm)(=CwVs;vtHb! z8(Nivy(f5J`3QNSY_l+kQvB7(G}iQ}XWJw{Rh!dbV;UeCP(eyS67`9(AOJmjvm&>$ zlAFXdqog{#Zg&OlxK}*-bZC9|lgrsqFXM(dbfl$&EaITOcg2A1wRA9|>s;nH7B-A;3h7$0;GOCM$ke znTned0rm$g0EK;N zDLIeIf4j~~dU|lsmuP;r(3G|gn)sT}*`Ie{1`H*kkBYZo{Da0SjiJl}@#nQ4HCTB1 z*ev>vS@?e*4;J6$pUL4-F`U>sXSMh%;F!^83$qK*nu*H!Spn#m2K?M`f4VidAc z964PLdw}u+G{J)IihQ#->zC5Cz&0Sm4}6}{*YPi3uh?S!^rTi>QJdLk4=~-7{QmA} z4usypjbj8c)}WgdJTLz({aR44rW)!b=(}?l55%NpA?+XY-4xE%MgFjYyi~y_UIw_H z5f;U*%QgQZ#-w8p;=|WtO{BNd)`}++rUNwaSKbG&Uq?iAq6rm37QfK3Hf8u1>9F_H zlYwaAtw6VV1n%)D_54O9xasz%W13G#^IPnDh4W)$^XK&(Ev6=yoqx86hIr{(YcPjqnS0dIglTK*jWdpr!eLkr;J&p5gns&Hb zc`F#s{4_L?{o>36d(v#65)*xDXY-LoHT7<3=vBza)TTL!wa1d^=By(Cz%w;b;g1@kCc95U9Rn zzI~K%GFGB(eMqj~a2Qcv3U@wx$6heU2BCF-EJyNxnruGA;cvtJbL!tlfVM=#lN{#) z4NK}~@~oVa?IvH+2w=%!tB7+bc0Ee*R-HnwFCL5!!f)jKj##!_aB*J>ygA}LGXF%f zm=XTk={<~2?$JeLLi3HD@^Wr|%hso?!~gVcGA7=`l1|sItgZ>L3yXP8Nc+#4J6iXJ zsWA!cj3s*FHLRd{5VSdvK@CW8t@5YDi$txkKc5|{c6a>2`X01E~3MgRA3_ws31vt+DENJiEr8BW+} zv%`C)s0`sD&%b}}b6{5l48Ko^Zh%fS(lKeqLBrgy2^mt-T+2y*@(<3}+>2{?xG5DM zl;?E3zf_IlZYqD41VTr(;C)6-CQ6#s=#KRpn;D{z{zg3BuOx4NyF|>LU?^S$VXN>- zdX?KJMwNO6QJuj&m!|{tYVcod>XJWAmk%Qd<1UH3e z3yX0ru`B%}3b)_}wFbrGL}5hZ($ThKeV%>Ausf!PTlF-bto&kBN>u&Fn+@jK8Q`Bi zh>v(+Z<>M%m*Z3Mea=a?vKn_$s@RqKUf<~$?;eKRnQ9HnZ0sFa!>-JBuk4G?m90Ps zmS#h0s9c7=;?ab+m&LOS*PfgHK)>ZZrKfM|tgJ*70C&1t$SWOFxaPeaQZiW4^Ka8M zTEJtc2DL{C(F|^j5%Iss5ZM?>WSS1XfMRl7_RwT)BF8rWuaxl8t_;SO<7o*N-Q3X} zfEytr(d6EQpers`Lna?0+fgJ!GyPDmUu?q7{{@3EzvX(I)H{W9kwO+fW++hAtP7$`Y@-OyKm|JCJij8#Te4JE&w3oa+S1`XXN4^!2|7Wsq?~-;?vr=a7N|`_E-FE zEPE&={pK8g?mQ4v2GXJ{W&?+FOUA$Vj_rBh=H_%mg{v8p6!%D*2z3>!G*rJqni7A8z;wiCOhVZt;3!|9xfM-^RWFyi{)#7W_zr{q67dT1+DxI{BvNk%ok zo@Dd!DU`@dQZ}=Lr0kY3d;f{0EX&*+^g&uWFP%PCZJ1PlQ@G**JQmp`#Wh3Tu>ZwN zsXigqr9eOo7g?vBcP8B|Z22-m{hIlvsc-6xW4$@6{Fs z=eX>H3uwH*eUQjtLAm1cgY83?^BG#+@(*~RibD}UXfAp4(F4PvNukrBruIW22l-~v zd>6Bg56qE?YpbrcT%KPP%7Xz%WWjA;2O_ zzy0!a)Wkby1BaVnMdzVNz(TRWN9GO2E%WjB_8W|TxL|G(fjY<^1qm;4#Ci9(1a7}F z$qz(1QUUpOICJ_7R52-pMh6<93VAyj89U9(pc}4&nT?H~c#cy@ECDB_5||$G_#1L` z`{>zqRgXjx2+a!sQehS<8!*+oyt-=ESJU)=Xv_l{H-662Zj_NQfAV`Kmg?J*xPjXB z6ga{9RaE#UMt=Upy$J%3zq4<&r))&V=vd268jsvXDONCeRcq6{4k%0v>&7}vVvY8G zrvWEdqe^V9rEqzoiG%Z|1Rx}OsCtJL^u5-b8f}V4!P8EjDSpd-3-D_i`C4;P4pR7p zt4KrKxV^f#xB5dO!e>_%~x1xshps8f^f6`A1 zTP$J76FV&k@?A=>+lptg7~$S$;Mrzq?RJ+=nzCZ3rZwAtv>S7GQWA2m?tIcvk>WT_{TrDw+JD;PtZ$m!g7EYLiyx-oe z=3)h5oijW@*_^?OEaK!N=h~;WDdL9rviT=0aeU0oy-&fDO_Ol-!vOWFDpK-4KFHR6 z#Z;%K5Gn9ablk@?hF=p6Y7>TYFT~+}PG80Xu(hE6>)zt_H-B~&Q+&dPbeu=0McUr} z$ukJY2TB!Y+&+Ngh*a8R=j(J!rBt=cGIHTVi}xyHn9Iy#=yQj4-)8NxnMl?pP*%%| zCnc?1o9QvN`z4`zQ^r)`jb>JMRUX5=4y=zpl*Uq|TGZ17gu7oSa4_ql=LyWZB&{%i zV0|rDaygdKrEc*zDj6o8^W_nDyQ$uDBgKFd0SXY#{ZTDJ6M9loK!q~=z7T=Hx?dzh zm_#@H2s=}R>?8pu?3l+Ru5X&tVo<_0$cK>>7y$n|x=*F`Dr3SzeP0ZZ z(@N7Pw6(s}73u7Bz4l9;AC5kvUueD~vDG4!vZ5c9r^O)KN zAn0{r2(q$0=p2>DdGg_mOv-IT13Ev9cFsJx*$*fFb%#aw)XnVQbO#S=zy~*MhwY)jvcFvf|jPcZ%$FHf|o0N5lk7(0qZrGNHD?@@na2O-F zV>$x}+&H0tgn%LGbn4O&Iek@S^><|WIsoyx?#{11JnqKlIOm{_w_bl+G$A9IrUsiWgU3vh@d+TIWa}S(L+8$>>$^$Frv*N4q^1ZC^ zTY}4;1P?jawj$Z$KYzu&lub|2mcQ*gAz%sf5FWbJik5d^cI>>!ocPMp->1T>6PXZWh<7+ z%lLTajSwXwY5XvA+tCL28YY&^W7y~kWI-vjbHMYf(i zQ{4-7L=Wk$pbzGoefNMPmn2F+7QS6!lAID!LXO=$+YD6Z#G#1{Aid<-D_a9`xXMx4QI$7Q$r6eMcVaGxt!(Uv8QJcVl(dBX#_m%**6G=*M4z9ptE3%c=4X~fj?BfrFRI7fQ zXC2rX^LVjAySbJh!Ogh|z`L{ky^lH73F*n(7a4ot@Gq$z?+T_d!*d!u0<6YO$dawkN;1(go^0Fo2ffdmob*hx#)5N$(+N_T9 zKm`A&y^7Y+Mr|QqKG?I>KlaGw^6!7jCLx>aKWTfTMZ36kpq6p9jgGvsELP!AB#BF!)?Z6 ziHwYt!-vz0%dgb$6zDmHY>2`K`Y2sLjrfoDlSGkoVWq18JP^@X@DqX4?%`N@)bL*)5)V`W5u-@Ws6>w8h~w@iDAk~=Y&Dj+al}|F=3<~6 zf5izR$#$rhj`sE5YMGAnZt0Qg$#72BOt&JVl(LXYk@G&`kEZussaRJS3pms3_^lua zk}O7D5EdQN=0z1Vsu`En&P$sVZ&Z~ zuik`VN|eO&Db7)6YtB{?Ouh_2NaXCku*)j)jev!p7~a3(Z>g5I~{f4I?|d7 zWt>u6pM}H+J{Mc+8R=B~J%i?J(msew+X@XuD>f-qNv@B;`t{?upw5a#2Q_3xRbIo3 zL&y+sPi#q++PvA&MX2dwTX%6o>s$A%O-J@s&I+TIKDcwY-Si#JpyMnyE+d;ImUVjf z7oV~-0eXpPrfEzl}FPi=k8FEdXH|ARpw5J_+V_9vTtP#b35y z-F`r>nXm_b8S!_)(Z4xgP0`q3MV8oLJ%FFZNS#<$E#k3D%SIzeG&J5gk%ZZ4tbBcc z{S3a+vP(i!LVda6u=R2hX;_g`RLg5w6VX;eBB2!JyhFMNhj+7P^L>PcTAzebQG`=E zIGl~XzW5!1sf_+_>yi_%0bITNZ4#FlEbvKZsM~aq;m+o@z*@iM(bJdOdH0yZ>(|HW z{O{iqMm~`4u4hZ^5zxr>g<)URP_!;*&2~`4QPBNIG!5y~4Y@KHkOxO0^{TyqSZ&ri zh+m`#w!eUO*k2Nl6L4vpAP&X!U^Wf}(}Kz%>@{ge!}^~(-@!m_;;lID43G(S zmMc7-3+4RkO_d4+Gx5f#R-6^Sgg?BWo+#}z_!hmUY6y}~Bb|gE?`~)Ncj*lF zxm~F{8QZkI#ynizt0&GOr3J(}{8!NjeJFxG+nTDl{j&V%&?{!Y}a4 z-k=?%dL%~3X|3!Ujizd0W49PgiW@dx&<&#sMhU;gwznSSmAL~oaagI^4iJ_vZf^ZZ zsR0fNiWz>Db3GTbD&9y4I5pbR11{945~N_e8*j5t?oZva8-QS^LzL=H(f5#6=K}I2 ztzfJQ5;F7qR&6kT+_XISl_s1wWe`W!56|(zm_*%I@9z`)h5E=Nkn#DVYOdSj>~#@xg1do>VbZ3I&YPiX=G zsF3stE0q~1#!aADQwS@(`{X?%sFXa~U?8wU)0t)5N)?%+FT3YI9uz<^C?oak4+>pK zta-`Z!I7VJ6sgs_`A%m877UL*aw2|-BgADd8Ie@6qVTI&um?2X=y#4@YlUDj zNdUPKY@qT<86Qy2H?f){XVWtPDqj4Mk2STiQn>SRX5NzXpVV`uOR2Mv(A9vXiL9gKK&|P}GAM=|0^Aas_|a1xvpUdfwD!d|-FEB;lV|Fpu7>qR}qU$cKyILbUUp>{m5#j-_t zX!@`9!3)7e?1)FmT>xHZZ1KO560#`|moyt<&P5o}n_P8n=y)8xj+z&~H6iw$M+fzA zd(4!_%^U~?;a1v`KQX)tRl2PipwR<5lp}Rh*S7BtkZ4Hwp`uPKg^p9sdqtj zL(-LK9GOj7v+8(m3c*Kv`eXHq{Pw%}K6nY2SLxk3=<2rn;toGa&HB?Xqy0yveNuMd z`0^}zC`rQ*sAA`mNlEUT`BV8wF?3=$Ofh2<1@J--CF9(bjP4w8-39tdO=lK6;Zhtr zc+$o-)Nbzq&C^Or!x( z8A*)EpHX`0UDyRat$#0i{`QqD`Zv;4ix4$&O_J3OxABRpnF~06X=-K{Wc;)(bbR^K zzl}s1h+jIw9~_r}u_}l4+IBC)hNh;9V~$%S)6F;~iUV=&{M4g>9+@bf!G?uf*(^w0 zhGN=>#};(&jw>mE;1q$5z-7^^DCpeZ+tMPPDy!4&pMTmERlA_#U~|M#0S#tZPD$qz z6BrvLt@%(Y1&05;su^M?G7)l&p|KS?6w&Etwkz7{N^7Ti>3scv6`hGc6aF8^UBx#_ zCCa&!tCF))WGh1CsN99g8Oa>EXH#TuIYx+8lB-C`S(|(A$z6`wm}_E(W7Ce`exJYL z^LTtd@AvC?uC}?z!xkmbYed%L7^70p18+^m_q(UM#nKW%-OT>n+Bb+l zSqH8|`QAur+(M-);uX>tGc|kis&JCVLCiFTcIM*wLY%(W#b3b1A(PkVD65)K756nZ zU!1QDD_T(#ojel4xaZ=|lnA2wdcIZqO_-UrL~QZFOjIuJ=a4CWL+<4QMr#Lb=G>r} za}UK&8?CNGz1K^f!ekRokg5?WhAa*EQLe@kU$}BRBle zl~PIZkT17oV7f;I@M%24qOn&T#%ZhjPw0jl$xH3&1x5sALWow&=#7V%$|iVNEQO5p z4LqBiwQ&839J^6njLC@)M&JB)*hQr1dF<4ckKyN~1foa7T)D+A&o$9&94Y+h*=~x@ z%Hks#N{-F*wd0&ON;QE|2u(KiE8yby>4YE5&N$D|BXF_KlYo55o*(+2bx2|I4LB~^ z?5FKhc*p7S1e)v6Uy3V~x&nX&>BuW0ARwK5fJL9vPRPjbRbE|Ra*&*Ts-Ylh8sI^X zr9a8Sjk^6c^+DjZt=6CSeiMAPb}$oR6K{YWK2Q-qOU-;B4YhktnZHXPgXvpBeN^)^5%}xrU_rdc%d33*q;Y20HZM&X0bm zJO(=|)FlC&4kyHGrYO&qQ%GkcSR^c`9UIE@a&8g&rXT?Mm70nBFOpIC4Ila78t!Lrq{E!Q#_v*6R__?`ZP-ZeUz8`VfE{dGtsw#QMg;-0?0H%LxEK6Nt`L@w4?%v%Y=A~fpKd# zF@^&oS2_Jc#&&4l{aSvq-Yq({;}!Vx^8NV;pkgF#kiD8YREuKq*yTFv_#>$uRW=pU zjs6ku^j~5Z2{|^MN+M$%cg{<&9V`Gw60eyyf>9JT0q{M?J44f}8|zzX2BOWQU#jjZ zB|5_0pjSU-kG*~F#e#VC+6^e^FkE`V45_yi3TkvcnDI|#e4*6e*=pr$npT26OV;; zGS?{NSCyn1Zh!e;`expBc6$a~E;o63zh|YEaX{ixwL5FU_#t}BhAE>7bSv29=Dj6t z#O$Y|?9BgL2aqJR{Z~TWnY*W5sv;Rr4=TSMHuwnM;ST5jsN-2%ddJWIu+8{Bk$6S^ z5_Y#~rQQcf)|MCnZ{8HVUtRBU*uDLrdr@Skvl<@YL9;w=DwlVJ#;CqnPrzc2NtsoP zH=GQacFI{CS`dc6i8?w`Z2B3h_r=R=Z7eD8Umwa?I^W0M(72{;AX9NroIOx$J-avr z3D}0M39HmE%>&R&Mc|d$V{B3QMxV$WQPtcb`ZMSJ7MmfF18xNsRAHPfp3b*p7&*Ro zMN}7QMXfURQxwV$TNL>GLRc?+i3~Smjo99t80Ffn=MMKZ?9VnWTd&dYhy66ayIFY) z+=%5P4WG-Q<=}k^1N;BAtI|${GL#rSkb4uTFedDTJp78JN;b}Xy?!$ z_8rsf9Kt?ghHm#EMGY=|eHL8EIYn*925V#!w_+K(KezLZrq>}Svl%M|e_ z+2yZ3ak4Z&d?KjQzauYB0|ef0?|ty<4moc5Tf|7N(zpN9SdDl8@N!qF90VGQ8|yzK zd5hPFE@AOHJZ|{*q-aV$)O3-j2}|31_uf75-w$4bQpzvzCbi4iMtC^7Cn=>Gy!^#G z4^aK8RPL=auT;#@St{gdl%cUWXl^4!VG*@5_VMXn?=@RJ$zl=xNH4wcovlDccc#*8 zb=#*nMKzMh(w=y?!DqN7uR^Wp8S7;63ZEIv+S6(ZO{IQ8DV^D}jwueTTtE$N;LufxV^OO+#+psO~ocX-5I93%G6mctSgcFPGgxBzwLYI5NM1w_~nX{A%- zQ~=hgA4ezp@&>B)N8%dXPMo`!EA+VX8YxrY?LyLm5k|R7Q;J&c%a8+He}}Y*d+7ot z3jm=ZNO5QRf+MK_3&U9h!ZqQu;(&A7wl}{Fe^n91bm|caHnK^A4akvWjmIw- zR>sehuo(GwESIH_SFPuRA`b^K7W5VJZ6cUi4e!X-WiK9hBCHFF|Gk=*bQOK?{Dr{p#W(XqZOk*8qrS>u z=a;5ZQ9DH_5r&de032c*a?-p7T6f`b9elxdonok5a6mu#RJd4)vgSlZ`Td=nHyxP6 z*_#KuQqrJ9kiH}ES)RHw@yeYEJ7g!A+;4LN%5mv9^=Z?Qv+d7V7Q-ABzB_zFrRR$XL;n*&xnB?%ty0QwqX8=6`=H97Add5 zgEhoA+cZXOo_Rr4E#}}EZGF>C2PRo{4Zu~+J1M_6 z+B|+8Jhpp248{tsGq3Y>pI)@V>; zn&kyfS7nZdJPeDd1v%9~SaTIr=2<`o!O@uM!(F0RBCM#=>0R=5Nm;rzvuj5^YidNF zR``BOU+00>{Eb!e!mcB5>#Gp68Od{|L5Z^aqVUT<8SabV_M>tJuJE)WP7dbDL1ONc zVrhMivCHag8PMlW$Tz(z4(CqBszunvuvkSD?%TVrM2XFYhbQI!`?&Yd(^WH7>d)!< z{nN-d#(qJd$V1mT9cFja#ZgNe&LIl$?+Nu#BM8v!;>SfU5iv=uhBI!-aZ>>^(A&U$ zHh&XKymV0>zYo?0R)&CSuY~j#cxv) zI9T@!Jw=tz?c=Szwvt53?o_uPjImq+t2~L48}ewuEXCV%0ZgRBE|^l}vZI2)d7pXt z9%rO;7gnwd%f3oGaOd1+fcc5Zrpv-tC#><20gn{Or+$3Vv9rF|j1_?Aeg#6WO!RUd z>+nUWHMda35L=2@S%G)_nl!mh|FWTrHisA%6RK}J9SMXYVkR`s?l1D*oumUChlgSr z87&u&&8+F6UA5d9`kmOKK4Fxd^77`nwmOcJN2~vKy6J}4bbl4Q!#8;XVdJMp1;!H= zlbbX&P^%=tQ4^8*7-?N+G<}NRJyp>=+Yxm8r}NQ1cdRf-kaajIMtE*W9u%mj1bZCV58=2k zE_ORNGYs`vC#>wgbSV_ZlOPO&UMj~%5e<1LsXu|*=|qfOymXIPRHu7kQn?H?J*Fo6 zmF2{h2I}8NlEo4;4THSQ}dFv3UkI?<)NqdlxK@_#9ti2PrKLi%2 zaO*zEQiWN>(O=fO{uF#=(YIAyJrwNVslH3hQFi<*pKE7?MU1TBV%)U$E=R=V#n_m; z$i7*Vo}QqVOJ&#Mqk0TY7cUxfzg6OyLa*}UQc+A{e2C*w$h}KiFY)>QB#VSZ0wrgG z;>i+3J!SO(9#C%Qsi1E0A@JdR1W^P17T2A|*;3Fq=H1s52*~M|OZ(}ydlZ}ZUZn!` z5F5&xsid-4*m*Dz*lieL8WJg{6>kIlYlr4|@DMluPQzK2;5~`H8=nWtH&5}3OYWSj zXc4BFp+z&`D-p&{s;a*Z=rnB`IFBnk*MjD0FDg4@aQrdWGAYjj9$1Xu#pNiawx%+) z72r+Tv>&Yk$i)z9x(hlQ#QY&iLNk$Yy8Sn(l3m!Q(sqC6`s=g>beQXeXvB+Hbrdoc zyhm8{^D5Oj=PN^d=DrcE*LJDq&uc=fKJI(oYW`r{fJ=>s2MR9uZlp^l4#0C(w0qF<3R$nCK;ldd{ zlP=_V)gQ@d$EF&IRls|+6<}&70V>5YYmGBL32tu#`!&IjD+D-&05g~7bGQ$KOJfDc zz8}HR6%D6Wr-G<6Uwokb@(9NkYE%+;wik0!TSQdQ#MhSg8)WcVvb-kZgMR+EvtTx1 z=rU{5g=y$Us(m=sX>%UkT1^6TY(_HB6u~&HRp5ma;R4gfg9}kWj_h{A;>E+bznO;% z#LOz0{rRc%?ug%?91W~E6kU59#om^aM_;y)&mEXhS=KEZn{TaP?0=ZA`9y2flXk#B zWqmjV&|1>$Z?#XbEEF{V#h&B~BzQm0J!{M5PC!fX(0X_6UZ^IDa#t}F;4Zx5N;GQ` z-sXCBVR*&*N}_rZ$^}e|GWszC51zdRwJF`z9yDVT=^BEni%HT(76@%nv`2lO>kn=a z$tBk=3=Xx|XfnSCEK?Q*b+x^=j#{i?E|>c6NQhvHwRZ`)%&WcK{l0~<6CZL_ zBDeE#$JH3kt2Tpk;HpLYj%ui78J$s@f|>wxB; zV!n?%v@;e4kNmEKwod3BDn)&KN^wls}WE98?}`ogG~W7%*AbR-Xt7jhfh z#SZhfOyVPYs*AqSg?BQvajV2uHQmw_{XMbau*^&<$fJ#GM&Gowk*KWJdT3@}`F$qY zcOShO9^A252-M?~mBO|gXFI1FPtUyP5C={U zr9)lL_vbJvs)8-94qU%-fy3#QN2&nm3n$?cc0y&!gBLDfXy(T+|FG1R`FXi%WAxnH z-aknn@`?cS^&nt4KM}uRBU7;Fgr;uyJwXAIKY9HzOt^lVi;7`_E{&aB;uZgUdwm>}*NAV4eKUxa}N8$*BzCE}DS3MX>>eMm>eeYEy}#QXlt zX#Y-;I-odap3l4-13llvCJ6FP44l!i>s?B~Xxth_72%pV(}+y!p$8nGsyIz>sXE`2 zsbL=P%ssO1GLXRL!nVO7BZ;|V{eENNehua4>#T#1Y}!^B29^U%9z1yvkl#LhMGTZa z&rz0ARdx~F6zstom)bLkc4{6DbXh85}FxVEdkLi z$&Z_E!$W6Nxa})i>;>^%qF}fFbfT6#5720~gTxR{yR|%7m?!hX+T4Sf1Kb1Lvzc>& zfKX6;q)Bgq!#E9#{s2!dhkM7NyedKEh~fb~Y;y2Jx5a?)h*+zb_a6hV*c)x`;Q1#w z3xJ56(Thc9qEygNA%C!{`z+OlzSo;v0G3r3-5A8zt)@26_A}r>sl1)8n1%x_X+x?CwjqDxeM_(>kwQ?t zckV}7=1c^~J^588R}Yp}4M4jApk6l1qYv;FWwW93p6V})%ixtad8WyhYqet~1Gze~ z-tyxnHlIp#r#^oN1g}D_%%=DS%RY)@-3r~NPw+$kWIO+!f&R0I?>bH;3d468s({1B zXr@3jzvZZlCd}va-txmQ#mS?*+%=J;8yQy+ODkHXNTM4f38%IZ)hKKzkGPv^6r~^`$$~7=Cv38mE@XnbOb-2psK<3!<4&L|O{_KdwXGc%4-3eqSPFI>e zbKSrNYy76<*wnj%8JhrK%_RWj$LnccB>%+M*IQ(rY37Dw&lvoZNQ}~|Fkps(^Ouy- zc0*+%G#^z<8yYAdf?f6s@t#^S=KAKrhoZQ5GEN}DC%iOuZX*XDXp}u@u0xsYxW_ouBxwM}`0H_=wyA| zE8)_i>OKbmw$;eho9to8`su9p#>P@i{m>v!HYrMx`by5{s2fgqV%IN2u``G2{;S#} z7(C_JHL#g4!TVKzH-;cqyTWYUbYJYD51;o&OW{neeF^8u{&=>3MOrA~?FdpJV zSYd`@e7yIF=r>t}q62JMgr{OifCEZ+OqL@U0qnPCM~vzAVAWSinbTGsoAj%8aAv*o zuWD3^SdZJGJp`)nD#ZmjSqj)I^?gr($f>AJ$#J))lJ(;mu}!}FFX04CDff;uyZT$@ z44yzaWcc(;REg2B-keS7+|){0hao1Ky6u~P!(lZL$EGcIp3i^I>#mUn%_C6l5a^P! z>!#Rsp#cEt6KG$x)xQV)s9bQ9Udl5Q!j2ysPa78L&HdLqdHuyUL@dr}NJnn_or0#u z)ho3h3FLS-gf8mRizhfvtzM0;@IyPk-^a6h9oP}I+0o=6~N{Rb6BX3y4 z5iV4cW^ZW|en}IQMT+TnetP+OC=>YD9ENf2e>0Cg{8J!oHPOl6dW}=^aM*Unss)1+rbRF+Sba7% zS^dsY{r8^f?G9m8-(u)oUlX_hU>wvBfuHDZcJ$scFzxx_sGe>&>$_MnNuJCsS&yi* z?S#{Ys<=ZKzX4zFL(&!$TFy;eGq<}lHtC1pKHZ{AsJ|Suh|q}G&Hj5`YQ6kg>-TLH z@Kyi8(;^duC=6+%3mPF4l)6`@ir!|39??Zz7I ztV%vhgYW=#7VO2Wemv>Gq}*g@;q;+w3>`V;kYxK;6FPKtq`3YYe^ONz(}&E_>Aq4d zi=*$Z4@FD3K~IDg#yC21E&p50#uK=4t=!6S^zF}6jtF|OY2C#@@z}oC8anXk#M0LC zd+<`)JID$k59QE^GI&PGf^LN=Mk)-?G zAp#plve>m9P|9#iZEcyjfDFB2Y_A!F^9a*j3Pm!I-(LKYNI0 A4*&oF literal 0 HcmV?d00001 diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/foreground.png b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/foreground.png new file mode 100644 index 0000000000000000000000000000000000000000..4483ddad1f079e1089d685bd204ee1cfe1d01902 GIT binary patch literal 12430 zcmeHuS6EX)+pUO#NL3(IK|}&d7YKwF5CM@UBE5tjTBw4Q5KwvxB2pw25vBJIB27p@ zOaSQt5eZd#CxmkF|4+F-=Q)?(#XNgvmzlk1)~tDFz3+~Fs;5bRo%8yoOPA=i9zS|^ z=@P~5f9V?4rAwDs!Yjfq4p(5Rx~i8hRVUG&*j~LT%Q>2AIqB+Nx_^yhg70E+c&i!%2~zqE0}mxIX= zz1$7|sWj&3yL#7D|4uLjQqV+x(Rz4WC{A9|^m@1A6`BNi38Cf3B^aJyqxF{TjS&2q=3$BC zB1Fu04C;%o9V_Yg;Ed;xpmge>%b<|5q52W_pTd9o;Qty2mQ+-Peu)^(K)RH^d5byH z>AGB-I7$|~9l)J0H_LPDsUUL#brIHpjO1>dJ9@_5&W zLV)s!AVn7*Hy{o<1zLA_Ky-TWzJ_^1=W=Gfyc#1ssqeY_2ww>;ANX%JT)(9uNHOtU zeqU2_{Wu6pLvCMBLgy+dx=13ZG-+cMrBf;#8KezD^}_F2x>_Nob0^iXEv>aML;8RQ@@sN(#bq~VsOa>) zW9RDe#_!zLkj)PyQ<05AjbPk5yJ^|B6q=sMX2L0JE|(P%=v2$6+4QL)cu$c*yt`EC z?)p#@xE12zK?QF2u^(xb0>KieYWS%DH`?=eOiFd!6)WRmCo6Joq6}7e=Nl_;oNJ{1 zu&szm^c0s*wAxfHSlk^+hb)aB<&B?9+_YvxC1LEy$(dDJ8J)d!>rwz?q zGTpJ5&uVwR#t4%B`T{*~RAd_Unnf&`*9c^zbZfsVc;v*@=BHOCX7VbyhnS5G*Pik} z@`U!W&dq$A-&GCYAWg@rG3W6ANL_2a)|;&HJSig{zyfyO87W{;ej&@-)yx~eu|G6S zO)U5U?QD)!ey@XcxEKX?m{R4VZN!*V9gT}6_lv@YD^}}y4OM(*#%kMMBij<9x4*by zCkGRQ3vqoZ)HvQ4oY~=kh{c09u`@Lzqk8)3R+$+hcYuhqajQqgq8qWy8X_QMy@1+T z0&yU)D$XzuW+GZpAB%%|^3*{x!r`8nOWhu6>t(2mvERH# zwD(@F(UyHL)A@d0q#?|SOaIrK7`~^_KhtD69y6E{G70hSpvkOuvhEmR1(|2efAmi@Xw9*}m%vZb>kVqe?t6*aL%179k2-;CD<(T2&{-rQ;%g&4b= zStwf@&UH8&T6lBt>jybuLy}~>HTF7(kmQuR6(8*l&xSQq79o~y=t@1Z0aSiA&-LWp z0NQ{@*q$n1m#1Z}?sFj0=6jxX!@eHh_D<=qD}vOG`kCQ^44In=iDu`srXYt8{4c&) z7G9;S9(*ydG({X#u#N%3l}&Yaq*lzrY-E%htNRQTrjCrX1NMi~a!soU$|=0*dXokbDxSFnm6OHLV@%5(K&ZQB%e+ZFne-TrP|veCOrVj;0pG zdbMMl{Z%MBfVA6b>SKLi zXyRQXFc}Krl(owbvDh?Um&9l0#P)rbdiZxK)8=RY8XvSG1@0=@vGxtW|3E{`T&9Zk zC0==A6=d?8`t>?}z3d12SZ$YU4KZHQPf~|w zJD7n^6bjSS+&0Kq6nxhj*9}9qDZC~A`nzEz{<+9lxx)v#qaCsGWko<{ahFVncU-R|715> z33|Jp;8Iq?Z)NXe;h$K{z8#lRB#JC*XUod!9+#hCfkg#-^FD5Jq@>Dt!SzYr@q0(& z;I!1>qg(PU*HMX7>G-#T5V;IOw~4L@XQ&5le>B4Va!sx0P1pm1PMa!%L##WB{CukUKwQLR#mw_r{d1DneIIJT(j#O#-det^FD zbdwZ-8R%84+Bo+g5iyd(a6x;*5F0xuclibP*ff{7PNPESiBNJu^Q2?h!4}38?XKcb z1cb%?RlBpM10D9~`7(D`#uzQxY}K)shcU_}%#WJZ`~FU)C1j&^b5i=Wc7uJW8^-NB z(rs3^Wms@#S~)+us~_(~uocjV^vU^euJHB^upc~CY%6gqBXHR3{FJ}D^V0uB8xrdo z%j>^}CvVUV6jaGJf5i$e;gXng&>{)uK?nWhEUaVrv+x8njtfCz>cqP8uUTn1`McQ;CD+jm zGle#Cefq~0!!v@W2XnNsA~8j@Gaaj+fT)QzP<&gR$L=bGEJ8^z*tHxS)sZ=vZPV!4 zw*)4rK3To_7<;de8PvEPu4Q5d;D=g00$bPnaG|sEP6(kDsxwc2+y=l@=8Gy3^DW?X z$=3@Y|B6^8mUadWxX-6z(Oh@9|3%Nv*Hz=bA3)}AiK3MrA@eOvp)YSd(Nf|v;6dz-v zI5xYnKImXz)PTM}jxK=GJh_OrE2HXqKgh*KB!U~;4W!DpXN6A98^kNt%~i7+I+`g5 zW}~Qod0A;Lw*Q@m73+!Rfuir!WXqcTd5mXE^DWV3AUSVk>5EA&b6Svd&!yh*!z+6( zh^>CvoV~2?y`UJ#Jho<+PlUEw=Y?Hyd8C#Oj$c!5d!Du*w4OQ9G&OxhDmQ=)tzD()srM-?#=f>aw-$x}3Z?qLOIJ{gnZu zd`Y3Pu@-6CD7)$*a6189&`vfy%c7^DmCj90Mw>5FgU_yh15-*dsMPOLpn%G&Gbq@c z)NN;i4jF!g3-}@w-}i(YUbp4WY;xYi8`sa3ep2V_UXf_!7A{;Fhp25CGF=6{xLd&d z!Mvrklt74KI=0hsCRMYBXM0Z?v1sDfN=Y&W2dW!hUyqiiU@A}R-XCxbIudes32?<&DQ!Hr>qn`aYQ?jSq?4X|x(CCDAB;b=wcWVCH1CfwqU1di z!|LlwpE@R5*{9XlM;`OM$(VZBN$c{`%$ZT3S3aYJwVO}kw)@4_EyP4SXgXkd)Q z7PtWeexnE98(N{TMKt-aG+YpQs`a~e_Y;}upm;CRXlTWI->sMI?cj%D`$7K@mQ<-e z6c3=23v>}kQ!+Z{G2&KQ99s+el!e053~lQJc`8%`$;xt_RQ&16M-jjl$HK)VZG-0esPL)%m(*xgTxhvj>YKkE?dOv3G%g-W9;dgR&pG1FoW|wrm7v|b_Y-VU zKV&S7NcSkHSjm4nrPIy#Wvwp8(lbN>^x7o60ICQ5m?QwOuUY9q(q~<6`0+a7 z_`Zhdli4>YUiT%XT1&z74m|S7pZ;||I*2@$Zd5=|9{V~xFLGS|sAE`ZQ=toXwPUzSz%(Ar!@#M}4%I2r*Ca<9 ze?7@cjo0^QC6zocYls~PXjm{I-w|^|?Hpmvl_!6;&?vERiS^(A2e-)2qxQ#IfuJ_M zgEhyUo8K;fE}w8OE$6nq26w$M-YgMyeYnhwguXF-@5ca=0xYn%I)Rl=_lZaUn5tgl zq{GPw`_E=ilA8s)Jy=%ks{*^ijmr0SqHYg5D%zYfzlqy~#fp6GHI7wm_SN!mo*B=(4jED535Cy$0WQgpMk_!VjQ zhjwgVnse1csNUVP_rkF)3q*bk`=D| zRm=kyT3qxBA7a}d4b433h)JR1r_zBVy6)DMRyM?5%=@^}YMnjurETi?w8)8Y2lox+B2Mc9(WcW709kmg&QO^PydT;QZ_K7tmYO8aA8M?Y);N zSn^>S4^jpy!tF}ZAn_;hcCNY$eyakky`&>*Nh{Yf8H17GR#{9&%f^ps6IAlo`0a7| z-5WT~hwWze!uONxb4D$Was0UyM#f|Al`@rMWg(+oyWOL{(2>P6$`ht&d;q3uD6W+D zQQKN!nzWpx$Ya8CUKa3dgn={(ad!Lm7qDcu`SB#dKHvAM#GW}Z>EZmS6yG22dWcVi zef}3H%>*xQE6XidovM|h{PD;~31ijm0ia9g=-tnlFk!0PDn12luSSt7gWP{nbUK-G z_;*xp66cFpR2OkYg+1wGZF$3SCHuNOh~T{QxmE}&DI?a%s+Q&BqRkJ^37TgbKmAKA z-lXW9)FAv@J#Z=C2lSk4@W5q7S0~BpAs>m(p{^)b2MCFka=_0~yTtPvSKJEH%6&GW zKv;f{iTBYXA0^wmTAmssRXI(3556s-FYRfgXSs2F7D?)Muw3X(n96>Fe~#_y!;5dQ zdOQ?Kp<{m8r8ee4PPIETr3Sr=L{BgNp=Hl~>nSiYS!vY-rs7>zJE&K9>k00!&bs>P zD`CMT*(GNFuh#^fdZE?R`V};&3K^rq3z5UT^^KE~V+Yq@nxU<{+Ug^t(FEIk@f~5* zgnEN(6_Zcdmg55!i|T1Xn2NBcinnnFghvgYxT5oG<#r&$ky|k5SaFs(+Vr@W6W!wc zhr8=;xACvw0kVQ6m+uK@w0M_|3*`l1D1SbQ1B%k-HMIa!=~kGkCfuQ8^C^ZQ&7xn%?zUs@ zJv~f?$}gE-(aEgrt|vKx z;}Q@0S-w8jTszP4_+Em>MvCg@+IT%eNk_MIr)gA`;*lhuP%vm}{=>pIah-$r^3{Da zp;l8BZIY#N3v`sN%POMh>Q=e-o^BM2OK_7-ztamrbZ{m49XWXIgg1Gqa+C!XfX?gxVvl@Yc z?lm`jKKariU3($HdVP4LPtp4+4mV=+tw*rjI~_q%R6DfIW|6`<`}My)W_VK!6c^i* zIvi5RI=c%+#{fOc1^%pnKBkmGk{n2 zC<)woa7^dmGd|$2v77jNVg{v9cP;?R<5Hz&w)i1YTrbpNc6%p0{Khx8hi!J94klTx zC9LuDS+2u)()U%ug}~voR<>Cq}#OQfXF2)TCm)4nk4dkJK<{Ji<% zcP30SBMi`eN&Lves%5zi8b`z0j<83Tc~cBqc7F%;N9zZcNAe!JR3!n;@j1h z1lCS;R&Xw6EFbwYNCw_`r4_DiPb}ogRDYy^watxfz7Xy(zQ=RKaRMV#RY}`WgLrrF zVY?S>T2T_0_gmfEc1P>euBpQk$h-TAw(GijhS$+YK=Tg$zQ6?>D}F1vFkHMoukc{a zEy_ED8Uf0r#&yr0HH7|2|B-{vV9-6x6%+AEp3Hd}4fvb`f5|t#1a^r!L``xWv0pYp zK_sWYo?M7Ka~?Ti?_2#VSWzD;+NOTq_0`+=>-+<27aH>r;wtxc2mAJdsVzr(62hGT z)&mW2D1I;#ot)2O9iIWid6J}Na=-qm<@K(sk9ppYVwcO*IkP(P8P9ER7!PsMfNBn& za^K3zdtRPHN^c^l9lmBs5m>rjxgOV7Io|5p!v}X)j;Ax&u7K?;q%XjX_~o%@lPr_8 z*9Uqq$6~D2?gL>l^=mP&+~8z3yT!99Io|+z9QCQwYR2S? z(t}t86UG(B`86l3E&Y`O1p($K!sj_~Szh|(peg0h(+?ymZ?)sk6C*iUD89q@SVAIS z4_&>H|FtF3pZ<_*-;w|rv%!y93`xISUXVWp-T~!8n*#@16?Q}v>{P^~9I69_ z%n*6qXY%Yy!%fWkW5OADjlkEKjP5d$8>`wRrhp=ra6@iEL)prjHQ=o3@+N$WN7maZarII1Zz-rqUrBVRY znukG8!4Q$))$$`IcgoPA;izr~)m2%Wl&%&EHeRmOXUJsiSwge{CQ5;l6K*f{(Y$dK zr+Ms$jZr918R?`Rysv0Z+#6wT~L%t0b;+Q^{rT$Y_J%=|3^Wd zt6$*epNax{<>cRLLyEm2t&MjM8j1U)pYxwc-MDWDwN~$V|G#;ney}e?-YB~f0-n-M zw?G0{JBvufZPvKoY*5O85X8y3)1IFwLkMFr+5G1knQdDje8Y{BGoelP12*9EUN%KY zxk|^L1xHs)rNCp_@p0*`=#9{%r)_7IsX3T&x{b&X;mgnjUOMtgKs#ylC}%kSdtkjl z8!FE;zg-elNMzzYzDjZ0)^Ieq?HW_G)|Sg=4mBA1EloCGZTG(+tr)OPwRZ{J7OY5O z-u^rg$|QACu3Cq*Al+><3gPrW!35XM#YAriTfXw+!m_NkpMN$HY+wKfNr4L9PYUX6 zzlS_jplR*TFaNt8ide7lbsipOGdSE!+zhi$@D8y%FCwjQ$r9L{z>FOk9`c^?Kjmj` zMuYzJ3lU=4n6Q;tr@a$L?%8~af{fraE2*s=hn>Cp;YCQ#>re~C6xoCO7}(mj#Xh*k zba*^&l5yo%qnHQd!W*<-IXZ+8vnMb>c^cM={07F5{v1ulw!aVecf>C42Ir44Vz);s zT-%=b<-{YEZ*nD{U;m4uIi#wyf4G^ggB0@5%#DRIbN7hz&!Bb!hl?A6#(~|dZ%%iN z%o^Sc0oq?wn5_;1HQ*s%km5+`HK!Bq9^dL$ZL7!o2j@&piKs-)bi>dGD9BCC4PSIk zrGJIk0P-Fv?{`4G0`eU>*i`V_XN2xXw%*xTUlVENh%_|iZDkl5p@Y866#=@Xg{cbE zjZtS75AB(^xEogv2B)1x^m!0XZdCqOZ~=~2%7kuI!6E74!u_j2iau*{do^aD^2Vk^O2eW~KSv(BzRD>xw` z&*Gb6ksujl^_Fg<9{Nxn%B8jSv6jcmU+Kw5-Q&psk7EU|G|_)%rogKwNzemwy6QX^ z@ujX`ZkT$alQ%3oWJ2VOJGz{G(ukN|LF&Ga)nKml$M>IY@1F)}2mL&m6~?A)CN|YS zLi^lZj;aN$DQnmlc~AgqcDB7)?<<0=D*JMD zM3%;`BX_AsO%3+;YjwAbOnkT+m^;*q5X>@S2hO@Aa1J zJCCx~6B|ewT}HQECVls)>JqY95!(x8tJTl^D9t}c_G8p6;&167Z{2*+*qbjZdPBKR zwYTwFdQwnL?Q_fZ1S5+O2`Bi&@(s_P_cQY7?>NOU&FL}U5YmlM6yw@TASK}~;pon& z&{?aE)kw+rf)rVR1R!KIA&R@6^&5tt+oJ8h+P)7GWpbZ0xhG1hCCSz8pFjdYT5mJUum4y`e6ST z&@%+@8U+Bx-^#X6vpu~G2`=~;;97zryltTvX_;q&`r%A)oV7(xhxX1-Obw!r%_aBq zXumue@LLi`iFY=9t~-zHYJC&!zW;W6TKK3YgAe-4E5@wu_HwjtlH4Ep5vqLS-2C5$ zSxHdkc#a7g$_vSgCJ_dxxPL&~SeaPflc=j>z18KsBxhHfhSRvim6wzyuJBI@*m2g@ zc2$Hh#1|Nide`x;s zFEY{lfS)AO1(&M2`md$eil6mNBxu2_M(#la)vUt>ub2uO+!3=jb#6Ic2xq$*jBF`n z%L9sP{NK&^17myQl!*yca`I%e*{%{^D5ld#5&5Dbmw2He%xl{Z?Bv@+UmIbjXEHB5 zH5Sh@UPidw19)2ZMmXkn`O@)IsF`Fbj+RLtb$qTJ#B-vXrZ?7??}cA6N56t|TzFj4 z=rAukcL+Zk?vE$J3_QP=HeaZiJ>sPUrar&8Ao}%X-FpDz+o?UsRbtr6!(ES)@vCo94^P>R%u%q(-9wy%Duenrn)jXuW z+2hV;WWLbrH-awRI4^BBwkb{USY=a|U+=L6IJbHc+!%aSb|KB}H$ z?;wmaMfCf`2o^LLsVRHayM++C2aVlLWRbMjawRSh!|`u4I8tjLx>H>?ZR&ba(LJXj z?DRP5gyUNUnznwc)C%qsQ!aTlw6i(@viQ+~|0fLN?FR=&Mz z!m?8%ms9Zm`@?A{S+a>p-JQ}TICnZa{gktp_;s>#3Wv_=7#GC;f$M! z&TRADKS2F7Grq42P=N2(^g3PHSv9Sr5khe~OZap~yE3UUWM-{Fh{H-BGK9MOV3L#y zw*TZQX^enrYRj7iXkEaCLTZF5z%T)MU*{_RxA-*;G{sl{7ry_e1h+X~HM>NyBnnV6 zzcFEEZvv5PId&nY^VG0nqu!l%4Ln9L8OVmkfQi1}=-j_u=t%I1_~|`SZ_zv+SV@2>e1;w+Y$vY75F((`NKQU2vax&tTw!~HE>c2M3z3d>g zk@W;ee$-qtx3IgJ&cQ;-5AmGPIIdtV0YQvcV7G)N!(PWkx#qq=;AiOzb$C@x+Z zu##CR=Q`hVF-LGTr?w9-umq+&6PrkTr)T1CJ!@XV9i+em9sS#E=UO}BNMwuBrCayH zAub{V#`%5ecrycz1$eSV8<2Ikv6CQ5E=h^K%3m6h74APzqFYP{oejD^Y7o_E2b3p| zeA*LbkS?zNs8`f>wX`CuZF=Vcnc?D9l|P;QF8KedIQiHkm!f>Y3}# zl9AL|w=FC#e&CG1Vj1SX@K&6z&wEdwI}i+9}=0 zD)hP8t2qSqGq-zz1>nRbHpsOX+Ou&rc&B>1K5Z`l|60?OVRG!%y@dyXhC`Y)1x&pBnbuTa%|7f^nM;OIHu%(W6&Ci`84e(2e5z z*ThM)rgG_sjP#cQ+Xs8;_5jS%p3?)1Cd0epUI+qH6)RAoaWyIr#O{wWN#wI+_de=e zPHAv`+(8DcYwZezvF?o<#{{xGw05-!dGx*J-i6B-YsG?>W6ke;g4Hg#P+$=@?s0UEI-*Bw6RE<{1I7> zjBlz61z%K{w(Fbs@*+5i`|zyRlh@qP_iu#(*1Wcpz$is&$q|YHc+dRFT7N)#@B@znBGn$2wXOi+ggc5BJ<+2( zlI3ksg*I$2(gaUp4h9pJY${1?hgh6#mU-3e=N{4cTb2V_4R`HbSASd)X&1AJD{hd8 z^}36_R=S?hhh>k{b|Q{V4g^$!<)__{4ZCIAOzE}*nn%8FpA_Bmaub%88)q94qdSj& zU&K}EwoAH(N;V`V{ZfKgP}7P8xX{2STb>)D)y3#SF&&=+6Jz=_o8pqGbBI1lUdL(1 zD2L567hm`YXfrYLV3fz4yv?7yE!3uaicqZ7ufRny<0U&B6qh8bcqsL`r9)-JOxkXy z+l@a1(ptpJ`{M2l$g!g@DX;KZcoPP93JT=vi}|dQ!tn5*k@U)brT5a*!NEAJ2Apj0 z3jNsKvYjiiy-sUG06+A3T)f+N_X|`ZAX$1+M8W1ZaK3Nm6Dd}Xw#CnL+A?Xi*n>}B z+g^J-yeBCQ;(6yjA1~5bLwIzXXp>6syw2d^&DXBrf$G@}~y*QOne;u_UdZD^Cl zXxza$QKpgXzp22W4GZI|8N{0M2?78Z`$wi+S>waN@uSr9`u5+ghvrjfhcjQNuoDp; zk9szfi0j_VBAd2M+55}LBoF!BASF5?QV6q5zf94lQ$2goh8#I@&N4tiMK&5WOgt0H zRiGPL-7G)N zj%2#teK$kweDwBL1+DK?B#>r?tjR02JIr zUq=)|zME?3CA9?-DRGfqM+;h7w&xgGmLjhTAOdy`b%#?iM;>=l7v)^GADOA64 zy}x#1eDIpJ^iQ-mHzp5#R2_{6(~wo;npi>z4tuCy@Z6Ovw1EGFOaCWi{Qog*{?+*F cSLciz6 \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/icon_doc.svg b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/icon_doc.svg new file mode 100644 index 000000000..ab6b1fd76 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/icon_doc.svg @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/icon_mic.svg b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/icon_mic.svg new file mode 100644 index 000000000..0aeb30d63 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/icon_mic.svg @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/info.svg b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/info.svg new file mode 100644 index 000000000..2210223f4 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/info.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/layered_image.json b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/layered_image.json new file mode 100644 index 000000000..fb4992044 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/layered_image.json @@ -0,0 +1,7 @@ +{ + "layered-image": + { + "background" : "$media:background", + "foreground" : "$media:foreground" + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/startIcon.png b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/startIcon.png new file mode 100644 index 0000000000000000000000000000000000000000..205ad8b5a8a42e8762fbe4899b8e5e31ce822b8b GIT binary patch literal 20093 zcmV)JK)b(*P)AsI{U0tD9;7S&f z3`9H(<`G*WCN>bN493AFOi{!!!L|afI7%o`6&6lXK&2`L1YumJiZTQ+5doQ^Fu|gz zI6Nvw1cME>!8`;4iI*N+z3;u_gZtzG5&vyF~^*1 z?S1yyXYbweAFzGO*PdLxe&gE9j&{c{J=rY}9i1#6cCzdq+ASx~UzXhiC(H6orN{Ar zj;qq$yDTU7NWP@ws1J2_*G}Ykx7%{iE$G@-7-eF^Y3#}`(v#ySiIZdTj}`y+a>=Im9Vq=f1W5yxR*!@kj+Rxz&v=+4_?qb>2v z^P8^zTt$BB=j8B|JpIS7`QY>Jz4z#w<>ZT>lB09T6nS2-t-LNa`Yg!ixr}^gvZsB` z{B;rQ@uVEqwOt7oA8%Sn=e2VBs;^`dNc~|xx$^LKH+*6BuO8<1`K9&UDuw8t_%!FY zoV0NZ!^eH~qhBH?uakr4K4~ZC5VHnAA|L9#J5r^|-)7;Y zUl$mM>pDMqeipwr+7#N+YO&F-3t!twD#tH9_S*S{wQ+C`@f*(uNuw}s=xXMh&DI;Q z;_u$0c(3`5*FEq(O?pz@6#ee_pZMDAFS)(D{hdnlGw+UhHaZ&vMC3y~_HorR=oT!) zD&Jv0*w5!@vBS?MX~$>r(d*!xjZ=9%U3__Gl0?W|%cDAF&TIVSk@)+3cqc!3boGhhYzil=`)k_5%wL2pqQz`Ju@50G)sNfVj zoXGZ|Q(f3+@xx0`O2~K<`L6lJ-SXStp$#*Nk@$Du%RKJ9@n>4_fX zCq4RXG{SB86?4nquk-Hy-E#B;AN86?zpBs|J16`d(I5ZXNB^!~KL7eV0uKN-_1L$Q zfhXMkzP+y=*8|%=cJL*vJ8JS$i*h!V@e z?gp)OZL3q^qPRQ$mTS*l z!1Lo9sgwA)pzOQd7ry0nSAP)8dF^z>J#;@|{wb*sK5UU+HV4!!`0VEJLKou6^E1;q z{-F(t{g8gMTs+F%4CL8B(dE++Be1u} zQa1d_@^?2B{4?(K#G2gBZ2YKxYj^wS1vv8wb2h-K`rtLS+C4j5oS5zZQT6pjk(( zJ4B5)x)C<~DS-Jn#3lX27u>p0yp_M+jn)mGYaUy>+T%Nnb1#0!>tbyAQ%)nklRSgJ z&7=Ic?ks-hoA@5fJ^x~JiY`PYkDmW0C(plGd!Q$Ex;t|N@d~qieC9rdJUa(Jbmg%% zxJoLcUW^RY7oUugb$iXkOVyLI8AJG+ zNchYly!4G7Y^6~5nrXo&e$8p}lUVB0m<1UOEOBY-ht5+)-??6hPx|GZjRV(b``>-$ zM|{PjUt-09)0*964ZWy4qG3A!iZuCL5J4vSq$?ol?wO2=1e&!;9t z{HK#&d2T{`aKZSSV$8nw`5IF+b?d?_&_RB2Nn@S=KEJHRZ&{wfFD-HANt+d!8=g@V${FeVy<@Q=p|RCl}k1iW;RIY+rXYw+ro1J ztScYrS3bq4R+FlcH(!!*-yB2t`NcV#59x0CP?FiqC-VdG1vMIuAg3o=Td=#P|3Z0B%|-@17rLGk-6p<6~!$6~POh1kU3(XXZO`=|>$d z!lw$=5_RyEi#Jr~RP#^%iC^4A^2m;K+VClBHe2;z6Z14*Mk&|$%X0f<_lmdugY8>E zPThfcKaZ0b)2b2Pn1`Dkmvb_pUZ*zC08jjo)ep|hccB`;;R{6kL;Ts-DL%Zk@M}Ec zYe??S-~5VIlRb~$9A!25WQb$>P5#6re$4=RZ7!m^$ICJHQwLq8^3qO zSIW*0ziJfhY2#Np#+5qaD29V6USiSHHu0r%dVQte1>d!Te30L9h<8T(gM1~;2HMmK zAIaG=K2h~u$+A`Ao#yL~^C@rnmi3*Dn>*0%_Q|VFij#Is9D-CUfq|-t52LPSO>Mf;|h8QzG9r>i*kxj)D&%wf12-@hxpQE(boL;`OLW% z&4ra*97R9KXL{m{MVR>LH~jeO-Z?hkb&`yq#K-O6lT$@0DD?-g)^Uzc7T&5n8gw__ z0DpXP`45D@vQE5>CYLA9MXJba02$ioVhjTWVS5bZ6(4zN`ENe`p5>!H^k})NKh(Lb zKhik@lUA-Xx~smjY)TJqEB4J>%kshNC(AGX&hhfC|NQ3id+))>f~iYr%eBS5L6diS z0c(T7VNUk2yzB*+mM{H`dzO#=6GzJf`m=$1G@nblG}%hD(09V$W~@UCQLSS;5BqEV zWae*vfSYo>EH@?Gc;aOFp#GTWmw)f}@_j#ZYkBJ*Le`;RxE%9>G%3oHFxKHSfF_;E zFF&fw_1jO}dg1SWTfI@g(_fZ9_1ee&mj2x4J1a|pX>wLqgaW;Whu>GnNZR9Y^4s;%W zx4i1NzvUU8TZ6Uq$a?oX>%J5^9jAU9em|0;-_C;e(1}uEYG}e zr$t+qTP`-spu!U-M~AgevS79|o^g>`wAc>y@e7Vk`?z91a^qxq>GOBXzxbc8ET8gX z-7Xxv6CigTGJZUUv*`9=vmA1gzg4h49N+Y^ODZ8#@KI9`q-_X zaPu5;fuSS!*@le$mhP;#HK&jK(B1NbUvXvmPhY0_kiYDk{5AHRoIkT@vw@Z8z;F1q z7l7fCCi(MA@@nf@5q}|i{jv8-IsM&M6%o3LI{BfEQREKp4HG$@wUJ1eYx}Q!%BAIh z`K$LWk8838tEq&7|H$p$UeKq__MwZg*U!9Rnw3=(J#1>imzU))z3%$*uKvrZuZ{Wd>ES!5dgNmrfBPTZ zSl;rks&UNFhD?$g9J)KT33%MPXFTyAfBeSP=e+&fch`Iedi2_(FPHhgB&G`tFhZFY^iGZTPO8%A6S;JedWE&6Z7VgKJMLTtbV@Au;oe}a$|fo@8QFpeTE;~ z=(!{4cwATZ_x+vv)3p?oK6COMai}`b-FNw9`G;R}pRW2^Ajgt*_)SjojgA<};ZV-D zH)q&q4iEL*eWU|BFmM=S?>NY;&)5I;`<6?(5sl{jyXGx}^8>dxQX%Vtv5PEo8w6JK zToHH6efQkYp6Q3Mqvhz+s$i(tXF7XpLn?CV%Z6Oqu_p_+nw!5{zT;K*3%heMNzF;f zzun5oTzGVll(CU?9of+U+nP1y(OpU zvv~w9Sr;nLG5?3p<|70ueyyDbUY}Yd!E0=`V+1F2S@%7DUU z!+3G5v_Yp@FhhD(9o{OXys6YM@?dLP0LotS!( zZ~o{ThY!62s*m!Sg&e-XdU0#<$S=0*Pb|w{eYqaXoLkS+K6Rp~Y^EN+{G*Qi6P;tq z8XuKI#YV0>%Nz^2?6yhv9fh2b=evx?JV#`6&=bQOMZM+dz(~P{OOO4g=JV%2_LA3t zIWdLGe~6_L*6U?ZoidN$t=;E~mp$XEY0L*5)a)#9%C_**_ejXj1}SaGL~lF&7ro-L z5_Il{V)fCw*fu?YZqYMj%cgB7z3S~eAahn{_@cQMlFic3)%3UY#Noj!JH4cEvRr#S z^9EDCiHH1&FTSjo9Q4r{^K&2ha-QnFK^=vKuFYqvdxW=7K2uz)M)&XO4}*2S)oU;32*?s`tzhPoNdy zMK~{~T*=4;PVlC()T`0MfB8pTs;kbv+GgKHr(Rq!;3+S|5(B&y+n5*@z^5dLrcGjDVs3` zF=w9B8T=Q$;LA>~9`X4+qVFJ-liI=f8qb5;adlP9$i*t%;M>z~dBL;M7jh(|v1O@a za}jzx7Y{1+b#a=fVe#WfJ$C)~F&^GD!hg8&3xD97hwY{wLOxnA2;wJqo|?br07>n| zdc9}P-SQkmio~mhtX%z&MJycY7!O^|^}~~L*w+vLY!DscBm0>6jPaAr#6u#lPtl}a zn^g8A4RF_SY<9BpclX?P?PZtsH(oFGD^X@u>A2cxb^Xba#{f#>E7Bp? ztFxkR`P@dmpq)Vyx9`@uFnA8e#&tpr-DGb_G^IYIlqLQGW*i-bW1&6e29O6Y4AR#5 zvw3QcRQo|aIrZklmvExE$M4X$oUyA07_9mhM=sXuWE_~5;nT=?xmN7c}VZTZ(}?rL~jVuDCHDd zW0I>4RkJL)P{rpZ{mdS{51lA{3Pf+T`jPlbs|k>vbZN6ZbRkPI+fmPp0DeI6t7Nc~ z$NhZ%nT)>k;6(Zz50&~yf1iG^fs4sKviK#}-Dl{r>Bu~hY2DR;F}T*pmL9|4wUTbw z@xnlPQdFhr&E%R&<~6QfTI+#VgCJrYF+`(acGqTfD_@rASLH)IiT<#`a<+xCqjpL` z>#D>_%Q%UnL=``~nBcrnhfBLfp$0UGM~}`pY-%%xL2Su?1!0>O+=jhV^Q|SHHsi~S zD~0ov1zlYjfNIlt^GFNNb-;qpg1EPAM(ME^ps)?4i@M~QXic5q&!wGA8~zyJ#}kr& z^`4JJ%2R4dCKVL9!V%6$c5)Gv^*q_xt7|K06))bGDUPP7^FtSfX;?h<0|XKb062A zIY|b0!pj0C)Y$7;i^P=d-~9Mh&zQKh^`h&1%>hsw!5hUsnpx4t z<}nU3;cAnu{B7X&Vn5^sgN95?k&<*Nw-dMSz$p_Pc^$xvIFk*X^*T}DEO_*uml7(B z&nEcAJ#m?Xu}#P#5u(vuOElFSM`G;J(?_?d0s0skGYz4+p=0BMwY@=f?C04B`6n16 z7Y+?9wH$J zAxS-==YiY@80*`{n1+s)KEk056AV77g?$%2H0xq(Q))9XS&VWbRL_G=l_J9>UJl0D zL}N3`NDj2QCw^L+J)AKpGPZ04N*&EdoH2o<_uVvg5ExqK?h8cD!pAn(v{$fP*#~QU zh>wrmGmlPAjvv4qPUcCCWLhX|Ka2&~1>W*WY1;yK(tBoXnGCEf#s(&kaR8=O7&`Rb z4)NokexjR!kF~8MOFmU5aQ$lW3aOlWOo#8pn)8ot^lQLVQZO5XoZ}x``u%x;$Cmjs zwt{}jE1RV@QuzczTVvNF(%{QMY#aX3$pievr_W(l1ZA{3C6z9Llh!WOKW`#3*AYhq z-tucRhL5MYjUq^yq;P4yz(j=;Uhu<*6tg}0;12PFp$~4~hxPm_+Zg8Ct>f7*BneZNsSb8?%&Jh@KlZTTrOg zc*d4a&)A=--&QSt^&=aCKtMfi2RM(tjY0_3lN)$zC%(pMOo(G{xaW#VQD)ml*8}*( zn%f398D{+~2NGYgRbLr0gOY-ta%{uQ8}bVGoMs=E!xb*`2zR1d+}H1qgGY~B`-@YJ z>*a;j$od&444i_t&M>U#WibY2>CmtI+6%Qc>JFq&fKMxFac!J|LFhSyp@oAfvh|$Q!ky#K zhS(4BtuuI=bE{5uez>A2b4!3M+hm`g$1$&w|CB6iS~rUj(~}eO8bJK3dJ?_67ebx{ zSHS|R%y8%`=YQMnAR>?_}JgGOix59Mum~lwBBOj7l{Dr%(^B9~CeuB#Ukb0`^qvuU*Y(62BICR)&Tg!A&&-M+!2eTcS zQp|kcb?_I5@TRuW`$zm0SeN?*o>tHfJx!tLIT3p}glz!EcCx$YvH;wLhF24aiOPLh zoyM4vMhXD7pn%KA%I|SJ3pjFVbc&HshPKa%R-zM#w$p3fhA+q*C$x=DN^`o8SMD%{ zlYy6XyKVf(AvWYbX0=U|B7A&%L$qy^lSpgCbq?mNVK#inCYah3&VIO?=1DXw=#`qC zbt3TAho;;JwjNhLV1kW_T;f+5&f5zw$zb{>8{!V`+%h~%KVy-DqlO+=H=VZ=FkY%TPJGOKbO-eUMZb@k`Qw5*kXQI4 zNn-VY-V}k{dvi=NgDj)aFv2b;9&Lhj62jH0Xgt5%4NV`a$nS9VFeZ8jwL3ZT-35mn zvUwAUQ9a=cgBJ%U^%9B`*>UXEt~NPJ9a#K=jILPgIq5_LF4);`bivL2J}%hVmz_pI z&(zfWn4ASNsVrtA?CTky6@SLgnCP>dnQ&s$k2bCduV@v=0M<$2v&?X_w&f?0 zdVL4q!ob4O|06wo;ixOrj>l#y;~Gg=-=WAx*pV-hTSqte=+)3!U&FCJJ(R7IGj_tH zSk_m_@)csRD}7KQl3@|As*N?`C_c!U@vo=O(oUUM9HYTXr$fev>%5uanu%NzjR zCb4pse%58Ff_FbT99ZTs=22SCWBp8Il>D>{j4u>gKeWxhWg0&$HJ{gkdPXCf61P@& ztiI#OvjYd~D)hvhL4pdPanYqKH?T(AS0xsJjcpoa4(T1TJw`VIoTCqRpI?P*;>dsN z5f0BOf=znyxkaZ2tJWn8N$N>lK}c;lWS?W5vOBR=JKko}KC|$3Z%PH$J5|jKJ-NqE z_ZknrZ7W~D$^f(y8P~onU3Oty2J4NY*@llDx%i|JpU9&wHDK(xtG@VU#^kYat*h>i zdSLC^jL7(-#cz$a=M=p%&kPDtW4)wR`B-^()-G4{E(m^LY+5LRq%6%7l<6vOPNhVCyvY=4yUI zIx&MxLE28(nmXlm7viLOLSs$b4|GCD7I{^>sJ)bo<7qB^r=YAS^^JFY6;xwEh zZpDM~;ZEeb0~BvkTQTEG0U3VZL5j9H_mXvxdHwoPMGk8H%GZ$DSUoG};o!Bp*+kXX z`qy7&0LlzDGC5UnIv&!hC5g%LKEG*AaEI$`J|`zF9*~_UC6v2ef%Yt=w?iGS=`x{m`*tc1v}Pz zf~slY{K=p-7He#u7L@_cNMwKhd*f^(-Vaneam*r{gTf>LelwEqaEL>^IXTI3UTi}^ zZkltHCYX)!fRgkGlZFWF0F?CZ*bebcbNh5(fov2_4=P{4lkUMPb=`l~2uhFxu>7&DseW}mFpI(L7m<98w3m<&s^gYwzKLS`@ ziH2UU5yjHI=Sa0E5;z6n)mm>R$Iaaa0HpF2H=cyKrST)6aY5j>Y2EFa4KyaOJpi`Y z0cR0NFVNX;eH&s&2RLs_Wk`!X1Ktl5EXMuVY^M5^Na4ay{PgzMr(hU*GqwVm<`|tx zHqpMHc}$IYj}CnPhO8RSa9ryZ-xY7p0CWe2u`wOua|f#J0CPySsjO015zUoj^|=$R z&P!8a>m2?Q`plg2TfXWox!mch;lqB)b!%4}(i&%-8hjt^C)?8v8krgXwGp&JSbXUmUuKNKj;seLQ@+i{*gD4%I@RALNg?5Nv zHQN3d?-dcg{ZuEQo!};N-E}JHlr|#Z=D+=Y^?ah~?(8cL)5{VsbD?G)a@Zyct*NHxP>~FNNVt39Nz-u{udkt;$vC~g<^Q~(o z@!$ErW946qkAsrqYR=YH5b{$F!kam>41*1>C($G?Qu;QuA8=!KcHIVdWNDr-8-7uK zNuNiULdrZEx{d!~v71dXW?a|C=vhDe#uyuYWb4hW)6k0ypF8ER{BAwTAx;YE-wb!) zU;16Was^(;$OUp5dXvkJY0hDAS|8fn=gyP6&xSuan8cZ0vW)z(=x@DiJPDG%HphC= z- zpYdSh-(EFF=R=BYI@>x#_%jYWdLEjhM|USaBzVpNLG3+y_(R$BD_RmMas$MWs~oG^0ClV~+&9ED$w?cD|Yz+=nu2k$xd2U}uu6PP0V zCo+iBf#`{lqWxs#{-;()(J&9)cV& z*MIxg+j{>(@hd`~jcXbH;1z zth?n%0u(-3tD58KJI#tQPuPp_{T#@NnLsv#(utmIWON>=r)G}FN{F5lNBD@6U;Bn9 z>MqnKn+0+&Jbe!0Sg#XY1|IL>WT_VXUT;oA+Kv6ir{@DlMjpC8`1rDX*N^ifn3Oa- zP>v=r{|3wSjsMrp<+?rvZ1#&IQ%o*?Q%fUy9{OfIvd7w82leqs-`IVe19y5!^8?p+ z%lE(O);9mymq@O`lr{MH-Gap%a!lvK(+9_5!wv_d}s`<0wzR2F;-6sG^f)1 zfAhBE<$Hhn)^a}|--)B-fGBwkg|A}DfUPxB;ADB-k7x(+!4Wu(Z^V|l+qB6&n>1q*9dcD_jHBlT z*vR|+hTp{?KmT(AyX9Nn__#hpI{B~9Yw%ik6(uW2wP}cuI}>`1H0k-6=fBTqX`C$v zyXpzH+GeRX%|8xjW>_S<&=S+Pnr``~H$Jia)W5&2PruNUE@20Cie;tIvIjt59r&b0 zjV=c|+__#ALk??qI+k=+1B_gv^QeSsUl&j? z;p|tZ|KgJ`FMscq_bfcG=0&dhz{tYj7c4!e`8Av9+C(?nNM0J_+A`~hL2+5Y%lGV- zcj`{^cVGXwo}+cX;<;dQvT7u2?0R+qYFq{XM198e*L=}E%d_>lL3~zo=0om&Voy%^ z%h9>f^lD0ytPpr zg~{1jZAiO~^T97J@yeh09w`1xwSh24F`NSEhCjRLSXJn`%mH@4#+$x@;up2ebwIl&_3snm%EJ(YEoj{-clclgY{Q#$UL- z{G^^VuQM1Gu)n(U2vif97a;}2J2D&cm4Ei0<mZtf?9#n|`tkjxXn6KX&EI1=R@*$+Kyw>;|^ zN6TfsKa#H^pu#R*_}$O*#n-X_6q!ggu8IzGT!q@a0d4&GoYsxW{s08 zxcb6`!zl91*VjDiv#}r4pKJ1goci!UFDRc`2%OJ$tT_0@2dCnL<$j-qr9L&M`lL5D z(Jg%h*(2AFmk(S^Onhux>cB?H;>YJE=cKZwR~3}pmJcYob}zo~KupBx=(Nh~M4*nz zFreXsw&7fy?>G)Rb7uLh_>fd0az4fHf;q3Jlg~yVw=Ucr;=5V{Uqw2b-#L3OowL9U z9j+Ix`1q<;8v}WtQ-xXig+I)9(3;nXc|pGNB1^pvR0~0A$kl-?YrweTR}h1GVi

c)ijgxDm}8EsRXFt3h@+Ufr7@DN z^55r2UpdZvo*$)c`MJ_3zXBARbH%T}ifygzYy6g*WBtspGU<*Ccb`wpyW!Ui$gZ}y zo>MwK`K>f-62KfvO2{S zXF|ni6T=gB=C>=mF~5ojWS?I%DBt!ouB^&}v*S8G>5&(6>bM<0W9)PIeSXbv;v2lq zgZx&0)nJZqzUPEz=3RZouldy~VSciFe9|fxrs_KoD#u$hYz3BTu8Twxs@yt>*lp{< zm_XbpVEfL5#v}%x;+@AY<0*cV$ZF-248A&7CXCUG-9e@z7Va=V8J*&{q4I$n{~M-~K{qUmg-Y{N~tC__Y!6wZ`uS zAN=8SKnb`wARia}P{>}4q*mFJ2rt$xz9z}40>2@prKgMpJ4y?1MK zsu;8LLY(s8tNKp-L`??i35r}^567PuI=u8S&*EdFoy9Nf;48%{S#m8d=h|q*N!*Hw zE&QzCc2jn4u4(uar*pTPKCQ7DC)&Cs49?>3$7+X~)XJA`!=HT>p7`~r%@S~FvIWT% zL)t28t$h|BY!xpHnSQNXihG*>p${(0U;hi2mrwZcOUrZh0ee^UiT1oYO{3$5Hop*u zLXEN0l1qM=vD`rN)XOLJdon_5oHz3`AzpsrE1f=|*Mk1={U^)6{EcJ3kodUYZmX=p z&l4~2a)h&L*mG4|<3d+3_?Prr)`vgu$Y1U7EWIl2?@iUEd5K>;n9zxxlFNU^0vTLl zH@o9AcfQkuuVr{d?>6N1tv`70$?|*eKGqA1!uC8^rS(s+P1LOQ9lYFac+7nk_^^=}_9|LQHrRm;gm z#jgtmwd-2xd;fSm;rGSZd-@wbDeXS|)%sP&lv@b1qs`Sf43!0V?3qvsHeeF4^Q(*h z^}o7zxuRcU@`@_U0N4FIMxo}rPTLvJc{K#}XhYWmowJJ2$Yjbl`u)zkPnNIv?#GvR zeQ>x@oZ)FOm|m&l>_ivC(ek;URCk@4f5BINBIPcJedSknv#$7sL09O4r%@qb_M zz2et2d?)PSD|vhJv?jf^coe^7;*5D_(i{GoNjc@GFgNZjMJ5=HK91L-#6s_k5ZsDS zGS%RQ&sF+5eNE*3{W~3);ByDsjH9O)4$S@$?yR>?gy?){V`EPI$n>{$7kZJt&E|jq z@9tl&>KhB0wjiX?fvux_ph<@^P`xU#l~@YcVmvoP|52 zFCDST=db-|m-UT`(xE24+%n&4gZ%FnLi&Yo)!)!<`8*?XqEn@~PlG4oI{hPQc|SBA-3UqQo@Ok7n} zIAZ21l@78Rn`X^sw|ukiJP&AnypS?sjm)BYgRrvd_2vm*-zj>cKd@`Ab&91Yp=>6{)F%4)7auKu@lUJhnvWozKNZb^uG+`E@Y3=U zeK~|@uUf1nf;jWRpXQgYuqA_|MTZQJmcB;TNR^GlS{T8}iC6rO{IH|tWqO{uY5h}C zK^05FmfvX7IMk$1hE*ehH{+tKyHIa1DdB;;rJvHi z@XysN8q8vy7k-&z&tLr~zqICPT-#vO+|kk)bI{UP%}!$rHS^6TDD1uXt~a|@W*~+c z8vo^wJW;Rw34f4ZJkG`2_D~Yj%WRNd2O^Mwn=s<$0*s{9@EYCPT5v)bA~e(n|~6M0EUxGtnrcN&$s(s zzN8S(XWAcol9+ za@NCPqQw`HsBTqo#8>DWj&U^~+CTP~&69^IHqX$ty#E|%_>m7|XO7~asM|V+|Xy_l(fh&fm#RNST>VcoN?=6S_DPi%0~BG=sQt4-78)-@|b)lahBHa~PL<9jHj zNE~dl9PG02qUPM@QPu+cEDu-Af8%z}zB%Ihfge*{9Wd$&G+)E(=&9+o!^CjO`cwNdjVRH+WU`h_MXAOitJp5x3ifW{$igPf9iBj$(b=HI#x==`-hy-E&gI#->XR(BW&pMdcoR19-nNcPkY4s2bR7uK27u z;T-wi{Jv$d3tg^Khr|3zu!D-f$3GV1rd-BjB{h8+psmB&uHFO}3e<>-KnIym}P_oSC zslstp61Dm&1NiV|^pEbaNt}ZX!rh1GA<@OoA~K`yhAgd{@foOROsg!`F}gM(u1!jB zP-&PeM7Vk8W1#d^)-p1e`o(13g|c~w?dj`;4_bZu^_E|g3d=E{cLES;rdxmDH283uG=7WUKG<2~ea{IxU4q0( zBCeM((XD0e;O571>R|^u&Ev*jpsQGwzvm-2(K$^ICifY)?_e`E(umG-isbY(H;sFS z_TV{-u;uIR9OWMt?$V=eCxZbQ9k$3lC>2^A@xz~@XvD&(_uWN31AO=Zpf(=jB!lHh zOT3|j8)NsuFr00(J`~5*Aa@-yCcZDeY#2MK^7+byjE?yuYo4B|14zoWZPTeh8BIOF zi#LZ9-0pPpQq1&2arSg`YF@vQoGhb26RLwnlb*1L_^M-Vlx>giHItHpV-y+pt6ZEK z556G7lZ4?GS?qbNp_S;OAM&IlDs9+mIL@;^vinA)D6z3H9OHAVWxzHP_n^luSJ#<< zbsIty2lS^g(Tp%sL>_Jx%DMrbLPR&IRuN*2au@Mv3b3wQaDyVnmOp4Ma3Q*l1@}l- z7!@6xqcC>X;&3#^WC@2>d~Pt-WCFI;DSS*he8-yHfN>hl!&k7gZRoJWX*}IU_<3Dv zFh%O=_d;$wPTu#$88_QzeaYlJH`gOD^~u}%0AtVi0{v!P<5awgzdH2uJ`V|wUL*2lawezA2~fq&{P;mfB?8T6HUC*4h6A&Uoa8O-j$RT~z$aZBVg6 zzF?cyl6N zdHw?sJ7Tp$XXHMr#>SS7hWS(q4Vv|F6FxR`qoAKa__u1W&%AQI4T^VKan^IyU>zfs zE|$R$NQPNwnbWKcmi{dLjG5%b9r@2i8f!K??SvY4H+*lPY@EblJRiC1P#E;CqroIW z@amJ2xy(A56v{9|GuaTpMMj+DK>H#%Xah4-!k=}#^ zneQH-ALI49-brtya+(0Rs?MoH;W4xa=7q~HKFb7Z1nBuy5&@vrkTKXDY=saRII;oP z3R%&P2^nF-NYearIVR*J3O2Ys934KH3%!qF8Ezacu`vg0S*Oab^yt!p+xLq-xy5gM z#Kw5jI=`XA!CkZ&zAqE&VEj1=NFmPhl*4MSO=PEas`~e2-T71-1sApc|fu*Q}= zsYFnC_DZcy+zSDb@&j)&>t^-n;oK7;%>Y=GI zf;q6^#lf=W>#ky4S#ll)lVVQT_DO*_|C(c%5cIB9nT$1w zdZdwu#x~{=-+@S!Al?*`YqRX_$W)w|mL<42l`iKk-%cwYqIN?eH8`i)kL=}d1?JZx ztLCs2KGwvGug#(X==ud4yo;s5T!B+uNNV9YMyc!;d~C+efEeaJa{IVw7aDzJFOkR6 zSlJt<<>?A3vyx@)YW!;#RD~3cJ<+yt$FWi*K*_8K6|i@y5t3Ja zJ+H|ads>I+vjj95MRGK=^x>=qv2joEMXBp_IFN4`AdHaye#ZCSN+T3ki zEEWhGJ-%>&Q^eAnKgqhuJba{|Jl+AxddOr{Cxi+(@50!IbHi4?hjyY5LQ=XVPTEpb zyqVjwx1@vOf~d3GC@cCi=V6PSGqd|Ua>`SZ|JP5mkUUL?=|EPi{@-nlH?JLkAw z*sMbLgtgvL+o_1?*wJfZjcXpC5>GR~M4yu?y`l7N54Pg1hB01ME2+8Z!14qfU-Yz@ zpP&@C_lf&Q^@(4j;1EbkPV$`KhCay2t@XoalE&DO(HG;)bGsV$(1$|8a365@r{WKw zNW$FkEp^Sm<|7b9uV3Ad{N#D~L@0goVuYqx6L^T_<{Zg#=0otZT7J0Sg93< zJ_mX2IquB#Bm6s#^rsweb>du#$y5q2icb}=oNpi;{UA7T{^iK)*yGw5d6=pq_?*D>mRC&iQRDaItw;A9 zUwyN}YMcO55)^&3H9%p>YklyFuHBgRqrZ5o{^}Fg-RyE2Q&BkPr4P7!;2dsBBY5kZ z6MOo=-HSke#!JD&S`O^!e_!8v^T8YV)+p1?{L!gB{K1puy1vT%sWe=-JBLXqC(&~o zh8QdS8g_rYT88wPo<6+$(H>5CKO8#&q^#c>*j4hprAvR9e{%Kyt8YGf`?u>?8Tz14 zS1k!Et{sV(!ehcu#U^0M9yMmukRS`=W<1D5*Xuj%0?f#3B#i1AuV%Dk0a#p(np`Z z@Ny<>{{ZDV5+@v)mOs>&&;9Vv>-)pHaOkS3YygE%;ePHnZ!h`bKx(H9HZuLnZ`piM z2ii=ClLN3rsu>=c{+jNjKd(=0rLpid^!u4*y(mWJPG6kjm0Yv8i=0jt@0q$c?3SO6 zo`T_+i0(Myt98b;JQvD(PJ8@c_^spR4R6xbATVp;gA^fWJoolt6Viy=aHkR(bL6>a z0*u#QIOR-CHs#1eI_@gp{LgMJH~1i?ZcMM{ufkCb2He+@V%l*Br$@ccN`(OGk)9u)8Cl^IS$70>cnNtJOD;^adIv1mfzOH@{j*A zpUGT+)Iu&-&YD8$81J|E-`Afpo?Sod(=~-f1KG?W4N<>A4H|trX(W)6k{Oa&+m(#9NV~FpO<-jgq5FpLo=R80h%`t-tc094&kfl2?<-(g>J|r?=r^r}OA> zmp&f(`pX~wSI3@L@|*kMoPV!t)up3lQ3afNHGkNJ?ukAA%&S+P!*d|=aQo0Nz5YfK zKR4s_UId|>uzYyqbjJt5=GTt(Ez-yS$U9G{Cqm(9+ajN> zgT~ide(a0*RMefm>R_qQXttNTKUJiWa#G(o>gibbxL(-&eO>l^>-4Yw{;}#f=Ndog zTpjgwLr5GKkp=Bm^VjU9%39U~*@|iCk3RCfSN<|`f4G7d?}tSDTy`AIwQL?;#$97+ ztSvnwvYK=4p}Io0?fv>@g@5oyeJpBc$rtZF^xS26hCWZ4#Yok->p2VeHu^YSPUGG2k^A|XtmgmW>+a9E=9)4OCk5TSW^(Rd;pI_JfySLre zQLOv*sbCN46V?6wuS}=FN|eBT_p(bFq*`MXpIA`Vg(EMp(umI{;a4t?=!xmyYV?&H2P7PMKv=d+vjRBWh(As6Lj0Qcn$#3?!%y6`&&<3aj!!;n$@xk0 z*`QFf2~yb7*ZgYBR84)J;s=KZ&x_vE!tWtII60`G5(@|IFyHPr=5zVG<@(X_<1hTc z_kGCwAo)o&!Uw+XL*A!{f;S*LxN;y5=0e-ZrK)pdNED2liw(!iVbw-%n7!XMpG8kA zGUJMmr0RBj5-MyJddQOpL{O*s7%s{`6u+WXrgQwlI?smCIg$&Q{AYgqCt0wKb7$_% zm%{TugWsEv_{Fa|uJO;}cZ_9uLpG0)>jq*Vhu`WPlbLjiH(IU~Fm-o{X+n|rIebs+ zBK*FBMohVN%r4@=_@qH>4)KXqe5CL#cK)Tu;+Dei@z-rsKEYOe;uO{W-~*^lGv{e} zg4af91r84J?WZul<4pXy&Q9bMAD7uEiayKu@j6WtFdw~+#;%<5b$dDfR;X#?4us;} z-~EhV6zs>~=Rof`?o~=VM~9%M_?8J+n!&AcCV)?AP=;fE71{~UeEA>#S{QucDki=r zzHybu$j{hvT>Nr&n2+r=zY;+&dlw*cHh$KbFJ$UN=-6jIG7AR2vDH_c$iN1FmhpRt z?{%2s!?BZglURd~-k|DP8~&9Flv)o?mLI$Jz3h>-Z8i{UeJRS<(K9vL#!-~$F*1Sp z9>4-|wb7EC2gB>kF9$2`EI#_O(HBeOdGZy+=Ze2BPH_+Mi?qgP47=j(>kB=mJ%oMS z9r<0iE@an9F`Z)KGra&4x%#2EIrCiSSMf=2pI?~4w>$UPbpC{gT;8zlrl=Bb2 zc!MuoiVfHWSDf^|NDlF(^ZW;&*`LSHX6X1EeyW$cIeN{P*pA<}=H;OUB#~>P2l%!Y z!u69#KlsSz*U2UJ{M*;+{q-Mwz4pdlJGFtZ-+TGiS1Ql<#B&y|xO2F8BP#-G95X!= zS3AtF&0v5*jT?Lk8~!j1%0_T}otooBko6is#Sgz&6@Aj7$ONp`$^7Ks*zOGN$=Vl+ z!3WfQyRB%BY(65Ff(S*v1=yWtyJ{I0gB$4W-~OP!g>&~BlI$ss{JeWJ0Y~lvE4La}LgwmJ{B^=-^LrxrR*K+!NY34Y z%M z<9FfUS32e(gAJbEtbl5ub8iasSIo+HYW6cI2(;PPCVrX9hj6>)HIID%gYPzH@6^%v zv^{*@-@5)2n!;y#NN$bBu|)+fn^0}89(_q=8AGE|lG!A3qm}-*G$sPd@g2 zSN`*ry_F8$fdaX8yu3>5_^=Mm3a>SxDq|(W496V3gthog+!l-+gI^0x3>K~U0B9_I z@g1v9#%%cbQY(J<)|7{e%NhR$c6@0R)3;{wt|Y5hT-qAn?23((Ie*Is_;P_4Gx3j1 z3^!RMCcZ=O#~*wM_}}BBm6H6+W|(D1K9`SA_)O&v{7zZehxLm7tBQH}eC`H%|3AL+ zwv$WC=ZSiwBbOHn*aasRMW->jDp-wcQfvqt$sDPv&GGOq`KuGkd^o;c>O`@?JJE_` zdU788%6;TNa;;()znFK!uf=i(n|UXb!}$}T5F5S&N6!Fu`(`Au^2Zij=Z|V?HNBZ# z{Jg_J&>P3Qlh3>HhAVHIXs5)?*?J{TB9TPPY-Gp32p`^F3!lv=`TY2MT!#Dn_EX5YDwXjm4@%zo zyA%j0dpPZ8aUi>rp!dHqyG~d+l6Q>+x9T-*oC&4dQmFv;TYcH~Spj>DJ0esIt zzWNO+#A`{>E5i(Xk;Z0`sjgNLsQM^ePYfMu`tZTDpWqGSgiZetwnduxeT7P8ynTsi zel~9SC}kpn5&t6m<~Z?*-@e9Xw_7%@1cxGiwOUv!*ZAgV{^YpI;WyoHSsAi`#H6j9 zt$aSe;%xY&tQ7Q@%CCLw|GfH*c7B0V=63;TLHuy07aBFXpK@e@kz6>#YSGcv3{ghz zzVXF3=^Q@()T&z5KP7&Q>i!XZTNu&$kfkNQnO!8-_aDL+?R~C8sjF4t! z6x@c9tB)3F@nK85F<=By?G&Gi4}X@LiXJ2XmM&tvDMDVeZJcH{s6W+y1bgFn`9~ZXTFjEjziZ(}(o3vn z`%X>ZGshK%2W48h%Jnqix>9=bSGbGC-{Va~Hp{r_k-l2)R5e=9GXJFTue#GuTPtHLO_kpoE;{;<|N8ou=yCIP zN<{A~WY5T@7mLhsKlK)EER*b9LF?v{dT-&+=Hpvd_~PVB{13->Hs|DD_AU++MKR^? zVbs#s_)ceV^X6!`7vaB08NBAP@4xarcZzYI{jMLv_MN@||G4r!x9+?3(b^}k&qm0m zIJo%3!Mf<)XVROminu6NX7e>E)#+h2O$}L)eu$)~=3}XaGUgyZ_V8KMnK#)7zjPHp z_Ts=j%wK(OAJ%4maf|Pa51wLAKZDR6(r+-k<@J}An;-pDHxE9y+0Rj)g#6$aUwirP zX!kYxQ0mVy-QN2yL-92;)+QS*i|kvrv|fAPK+-?Jmin%y1ZS6N0LGw(w2!|y(vgZ*y#F}>^b>-1db)Nj=f;xC|Ft8@YI zMIq1nn~#0+?)d1{!hey9e+8a5izk@{Oplez2GHqrSUlSN&@^wrvVyP!giSlmuO%9r zW`jOGD83?gYTjdlCEZT%G_f_YKb`yp!)N?Qcc8y6-5c~LFW-9YpKRX@b^v?Vs?#fW z*DlT`JnOH$|Jl3C_q|fP=kqnu&(d`7^YSrkS5(VraZMu&zIv_2t3qXyto_-1d=_pk z^vbJk!~$p|XLVszAW2V_Pv+Y=r{jaEb~--#@C&o@YkYyT{(x!uak=@SdyXFer}KN5 zFTlMk$hvZOMZ0@2f4q3@#*LTjFKs?eK|fUioJEMtmjUO-<02&yOE|p|V-%X=6Xv@X(oCxjr1jf2;npdQ$tQM<2QW z=azp~pZ|S`@O0`r&8O4l#eLPLy7n@?{`u15<>(>(HP?sj)ax^gp0C0^Q@=iWK*f2c zD)fL#sXs~F-K&MVM;neWi6M8@tERwteOT%%cv{JMqtu2a&-F?ld~arKwAH@y=LKKw z#h-2EA?L&VSjQ(K-_mq$Dl8u&b4}hKRXUGo8jtD{dqj15STlZy(C<7sI)2CQ_~fnE k9@EG3{4s5ok?kb>|H;3ubeVRY^#A|>07*qoM6N<$f~C=$asU7T literal 0 HcmV?d00001 diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/profile/backup_config.json b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/profile/backup_config.json new file mode 100644 index 000000000..78f40ae7c --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/profile/backup_config.json @@ -0,0 +1,3 @@ +{ + "allowToBackupRestore": true +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/profile/main_pages.json b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/profile/main_pages.json new file mode 100644 index 000000000..1898d94f5 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/profile/main_pages.json @@ -0,0 +1,5 @@ +{ + "src": [ + "pages/Index" + ] +} diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/en_US/element/string.json b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/en_US/element/string.json new file mode 100644 index 000000000..8679134da --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/en_US/element/string.json @@ -0,0 +1,20 @@ +{ + "string": [ + { + "name": "module_desc", + "value": "On-device real-time speech recognition with Next-gen Kaldi" + }, + { + "name": "EntryAbility_desc", + "value": "On-device real-time speech recognition with Next-gen Kaldi" + }, + { + "name": "EntryAbility_label", + "value": "Real-time ASR" + }, + { + "name": "mic_reason", + "value": "access the microhone for on-device real-time speech recognition with Next-gen Kaldi" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/rawfile/.gitkeep b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/rawfile/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/zh_CN/element/string.json b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/zh_CN/element/string.json new file mode 100644 index 000000000..e4ff4f4e3 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/zh_CN/element/string.json @@ -0,0 +1,20 @@ +{ + "string": [ + { + "name": "module_desc", + "value": "新一代Kaldi: 本地实时语音识别" + }, + { + "name": "EntryAbility_desc", + "value": "新一代Kaldi: 本地实时语音识别" + }, + { + "name": "EntryAbility_label", + "value": "实时语音识别" + }, + { + "name": "mic_reason", + "value": "使用新一代Kaldi, 访问麦克风进行本地实时语音识别 (不需要联网)" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/ohosTest/ets/test/Ability.test.ets b/harmony-os/SherpaOnnxStreamingAsr/entry/src/ohosTest/ets/test/Ability.test.ets new file mode 100644 index 000000000..8aa374977 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/ohosTest/ets/test/Ability.test.ets @@ -0,0 +1,35 @@ +import hilog from '@ohos.hilog'; +import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from '@ohos/hypium'; + +export default function abilityTest() { + describe('ActsAbilityTest', () => { + // Defines a test suite. Two parameters are supported: test suite name and test suite function. + beforeAll(() => { + // Presets an action, which is performed only once before all test cases of the test suite start. + // This API supports only one parameter: preset action function. + }) + beforeEach(() => { + // Presets an action, which is performed before each unit test case starts. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: preset action function. + }) + afterEach(() => { + // Presets a clear action, which is performed after each unit test case ends. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: clear action function. + }) + afterAll(() => { + // Presets a clear action, which is performed after all test cases of the test suite end. + // This API supports only one parameter: clear action function. + }) + it('assertContain', 0, () => { + // Defines a test case. This API supports three parameters: test case name, filter parameter, and test case function. + hilog.info(0x0000, 'testTag', '%{public}s', 'it begin'); + let a = 'abc'; + let b = 'b'; + // Defines a variety of assertion methods, which are used to declare expected boolean conditions. + expect(a).assertContain(b); + expect(a).assertEqual(a); + }) + }) +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/ohosTest/ets/test/List.test.ets b/harmony-os/SherpaOnnxStreamingAsr/entry/src/ohosTest/ets/test/List.test.ets new file mode 100644 index 000000000..794c7dc4e --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/ohosTest/ets/test/List.test.ets @@ -0,0 +1,5 @@ +import abilityTest from './Ability.test'; + +export default function testsuite() { + abilityTest(); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/ohosTest/module.json5 b/harmony-os/SherpaOnnxStreamingAsr/entry/src/ohosTest/module.json5 new file mode 100644 index 000000000..55725a929 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/ohosTest/module.json5 @@ -0,0 +1,13 @@ +{ + "module": { + "name": "entry_test", + "type": "feature", + "deviceTypes": [ + "phone", + "tablet", + "2in1" + ], + "deliveryWithInstall": true, + "installationFree": false + } +} diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/test/List.test.ets b/harmony-os/SherpaOnnxStreamingAsr/entry/src/test/List.test.ets new file mode 100644 index 000000000..bb5b5c373 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/test/List.test.ets @@ -0,0 +1,5 @@ +import localUnitTest from './LocalUnit.test'; + +export default function testsuite() { + localUnitTest(); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/test/LocalUnit.test.ets b/harmony-os/SherpaOnnxStreamingAsr/entry/src/test/LocalUnit.test.ets new file mode 100644 index 000000000..165fc1615 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/test/LocalUnit.test.ets @@ -0,0 +1,33 @@ +import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from '@ohos/hypium'; + +export default function localUnitTest() { + describe('localUnitTest', () => { + // Defines a test suite. Two parameters are supported: test suite name and test suite function. + beforeAll(() => { + // Presets an action, which is performed only once before all test cases of the test suite start. + // This API supports only one parameter: preset action function. + }); + beforeEach(() => { + // Presets an action, which is performed before each unit test case starts. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: preset action function. + }); + afterEach(() => { + // Presets a clear action, which is performed after each unit test case ends. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: clear action function. + }); + afterAll(() => { + // Presets a clear action, which is performed after all test cases of the test suite end. + // This API supports only one parameter: clear action function. + }); + it('assertContain', 0, () => { + // Defines a test case. This API supports three parameters: test case name, filter parameter, and test case function. + let a = 'abc'; + let b = 'b'; + // Defines a variety of assertion methods, which are used to declare expected boolean conditions. + expect(a).assertContain(b); + expect(a).assertEqual(a); + }); + }); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/hvigor/hvigor-config.json5 b/harmony-os/SherpaOnnxStreamingAsr/hvigor/hvigor-config.json5 new file mode 100644 index 000000000..06b278367 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/hvigor/hvigor-config.json5 @@ -0,0 +1,22 @@ +{ + "modelVersion": "5.0.0", + "dependencies": { + }, + "execution": { + // "analyze": "normal", /* Define the build analyze mode. Value: [ "normal" | "advanced" | false ]. Default: "normal" */ + // "daemon": true, /* Enable daemon compilation. Value: [ true | false ]. Default: true */ + // "incremental": true, /* Enable incremental compilation. Value: [ true | false ]. Default: true */ + // "parallel": true, /* Enable parallel compilation. Value: [ true | false ]. Default: true */ + // "typeCheck": false, /* Enable typeCheck. Value: [ true | false ]. Default: false */ + }, + "logging": { + // "level": "info" /* Define the log level. Value: [ "debug" | "info" | "warn" | "error" ]. Default: "info" */ + }, + "debugging": { + // "stacktrace": false /* Disable stacktrace compilation. Value: [ true | false ]. Default: false */ + }, + "nodeOptions": { + // "maxOldSpaceSize": 8192 /* Enable nodeOptions maxOldSpaceSize compilation. Unit M. Used for the daemon process. Default: 8192*/ + // "exposeGC": true /* Enable to trigger garbage collection explicitly. Default: true*/ + } +} diff --git a/harmony-os/SherpaOnnxStreamingAsr/hvigorfile.ts b/harmony-os/SherpaOnnxStreamingAsr/hvigorfile.ts new file mode 100644 index 000000000..f3cb9f1a8 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/hvigorfile.ts @@ -0,0 +1,6 @@ +import { appTasks } from '@ohos/hvigor-ohos-plugin'; + +export default { + system: appTasks, /* Built-in plugin of Hvigor. It cannot be modified. */ + plugins:[] /* Custom plugin to extend the functionality of Hvigor. */ +} diff --git a/harmony-os/SherpaOnnxStreamingAsr/oh-package-lock.json5 b/harmony-os/SherpaOnnxStreamingAsr/oh-package-lock.json5 new file mode 100644 index 000000000..f538ae290 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/oh-package-lock.json5 @@ -0,0 +1,19 @@ +{ + "meta": { + "stableOrder": true + }, + "lockfileVersion": 3, + "ATTENTION": "THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.", + "specifiers": { + "@ohos/hypium@1.0.19": "@ohos/hypium@1.0.19" + }, + "packages": { + "@ohos/hypium@1.0.19": { + "name": "@ohos/hypium", + "version": "1.0.19", + "integrity": "sha512-cEjDgLFCm3cWZDeRXk7agBUkPqjWxUo6AQeiu0gEkb3J8ESqlduQLSIXeo3cCsm8U/asL7iKjF85ZyOuufAGSQ==", + "resolved": "https://ohpm.openharmony.cn/ohpm/@ohos/hypium/-/hypium-1.0.19.har", + "registryType": "ohpm" + } + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/oh-package.json5 b/harmony-os/SherpaOnnxStreamingAsr/oh-package.json5 new file mode 100644 index 000000000..a79d5300e --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/oh-package.json5 @@ -0,0 +1,9 @@ +{ + "modelVersion": "5.0.0", + "description": "Please describe the basic information.", + "dependencies": { + }, + "devDependencies": { + "@ohos/hypium": "1.0.19" + } +} diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets index 2675c7b77..5bd4b092f 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets @@ -229,7 +229,7 @@ struct Index { .lineSpacing({ value: 10, unit: LengthUnit.VP }) .height('100%'); }.alignItems(HorizontalAlign.Center).justifyContent(FlexAlign.Start) - }.tabBar(this.TabBuilder('From file', 0, $r('app.media.icon_doc'), $r('app.media.icon_doc_default'))) + }.tabBar(this.TabBuilder('From file', 0, $r('app.media.icon_doc'), $r('app.media.icon_doc'))) TabContent() { Column({ space: 10 }) { @@ -278,8 +278,8 @@ struct Index { .height('100%'); }.alignItems(HorizontalAlign.Center).justifyContent(FlexAlign.Start) } - .tabBar(this.TabBuilder('From mic', 1, $r('app.media.ic_public_input_voice'), - $r('app.media.ic_public_input_voice_default'))) + .tabBar(this.TabBuilder('From mic', 1, $r('app.media.icon_mic'), + $r('app.media.icon_mic'))) TabContent() { Column({ space: 10 }) { @@ -300,7 +300,7 @@ https://k2-fsa.github.io/sherpa/social-groups.html ` }).width('100%').height('100%').focusable(false) }.justifyContent(FlexAlign.Start) - }.tabBar(this.TabBuilder('Help', 2, $r('app.media.info_circle'), $r('app.media.info_circle_default'))) + }.tabBar(this.TabBuilder('Help', 2, $r('app.media.info'), $r('app.media.info'))) }.scrollable(false) }.width('100%').justifyContent(FlexAlign.Start) diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/ic_public_input_voice.png b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/ic_public_input_voice.png deleted file mode 100644 index 37fa8591f5bc675e573a21b38e4556188230a99a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 474 zcmV<00VV#4P)Px$lu1NER7gwh)v-%fVHC&l&zl7b6%NtT(%2MaqeK!>l+aSqV1iK%L9>lwOZ@{i zB&gLAQfst?5HvK^U*Hfp3Q+Y*%qXaWyQfavFY>}j$;-Z;KO zK&NmZotK*-oo?P|n${dko0X=5gZNwm#Ca^XSY{t_Hg{2cA4<{Jm)zm({XX2pz0H-j z4O(Be=oEfn4tKDW0ruxDiz=ehxIGuU-MC(pyaSWCja@Aod`X+%GGDi^Srw6OXmA;a z((zr28F&7}zb?Ct$^6Whn#8Kt^|k>5^d_hJXz5q^CmPho>S|Zr%fF@k0%3JU8OM7| QB>(^b07*qoM6N<$g8z5cQ~&?~ diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/ic_public_input_voice_default.png b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/ic_public_input_voice_default.png deleted file mode 100644 index f842d57073c509ffc004a2a5e669b8b313935289..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 438 zcmV;n0ZIOeP)Px$a7jc#R7gwh)k|wsK@bJtFD4q0fRFe5)Sl)VN6fA#MxPJ7k8$+&Z$#X=T4b$OiehyUVu4V#(W|48Sn6|MX(1T$?xIF zNU|L~Yk~9t+{C#G(~npmS49uN158(v{Kk{R0z7Q!{OCWxKbPnN&Xnm$6MFz=a08#P zIoy+jrsy_KV+~tarn->Xg~vFBJNSaO_VqkK45BsBdjrXIK)&TPRtQg+<)dr5Gt_|w%=X{IM~{So85DF0nTlGM&c zjyDG}A0YLZUD&1vnXKbO5~Z1WB4agr0ERCQmTs)U^)$K0XyO3*vZd;$4GV9MY482! gxxei}GvVlgJPx#;Ymb6R7gwhl`ReeF%X5nRd5E*fFUSQ1dkv=;yC~p-~vbl9*M&f6pnBU+@K^mlhib?^G#=(zKoV5bL3wGBVb&Dy#h`loeh`+JKzl50|!-rLPZ22HsAR5}?FS^eag-h_)@Y=>Hb!pML_&Xezhh0CWNQ#3$^}&J5`WTq;5)Z+{-z thUGK;DKMxAkcZ35<%r*K0V?JV%U^bWOBqFh{fYnp002ovPDHLkV1mz-az+3E diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_doc.svg b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_doc.svg new file mode 100644 index 000000000..ab6b1fd76 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_doc.svg @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_doc_default.png b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_doc_default.png deleted file mode 100644 index 3a6e5f2c6cec2451a16613b462927b7d75981f2c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 295 zcmV+?0oeYDP)Px#;Ymb6R7gwhl`ReeF%X5nRd5E*fFUSQ1dkv=;yC~p-~vbl9*M&f6pnBU+@K^mlhib?^G#=(zKoV5bL3wGBVb&Dy#h`loeh`+JKzl50|!-rLPZ22HsAR5}?FS^eag-h_)@Y=>Hb!pML_&Xezhh0CWNQ#3$^}&J5`WTq;5)Z+{-z thUGK;DKMxAkcZ35<%r*K0V?JV%U^bWOBqFh{fYnp002ovPDHLkV1mz-az+3E diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_mic.svg b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_mic.svg new file mode 100644 index 000000000..0aeb30d63 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_mic.svg @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/info.svg b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/info.svg new file mode 100644 index 000000000..2210223f4 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/info.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/info_circle.png b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/info_circle.png deleted file mode 100644 index 1bb7ed86d79442dc9cffeced41d3909e4bdd4f14..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 636 zcmV-?0)zdDP)Px%Hc3Q5R7gv;mA`5gQ4q$zIZq%6dS=B!3&F;aSXkI3s1XC^0WOlpprsNJ%fwOy z(NfVyo&Xh&UfawJ2U4yg#Ua| zTmCG|t`tRa8NgWp=c9fTz!&G-TAt_cn{w0wy!T5)Gz(zd%svoNPDHyQnVB6ovy6x? z0oXRPWmR3O0#yQ-c^$wcfR|?WT2(iy7JKg}iRcD^n|t~V5t-`O9RM)1MbTXmdD>RO z5NGB)0G@@EdQ4zVC01BWhA`2q|JO93MxHP>=tB7Aw+H`~2osNmW)c}Io_1Jc;T{oECf!xXDLZH#E)5>Jq zLmoIiHl1<}b7cMk5O#doGBZ1N=p*yr8U&{WFl}b%0F1@hXCm4GuzKi2zhy@c9R3Ri W?Hed!u$H6%0000Px%Hc3Q5R7gv;mA`5gQ4q$zIZq%6dS=B!3&F;aSXkI3s1XC^0WOlpprsNJ%fwOy z(NfVyo&Xh&UfawJ2U4yg#Ua| zTmCG|t`tRa8NgWp=c9fTz!&G-TAt_cn{w0wy!T5)Gz(zd%svoNPDHyQnVB6ovy6x? z0oXRPWmR3O0#yQ-c^$wcfR|?WT2(iy7JKg}iRcD^n|t~V5t-`O9RM)1MbTXmdD>RO z5NGB)0G@@EdQ4zVC01BWhA`2q|JO93MxHP>=tB7Aw+H`~2osNmW)c}Io_1Jc;T{oECf!xXDLZH#E)5>Jq zLmoIiHl1<}b7cMk5O#doGBZ1N=p*yr8U&{WFl}b%0F1@hXCm4GuzKi2zhy@c9R3Ri W?Hed!u$H6%0000 Date: Mon, 9 Dec 2024 19:23:18 +0800 Subject: [PATCH 096/183] Add speaker identification APIs for HarmonyOS (#1607) * Add speaker embedding extractor API for HarmonyOS * Add ArkTS API for speaker identification --- .gitignore | 2 + .../SherpaOnnxHar/sherpa_onnx/Index.ets | 6 + .../src/main/cpp/speaker-identification.cc | 22 +++ .../main/cpp/types/libsherpa_onnx/Index.d.ts | 15 ++ .../main/ets/components/NonStreamingTts.ets | 2 +- .../ets/components/SpeakerIdentification.ets | 139 ++++++++++++++++++ sherpa-onnx/c-api/c-api.cc | 33 ++++- sherpa-onnx/c-api/c-api.h | 5 + sherpa-onnx/csrc/offline-tts-vits-impl.h | 4 +- ...speaker-embedding-extractor-general-impl.h | 16 +- .../csrc/speaker-embedding-extractor-impl.cc | 32 +++- .../csrc/speaker-embedding-extractor-impl.h | 10 +- .../csrc/speaker-embedding-extractor-model.cc | 39 ++++- .../csrc/speaker-embedding-extractor-model.h | 10 +- .../speaker-embedding-extractor-nemo-impl.h | 17 ++- .../speaker-embedding-extractor-nemo-model.cc | 38 ++++- .../speaker-embedding-extractor-nemo-model.h | 10 +- .../csrc/speaker-embedding-extractor.cc | 24 ++- .../csrc/speaker-embedding-extractor.h | 10 +- 19 files changed, 374 insertions(+), 60 deletions(-) create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/SpeakerIdentification.ets diff --git a/.gitignore b/.gitignore index 18178f5fa..cfb6fa57c 100644 --- a/.gitignore +++ b/.gitignore @@ -123,3 +123,5 @@ sherpa-onnx-online-punct-en-2024-08-06 sherpa-onnx-pyannote-segmentation-3-0 sherpa-onnx-moonshine-tiny-en-int8 sherpa-onnx-moonshine-base-en-int8 +harmony-os/SherpaOnnxHar/sherpa_onnx/LICENSE +harmony-os/SherpaOnnxHar/sherpa_onnx/CHANGELOG.md diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets index 14dff071e..5132df5f1 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets @@ -51,3 +51,9 @@ export { TtsOutput, TtsInput, } from './src/main/ets/components/NonStreamingTts'; + +export { + SpeakerEmbeddingExtractorConfig, + SpeakerEmbeddingExtractor, + SpeakerEmbeddingManager, +} from './src/main/ets/components/SpeakerIdentification'; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/speaker-identification.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/speaker-identification.cc index a08a6ed66..21c9a89e7 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/speaker-identification.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/speaker-identification.cc @@ -11,6 +11,17 @@ static Napi::External CreateSpeakerEmbeddingExtractorWrapper(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); + +#if __OHOS__ + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } +#else if (info.Length() != 1) { std::ostringstream os; os << "Expect only 1 argument. Given: " << info.Length(); @@ -19,6 +30,7 @@ CreateSpeakerEmbeddingExtractorWrapper(const Napi::CallbackInfo &info) { return {}; } +#endif if (!info[0].IsObject()) { Napi::TypeError::New(env, "You should pass an object as the only argument.") @@ -46,8 +58,18 @@ CreateSpeakerEmbeddingExtractorWrapper(const Napi::CallbackInfo &info) { SHERPA_ONNX_ASSIGN_ATTR_STR(provider, provider); +#if __OHOS__ + std::unique_ptr + mgr(OH_ResourceManager_InitNativeResourceManager(env, info[1]), + &OH_ResourceManager_ReleaseNativeResourceManager); + + const SherpaOnnxSpeakerEmbeddingExtractor *extractor = + SherpaOnnxCreateSpeakerEmbeddingExtractorOHOS(&c, mgr.get()); +#else const SherpaOnnxSpeakerEmbeddingExtractor *extractor = SherpaOnnxCreateSpeakerEmbeddingExtractor(&c); +#endif if (c.model) { delete[] c.model; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts index 057d5af25..d2b6d6ea4 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts @@ -47,3 +47,18 @@ export type TtsOutput = { export const offlineTtsGenerate: (handle: object, input: object) => TtsOutput; export const offlineTtsGenerateAsync: (handle: object, input: object) => Promise; + +export const createSpeakerEmbeddingExtractor: (config: object, mgr?: object) => object; +export const speakerEmbeddingExtractorDim: (handle: object) => number; +export const speakerEmbeddingExtractorCreateStream: (handle: object) => object; +export const speakerEmbeddingExtractorIsReady: (handle: object, stream: object) => boolean; +export const speakerEmbeddingExtractorComputeEmbedding: (handle: object, stream: object, enableExternalBuffer: boolean) => Float32Array; +export const createSpeakerEmbeddingManager: (dim: number) => object; +export const speakerEmbeddingManagerAdd: (handle: object, speaker: {name: string, v: Float32Array}) => boolean; +export const speakerEmbeddingManagerAddListFlattened: (handle: object, speaker: {name: string, vv: Float32Array, n: number}) => boolean; +export const speakerEmbeddingManagerRemove: (handle: object, name: string) => boolean; +export const speakerEmbeddingManagerSearch: (handle: object, obj: {v: Float32Array, threshold: number}) => string; +export const speakerEmbeddingManagerVerify: (handle: object, obj: {name: string, v: Float32Array, threshold: number}) => boolean; +export const speakerEmbeddingManagerContains: (handle: object, name: string) => boolean; +export const speakerEmbeddingManagerNumSpeakers: (handle: object) => number; +export const speakerEmbeddingManagerGetAllSpeakers: (handle: object) => Array; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets index a60a0e748..556877489 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets @@ -4,7 +4,7 @@ import { getOfflineTtsSampleRate, offlineTtsGenerate, offlineTtsGenerateAsync, -} from "libsherpa_onnx.so"; +} from 'libsherpa_onnx.so'; export class OfflineTtsVitsModelConfig { public model: string = ''; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/SpeakerIdentification.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/SpeakerIdentification.ets new file mode 100644 index 000000000..e490ab15f --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/SpeakerIdentification.ets @@ -0,0 +1,139 @@ +import { + createSpeakerEmbeddingExtractor, + createSpeakerEmbeddingManager, + speakerEmbeddingExtractorComputeEmbedding, + speakerEmbeddingExtractorCreateStream, + speakerEmbeddingExtractorDim, + speakerEmbeddingExtractorIsReady, + speakerEmbeddingManagerAdd, + speakerEmbeddingManagerAddListFlattened, + speakerEmbeddingManagerContains, + speakerEmbeddingManagerGetAllSpeakers, + speakerEmbeddingManagerNumSpeakers, + speakerEmbeddingManagerRemove, + speakerEmbeddingManagerSearch, + speakerEmbeddingManagerVerify +} from 'libsherpa_onnx.so'; +import { OnlineStream } from './StreamingAsr'; + +export class SpeakerEmbeddingExtractorConfig { + public model: string = ''; + public numThreads: number = 1; + public debug: boolean = false; + public provider: string = 'cpu'; +} + +export class SpeakerEmbeddingExtractor { + public config: SpeakerEmbeddingExtractorConfig = new SpeakerEmbeddingExtractorConfig(); + public dim: number; + private handle: object; + + constructor(config: SpeakerEmbeddingExtractorConfig, mgr?: object) { + this.handle = createSpeakerEmbeddingExtractor(config, mgr); + this.config = config; + this.dim = speakerEmbeddingExtractorDim(this.handle); + } + + createStream(): OnlineStream { + return new OnlineStream( + speakerEmbeddingExtractorCreateStream(this.handle)); + } + + isReady(stream: OnlineStream): boolean { + return speakerEmbeddingExtractorIsReady(this.handle, stream.handle); + } + + compute(stream: OnlineStream, enableExternalBuffer: boolean = true): Float32Array { + return speakerEmbeddingExtractorComputeEmbedding( + this.handle, stream.handle, enableExternalBuffer); + } +} + +function flatten(arrayList: Float32Array[]): Float32Array { + let n = 0; + for (let i = 0; i < arrayList.length; ++i) { + n += arrayList[i].length; + } + let ans = new Float32Array(n); + + let offset = 0; + for (let i = 0; i < arrayList.length; ++i) { + ans.set(arrayList[i], offset); + offset += arrayList[i].length; + } + return ans; +} + +interface SpeakerNameWithEmbedding { + name: string; + v: Float32Array; +} + +interface SpeakerNameWithEmbeddingList { + name: string; + v: Float32Array[]; +} + +interface SpeakerNameWithEmbeddingN { + name: string; + vv: Float32Array; + n: number; +} + +interface EmbeddingWithThreshold { + v: Float32Array; + threshold: number; +} + +interface SpeakerNameEmbeddingThreshold { + name: string; + v: Float32Array; + threshold: number; +} + +export class SpeakerEmbeddingManager { + public dim: number; + private handle: object; + + constructor(dim: number) { + this.handle = createSpeakerEmbeddingManager(dim); + this.dim = dim; + } + + add(speaker: SpeakerNameWithEmbedding): boolean { + return speakerEmbeddingManagerAdd(this.handle, speaker); + } + + addMulti(speaker: SpeakerNameWithEmbeddingList): boolean { + const c: SpeakerNameWithEmbeddingN = { + name: speaker.name, + vv: flatten(speaker.v), + n: speaker.v.length, + }; + return speakerEmbeddingManagerAddListFlattened(this.handle, c); + } + + remove(name: string): boolean { + return speakerEmbeddingManagerRemove(this.handle, name); + } + + search(obj: EmbeddingWithThreshold): string { + return speakerEmbeddingManagerSearch(this.handle, obj); + } + + verify(obj: SpeakerNameEmbeddingThreshold): boolean { + return speakerEmbeddingManagerVerify(this.handle, obj); + } + + contains(name: string): boolean { + return speakerEmbeddingManagerContains(this.handle, name); + } + + getNumSpeakers(): number { + return speakerEmbeddingManagerNumSpeakers(this.handle); + } + + getAllSpeakerNames(): string[] { + return speakerEmbeddingManagerGetAllSpeakers(this.handle); + } +} \ No newline at end of file diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index e25097809..7748b9fee 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -1328,8 +1328,8 @@ struct SherpaOnnxSpeakerEmbeddingExtractor { std::unique_ptr impl; }; -const SherpaOnnxSpeakerEmbeddingExtractor * -SherpaOnnxCreateSpeakerEmbeddingExtractor( +static sherpa_onnx::SpeakerEmbeddingExtractorConfig +GetSpeakerEmbeddingExtractorConfig( const SherpaOnnxSpeakerEmbeddingExtractorConfig *config) { sherpa_onnx::SpeakerEmbeddingExtractorConfig c; c.model = SHERPA_ONNX_OR(config->model, ""); @@ -1342,9 +1342,21 @@ SherpaOnnxCreateSpeakerEmbeddingExtractor( } if (config->debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s\n", c.ToString().c_str()); +#else SHERPA_ONNX_LOGE("%s\n", c.ToString().c_str()); +#endif } + return c; +} + +const SherpaOnnxSpeakerEmbeddingExtractor * +SherpaOnnxCreateSpeakerEmbeddingExtractor( + const SherpaOnnxSpeakerEmbeddingExtractorConfig *config) { + auto c = GetSpeakerEmbeddingExtractorConfig(config); + if (!c.Validate()) { SHERPA_ONNX_LOGE("Errors in config!"); return nullptr; @@ -1983,6 +1995,23 @@ SherpaOnnxVoiceActivityDetector *SherpaOnnxCreateVoiceActivityDetectorOHOS( return p; } +const SherpaOnnxSpeakerEmbeddingExtractor * +SherpaOnnxCreateSpeakerEmbeddingExtractorOHOS( + const SherpaOnnxSpeakerEmbeddingExtractorConfig *config, + NativeResourceManager *mgr) { + if (!mgr) { + return SherpaOnnxCreateSpeakerEmbeddingExtractor(config); + } + + auto c = GetSpeakerEmbeddingExtractorConfig(config); + + auto p = new SherpaOnnxSpeakerEmbeddingExtractor; + + p->impl = std::make_unique(mgr, c); + + return p; +} + #if SHERPA_ONNX_ENABLE_TTS == 1 SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTtsOHOS( const SherpaOnnxOfflineTtsConfig *config, NativeResourceManager *mgr) { diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index fde626e99..111aae779 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -1572,6 +1572,11 @@ SherpaOnnxCreateVoiceActivityDetectorOHOS( SHERPA_ONNX_API SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTtsOHOS( const SherpaOnnxOfflineTtsConfig *config, NativeResourceManager *mgr); + +SHERPA_ONNX_API const SherpaOnnxSpeakerEmbeddingExtractor * +SherpaOnnxCreateSpeakerEmbeddingExtractorOHOS( + const SherpaOnnxSpeakerEmbeddingExtractorConfig *config, + NativeResourceManager *mgr); #endif #if defined(__GNUC__) diff --git a/sherpa-onnx/csrc/offline-tts-vits-impl.h b/sherpa-onnx/csrc/offline-tts-vits-impl.h index 5ef79f69b..560576357 100644 --- a/sherpa-onnx/csrc/offline-tts-vits-impl.h +++ b/sherpa-onnx/csrc/offline-tts-vits-impl.h @@ -62,9 +62,9 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { for (const auto &f : files) { if (config.model.debug) { #if __OHOS__ - SHERPA_ONNX_LOGE("rule far: %s", f.c_str()); -#else SHERPA_ONNX_LOGE("rule far: %{public}s", f.c_str()); +#else + SHERPA_ONNX_LOGE("rule far: %s", f.c_str()); #endif } std::unique_ptr> reader( diff --git a/sherpa-onnx/csrc/speaker-embedding-extractor-general-impl.h b/sherpa-onnx/csrc/speaker-embedding-extractor-general-impl.h index ca384c855..8a884e85d 100644 --- a/sherpa-onnx/csrc/speaker-embedding-extractor-general-impl.h +++ b/sherpa-onnx/csrc/speaker-embedding-extractor-general-impl.h @@ -22,11 +22,10 @@ class SpeakerEmbeddingExtractorGeneralImpl const SpeakerEmbeddingExtractorConfig &config) : model_(config) {} -#if __ANDROID_API__ >= 9 + template SpeakerEmbeddingExtractorGeneralImpl( - AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config) + Manager *mgr, const SpeakerEmbeddingExtractorConfig &config) : model_(mgr, config) {} -#endif int32_t Dim() const override { return model_.GetMetaData().output_dim; } @@ -46,9 +45,15 @@ class SpeakerEmbeddingExtractorGeneralImpl std::vector Compute(OnlineStream *s) const override { int32_t num_frames = s->NumFramesReady() - s->GetNumProcessedFrames(); if (num_frames <= 0) { +#if __OHOS__ + SHERPA_ONNX_LOGE( + "Please make sure IsReady(s) returns true. num_frames: %{public}d", + num_frames); +#else SHERPA_ONNX_LOGE( "Please make sure IsReady(s) returns true. num_frames: %d", num_frames); +#endif return {}; } @@ -64,8 +69,13 @@ class SpeakerEmbeddingExtractorGeneralImpl if (meta_data.feature_normalize_type == "global-mean") { SubtractGlobalMean(features.data(), num_frames, feat_dim); } else { +#if __OHOS__ + SHERPA_ONNX_LOGE("Unsupported feature_normalize_type: %{public}s", + meta_data.feature_normalize_type.c_str()); +#else SHERPA_ONNX_LOGE("Unsupported feature_normalize_type: %s", meta_data.feature_normalize_type.c_str()); +#endif exit(-1); } } diff --git a/sherpa-onnx/csrc/speaker-embedding-extractor-impl.cc b/sherpa-onnx/csrc/speaker-embedding-extractor-impl.cc index b9591d624..650b1576a 100644 --- a/sherpa-onnx/csrc/speaker-embedding-extractor-impl.cc +++ b/sherpa-onnx/csrc/speaker-embedding-extractor-impl.cc @@ -3,6 +3,15 @@ // Copyright (c) 2024 Xiaomi Corporation #include "sherpa-onnx/csrc/speaker-embedding-extractor-impl.h" +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/onnx-utils.h" #include "sherpa-onnx/csrc/speaker-embedding-extractor-general-impl.h" @@ -35,7 +44,11 @@ static ModelType GetModelType(char *model_data, size_t model_data_length, if (debug) { std::ostringstream os; PrintModelMetadata(os, meta_data); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; @@ -59,7 +72,11 @@ static ModelType GetModelType(char *model_data, size_t model_data_length, } else if (model_type == "nemo") { return ModelType::kNeMo; } else { +#if __OHOS__ + SHERPA_ONNX_LOGE("Unsupported model_type: %{public}s", model_type.c_str()); +#else SHERPA_ONNX_LOGE("Unsupported model_type: %s", model_type.c_str()); +#endif return ModelType::kUnknown; } } @@ -91,10 +108,10 @@ SpeakerEmbeddingExtractorImpl::Create( return nullptr; } -#if __ANDROID_API__ >= 9 +template std::unique_ptr SpeakerEmbeddingExtractorImpl::Create( - AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config) { + Manager *mgr, const SpeakerEmbeddingExtractorConfig &config) { ModelType model_type = ModelType::kUnknown; { @@ -120,6 +137,17 @@ SpeakerEmbeddingExtractorImpl::Create( // unreachable code return nullptr; } + +#if __ANDROID_API__ >= 9 +template std::unique_ptr +SpeakerEmbeddingExtractorImpl::Create( + AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config); +#endif + +#if __OHOS__ +template std::unique_ptr +SpeakerEmbeddingExtractorImpl::Create( + NativeResourceManager *mgr, const SpeakerEmbeddingExtractorConfig &config); #endif } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/speaker-embedding-extractor-impl.h b/sherpa-onnx/csrc/speaker-embedding-extractor-impl.h index 9465ab94e..6299dce4b 100644 --- a/sherpa-onnx/csrc/speaker-embedding-extractor-impl.h +++ b/sherpa-onnx/csrc/speaker-embedding-extractor-impl.h @@ -9,11 +9,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/speaker-embedding-extractor.h" namespace sherpa_onnx { @@ -25,10 +20,9 @@ class SpeakerEmbeddingExtractorImpl { static std::unique_ptr Create( const SpeakerEmbeddingExtractorConfig &config); -#if __ANDROID_API__ >= 9 + template static std::unique_ptr Create( - AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config); -#endif + Manager *mgr, const SpeakerEmbeddingExtractorConfig &config); virtual int32_t Dim() const = 0; diff --git a/sherpa-onnx/csrc/speaker-embedding-extractor-model.cc b/sherpa-onnx/csrc/speaker-embedding-extractor-model.cc index e5fa26eed..48d7f19e0 100644 --- a/sherpa-onnx/csrc/speaker-embedding-extractor-model.cc +++ b/sherpa-onnx/csrc/speaker-embedding-extractor-model.cc @@ -8,6 +8,15 @@ #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/onnx-utils.h" #include "sherpa-onnx/csrc/session.h" @@ -28,8 +37,8 @@ class SpeakerEmbeddingExtractorModel::Impl { } } -#if __ANDROID_API__ >= 9 - Impl(AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config) + template + Impl(Manager *mgr, const SpeakerEmbeddingExtractorConfig &config) : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), @@ -39,7 +48,6 @@ class SpeakerEmbeddingExtractorModel::Impl { Init(buf.data(), buf.size()); } } -#endif Ort::Value Compute(Ort::Value x) const { std::array inputs = {std::move(x)}; @@ -68,7 +76,11 @@ class SpeakerEmbeddingExtractorModel::Impl { if (config_.debug) { std::ostringstream os; PrintModelMetadata(os, meta_data); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; // used in the macro below @@ -84,8 +96,14 @@ class SpeakerEmbeddingExtractorModel::Impl { std::string framework; SHERPA_ONNX_READ_META_DATA_STR(framework, "framework"); if (framework != "wespeaker" && framework != "3d-speaker") { +#if __OHOS__ + SHERPA_ONNX_LOGE( + "Expect a wespeaker or a 3d-speaker model, given: %{public}s", + framework.c_str()); +#else SHERPA_ONNX_LOGE("Expect a wespeaker or a 3d-speaker model, given: %s", framework.c_str()); +#endif exit(-1); } } @@ -111,11 +129,10 @@ SpeakerEmbeddingExtractorModel::SpeakerEmbeddingExtractorModel( const SpeakerEmbeddingExtractorConfig &config) : impl_(std::make_unique(config)) {} -#if __ANDROID_API__ >= 9 +template SpeakerEmbeddingExtractorModel::SpeakerEmbeddingExtractorModel( - AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config) + Manager *mgr, const SpeakerEmbeddingExtractorConfig &config) : impl_(std::make_unique(mgr, config)) {} -#endif SpeakerEmbeddingExtractorModel::~SpeakerEmbeddingExtractorModel() = default; @@ -128,4 +145,14 @@ Ort::Value SpeakerEmbeddingExtractorModel::Compute(Ort::Value x) const { return impl_->Compute(std::move(x)); } +#if __ANDROID_API__ >= 9 +template SpeakerEmbeddingExtractorModel::SpeakerEmbeddingExtractorModel( + AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config); +#endif + +#if __OHOS__ +template SpeakerEmbeddingExtractorModel::SpeakerEmbeddingExtractorModel( + NativeResourceManager *mgr, const SpeakerEmbeddingExtractorConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/speaker-embedding-extractor-model.h b/sherpa-onnx/csrc/speaker-embedding-extractor-model.h index 83ef0cc0d..6c6bdd01c 100644 --- a/sherpa-onnx/csrc/speaker-embedding-extractor-model.h +++ b/sherpa-onnx/csrc/speaker-embedding-extractor-model.h @@ -6,11 +6,6 @@ #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/speaker-embedding-extractor-model-meta-data.h" #include "sherpa-onnx/csrc/speaker-embedding-extractor.h" @@ -22,10 +17,9 @@ class SpeakerEmbeddingExtractorModel { explicit SpeakerEmbeddingExtractorModel( const SpeakerEmbeddingExtractorConfig &config); -#if __ANDROID_API__ >= 9 - SpeakerEmbeddingExtractorModel(AAssetManager *mgr, + template + SpeakerEmbeddingExtractorModel(Manager *mgr, const SpeakerEmbeddingExtractorConfig &config); -#endif ~SpeakerEmbeddingExtractorModel(); diff --git a/sherpa-onnx/csrc/speaker-embedding-extractor-nemo-impl.h b/sherpa-onnx/csrc/speaker-embedding-extractor-nemo-impl.h index 7e0883085..ec1c44d68 100644 --- a/sherpa-onnx/csrc/speaker-embedding-extractor-nemo-impl.h +++ b/sherpa-onnx/csrc/speaker-embedding-extractor-nemo-impl.h @@ -22,11 +22,10 @@ class SpeakerEmbeddingExtractorNeMoImpl : public SpeakerEmbeddingExtractorImpl { const SpeakerEmbeddingExtractorConfig &config) : model_(config) {} -#if __ANDROID_API__ >= 9 + template SpeakerEmbeddingExtractorNeMoImpl( - AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config) + Manager *mgr, const SpeakerEmbeddingExtractorConfig &config) : model_(mgr, config) {} -#endif int32_t Dim() const override { return model_.GetMetaData().output_dim; } @@ -54,9 +53,15 @@ class SpeakerEmbeddingExtractorNeMoImpl : public SpeakerEmbeddingExtractorImpl { std::vector Compute(OnlineStream *s) const override { int32_t num_frames = s->NumFramesReady() - s->GetNumProcessedFrames(); if (num_frames <= 0) { +#if __OHOS__ + SHERPA_ONNX_LOGE( + "Please make sure IsReady(s) returns true. num_frames: %{public}d", + num_frames); +#else SHERPA_ONNX_LOGE( "Please make sure IsReady(s) returns true. num_frames: %d", num_frames); +#endif return {}; } @@ -72,8 +77,14 @@ class SpeakerEmbeddingExtractorNeMoImpl : public SpeakerEmbeddingExtractorImpl { if (meta_data.feature_normalize_type == "per_feature") { NormalizePerFeature(features.data(), num_frames, feat_dim); } else { +#if __OHOS__ + SHERPA_ONNX_LOGE("Unsupported feature_normalize_type: %{public}s", + meta_data.feature_normalize_type.c_str()); +#else + SHERPA_ONNX_LOGE("Unsupported feature_normalize_type: %s", meta_data.feature_normalize_type.c_str()); +#endif exit(-1); } } diff --git a/sherpa-onnx/csrc/speaker-embedding-extractor-nemo-model.cc b/sherpa-onnx/csrc/speaker-embedding-extractor-nemo-model.cc index 1b60e2469..3983e1cb8 100644 --- a/sherpa-onnx/csrc/speaker-embedding-extractor-nemo-model.cc +++ b/sherpa-onnx/csrc/speaker-embedding-extractor-nemo-model.cc @@ -8,6 +8,15 @@ #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/onnx-utils.h" #include "sherpa-onnx/csrc/session.h" @@ -28,8 +37,8 @@ class SpeakerEmbeddingExtractorNeMoModel::Impl { } } -#if __ANDROID_API__ >= 9 - Impl(AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config) + template + Impl(Manager *mgr, const SpeakerEmbeddingExtractorConfig &config) : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), @@ -39,7 +48,6 @@ class SpeakerEmbeddingExtractorNeMoModel::Impl { Init(buf.data(), buf.size()); } } -#endif Ort::Value Compute(Ort::Value x, Ort::Value x_lens) const { std::array inputs = {std::move(x), std::move(x_lens)}; @@ -73,7 +81,11 @@ class SpeakerEmbeddingExtractorNeMoModel::Impl { if (config_.debug) { std::ostringstream os; PrintModelMetadata(os, meta_data); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; // used in the macro below @@ -93,7 +105,12 @@ class SpeakerEmbeddingExtractorNeMoModel::Impl { std::string framework; SHERPA_ONNX_READ_META_DATA_STR(framework, "framework"); if (framework != "nemo") { +#if __OHOS__ + SHERPA_ONNX_LOGE("Expect a NeMo model, given: %{public}s", + framework.c_str()); +#else SHERPA_ONNX_LOGE("Expect a NeMo model, given: %s", framework.c_str()); +#endif exit(-1); } } @@ -119,11 +136,10 @@ SpeakerEmbeddingExtractorNeMoModel::SpeakerEmbeddingExtractorNeMoModel( const SpeakerEmbeddingExtractorConfig &config) : impl_(std::make_unique(config)) {} -#if __ANDROID_API__ >= 9 +template SpeakerEmbeddingExtractorNeMoModel::SpeakerEmbeddingExtractorNeMoModel( - AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config) + Manager *mgr, const SpeakerEmbeddingExtractorConfig &config) : impl_(std::make_unique(mgr, config)) {} -#endif SpeakerEmbeddingExtractorNeMoModel::~SpeakerEmbeddingExtractorNeMoModel() = default; @@ -142,4 +158,14 @@ OrtAllocator *SpeakerEmbeddingExtractorNeMoModel::Allocator() const { return impl_->Allocator(); } +#if __ANDROID_API__ >= 9 +template SpeakerEmbeddingExtractorNeMoModel::SpeakerEmbeddingExtractorNeMoModel( + AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config); +#endif + +#if __OHOS__ +template SpeakerEmbeddingExtractorNeMoModel::SpeakerEmbeddingExtractorNeMoModel( + NativeResourceManager *mgr, const SpeakerEmbeddingExtractorConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/speaker-embedding-extractor-nemo-model.h b/sherpa-onnx/csrc/speaker-embedding-extractor-nemo-model.h index af0623724..ed61ee8b5 100644 --- a/sherpa-onnx/csrc/speaker-embedding-extractor-nemo-model.h +++ b/sherpa-onnx/csrc/speaker-embedding-extractor-nemo-model.h @@ -6,11 +6,6 @@ #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/speaker-embedding-extractor-nemo-model-meta-data.h" #include "sherpa-onnx/csrc/speaker-embedding-extractor.h" @@ -22,10 +17,9 @@ class SpeakerEmbeddingExtractorNeMoModel { explicit SpeakerEmbeddingExtractorNeMoModel( const SpeakerEmbeddingExtractorConfig &config); -#if __ANDROID_API__ >= 9 + template SpeakerEmbeddingExtractorNeMoModel( - AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config); -#endif + Manager *mgr, const SpeakerEmbeddingExtractorConfig &config); ~SpeakerEmbeddingExtractorNeMoModel(); diff --git a/sherpa-onnx/csrc/speaker-embedding-extractor.cc b/sherpa-onnx/csrc/speaker-embedding-extractor.cc index d90b0b1e0..5d52fb2f9 100644 --- a/sherpa-onnx/csrc/speaker-embedding-extractor.cc +++ b/sherpa-onnx/csrc/speaker-embedding-extractor.cc @@ -6,6 +6,15 @@ #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/file-utils.h" #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/speaker-embedding-extractor-impl.h" @@ -55,11 +64,10 @@ SpeakerEmbeddingExtractor::SpeakerEmbeddingExtractor( const SpeakerEmbeddingExtractorConfig &config) : impl_(SpeakerEmbeddingExtractorImpl::Create(config)) {} -#if __ANDROID_API__ >= 9 +template SpeakerEmbeddingExtractor::SpeakerEmbeddingExtractor( - AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config) + Manager *mgr, const SpeakerEmbeddingExtractorConfig &config) : impl_(SpeakerEmbeddingExtractorImpl::Create(mgr, config)) {} -#endif SpeakerEmbeddingExtractor::~SpeakerEmbeddingExtractor() = default; @@ -77,4 +85,14 @@ std::vector SpeakerEmbeddingExtractor::Compute(OnlineStream *s) const { return impl_->Compute(s); } +#if __ANDROID_API__ >= 9 +template SpeakerEmbeddingExtractor::SpeakerEmbeddingExtractor( + AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config); +#endif + +#if __OHOS__ +template SpeakerEmbeddingExtractor::SpeakerEmbeddingExtractor( + NativeResourceManager *mgr, const SpeakerEmbeddingExtractorConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/speaker-embedding-extractor.h b/sherpa-onnx/csrc/speaker-embedding-extractor.h index 4d9783c85..068e6b8d3 100644 --- a/sherpa-onnx/csrc/speaker-embedding-extractor.h +++ b/sherpa-onnx/csrc/speaker-embedding-extractor.h @@ -9,11 +9,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/online-stream.h" #include "sherpa-onnx/csrc/parse-options.h" @@ -45,10 +40,9 @@ class SpeakerEmbeddingExtractor { explicit SpeakerEmbeddingExtractor( const SpeakerEmbeddingExtractorConfig &config); -#if __ANDROID_API__ >= 9 - SpeakerEmbeddingExtractor(AAssetManager *mgr, + template + SpeakerEmbeddingExtractor(Manager *mgr, const SpeakerEmbeddingExtractorConfig &config); -#endif ~SpeakerEmbeddingExtractor(); From 14944d8c815195aa74048558b433f3199293394d Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 10 Dec 2024 14:50:13 +0800 Subject: [PATCH 097/183] Add speaker identification demo for HarmonyOS (#1608) --- harmony-os/README.md | 3 + .../src/main/cpp/speaker-identification.cc | 2 +- .../.gitignore | 12 + .../AppScope/app.json5 | 10 + .../resources/base/element/string.json | 8 + .../resources/base/media/app_icon.png | Bin 0 -> 2777 bytes .../build-profile.json5 | 40 ++ .../code-linter.json5 | 20 + .../entry/.gitignore | 6 + .../entry/build-profile.json5 | 33 ++ .../entry/hvigorfile.ts | 6 + .../entry/obfuscation-rules.txt | 23 ++ .../entry/oh-package.json5 | 12 + .../main/ets/entryability/EntryAbility.ets | 43 +++ .../entrybackupability/EntryBackupAbility.ets | 12 + .../entry/src/main/ets/pages/Index.ets | 357 ++++++++++++++++++ .../entry/src/main/ets/pages/Permission.ets | 26 ++ .../workers/SpeakerIdentificationWorker.ets | 113 ++++++ .../entry/src/main/module.json5 | 64 ++++ .../main/resources/base/element/color.json | 8 + .../main/resources/base/element/string.json | 20 + .../main/resources/base/media/background.png | Bin 0 -> 57364 bytes .../main/resources/base/media/foreground.png | Bin 0 -> 12430 bytes .../main/resources/base/media/icon_add.svg | 10 + .../main/resources/base/media/icon_home.svg | 1 + .../main/resources/base/media/icon_info.svg | 1 + .../main/resources/base/media/icon_view.svg | 13 + .../resources/base/media/layered_image.json | 7 + .../main/resources/base/media/startIcon.png | Bin 0 -> 20093 bytes .../resources/base/profile/backup_config.json | 3 + .../resources/base/profile/main_pages.json | 5 + .../main/resources/en_US/element/string.json | 20 + .../entry/src/main/resources/rawfile/.gitkeep | 0 .../main/resources/zh_CN/element/string.json | 20 + .../src/ohosTest/ets/test/Ability.test.ets | 35 ++ .../entry/src/ohosTest/ets/test/List.test.ets | 5 + .../entry/src/ohosTest/module.json5 | 13 + .../entry/src/test/List.test.ets | 5 + .../entry/src/test/LocalUnit.test.ets | 33 ++ .../hvigor/hvigor-config.json5 | 22 ++ .../hvigorfile.ts | 6 + .../oh-package-lock.json5 | 19 + .../oh-package.json5 | 9 + .../main/resources/base/element/string.json | 4 +- .../main/resources/en_US/element/string.json | 4 +- .../main/resources/base/element/string.json | 4 +- .../main/resources/en_US/element/string.json | 4 +- 47 files changed, 1052 insertions(+), 9 deletions(-) create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/.gitignore create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/AppScope/app.json5 create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/AppScope/resources/base/element/string.json create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/AppScope/resources/base/media/app_icon.png create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/build-profile.json5 create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/code-linter.json5 create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/.gitignore create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/build-profile.json5 create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/hvigorfile.ts create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/obfuscation-rules.txt create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/entryability/EntryAbility.ets create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/pages/Index.ets create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/pages/Permission.ets create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/workers/SpeakerIdentificationWorker.ets create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/module.json5 create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/element/color.json create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/element/string.json create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/background.png create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/foreground.png create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/icon_add.svg create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/icon_home.svg create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/icon_info.svg create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/icon_view.svg create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/layered_image.json create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/startIcon.png create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/profile/backup_config.json create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/profile/main_pages.json create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/en_US/element/string.json create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/rawfile/.gitkeep create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/zh_CN/element/string.json create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/ohosTest/ets/test/Ability.test.ets create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/ohosTest/ets/test/List.test.ets create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/ohosTest/module.json5 create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/test/List.test.ets create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/src/test/LocalUnit.test.ets create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/hvigor/hvigor-config.json5 create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/hvigorfile.ts create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/oh-package-lock.json5 create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/oh-package.json5 diff --git a/harmony-os/README.md b/harmony-os/README.md index 566e1a74c..5c1fb9ed7 100644 --- a/harmony-os/README.md +++ b/harmony-os/README.md @@ -13,5 +13,8 @@ - [./SherpaOnnxStreamingAsr](./SherpaOnnxStreamingAsr) It shows how to use streaming ASR models for real-time on-device speech recognition. +- [./SherpaOnnxSpeakerIdentification](./SherpaOnnxSpeakerIdentification) It shows how to use + speaker embedding models for on-device speaker identification. + - [./SherpaOnnxTts](./SherpaOnnxTts) It shows how to run on-device text-to-speech. Please see the doc at diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/speaker-identification.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/speaker-identification.cc index 21c9a89e7..bf49fef9a 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/speaker-identification.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/speaker-identification.cc @@ -764,7 +764,7 @@ static Napi::Array SpeakerEmbeddingManagerGetAllSpeakersWrapper( int32_t num_speakers = SherpaOnnxSpeakerEmbeddingManagerNumSpeakers(manager); if (num_speakers == 0) { - return {}; + return Napi::Array::New(env, num_speakers); } const char *const *all_speaker_names = diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/.gitignore b/harmony-os/SherpaOnnxSpeakerIdentification/.gitignore new file mode 100644 index 000000000..d2ff20141 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/.gitignore @@ -0,0 +1,12 @@ +/node_modules +/oh_modules +/local.properties +/.idea +**/build +/.hvigor +.cxx +/.clangd +/.clang-format +/.clang-tidy +**/.test +/.appanalyzer \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/AppScope/app.json5 b/harmony-os/SherpaOnnxSpeakerIdentification/AppScope/app.json5 new file mode 100644 index 000000000..650f50349 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/AppScope/app.json5 @@ -0,0 +1,10 @@ +{ + "app": { + "bundleName": "com.k2fsa.sherpa.onnx.speaker.identification", + "vendor": "example", + "versionCode": 1000000, + "versionName": "1.0.0", + "icon": "$media:app_icon", + "label": "$string:app_name" + } +} diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/AppScope/resources/base/element/string.json b/harmony-os/SherpaOnnxSpeakerIdentification/AppScope/resources/base/element/string.json new file mode 100644 index 000000000..6204133f2 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/AppScope/resources/base/element/string.json @@ -0,0 +1,8 @@ +{ + "string": [ + { + "name": "app_name", + "value": "SherpaOnnxSpeakerIdentification" + } + ] +} diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/AppScope/resources/base/media/app_icon.png b/harmony-os/SherpaOnnxSpeakerIdentification/AppScope/resources/base/media/app_icon.png new file mode 100644 index 0000000000000000000000000000000000000000..a39445dc87828b76fed6d2ec470dd455c45319e3 GIT binary patch literal 2777 zcmV;~3MTc5P)9*YHQQH znh@I(s7WDIN`nJ+5@|<)iZcg=qN74U#DNnD1Se7u4fs(|1ivr?9ayP|B3iYCD$mfQ zCQ{S1n2)}^yxe#1J=_0pt-a1UPwQ^Z*?X_`Uu*sM+8<}X+baE^a`3seUF}?bEaiMO zrD`Qrd5@qw^epHZ>Df|p-qKBUEB%*?!m0{PHC6j|RplEgR~PkM5a^}N)Sfwi>W;Uz zdhwo_4HXBU%kRl^w@&7iKPx$e-n9%#IU!&oMI~iNsw0n19qSX;dS>I`G_G=WdcN9r z;_Rtv9XC<7kbL+HHxJ782T~pg05t)tf^>2vNJqfYt{YmqQDoBxkv+ra*BxxhcuK2v zm5%@Y)biQz)R8O%e=o%n${;ojY;EUP>`Qj6Cq)7GHm)C%2%^+hI;Z4T#a|oKIvshv z5H%!I+|I4PEXaXj04%ybsVolr%vhKnW7AEhC?eP!o1{y;8m2R#;}{6VZPc!+)ou0C zVWz$|1#2(|L5z%EYRxOzP+uLB>qYGuajX-<#^u;Kw&2uh&93)h>nHaFA%{&2PW=Nn zr?*a;gk3xvRhQIRa1de-!r(ss&?tRmZ=L2FMkhxI3lK6Jn<>5c*ID|@KU#^MCIo6> zpFA{|R(4fsBwHIW z9v!7G|7enadv4}~*8q_h%tD^j$7=PCnn0=dR0GKA(fgb9`2IRg6ksBIo+Gdw#|-3eSe=3tmDe zIqVN)tScM`0W#Z>2wc>~2Uv=3L)~D4gXqZtPQ8rifbYJqwkG>bv}95G7+};9Br?hF zWSa3b)X}z#79W9kukM%6-b_54WDJm~Ub=gsrJ0lz-8&lrQ7zfK1qzuZQkZvcE3|~S zZWmk0ETaNIHnMALn>akuvHLf5c4`y%!f+u>ZGp%@q_;T!`76_snc_?K;Wx%YpF;5K zw^F+BCYUPy`fpRif@5O@Im5cf?evD$>KlAgX;D0*HiO0`Yg3j;R4jT(9h(L_TsY6yxk*@ZBe%+dMqY=cB5oGs{D$QwOFbH)G$iVf<3Olcd7^#fr- zM{!ILWt#coT)s9ySkwDCPHv0oww8g8K%Yr{aR}msELVX(}JQr%F4Q8=KKn*OjSO*uSp;JK%GwhRF_K??vGC$ZqmJX z@+}8sQ)9Z}3*DiWl+L_7OXn_^{SW~2&C*b^;%IP!j$lkre7H&bMR1}7aTT*G8P}|G zHM1)hZDe{r_E3{{Y=d}}_PxJO_w4MaE4)$<<3JwzPdwPzfNemK(-X;{UCzmVr0zu5 zEnT}fzx)oVd!*W77`1Ig`DFcZ6TkPaI$hO1+`cGb$({ukz&{p4Ic-Xnwrg-KEkDqW zW3l$7Q`V$!1T(=QL1jgjIachdr75>-8>1A^h+;rTrD^nnwf?bw(Rang!*16Odj$Pn z@)JN5&5w~}ae6d};oa|&G>sT!)ixE#5;QW(u(=bqYHXcOflE%@t4A?n5fTUm0F~8_ zwpoz9rrU`@G=vsNjDRY(CrF(jIjqg8bd|CP02>eFag7T?u;C^ir+Z7YKmBYw;%%XdT2T}a$X4yR7EI;zaof3a)5Z;`OwVi%D?gbkBj!{;z2tOBSFk&E1DeiZXD**uvNqL}+|pO{ ztO$}2NMRit2ddU?)7Prq&*&H3X>&=E{-+j4iUz zrvL;?0$^@lyl=LHz9G^$SJV6ID__@7z->Bh>Vm=6AK&5bP%@heveHja5F@agGgUsY z@L@W2+^*NVoId0!kS~4XkWb%y;f}XBf>S+NIw9aHK;vN+4mJ|em)_QjIVfb2$;bwv zDKmoq6AThgKydS6Hs+UpKPWq|UA}s=UOEBZNM3oNT5qTAabY)X>L6jxfGDuu7&GD_ z=@@m?sJ-o2GS}&hNRW}-zHkr>o4&138@a8IC-FjSBxzjx?(*3@YmdmWGAd%0QvXzS zJ53JpX%Fp!=>v&`Hd7F@+Atw2vx9%^2M-APg0Jd|ePsRn3*B$#9Z5hCou4fo7W#SN z#}-@-N=##yQDh26pNzr9f*Q88krhI5@DHcf{dU-~PLSs}MvI4s1i|<=qxD~9`7>*~ znlw5lr$_6mTG4XbBNF_79BzvZ!TeIP)exdk3)kSHjYdW1P10ZJ_NCJSlrCuIU#gqw f88(SSw!Z%ZUzhC#9QlKF00000NkvXXu0mjfG$}gK literal 0 HcmV?d00001 diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/build-profile.json5 b/harmony-os/SherpaOnnxSpeakerIdentification/build-profile.json5 new file mode 100644 index 000000000..8e63d9768 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/build-profile.json5 @@ -0,0 +1,40 @@ +{ + "app": { + "signingConfigs": [], + "products": [ + { + "name": "default", + "signingConfig": "default", + "compatibleSdkVersion": "4.0.0(10)", + "runtimeOS": "HarmonyOS", + "buildOption": { + "strictMode": { + "caseSensitiveCheck": true, + } + } + } + ], + "buildModeSet": [ + { + "name": "debug", + }, + { + "name": "release" + } + ] + }, + "modules": [ + { + "name": "entry", + "srcPath": "./entry", + "targets": [ + { + "name": "default", + "applyToProducts": [ + "default" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/code-linter.json5 b/harmony-os/SherpaOnnxSpeakerIdentification/code-linter.json5 new file mode 100644 index 000000000..77b31b517 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/code-linter.json5 @@ -0,0 +1,20 @@ +{ + "files": [ + "**/*.ets" + ], + "ignore": [ + "**/src/ohosTest/**/*", + "**/src/test/**/*", + "**/src/mock/**/*", + "**/node_modules/**/*", + "**/oh_modules/**/*", + "**/build/**/*", + "**/.preview/**/*" + ], + "ruleSet": [ + "plugin:@performance/recommended", + "plugin:@typescript-eslint/recommended" + ], + "rules": { + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/.gitignore b/harmony-os/SherpaOnnxSpeakerIdentification/entry/.gitignore new file mode 100644 index 000000000..e2713a277 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/.gitignore @@ -0,0 +1,6 @@ +/node_modules +/oh_modules +/.preview +/build +/.cxx +/.test \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/build-profile.json5 b/harmony-os/SherpaOnnxSpeakerIdentification/entry/build-profile.json5 new file mode 100644 index 000000000..572de83dc --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/build-profile.json5 @@ -0,0 +1,33 @@ +{ + "apiType": "stageMode", + "buildOption": { + "sourceOption": { + "workers": [ + './src/main/ets/workers/SpeakerIdentificationWorker.ets' + ] + } + }, + "buildOptionSet": [ + { + "name": "release", + "arkOptions": { + "obfuscation": { + "ruleOptions": { + "enable": false, + "files": [ + "./obfuscation-rules.txt" + ] + } + } + } + }, + ], + "targets": [ + { + "name": "default" + }, + { + "name": "ohosTest", + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/hvigorfile.ts b/harmony-os/SherpaOnnxSpeakerIdentification/entry/hvigorfile.ts new file mode 100644 index 000000000..c6edcd904 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/hvigorfile.ts @@ -0,0 +1,6 @@ +import { hapTasks } from '@ohos/hvigor-ohos-plugin'; + +export default { + system: hapTasks, /* Built-in plugin of Hvigor. It cannot be modified. */ + plugins:[] /* Custom plugin to extend the functionality of Hvigor. */ +} diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/obfuscation-rules.txt b/harmony-os/SherpaOnnxSpeakerIdentification/entry/obfuscation-rules.txt new file mode 100644 index 000000000..272efb6ca --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/obfuscation-rules.txt @@ -0,0 +1,23 @@ +# Define project specific obfuscation rules here. +# You can include the obfuscation configuration files in the current module's build-profile.json5. +# +# For more details, see +# https://developer.huawei.com/consumer/cn/doc/harmonyos-guides-V5/source-obfuscation-V5 + +# Obfuscation options: +# -disable-obfuscation: disable all obfuscations +# -enable-property-obfuscation: obfuscate the property names +# -enable-toplevel-obfuscation: obfuscate the names in the global scope +# -compact: remove unnecessary blank spaces and all line feeds +# -remove-log: remove all console.* statements +# -print-namecache: print the name cache that contains the mapping from the old names to new names +# -apply-namecache: reuse the given cache file + +# Keep options: +# -keep-property-name: specifies property names that you want to keep +# -keep-global-name: specifies names that you want to keep in the global scope + +-enable-property-obfuscation +-enable-toplevel-obfuscation +-enable-filename-obfuscation +-enable-export-obfuscation \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 b/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 new file mode 100644 index 000000000..e57a8e40e --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 @@ -0,0 +1,12 @@ +{ + "name": "entry", + "version": "1.0.0", + "description": "Please describe the basic information.", + "main": "", + "author": "", + "license": "", + "dependencies": { + "sherpa_onnx": "1.10.33", + } +} + diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/entryability/EntryAbility.ets b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/entryability/EntryAbility.ets new file mode 100644 index 000000000..679d91453 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/entryability/EntryAbility.ets @@ -0,0 +1,43 @@ +import AbilityConstant from '@ohos.app.ability.AbilityConstant'; +import hilog from '@ohos.hilog'; +import UIAbility from '@ohos.app.ability.UIAbility'; +import Want from '@ohos.app.ability.Want'; +import window from '@ohos.window'; + +export default class EntryAbility extends UIAbility { + onCreate(want: Want, launchParam: AbilityConstant.LaunchParam): void { + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onCreate'); + } + + onDestroy(): void { + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onDestroy'); + } + + onWindowStageCreate(windowStage: window.WindowStage): void { + // Main window is created, set main page for this ability + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onWindowStageCreate'); + + windowStage.loadContent('pages/Index', (err) => { + if (err.code) { + hilog.error(0x0000, 'testTag', 'Failed to load the content. Cause: %{public}s', JSON.stringify(err) ?? ''); + return; + } + hilog.info(0x0000, 'testTag', 'Succeeded in loading the content.'); + }); + } + + onWindowStageDestroy(): void { + // Main window is destroyed, release UI related resources + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onWindowStageDestroy'); + } + + onForeground(): void { + // Ability has brought to foreground + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onForeground'); + } + + onBackground(): void { + // Ability has back to background + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onBackground'); + } +} diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets new file mode 100644 index 000000000..d2c48b421 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets @@ -0,0 +1,12 @@ +import hilog from '@ohos.hilog'; +import BackupExtensionAbility, { BundleVersion } from '@ohos.application.BackupExtensionAbility'; + +export default class EntryBackupAbility extends BackupExtensionAbility { + async onBackup() { + hilog.info(0x0000, 'testTag', 'onBackup ok'); + } + + async onRestore(bundleVersion: BundleVersion) { + hilog.info(0x0000, 'testTag', 'onRestore ok %{public}s', JSON.stringify(bundleVersion)); + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/pages/Index.ets b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/pages/Index.ets new file mode 100644 index 000000000..ec12f51a6 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/pages/Index.ets @@ -0,0 +1,357 @@ +import worker, { MessageEvents } from '@ohos.worker'; +import { audio } from '@kit.AudioKit'; +import { allAllowed, requestPermissions } from './Permission'; +import { Permissions } from '@kit.AbilityKit'; +import { picker } from '@kit.CoreFileKit'; +import fs from '@ohos.file.fs'; + + + +function flatten(samples: Float32Array[]): Float32Array { + let n = 0; + for (let i = 0; i < samples.length; ++i) { + n += samples[i].length; + } + + const ans: Float32Array = new Float32Array(n); + let offset: number = 0; + for (let i = 0; i < samples.length; ++i) { + ans.set(samples[i], offset); + offset += samples[i].length; + } + + return ans; +} + +function savePcmToWav(filename: string, samples: Int16Array, sampleRate: number) { + const fp = fs.openSync(filename, fs.OpenMode.READ_WRITE | fs.OpenMode.CREATE); + + const header = new ArrayBuffer(44); + const view = new DataView(header); + + // http://soundfile.sapp.org/doc/WaveFormat/ + // F F I R + view.setUint32(0, 0x46464952, true); // chunkID + view.setUint32(4, 36 + samples.length * 2, true); // chunkSize // E V A W + view.setUint32(8, 0x45564157, true); // format // // t m f + view.setUint32(12, 0x20746d66, true); // subchunk1ID + view.setUint32(16, 16, true); // subchunk1Size, 16 for PCM + view.setUint32(20, 1, true); // audioFormat, 1 for PCM + view.setUint16(22, 1, true); // numChannels: 1 channel + view.setUint32(24, sampleRate, true); // sampleRate + view.setUint32(28, sampleRate * 2, true); // byteRate + view.setUint16(32, 2, true); // blockAlign + view.setUint16(34, 16, true); // bitsPerSample + view.setUint32(36, 0x61746164, true); // Subchunk2ID + view.setUint32(40, samples.length * 2, true); // subchunk2Size + + fs.writeSync(fp.fd, new Uint8Array(header).buffer, { length: header.byteLength }); + fs.writeSync(fp.fd, samples.buffer, { length: samples.buffer.byteLength }); + + fs.closeSync(fp.fd); +} + +function toInt16Samples(samples: Float32Array): Int16Array { + const int16Samples = new Int16Array(samples.length); + for (let i = 0; i < samples.length; ++i) { + let s = samples[i] * 32767; + s = s > 32767 ? 32767 : s; + s = s < -32768 ? -32768 : s; + int16Samples[i] = s; + } + + return int16Samples; +} + +@Entry +@Component +struct Index { + @State title: string = 'Next-gen Kaldi: Speaker Identification'; + @State titleFontSize: number = 18; + private controller: TabsController = new TabsController(); + + @State currentIndex: number = 0; + + @State message: string = 'Hello World'; + + private workerInstance?: worker.ThreadWorker + private readonly scriptURL: string = 'entry/ets/workers/SpeakerIdentificationWorker.ets' + + @State allSpeakerNames: string[] = []; + private inputSpeakerName: string = ''; + + @State btnSaveAudioEnabled: boolean = false; + @State btnAddEnabled: boolean = false; + + private sampleRate: number = 16000; + private sampleList: Float32Array[] = [] + private mic?: audio.AudioCapturer; + + @State infoHome: string = ''; + @State infoAdd: string = ''; + + @State micBtnCaption: string = 'Start recording'; + @State micStarted: boolean = false; + + async initMic() { + const permissions: Permissions[] = ["ohos.permission.MICROPHONE"]; + let allowed: boolean = await allAllowed(permissions); + if (!allowed) { + console.log("request to access the microphone"); + const status: boolean = await requestPermissions(permissions); + + if (!status) { + console.error('access to microphone is denied') + this.infoHome = "Failed to get microphone permission. Please retry"; + this.infoAdd = this.infoHome; + return; + } + + allowed = await allAllowed(permissions); + if (!allowed) { + console.error('failed to get microphone permission'); + this.infoHome = "Failed to get microphone permission. Please retry"; + this.infoAdd = this.infoHome; + return; + } + } else { + console.log("allowed to access microphone"); + } + + const audioStreamInfo: audio.AudioStreamInfo = { + samplingRate: this.sampleRate, + channels: audio.AudioChannel.CHANNEL_1, + sampleFormat: audio.AudioSampleFormat.SAMPLE_FORMAT_S16LE, + encodingType: audio.AudioEncodingType.ENCODING_TYPE_RAW, + }; + + const audioCapturerInfo: audio.AudioCapturerInfo = { + source: audio.SourceType.SOURCE_TYPE_MIC, capturerFlags: 0 + }; + + const audioCapturerOptions: audio.AudioCapturerOptions = { + streamInfo: audioStreamInfo, capturerInfo: audioCapturerInfo + + }; + audio.createAudioCapturer(audioCapturerOptions, (err, data) => { + if (err) { + console.error(`error code is ${err.code}, error message is ${err.message}`); + this.infoHome = 'Failed to init microphone'; + this.infoAdd = this.infoHome; + } else { + console.info(`init mic successfully`); + this.mic = data; + this.mic.on('readData', this.micCallback); + } + }); + } + + async aboutToAppear() { + this.workerInstance = new worker.ThreadWorker(this.scriptURL, { + name: 'Speaker identification worker' + }); + + this.workerInstance.onmessage = (e: MessageEvents) => { + const msgType = e.data['msgType'] as string; + console.log(`received msg from worker: ${msgType}`); + + if (msgType == 'manager-all-speaker-names') { + this.allSpeakerNames = e.data['allSpeakers'] as string[]; + } + }; + + this.workerInstance.postMessage({ msgType: 'init-extractor', context: getContext()}); + + await this.initMic(); + } + + @Builder + TabBuilder(title: string, targetIndex: number, selectedImg: Resource, normalImg: Resource) { + Column() { + Image(this.currentIndex == targetIndex ? selectedImg : normalImg).size({ width: 25, height: 25 }) + Text(title).fontColor(this.currentIndex == targetIndex ? '#28bff1' : '#8a8a8a') + }.width('100%').height(50).justifyContent(FlexAlign.Center).onClick(() => { + this.currentIndex = targetIndex; + this.controller.changeIndex(this.currentIndex); + }) + } + + build() { + Column() { + Tabs({ barPosition: BarPosition.End, controller: this.controller }) { + TabContent() { + Column({ space: 10 }) { + Button('Home') + } + }.tabBar(this.TabBuilder('Home', 0, $r('app.media.icon_home'), $r('app.media.icon_home'))) + + TabContent() { + Column({ space: 10 }) { + Text(this.title).fontSize(this.titleFontSize).fontWeight(FontWeight.Bold); + + if (this.allSpeakerNames.length == 0) { + Text('Please add speakers first') + } else { + List({ space: 10, initialIndex: 0 }) { + ForEach(this.allSpeakerNames, (item: string, index: number) => { + ListItem() { + Flex({ direction: FlexDirection.Row, alignItems: ItemAlign.Center }) { + Text(item) + .width('100%') + .height(80) + .fontSize(20) + .textAlign(TextAlign.Center) + .borderRadius(10) + .flexShrink(1) + + Button('Delete') + .width('30%') + .height(40) + .onClick(() => { + if (index != undefined) { + const name = this.allSpeakerNames[index]; + console.log(`Deleting speaker ${name}`); + if (this.workerInstance) { + this.workerInstance.postMessage({ + msgType: 'manager-delete-speaker', + name: name + }); + } + } + }).stateEffect(true) + + Text('') + .width('15%') + .height(80) + } + } + }, (item: string) => item) + } + } + } + }.tabBar(this.TabBuilder('View', 1, $r('app.media.icon_view'), $r('app.media.icon_view'))) + + TabContent() { + Column({ space: 10 }) { + Text(this.title).fontSize(this.titleFontSize).fontWeight(FontWeight.Bold); + + Row({space: 10}) { + Text('Speaker name') + TextInput({placeholder: 'Input speaker name'}) + .onChange((value: string)=>{ + this.inputSpeakerName = value.trim(); + }); + }.width('100%') + + Row({space: 10}) { + Button(this.micBtnCaption) + .onClick(()=> { + if (this.mic) { + if (this.micStarted) { + this.micStarted = false; + this.micBtnCaption = 'Start recording'; + this.mic.stop(); + this.infoAdd = ''; + if (this.sampleList.length > 0) { + this.btnAddEnabled = true; + this.btnSaveAudioEnabled = true; + } + } else { + this.micStarted = true; + this.micBtnCaption = 'Stop recording'; + this.sampleList = []; + this.mic.start(); + this.infoAdd = ''; + + this.btnAddEnabled = false; + this.btnSaveAudioEnabled = false; + } + } + + }) + + Button('Add') + .enabled(this.btnAddEnabled) + .onClick(()=>{ + if (this.inputSpeakerName.trim() == '') { + this.infoAdd += 'Please input a speaker name first'; + return; + } + + const samples = flatten(this.sampleList); + console.log(`number of samples: ${samples.length}, ${samples.length / this.sampleRate}`); + }) + + Button('Save audio') + .enabled(this.btnSaveAudioEnabled) + .onClick(()=>{ + if (this.sampleList.length == 0) { + this.btnSaveAudioEnabled = false; + return; + } + + const samples = flatten(this.sampleList); + + if (samples.length == 0) { + this.btnSaveAudioEnabled = false; + return; + } + + let uri: string = ''; + + + const audioOptions = new picker.AudioSaveOptions(); // audioOptions.newFileNames = ['o.wav']; + + const audioViewPicker = new picker.AudioViewPicker(); + + audioViewPicker.save(audioOptions).then((audioSelectResult: Array) => { + uri = audioSelectResult[0]; + savePcmToWav(uri, toInt16Samples(samples), this.sampleRate); + console.log(`Saved to ${uri}`); + this.infoAdd += `\nSaved to ${uri}`; + }); + }) + } + TextArea({text: this.infoAdd}) + .height('100%') + .width('100%') + .focusable(false) + } + }.tabBar(this.TabBuilder('Add', 2, $r('app.media.icon_add'), $r('app.media.icon_add'))) + + TabContent() { + Column({ space: 10 }) { + Text(this.title).fontSize(this.titleFontSize).fontWeight(FontWeight.Bold); + TextArea({ + text: ` +Everyting is open-sourced. + +It runs locally, without accessing the network + +See also https://github.com/k2-fsa/sherpa-onnx + +新一代 Kaldi QQ 和微信交流群: 请看 + +https://k2-fsa.github.io/sherpa/social-groups.html + +微信公众号: 新一代 Kaldi + ` + }).width('100%').height('100%').focusable(false) + } + }.tabBar(this.TabBuilder('Help', 3, $r('app.media.icon_info'), $r('app.media.icon_info'))) + + }.scrollable(false) + }.width('100%') + } + + private micCallback = (buffer: ArrayBuffer) => { + const view: Int16Array = new Int16Array(buffer); + + const samplesFloat: Float32Array = new Float32Array(view.length); + for (let i = 0; i < view.length; ++i) { + samplesFloat[i] = view[i] / 32768.0; + } + + this.sampleList.push(samplesFloat); + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/pages/Permission.ets b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/pages/Permission.ets new file mode 100644 index 000000000..40ef391ad --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/pages/Permission.ets @@ -0,0 +1,26 @@ +// This file is modified from +// https://gitee.com/ukSir/hmchat2/blob/master/entry/src/main/ets/utils/permissionMananger.ets +import { abilityAccessCtrl, bundleManager, common, Permissions } from '@kit.AbilityKit'; + +export function allAllowed(permissions: Permissions[]): boolean { + if (permissions.length == 0) { + return false; + } + + const mgr: abilityAccessCtrl.AtManager = abilityAccessCtrl.createAtManager(); + + const bundleInfo = bundleManager.getBundleInfoForSelfSync(bundleManager.BundleFlag.GET_BUNDLE_INFO_WITH_APPLICATION); + + let tokenID: number = bundleInfo.appInfo.accessTokenId; + + return permissions.every(permission => abilityAccessCtrl.GrantStatus.PERMISSION_GRANTED == + mgr.checkAccessTokenSync(tokenID, permission)); +} + +export async function requestPermissions(permissions: Permissions[]): Promise { + const mgr: abilityAccessCtrl.AtManager = abilityAccessCtrl.createAtManager(); + const context: Context = getContext() as common.UIAbilityContext; + + const result = await mgr.requestPermissionsFromUser(context, permissions); + return result.authResults.length > 0 && result.authResults.every(authResults => authResults == 0); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/workers/SpeakerIdentificationWorker.ets b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/workers/SpeakerIdentificationWorker.ets new file mode 100644 index 000000000..9dd97d108 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/workers/SpeakerIdentificationWorker.ets @@ -0,0 +1,113 @@ +import worker, { ThreadWorkerGlobalScope, MessageEvents, ErrorEvent } from '@ohos.worker'; +import { + readWaveFromBinary, + Samples, + SpeakerEmbeddingExtractor, + SpeakerEmbeddingExtractorConfig, + SpeakerEmbeddingManager +} from 'sherpa_onnx'; +import { fileIo } from '@kit.CoreFileKit'; + +const workerPort: ThreadWorkerGlobalScope = worker.workerPort; + +let extractor: SpeakerEmbeddingExtractor; +let manager: SpeakerEmbeddingManager; + +function readWaveFromRawfile(filename: string, context: Context): Samples { + const data: Uint8Array = context.resourceManager.getRawFileContentSync(filename); + return readWaveFromBinary(data) as Samples; +} + +function initExtractor(context: Context): SpeakerEmbeddingExtractor { + const config = new SpeakerEmbeddingExtractorConfig(); + config.model = '3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx'; + config.numThreads = 2; + config.debug = true; + + return new SpeakerEmbeddingExtractor(config, context.resourceManager); +} + +function extractEmbedding(samples: Samples): Float32Array { + const stream = extractor.createStream(); + stream.acceptWaveform(samples); + return extractor.compute(stream); +} + +/** + * Defines the event handler to be called when the worker thread receives a message sent by the host thread. + * The event handler is executed in the worker thread. + * + * @param e message data + */ +workerPort.onmessage = (e: MessageEvents) => { + const msgType = e.data['msgType'] as string; + + console.log(`from the main thread, msg-type: ${msgType}`); + + if (msgType == 'init-extractor' && !extractor) { + const context: Context = e.data['context'] as Context; + extractor = initExtractor(context); + manager = new SpeakerEmbeddingManager(extractor.dim); + + const filename1 = 'sr-data/enroll/fangjun-sr-1.wav'; + const samples1 = readWaveFromRawfile(filename1, context); + console.log(`sample rate: ${samples1.sampleRate}`); + let ok = manager.add({ name: 'fangjun0', v: extractEmbedding(samples1) }); + ok = manager.add({ name: 'fangjun1', v: extractEmbedding(samples1) }); + /* + ok = manager.add({ name: 'fangjun2', v: extractEmbedding(samples1) }); + ok = manager.add({ name: 'fangjun3', v: extractEmbedding(samples1) }); + ok = manager.add({ name: 'fangjun4', v: extractEmbedding(samples1) }); + ok = manager.add({ name: 'fangjun5', v: extractEmbedding(samples1) }); + ok = manager.add({ name: 'fangjun6', v: extractEmbedding(samples1) }); + ok = manager.add({ name: 'fangjun7', v: extractEmbedding(samples1) }); + ok = manager.add({ name: 'fangjun8', v: extractEmbedding(samples1) }); + ok = manager.add({ name: 'fangjun9', v: extractEmbedding(samples1) }); + ok = manager.add({ name: 'fangjun10', v: extractEmbedding(samples1) }); + */ + + if (ok) { + console.log(`Added fangjun`); + let n = manager.getNumSpeakers(); + console.log(`number of speakers: ${n}`); + console.log(`speaker names: ${manager.getAllSpeakerNames().join('\n')}`); + } + + workerPort.postMessage({ + msgType: 'manager-all-speaker-names', allSpeakers: manager.getAllSpeakerNames(), + }); + } + + if (msgType == 'manager-delete-speaker') { + const name = e.data['name'] as string; + const ok = manager.remove(name); + if (ok) { + console.log(`Removed ${name}.`); + + console.log(`Number of speakers: ${manager.getNumSpeakers()}`); + console.log(`Number of speakers2: ${manager.getAllSpeakerNames().length}`); + console.log(JSON.stringify(manager.getAllSpeakerNames())); + workerPort.postMessage({ + msgType: 'manager-all-speaker-names', allSpeakers: manager.getAllSpeakerNames(), + }); + } + } +} + +/** + * Defines the event handler to be called when the worker receives a message that cannot be deserialized. + * The event handler is executed in the worker thread. + * + * @param e message data + */ +workerPort.onmessageerror = (e: MessageEvents) => { +} + +/** + * Defines the event handler to be called when an exception occurs during worker execution. + * The event handler is executed in the worker thread. + * + * @param e error message + */ +workerPort.onerror = (e: ErrorEvent) => { +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/module.json5 b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/module.json5 new file mode 100644 index 000000000..80e93ca6a --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/module.json5 @@ -0,0 +1,64 @@ +{ + "module": { + "name": "entry", + "type": "entry", + "description": "$string:module_desc", + "mainElement": "EntryAbility", + "deviceTypes": [ + "phone", + "tablet", + "2in1" + ], + "deliveryWithInstall": true, + "installationFree": false, + "pages": "$profile:main_pages", + "abilities": [ + { + "name": "EntryAbility", + "srcEntry": "./ets/entryability/EntryAbility.ets", + "description": "$string:EntryAbility_desc", + "icon": "$media:layered_image", + "label": "$string:EntryAbility_label", + "startWindowIcon": "$media:startIcon", + "startWindowBackground": "$color:start_window_background", + "exported": true, + "skills": [ + { + "entities": [ + "entity.system.home" + ], + "actions": [ + "action.system.home" + ] + } + ] + } + ], + "extensionAbilities": [ + { + "name": "EntryBackupAbility", + "srcEntry": "./ets/entrybackupability/EntryBackupAbility.ets", + "type": "backup", + "exported": false, + "metadata": [ + { + "name": "ohos.extension.backup", + "resource": "$profile:backup_config" + } + ], + } + ], + "requestPermissions": [ + { + "name": "ohos.permission.MICROPHONE", + "reason": "$string:mic_reason", + "usedScene": { + "abilities": [ + "EntryAbility", + ], + "when": "inuse", + } + } + ] + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/element/color.json b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/element/color.json new file mode 100644 index 000000000..3c712962d --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/element/color.json @@ -0,0 +1,8 @@ +{ + "color": [ + { + "name": "start_window_background", + "value": "#FFFFFF" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/element/string.json b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/element/string.json new file mode 100644 index 000000000..d2ba01212 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/element/string.json @@ -0,0 +1,20 @@ +{ + "string": [ + { + "name": "module_desc", + "value": "On-device speaker identification with Next-gen Kaldi" + }, + { + "name": "EntryAbility_desc", + "value": "On-device speaker identification with Next-gen Kaldi" + }, + { + "name": "EntryAbility_label", + "value": "Speaker identification" + }, + { + "name": "mic_reason", + "value": "access the microphone for on-device speaker identification with Next-gen Kaldi" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/background.png b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/background.png new file mode 100644 index 0000000000000000000000000000000000000000..f939c9fa8cc8914832e602198745f592a0dfa34d GIT binary patch literal 57364 zcmYIuc|6qL_rIk#Su&MMQlYU)cz{|$Qc0x~A^BEf( z`{n=HaSk>%wsfNM*uUkN^8dI{qxxW z*@b_`#>VlLWSG9 z0>QdPQ-&i_RCVdp2s$-u%S362^SHV0`EO6;@n(xK));G>#qwhPWrDXGk@OBMV}H!J za!48&`xhWJKj{_+f3ir<>Jg6Ax<&Xgn;)U7UJyAw{(u?zlf{oLsJTS-_o1?+lSg-j z8fcZj1*Ad(!X>WuuxM!H5t@V3*8vLL6`QnC!q!BwQjI{yk*;~@|3;B)`p$WYcDmnZ zt`R zr=oS6o-D$WZsYKh1PiOdhhX&YWGOzpc<6ITKzr^zi-#>z){t;yz3tu_a!>)(tTU9d zd}COuy~Tb}UIRNX@aVGJqEKUa)1#E-u}pl!sY)z4cu+Hu9==`6=0Ob#x-%q}t@jBp zmoiZDcfF1WL{PB0ZO**8yZ+%;LF6K*JDUoHrJkl0Wzak+Y%E( znUmuA^p@Jv6{%Y;MsiZ4O?#ID2b2ssEq6_KGL z8T%zdA3YhMnkBu19bNsa_$$_1^16jadx`0ZzPx`M%T>qZpYyNYOeDdmqLTNWpR5T% zOlRrW_xNCD+*3_WSxvt4P-@qQ9g_$aedDk-hcV~t>Oxw;UaAk1V?9m5<2k4%VrM$- z?{KH{)m_U~yJcBbX+vqVfq&4)Vf+FvAHd|s{V34=f#uJM!Tp?b32THmfzNn1unwY& zPNtaE{ZZ=OkZFh*xW2FT&fDF?64Q%l>dwdZ#Bg;^v;dAbU*QLEQG@_|ucNXFyx~H( z#h?kJKeI3jD^U~`e`*^zcm?PlIWj|tL_a8NC?HVl*gX%;5PW5Y%ZZ*G=jPn5#o+Sh zhnE>D@Wb!f*O>cZ0}ZT=HlEdoWVWk}5H1S;$vxe#Rv~;l5rJ=w--wPl621jCW}B|gxECKzT9 z3FKlD{=OfN5$J3?Ag0g4F5t8_D(RvO8W!*~?#q{Dhx(Sj=)^9ZlE|LyI?p1NXMWr| zGGbzFN^3)5?P^vfnD7XZo*8yf&O&>7XULUUvhJT@rHcF>PmjodH~u4RSmX4TH?v`IKg2cy7 z(T@e4&pPRHRczikEvwvO?jbblSVp z2qpyT+LHUFhHwcunP(^h)G#uA95vF`Gd&1k%F@wuCk3DnjNjw;b}*;dY{F5{7tNsg zLf4y|)RTV`PjQ^!NoWB3YA@S@Cw zUAr?mUcn7g)n!3J`D7*H`y{%TuT$wNY;))rP(y@kdFdPH#h|rjcW2#oRybxTchXlQ zwMW{bVcqRRc_2r^tI)Zav_+qLwdd|Bw=*pM!|pflbT%K&Eof^{6+|k{2_;HcrWd3? z#z;>@Y3dp#B^R5c9RhH8lT5MRr*;>xd<%C3sV2Y}>{On@a*oump`g#H<6V&DKeZ-?Zic$S$>ulEiZvJG8kHMeSzVE(R|E-<}cEG^n2E*Cp z-25-DQv_Mf+&WhT3r?23Phid$q`Z3HE($RgC{EJA0Yc1SP6(a(oZ4RU2L1~H6k0Q< zHY1Mj{)b(ll3Wr=HakbiEk13zYKN&f#9*}tMZiQ7h@Us+N(Jk`aWQHf)r!ObZAT>_STJuzjuO{qHMlTjN9^hPZ8sZBMl zl&MX}xk{d5VUEInRK9r^Tnx#HE2;hFoa7?NDufAxZV6Mj9B^NaAt4;oStAtWfVg8< zjQAfLPj#u>Xp*sALAi;M(f1>la|_-k(E*-1Sa_Vdt$KsCNAwAbm8CmvpDbwL$`Cx8 zkBC0&3#@q@7E3LVtGQcrGS=s-uh6FHuC)WTtU_@t5c_T~`Wv+F0Jd$a9s(?ucd&l{ zWThjQ*u4YqU6Wq{+^0sC%S;vXx~qO|+s%Am1m-N}zkd84>GT;5u}a1*p9&!g%3wk2 zl=rj+H9g>!z4_zdU1iItL}Zox?lwK^ykQ+_#Ym~p>s8CgcLQYV4wezL^K-_HzM$r! z1m$U&G13HqDckgHschNcoe73o=)$P$j46Y)SnaZK(U|F7d#{AGb%>@b+jX#5*Rf5x zq}@ejPTyyn&&@n|dDGl-o-=XF%6dndW+}@7JDd?6b}Mt-SX_GV^3{!3Yz5a~X@$Fw zyDIkaWq*rtn{8knumG6=yF(6lzQnq)&M@%8RzdC%{%-0Ey{v&0pp-aIPP$bTrF|=~!MvLftx2pd=0-86i#@A;;(b^r-TzBJn~W4d42|-V*)} zt}h95!TwDQ%xWD9TFS{BwGO@d9P>kia=+LQ@r>0>5VvEV8(&tEuw%+YP*Qm6KzZs9 z#qL6SPwl9DtPZ{0`)Vph`^ryNV|=I7r2Vf@LrX3<=$f6zv1^z*!<6j{f$|6Jw=%s2 zb)|d{?()1m_Xoab$B5r9#&taTI^E@0yTQ$UB1_f0nc<oQhFOi;b@!o=y6w&Tsrw|K5XXEJEA>@Eb?8hi( zlT-*bXZd6g*C+W9V6B5iF$2f(_-ek(ko^JW%$@}`#GJVV0S8A~FwzM(JdY)c1B&ls(qJ=bvy&S10cqD8@1Clbooq|3kmbD_she z@O#tu^ibgYfM#HD%WIF%%uf7+)sc&Dejs@WRQE+Q1jXlN2z>9dB;X9e>Y3a-&-A;T z>||D+o$j^$E>F`4y02DTELRMYH*biv(5+ed(cQq&82Gu z2~UNnOcNc&MwT3lD@S}nPJMsvOT%0L{`dN}DU&^Z#6?2^aE!5ulUV_Zct}2~K6R!_ z4ReuaX$@AP?M!XMpi&ZJwsY2up5F-xe0{ym`9#@pr%63v->d&@UoFthcC1`k$L=ze zYX1{xl49Q=z953h>NzyMc3UuH96t7)-k|lRw-P=T%Q`;dC7@r`uCOq8Eqi7gKC)~7 zb(*Q>H|T2(e>5DVf9nswM~C%V2G2 z#B|VOitZm{FlV>EydvsFF|Ue~ium0%0KOaFiMOLk(X}jHq@dI@*AM2G6XzCU zSpFR?#U4MPz~VZR>RA@a!CZu45#f<)^f#kJ+ULtRLJKzSj=cf+NxQ}Kw)Yme6wJz; zu3W=Jz<}rEm$g7sNy>yr-Z|OiI>qQ4m37~);`_~Xgr~N4wOAssk(HTh5er1XtFm+! zb`5FT&FoKA{ADaUP!Y#o^sGPb?mT2wBY9ZfQ}ujLk`C_dyTvT&)34sj!RXJcZ%lCzF?kE~i-xCSGh{ zy%iUR0+S_JP(#%W9!Npk=RL(8tFB7(up1ms-Q#8 z$-{dva97!EQB<5#@0KgW&2S|ddKN*<(?}37-=O@1bF668sG)3(D61=Ech&sJ;j|An zqu1a;`}bcMj;#tF3l~&Ue9ES7GRw~kIPKK&q&^No_3M#yjp?ygI;To&wcXbe%ju*z zpMI!gbi8@{AJVkgXR+py{dMSfko}H`^q^elZQ-5<2bG-K8tYq8Q@*4t)`Blvz!#v> zE;3kk_e^|Kew4?}eU;3n)q48yWgAm)d+F(;W(>jPB_glQLiH|IE=EDVFI*j_FBebS0vXyh5@x9LS?RNi7vXf?RckfXjvy^Pifki$9G zzwp&k7S+aNOI8%DUON~#xxv+a5rJDE+^6;@RcjnwKZ|%#%Ukq~@&vL#Pts;`f?jwYL)Y zDOROB^T8hlFfA@(=$bFYKWy{F^5$#{h*A1FG5GZZ1?>Y+!}UULap(oEekfHZCJkpC zppRS@+Uvrs>_Df!YT#HWpuaEwRq)V49)TgZ7Jf{A6@tpv&>tG)c9F&eZWo)(tDPDB z4Fkl6@ov*S4!gboeokhZ>My7@q%!Z93-zy>Y(_9axnH2W2Ie&#X2Z->o1A6ZoV(OgY z@PpdL`E%U}QN-vzdLCdkVX)Vp-z|CGg)^e06LvMfbj%1)ZdXNB>r>{Jk&ApwTkkLr z-2C5e31{3c{*xsm?)EItQ%pSW(%723B}AHgke#M{7KJW6TT*>9^+`FIe4;VHRwSF$ z9rBta7_>vwCuV;vFY=|NZ2KlX$A`EUk*phH=Pd~I8Kkr|v!j3sBAD^fPD!FoPpnHf zqP&jc&^s{jm0M&oBNXjUol2${7|G^u7UtOd2kxA0b?japS#xlwo_TaY+jh-`+$sfO zFLgfqb~kaemX{ErUn7}?_tb>g?G@UyT99HoY^;BG(5|gh>F3J!9J* zvrz6TP+;XdE$<41%Vony^Y}i*aCz@+4v^38p)5?Nhw`m%Cbg5Lpz%VOxaBnlA9P;N z9D=#{(>`$N_!?&CKf9eJGzIc>dhWes8XtpX`{gOhP;HMklZ8~@Yu~YT1bZZ{VwrAffDNiZ6Mh5vEzpq z=5A;0ff@>1MG@vbwRU!?7ZFD-SYng>JN(=>uwrkrl@4u6M^n6jl1shsk;DM`t#|F? z(H9W(@&~b(mmUR)30H=vAZdIrX%9iR7rMruZ_I4$Eq7YnBI4Z8T zj5;RTUu8?(ZsW>30%Hk#$^zfAtgZ&y!|p@5%e_4oe7)3{Y6c^x>zv=o_XPiF*wI1y zNe5L3p=L;8_D7-+5I+LfNgDYrOIUD_Iu_VJQD^=4v=Gd z_u%h$8{Lobyu6%VkeZI%T_vssgc#J4yD+&6pVkdLYl@3@NdcQbwl!J%4{RC80oF1z z`ksIXyrZT=Apq3kOR#m795+y}-8NizKBNESZCmBS#jqG`n4kCydp-4DZ^BF-zWD2# z1@F?p*^9m)EPrkd^E&cimk<1mN+iwSCVTHpqz^#`_Dj;-5xURqxK*!kp5asE##*=< zc{bFC-`m;q4VL3=| zKN6@)%XIu=yS*-K-9Bw`jN+-lWBttd77x>|g)~$UgPB_qH0h&bm}j3#sdLfV&xcR^ zQFk=d3;U8~YLqm@^61C zmaLbHw=dJ0oLP?>eyJ&=wgtZm!2mS9V!i~62x+n`%jyesf0bKruxRDH-)c2uF;&qT z4Z0drBbHg-G#ueH1vVaEJFTw$U))8mlUjFz?!PDqNpcIqZ%B6$Ju$CzrK@_na@?na5LpJODS}`)`8j7i#>C z0RNEb>nnQ8v$qXrgh)-(=VVRFwj4 zZKH}5T4rlZ$PiI2z3_*{`av5A0jPJY!Y*RQ?XbKPZmNdwp6ufAH4m~1%r{gYeOJBR zai+gl7I{Z35P0Q7EoGmkkLGHe5rR^{bdxWyMiC1~&kI@I-bYJrdGv{=O7!p&kKxN3 ztOoyzWj_asX!zA>`fa~&>#$n@3{c@VVcl3(1m5=dCI-~1uR+4s;@87ozKCU|Z(EhE z7~Csbr}e|&-zPK~*W}WcKqB+rv-rNRzvqfY299AvP zA5u^Rs->xN6b@MzP_f(M+}|~RxUHs#zO%D772V@B$F;5<%Jx|0#Oh_?#%yrHfV>}I z!Lfe59_VCjJ!pEQOWyUr;CdyL z-RzERMQjU_j%}N!Av?++44uVMc#r_KCTZxxSZL>4`xbm)#)*?4I#nFDOZLv10s^{6 zAyo6zfA)w8n^jk|KBb4J;|Gbx9)grFflY-Nyl_v8_@}gizDNn(Y2l6TqM&aN(+9Qg zTBo#J4N$h%f!;K&2NqBlT~J6aqHGy6HI`Xn*)UV$w2>iLk~P=l)VTdah9Ab`z%}dg zxIvG$xPG=H0NRw|6_-~Bzh+BPv9&C;z)58?`7t~$HupdHcF!F5dirrGrn3d}wAHr! z^@&!aoW@3sENjl#i@LzRYOZ4b#v|Jk_Mo$-VYlgbE3LQVKniS1mH)uO`90X{bc~{1 z*%Wm4$E_2-W__`4`mDu;Ld(wv8e147=mMu!AKSC=mw*4n^8S>~fm9mJgf4~8t(bb> z^_3WSK>aAZ6lK3OZ#_7g@)?z1#pZ zoR2>rm%_enbG!+Y34#Jmal)V9@-s8li+_Le^~z8cxHeF5vR%p~{93TJv%YmeTB|@^ zc=}q4Gofbju_Z#%Iv9|44|pawNvh^mFGBA_KZ5C^rx-l~Ytqf4;%SxezE8%O)aJh& z>2it7b`epB=P&=s^y`mJMjMq&9Jvpdhn}6sFHk)q%d zE_RV6%-}?H)w7yAW9TA)&7XbMyu=N}tRA-JTl2iG6u8;@?;!BW;ykyof{i+alo zJu1v~ITow6y^)5crWdi)&;yNs0d)3*vN+aSszJ%`1`(%9X-Hi}3gH#iRg@{Svm?cP zM}T*)U{A8FTQ7b@oc$7vr_EeTIj6N%Cr}VI5VcfZk+@1UFc>zpJkm3S%cb<~=~`BV ztbyjzOPJuDkTJJ!hL^nLk}*=2EXd?->%+3NWrq&5a$%1G{r2~cLQT2W>8!pd$9G;K ziQIDUErsVk$XQPRm)pDFYVuLFlx&eiFlnoixT|jvAoB)ryM_}euaYFXrdKLqi|4AL zG`rnvWi4Qa>Wvo=;Y+t@ecMjl{#37K;?VkYdoSbT(2m}8!k~RT{yv0l8cPp{jtiXr z$7KAJAvM_g4ak}0Yo*q!sO%PN_CK)Pv>lC7xoB~vG1hs?Wv>^kpOBU0WV@$|oL!cE z1FV3%^4Pjr5Fqc)|Sv+upxx8BCM z9*cYQYi3jY(^pUL8`I|3rHf+5>sq98e!hkPsfNMQ1@y7Tnf4{F2p zx9AO&@zYO;WpCQUf4G@!d<{t43@&RHh2Ukg^D-8_;De`dc{hz?yPS_7BzU!x^P-tj zBWt_uk{g94M1uo_&0l?m1qh!Q>=dKy5cx zRa7mv(}`xYKJOm)h3s8goQ*XK1OT<#&Ozf35uTB^VD8m)Z6Bnlal5r-bkso}J^TcM zo)ZSc#2@`h0Si}lrnCFt67JFa*e&}2avKCL|IIk<$R2*5sILkv4P( zesTX_tP#NqXN#>Q{4oe!N=G{SZ_I#~%^kq5ilGc=Q63_5uRt!D^j$k=&$`Ha&bGlAjZ2&hWa=M};Cw|5onME2e;8le z)-hK+mgNbGw-4puLN6g_q5p6T?0XM^dMo810rSBSw7Rrl(jt2JNVBwhB0o3``lZ1y zBr`Dy8LdVilxv`X5b0N8#{#(y<2vQrLj;qv`XA#RZ+@Q~*aYa^UY~;#F>6BL>75+E zeH2(L#HhLeI=Mz1#%^96zY$Se;@N)biYOvM6H1p6-4LcvA=&GP()#?u=_WXgAoZl* z+bR{6BA52?12Rex)v?(LMRsKvf9{KzP<^4&NISV{2!a;wEhr&E)EloHqSR9%ezb)? zl9X;qQSTg@es%UevGs9-KQk6RqJ;Ui(v@S0=JpkXQVYgXlRKQcfFLT2A%*#c?7(b} zjki==Q^Y#Qf}ZVpFtF6<4SbGKkkU>I6wY*Ps*EAzemS5Z0r!-oD>~r!<<+c~fHK+{ z`u4nWcW&4!()0%2>r>@zr$F6$;5*IAuq5bc>cn-IEZ+B|hkO&NPeBi&47YiU-<$w0 zq-j9aGH~K;Y%0{D&e90RZ(J_@o*`(e0TgqWM zz>V1_2|7MMg_6zbeK`A2oW6>`dUuDIll*?4hKaK{^>2t!B*N9o7_!iC51?A=hss#S zTOD48mGM}}JkMLeB>f0zNw|zPj8Efyx1Qh?QyT7Bp*PsC1%+$kgboSqDR=rTEs%8X z-t2|68n3XC`A-sBYO9tXuQqE7{}pE3mRASQTvScN7(%JH0{M|k4t%rE7xh`qUf4A- zgEE3f#zcuMyMYyiu;w=#PFC-_W0rb;u#{l@E}K0uMy~Ec1MBz-KglT}I_AG%m9nb!XAkpoW-`_85Umy)5g0j(3(>`;o1;w;CKp zLKdGc@@LrE*Y6B#H>jMeTcD6nZx;FZw zZ?8nd;T;sv#~t>9Stu`V2=$pLBHrDq3VNw9{KZU-50LlNLK@?o*hLF?1Kjl3op`;u z=nFLXc(CuUKp%gcxwwBm08`iDki>51cyobB9Eypc5@0Uv%$x+m$P}vtzJ@yXv2Y(6 z%G|Dfw#*GyPhoZ)9Obc;u$h*k0~W zv)EW8ChYvHNP~Ws5(MQk4JSGnG!l*4I-odrw$8E;u9uTN)1sDTSK-9%H|jqRi1XpO z_RLbdR5?V7FZiM9a@_RLzrIa?o8u(&ct}&dJFEmRO#py=1J(LW)$S@B$xLi6T)SOw|;fa7Myzv z?MOZ*b$o!rCg?J9&v6SsP#m&goHWvlC%0`IUKT~X&=s1cU$O`0Ea`_f|aU@(<=bXW{`6+7W#cu@H9t zagx-Usc&&vez&!Mjqpdk+Ol(}Uo_B;A&JhUaOe-iG9|*Z<)SYRZ;!m{$5X=V;9Cl+ zs(#H}WR`823f+9`wmRKF;(;wyt*?b3@Y`H^;&@1GipUF_{Gb_RzIV!3$qMq++{iyr8Th+msVi*eA69cY1K|TmaXNA-rCXT%k z%$21aDiQY_-+BI`52BI$rv}FI)tg7-CaaD7_O`l9ngVYH9#Xu44ly2flHy-xuzEyCWC^6c-^K*QrZW zNG1PL`B#xfh_CD57q**Q+=Ty9EEolHUwT`)Z`SWJPvsxa-f8_iHO;AQOj^^?v$Pd6 zy~3pjahT&?UwB@2zW1)s8+UfK$SFAL~tHHx3whuvPyW4mh3w z`_Q5~nHOsoDT0sx@+N~J<-Y&TvqV4MCkgXgo^ntecjdoSopR%@?wkEfAuHDOIVHQe z|K0}d$IAWT3jC{=QJCD$*L3=%k#f)T)tT7R=nTHqn)i5$Q)sm)53ZV1w&{swK_X#n zpD3;2Eb$r)$CDg__L8tv=0-5U5hB))B~SI2(6`QM95phAkktAVs0hU305vOGT{|^t zH`?>)3!24y5TBnjRfAJG|J9jjj_JYwB?gujfD3QwPf@~K(A2Z4KynC|m! zMt!}`yx4=~u?@-#ab5-T?In;dGAUlGajcN(yFF%ypy(av6(B6-=d(A}}k7wcgUJ%c_TA&p~<@ZA~EU-mvx5S_ykM?O8{R|mH|RE75BR5QQ#CTpy{;f{(N zFpFjUOJ}EEwov(%eX6wm&~H5dD|PO&*VQvG&6Br6eo1I>i7L)sk`T?{8}`lQfCB2R z@nDF(51Rl?^;uv9K%Wz-qpmyIoZjoO+tGhY)P>lU7U1Rpv;b{^)mu_I7=1e%POI7M zneWYe`!E(sG!D4Pm@9XD2!jhItDw15w=Vl)ioN}tjFK(3~fxy=!h!`6@!cQuCP6#aH;{{dyV2@O1#ZX{Zl4pLmD z7*{Ip)`V*gV-QVaE+>|4R`><5Z1*;n%pfkb3AiZ1s39)5f5khONJ{XZ5dEj{AwE^i zj6G1{WVlyMNlC3!_Nyk^Z0DjKo$ha)xbx}7UO*rnNj8he_fyO?v!so#$d4^uhxAXf zZNG(a)^5wM7^{-xB|`JITdre*!q^0$>^GMLKm@oauH?5G^;l>0Hp)xxzomAmYTE02 z+c%CPd*0$Be%v~(u%mMywt>EgIlKPOZH{Q%Y5c6=;F0usNLUPph9Xez1H1>s1YOPG zz|s4D9}W5qUuupaM_2#&;|@Jl=mK~Bc0i~OYb643=Gzqz>i%czm6IJ}e-nj~`8ZFe zGWf#c?5=VP0hlqMCIlRJj0p>6ob8O5e(*AYuP~QI>C$d^Yi`)_a|r1LwH(~NZ9P?Y ze?ts^N2upq=Br??YX8%HZ%xopU$9Z$(sjX zPFNIynnhW{IRi^L#G9#+Ew!gHJ%T1dibisJk2~6dM4r$&WR1@Yh3+PZbrp7G519Z>UKXw(mZMT+M-ozzkggshV_x`b zthj%~?f*E&m2-P{17aTUsk&fyuduoa3w}G`Ii-fByRE*XlORaY&Ax;2q^Y}9DeUiq zyMK?>G}eX;GkTjbS%GZr z5T&~;Y#yW|>Ep#W|B^P_r=X1$4uFNPGyw?zjr2lT?F6>ZQaaY;=%~?w4R^35Z=yWu z?(pW}`Hbg{7^L5u3abb48R>Wz-8&e~ld& zG34mkg*Nsz8LkANRe$e1~y0OAYcFkLVXfFw#0X3 z=EB)RkCjS-zhk?;_Eww$ZWCeYf2AIt@_v0+O&5H%+nUcKQQZ*-D#Mj9~nh zx&c!=`cApy)!}O~mTV6{@dbum`*7{`e8wKXQ$qf(L_&%pEl%&9Hz4Ua`%w=5(|{Fe zG=KtAxRHvVR%isJiC+qS)RMDX`xiqORyFg!x&NkABWs5}rYfi3W6r|&5P*I>{#$0n zSspPdl-FAPCWDVqU+`hp5SJ)}U4;QxQ*A|gM$`7-D_HnBBw1Px+%y8Fr*ZBkK&P(5 zLO)g}sM)3#vqJr|zOLiUYMzC)Ip0^+BMHE(YMU_d9|WolPeKCgmx*JYTE6;S>Wa~2 z4x7~9yMFQiL85QHvJtCUi;sWX->6#j?bP;4-B$$B=t*-7v~dwa7d_l5=?cxUgm6Cd zaZr_|B^X5;{k6{%BEZY5G{tgIXaw~PMvhi$_PDnHbyno3v;_gj5-=Qm12)lz+O@kglm5{q;M_RZxMCq-* znMrLfk)rYkS^lo@-6`Sd+^FUeRw9NYH^+}naYE(H+Zh38KI`SA9vUIYM`w7n(({Fc z<0<5oW06nE*}@UB$5AV7a^dI2srSJRcWrClmn7EQdBmJ6?_NrBl@wo_%pe-;K3ph= zN1j@y%^Bw-|7I#-OsQL<1zRV2i1N8h%Jz zJwR0GxN$z5cL7T2`h@=Nn-d!(GsG9!?+6zh=pQ$E{l5S3TiBHQ1&Bvy(*8{} z3j>EOJw+p*2|#VfcRh@u)N+@NMx-@QrQhRg>Tr5cY}aHl3CA*moGLkK0}rdRVR=E^ z{#;gyR7l*RccCrEo*H}%3X|@5YPQ+FM>u|=k#sp-M{J+EGRGl7LH4Z8UIUZqJ%O1S$-a-TXZC__K^ zV}HQ($I)a#fHDGwtEVN4+}*Rszq5|ewZGZEuA5Iw2OpA6%g^thr!`g2lSe?v{V!Zs zZR|Oezz_e)(WIs7nejBn3Q;m~{el(T15QaA3slu+pDiHa->pWfN1C6rVtf%}cuYmO zgKLKj2iNqdxC5nzUkN5bWkY7QyW{~Jm`(yqq=456x~COUo&to>DhmwrE0T1u8eLBX zmGKaO;crc6pm6&VjM@?bZCAXTbba*pRUvkbglVZYwEkF8YfO`T(Y8Hj5McaI z|C{H>yx3qKlRMuy-lc?Sc1!2)CVr8jr{HCfqbxH-_?m>w0h)fl`U3oh{a{=<4u=GG zzB1dSG{rJNtgG}nPU<2q1UPrW{mUkc8)_`L7OAnol7dZB_a(SX@-|yK8Wwm(0F1NEm_aN1wVsURw>% zPcJ-K`1h9E5@B%#7Tn`q0}2)m8v1N;72R}2#~JeoV=z!u6nMx5Hh%7WcQf@>B}s}j zpX2a$CtQcsC3W?=6QyG8m#bS^7MwKolNJR0blaxwZnvS?S;Zd`$Td4sdlY4B=DpVj z;GB--4WcwwL>bZgwia+-FoH)nTd?9m$)`kWfURntsPevI9OkDUq}At_Fhr2*m>J<7 z|K^#22*1UDq{{(|XIx*ulqtAAdQ3OrRygED^IBKe*@i}bZ9_@AZve0qu;T?J2LZ}j zw%cP}y=TD%H^Z>GUW2*063o&E!US9==;FnvZpXFNHRbelmmD_~T)}7{w z&e;xBEsak%$=pypJ3t9=dtnbS!6w40@X`hEdjEiR%*$gfB`8X5t54B?{Y@k+{O-C( zyWn|kD&H^1e6{Z}+mjH!-{_d1n-62-&sj0eAIe`j`?O4m+Khn*F7;(ko`grc}wJs-Gcu{X=-q9>JtlE}duQ+wL-kpryH@ zy?9QcUQwlU%a{$3@vO{6uEg-;vQ6$i3UQK;nO(8qR*T1<;wvvr-5aev6Kzq@WY?yI z8CkJ-_v2o5#Cy<>1tkp7W+umyd18ce*OX=Fs(i}ooB^lb_(Z+B(#0c+peWSQ7vamb z`z_V8WZ6ITb0VsHVCX3uI!$aMYq+2H_VJv|H+xOae}8%g0Ho5T!|3N(fPIQlqqpY} zehINqo%!U~bwZHmWWWQHbG6yOu;gWGMqLHRHz7_bwPG8clq4AvuJY+yO|fZb!!O?8 zu}-gsTJ7>_YGOwb9ZP{7Y~E_-54t0uZ3t;;kkys%#n||9@a5P2V=teS?-R*n9l4LU zX`b4bjK#bVZd&U8y01tpmu%od$DMxAMMv9l&MoL=#mqz@UrVGR_l0_DR1(?*60e1Gde-2*c+IsqkdsUBQplCu zbAh}kVEU~E+wWc#ljwacB1;-}=6;qO#+T9U6+R*7gTqwax52TW8BT?9baXZbe&3!{KI_6)y4?e%W{LkWI2jCl?{Trz8L**uH#O^Q>E0F; zvZVDQPmj+y3P_#pP5&8F;btP7L{R3-N@^b&z}P6C*IselB-bHG;@9&O))tmx7<0R@ zq~8V%kqZ)eZcoE~O~sQ8B8+i&1Ue*r4H|9dY8S&zqWooS;5LT2)V0emG9SEr9t7AM z08Kh_ER&MkZz||l>!~yU@mi`?QQ4AitwkZp6F1DCU$U*G8x922-bf6%3pYrD#i2*< zwpz(G$kV;(&?c|bI?kVkWtK(xu`&B#;UTMoJn+{-FXYMJH&~sfC%3D^A2%%pYB~Fx zYFb@KR!L)a;xpqnrzd^@O_;-5c!|es9)R%NkQ;Y{;h&+Ck8^jTn&jZ}P=M)n>!7A9 zbI=`ms%#Cn4 zcD|SP<@REH*!8~greM*drUAx|97aK~i?nk84xe+fW zZ{VZUt^WcR{^_IyCA?BgZ6gdxVu5?G1|-aEz1&EUsaWP+cJ~=7?fk17Km5W&X3{&= zr6*juZl+Xa>izM!qk7^<2X1*30KepqIdjyV2i+e+zNXSxbK7Tpa}Fm~tK0+5Cmz|g zd=qVePKdNVx^>DVw^plZ?2M6Lxb`!8Ti#RkyDG;^w5l=4mTJ7GuF?>G>j?|lQi82< zNSi&Ar21!4wJGm%haIm3(&qHRaalgKQ+Zo*VUmdvO3d*r$tQiZdevGg?sUI{@hBMB z#c4dG%$ziRt^bWNf~3wy9fsIN_Xz#^hwnqZ)3n%{%nU9mIShVGJbF@_aV%R@{2`Bd zRRV1z;iLf8vnhQhV!*)}h_XFiU+=HG5zruPk-I(^EEW2+SP43iUg88Ktt+fn{a3`C zxH5^rzt^)}NibifBptLnWW>O$q<;o81Ytp^|JHO2c^)R9nQizz@=pua-L?WcDwzsk zqLYg1NS}l0EoS1SEwfU_n>3wtIkq4r(>>1vzP9Z)u* z7!cFZk(y94Ta9;@KGI}VuVTz%OclFRP84+NBUYBAN9)j18h-Dk(N_YxRc+#$@;E!G zk3>;{dx`$+A4-y+OCDz=U?O~&oq10pF2=@SEP`h*hn*uC*BdqRBV;NUWL%7GQwvf+ zy^@Jg8oV=aF&&>FIZfBUhPx!`mVdKBuW_kcOjuX6o{4h~GUS(Oc#=*IhjnUUK6V>q z3|r^NJ1i%lyLPs-RMaW{5i$=F!>FC4M0Pj0<<@G%muXC?eGi&&ai*KS|^#9Ba>V z1r&49PJmi&clkkAhrws5!q)&@Ng2>63rG~VPQPfM6P3_7JQhw!k2;x7`97!rb;o&f zj*N+5e^fk>D^vzYxcBT!!vc`_!+5f!_>XV3z@oz}r2l;7v?ybOOoLg1yQEm1p==et z8!M{V&DaVz@Xg1^2sOzN<|B~4p!Qqom;IvMJuhY^iq(pcg1vcJBD)9j$F|MVwyRM%Pf=l_jD+NyPHL%YE6 z$(-O5y>IX=Oj2(?JA*YBgFzC#Ok z9`8k0Tqim&9(eUu$uOl3X@wSOFmmcm0q`1mIA64Ve_<%3$nNID@10j(FXICMN0-)z_1h!Y(wFt@%rzn&KWkzAN|(aV{DA=J;-G z#?ZdfVo{uhmv0)tmnXPt7NlYVPN%)+Ps(HATs zB#a;EeCAVi=f9W$o`(OvXpJzf;CLh}-04ibR;6BeF3%HSpb7P|@BS;Ns&;?bSOo4F z4DlH!B~h1(AX80$!u6fC-}OET`Dlw`(|?}OMDd~ z>qFr8tnPYIjcmoZtVUn^-ei%&OQA5Tc=Z`Iz9m6b8v)SNDYgGI z&ufpuaQSeQ_2BtH5K)eKXd4pr>O-P(?zf3-LUZVNwLsusL-~7SqM_*WS%%V#M4_TG z{P&M5x)q1sQS4zgx}C=u@Q?t@YU*P&n!}ih@#Hx{2kRN*I*QhP*keYtJ=k?c?y9!B$5bcgrQql3d(MDOE& z$&4)a62X+@f)63w)4wmU=x5`h3F6ai?c0HhJ~iZLYXK!aa#)hyA>2 z|mZaulq=2%a+*w}~-#`f<0;rmBC$8kUReVyk83I8Vz z9h*!SORnHE+X=(t1767g6#NDfz8iGC>whkQKj)G}l@4r;Kv22N_b&h+TX2u|j7#Oj z(K3uiNL1XY*yk@SMq0V^nF^C4tY7F%Xkl1!XVbIhi9k&fR@zT?lM-aSH@RdqE*fzT z0x=nU5YhN`oe2_Me7X&Slwrh-emZTam}o^KV=~utowP0%qBQVdeF^BBD(JrsnqT=g z0Kw~8J^_6p*PaLgV@w0$mjgf4%j*&bCxW;?u04g`wLQC{3<iiFFlUUNQ@-0`3U0PTr^* zMu`6+{ji*^jscj}HzT-Ix^mFBSE+}Zet434IpXr-z;GbHM|<6Z$ud>QLOHm$q>Yj? zi=X^?XVKh5dmh63E6q?c-(MkM>f(9y>kJ)X*W=($$*zh%V_IowxHcM_Px=q^tBS~D z^CNokYN*qIzqTFLw@*J|W1E6Y93dEjFM7bVH;omm!&C=Z%kF zDZ!5^rmEV)HlD6O6Tr*st_e4;^fb1cMxb2+e*K7{dMXd+lY~LT*&%qoG(^LQ;xu2U zlX&3i8OG86!Vntf_USh9iF4*U|J`}Z=mVM)PeAs{D4WZ*4$7P zB%t)P&$2Kr04o8Xy;J`g@KPzWe`1T}m6IZ9MOy`GPfato?=$ik(>JsouPv<{^B1k$GpotiH# zAFc}^jX-(p!24l8(M&7@pUe|Pfm=;J8d^`&7M`y}lC2ikiklLO3&7s(v`TZM_wLvp z)BGvu*V#(5myOg0-#f?hZM~gOm)pbI4r6l2`c;x+BoKN zlf8pTUa5LIE_z>y*IP(5Wwu|3hR`D}LJe2Z{OO%LwF75itx_bm2;*V*L_d!<^U`113iZ?AUR2fo{~@G!O7S z8ry*a+L@ya1s~1tUwKIw=9Y$~W4(^vWXYd@p8Pzd41rg5Et!ZFn)0i|BZzsFQS{Ma z45FpX$A2OpdxJDya+vhWuRX!EMr)~=G60EB#(9=Cm{yUH#1~9tH^>Jf<0R6m#c}G< zi(K*ezx7%l*|KrLE}7Nbi?ghND_o~9`pZ1q-*}Q*Q`{_{6rWZ;i3So3-$FI8e&&NC zWaY{pZS>)b>-mE2`c_1^jB#|!C|63e+q*hQFKyk1RQ#UTkJI!M6}>*G=VmpY(8bq{tn;^1f`?7^Zc-BLmxn4n zI7ms3JW&2@wCq%Iun#b{=0FF4fUU|6)~D`fAdrMDf-%qb7}(_}O-Q%nk`;V~i0&E` znTDr*@a5IOZ9_&vz`~lLmNpX8``JG1kxEJD;}0!4K|3<0TVqBa%r23*zlrBZWH4U0 z5PQ(DoTHN$fb7YEFYgjdU<)3`W~2TCFZR=#A)q&Z+nJ$iP35--s`>pS@B(Z1_+$t{8(iqnGXFSA(Eez$U z(rAcMIv(%#M&j7W?q4q*k#Rn$E zuip+NtT*wwH#{;4u5GD8u}hZ<6@&20Q`j4GxWAW}!MyTY;KIYKaj~9lLj|ADb-{w> zXQV5^!qH%Z=(nxMKm85L9tLs3cFQNel6fR6KmK|2x@yy>gzqqyx%l2?3(eDsLCocG zdslQ2dcLqbO%Nc`$|v^)KCTKql8YQ&?l90WQGtlNjj$*dWc`kau){M=;cMhq|fFjQ_6$TE)+((=L zN}9jU#9gO~MwryIRsj`Atd^e}?`()lD^;B%s>2xr9u$3Ux0maqBQ-M>|74?_%Xg7K z!Rj9hvpde``3walaYgh+!5Q07qw5!{qQ@py4<7ToKiaHbesEVf#mwc)!Ha{sUwaYR zYil{4w$X?jszTm52%aZddax+>6ZVji-I*L2fukc8YS$2F;Fp7qW|#QMx9#UKh&WC@ z@b|j|WKkGzxI%6W_|)$N(vBy^<2S&=M}T&+nZ~}8nxXRO<)lH7nb=UnCA)@o7GYXG zo3mta!~WY5Dh@By(QrLSG!7x6di% zS9=>}2G(da?F-j0X5}QM<)9<2P^&l*D$0iYCMgnRBFhgP;FHiQ{{xc#7njIn&F46G z?iOCDCSZ+j2-Bt2p^J`aBdnQ2?1U{L4m?WeF)8Z<2czjUtR`T$m;{Z_29g z>0R-hEnP?RcHD}C;UCvlJW`!Q#=eH%5m;&(#~y)~Xxx)!XmTP*e;VXL8x+aO(;`p| z^Y7W=lRA)%A&Qg4Ci82P=5l54I9(e#7KD~f&prgcc-_0=Y$*(6kGR#%a+Hj=nMsHH z{nStbI?Mq~mcO0m3g4GMOW%!sg=~(F zHo*;$bSAPDVg*dJd-V~f&<4;QrUGPQ6G10(WzW(3hbT`A_0#Y>R2$q%MZMcYywII% z>aI2%Lsu?S5d6~Z&+thwjJ}cHCua1T#4KIVsE)J)J~nf3t4Di|CU2=n)FGexBvJ*U zcqjy-l@EC24Xf1KX1_uW^(#D5hrp2oIs)xY*_=Xl}7sic0DaxuVQ;Vj(H8jl6{ ztl@;=7&sO8d1Gy79NJS|g5yuZoY}H4{hxfL0oDiPGb?VB&s?rXwe~sbb+Sdvx96Mi zf7XvCdY<~>#8qEs6=adRIh)T#cly&iVqloGZYgq2DE$sBY(0R;w#HyO5m{Xi|j`ryzeJhFvObXi}zQ$^dkUa z8-=*j7t{_XJ~$Hv+WXY=obm2O&HfejylNDi~KEqaO>WLW#z~4D&S_4?L?|I7O zd9bOA>y97h8sWz}k$zJxC8agx00PU z=&q>}m9ckFl0H+8hHU7@QXQTDL?Q5QW~dH6U!?M-P2yvDhHyR=*S$jlFb&0tEg}In&YcQjdt18>ST2pa1*s+G_eQ z$i_(cvP~<#>q^Bp?-6%4Xz=QHw?E&1dQfBsGqE1{N7)PW@SLg91&af=IdJ<2o23%I z=B3MHDwg?zEY+b7?2pWuog5RCD;Ts$p6L=wk|sWaAE$aA+6Z*uB?%5v$opCbw9)s| zLe|cu36WL79#gea+kAOY86xuP@wbA8`P>mQkI<_463)vU;mhz}ev%wYe9GJV8DG zsI*WsdD7gNyjS4W75N&vocg7{z5xOXo$IkwyV2@+8uJ0z_5FJ|yr3t0HolQ8DNX*! z@UtBrYSwpRoJm))>Ui-&I|GfHtg}9}+AglmSHBzP+5p0(>?gKNG`pAQ!o9wA#@CUV?kk=n|xk;NAC7^On%cCA6GUg(8h74Mx zmW0D{fTc@BUs1k3M=8z#svN%Ei)~)D$!SRh)g|_VkdkQiW;lkt?N}oDiND=P-Idjx zkXC>GUNXXJwB{;*6!`ng08u+T37|1I=G#2R0wvra0A!Sc!<9r=?}l{$d_EW{5PB5< zwUrHoXWjP(om^Xc&*V*LNj~HwO;dHpPQq`eu13BY+nHVMI=pjOlsk;VH~8AK#p3E# z1Ayw~&8+%!P<)FVQz)NqdGfTyNTcPU!_)~5lQhDRYkp zC_%1KG3Srg*YlBCiN@6Rz58(IAeQR&A_FooBDOZM83P*b{nB%0neKaT#g$Y7rGmbH zHMCz_Yq+w?u72_rRDz6F4}2GfvaFfx80_zu;fIdvk1$FYLSXCbPQ#V%gzb)_Nq(}y zU3ZOC)Aq>!)bT44i|W`IwFgrG;@_%k*I%D4G6?l|eYRk%UGdM|8h^+cnFz~LymyV5 z5h^5j|4ieG`CvT0^v)hdx>x$4e6v^czfVQlAfgj#Fy_(pxneG?yXsOU8$@^>PX-We zw`wab$am3g+C&Uz4)|>7a*fvwKsEZ&?Ybqt9)qDXf}-cC5E22Loax}F)rj@7O7$(2 z?!By3nfztcBnGSUa1VZ)041(8iYs;m!`C^1Tiyg?|0l^IwgFc*BSY;i+Ru*Uh}%B( zpGlO&;XTgsH^=xdf>7^jmsz*4(_pfM?Wj~cXnBx z$yXh{O^XBq{@qVmy!3{Fe;!W@={=aK2j2UzP5%pMBJj0CeFX*AMz0*|e5> z0wrQ0n97T;j_W9N+s3LX;fTC8`{qy)IZ0K9riL!D!5uE5b9WPVf&!-Q=RVOjTSwBi z;k8~2s=sRnuy~C3mJ|d`StNjPSpD|gN1T; zzn|xTg~NK#smNy7NR@gBtcTMt3~%0kdbzV9%NPq6P)tbZzz0`C{C#mdv%>;Ao>|XF z9T!uW%f{;V^q70#wi`Y&^GyCG4UkW@$`FG>2r$|+R>cng%Ay@aip@1NWmZ1+gcN$V zGh=iq+^Iy7a|>y}@#KfqSDsgM>yr($WF&@~n1*KGhMF{vmm|Fakd5mo!~zM$Gew zn{T}s^aD5dq_;fJQ%))f`$5s3r1`G7tNu9Cv_YzL=G)n86=SkQN(esj_>Q{^f$Q0l zj$sILcM@Rv$kp*t$s4ktEp{iiV&b;eWR+O7^3?$9y^dc_N(V^%wbpl*ZmZW}s~61t zC)3`KlBcpmunVa)|J8NwWr3e`izfB^AQkzeKpWXQY){k@)2p5_!R@8GcPFT#3p_sS zU2P7<-pWbsgYLk%M&LUO#ycYKV59bKe8nkHyyH-9+I^Gtsekp|x9$Vh6x$K2JW4MH z?B97keW}HJL>CBgaJvcIuqZwH&v0t{zp6rmOjcJdt=5#U0gz%O;r5BPbli`~bn-B~x)jPcuX;Qa4p=fVKCY!AcXB)_9R@svcMQ3a+3Qf#anpAW6c zy`hp8b*Np5O#tA*6rhnIK0?8wYULw21)NewAS@DQyw=aryfmQb0zC~6F(8jHAmH%yD&YeYF3g2R$mBpYO8RPkdMs{f+{XJILUCPEi(lE9^uM}al?6z}`_pj_)mbUDDEc^i26 z^#|94ClCxrF#PNB6U=hBSP%DQzhg!rc^sg`bNY4$x@IgCJ_Sk>1Ce0sp47kZzXIY9 z|7!cT`@e6#M>bl%n(^E0X@sPdj`Wk)&2m9A|eG&Uv*S&;NUT2*W&tD|}H=7Wpy5$Op4C z;lrxxFPj050yU58a@~5snJrO;gF|XTcxBFwrycmk?zoNvu6Cu}Gr@DrqBwXLlharC zl1vBO)RIe=mBUAV+QtI_*stF9v3zwjExdyrp!b|Em z^Qi{xZ+SxKi*%CxJR`=belBN2@N*NRaj@ydsNK{UIK2gkP!gwG=z;sfD^oQzTA#La zO5vBp_e3}q=cE4-Kbqa{n-PV-zF=n@csZ2&dJ< zfPr0T)65}Y8PR7?#2yb`jv;P)6TsvSoOqenNdzgKy#1i7h!>dojt|V;PIc}Z;55sXdP=l9(^p|759HpLCBthH#}Aa`oZ`9GAO=*n{lX#bRAm^gh`ld{8~~gycM6iYEUB7zn&$9I}i%`)4W;V0V(Jht>^f zV!k8yO{{Cv1jw`yBk8d85UqHM5mK#FpJ3fnn2WQtrDy9`CEQO68Kxw??(_}4`m&iQ zn>(Hh5S=F6y#FT24V9j|Trq(4`!-UVkr>`Hu!LD=3vz0ks3PQsHSoStgeYXiK=vGzZpKaR8a6rQN!4etGo|kBLTOdJzt8YADqF*68=L zY+4i#i9+9$xs`EF*s$V5G6!#;J-EZDvfDh2F4xfkUa^ny{IpzpCqRC?vPY5~C+HEo zw2A<6CfR4qiAr<&J`>#S`=sNLi@g%rg=i@z|;p+JN}{J+d~3!bwR|1_p_WZ*zFg8JdY2H&$(=>qm|h~`0d88 zWfyZh%%J_j4Dq6hl=rxTCAnU4frH$_ytGsCU*D1mn`Z+sw9>F*#!002LkOF@J|RgG z&VYXmonzYG{uD{CvS4 z2zvgHZG^kGrEZme_YMX^>Jp5Ekly?SG)UqM2$JF;2kQZuO3HlZJBAWt5XB?QAtk6p z;PZBUYmLv}O4#vA`t8Ta9W!j|LYfuO*R{kX~Gkj&k=x{OR zgyuxc7eyW4QKwM~Y;XaJ4k9|Rj;;=@E%@FF)P+@9Wx#6|HcbPs9Er>v%et4vJrx)Y z3O+mlAgaHtAg>Nf|0Z2za?+B6+hfpony5lDAE$d(o?L1}N0%V|tJR#e1J<;%&1W}W z4sdoDCj#!=VGrjHHMfK~!Aastb2s_g)o|qjTPwpxh%bS!912Ze_R1@tsT?0hUX>l= z0g~f3qq>IyyT|fEsc3UU%%e9f@6tYuSbu!PUgly3^o}%#>ptxjwWfP1pM1AwR0`_Q z%ul*q5UsD$nLPe0@(4Nfp56?GD!KCH8Cq7Ut-*bUr}KB^_liJCg=aP&2w@$IA|4wz z09gyWU?8N!5TMlMU;(rK)zk;6jObF@{cH>4aH;$*7AvDf@#!;Um?R*(8&!b z5TAj!VC4&7_>dCm<;$(+T{TeoPk0>2{Bi?uVfbTXN!yb(S#~8f2){1p713Ty*{jc_ zRf2HseOZT8+!fPXa&@%N3i994vCh!EtP(;}!4)kKE%-$Ir&(6wqjxugE|6~v?;rNi z^h=ZRn^;Nzm0U~}M7eO*=BYA-tWFv8ZnP1qe?Ete!mwVw)ZOGc|2qNyR1{vBFqdt9 zt8xG7xKiWPD||`~g42zB1A?)^}Kb zHZN&k&5<=QopZ~J#!ma`OZ1?J|EfUB-SQyjl4>N4fd(x7L!Tv?k{Xl|Zi zj!2NPdK#Lr$aN7wpAeRyx5Er=tJ$^W!M|(Z|tTlIzdC>lf3BIlUt5Nq<^Tm~-|%FF_W;5qeHfl!yrS z9V6$z>|&Do^kuvZw?FH)k}b0zXk(QJeS<=)fX#LP&{-( zR1mXZ<8?!2fYl{@0Ezi8RS2-g=bTa3d*Q&5p}B_RA`OEM>K{D%u@0Na==gQGyV{eE z-kFU(OR^Kv7pt2ORs?Lq@qv7IXi2vKqKf33 zR~4e`{tcY0mG_o&UQI&*yPiUi5dRcXr0|&)XZQi&;?5gVlgjsGONiCF!slVgk!>pJ ztZJM|yhmK~(d5AOK36q1cB9m~^hW}b?T;y(@{Wy2Pli96zt0DS-1xLeo%g87+w+(p z>nEs|=n}0MPb;Eh_?gkGvf)rv3^I(x!*_Q~yK^$LoJi7p0jnH_?F3AMe?u6qKfACz zxBXJe>2EQe*q$tu`?_BD9)1(HV@WigmKpH)8qa8vN?apP0c^wh78>C_RjVEiq^C_M ziLc~F=qyRnDrNWFk00VNCHidqC;&lO-YJo^ilZH&&-2-nnG7s%+mw0h_s~!K*O8R3 zdXceMp|+2$u<*a4dybOy{rsWgc1HcLhxIs2qQ3&MoFc#~p7=ka}> zSXC^xPkO?8?qUqhJM_C!S!&(m8G3Jwc`Rc0Lv(=16$e0NUMq zg&0AcMq)4ca){?MH15c7r++038WzbRm^di@BInT7Q-|RVTyl#F$ zN#cH-@iNC$)^ouQ!q6}$)J3U?09q+e;jv%7R-)S-Tg~Fv-s)g$Za{wkkBTK+0U;hs zJXGJte6PM&iTX!8$oZr`sB{db{2cefDoJ1AZ*D#m-oYZdmG{q?_rL4IK4v0^_kBK= z-j#xDpZt3e8`$7C&CK}3T!m8lU>~eN6kQ*41SgS%V5hKZw=j)Y0#FP)dY2(Th|uUH z*sKv>v8vZVEx?Sto1+TzzFaFnv5g#17WrL9fQ9+6OXt`vpdPYF5qWs`#godJitEns zqdqueW_c6LUNyQ!6e)bV(zIh${I@c-qB98Qqq!2VR${EvJCyR!=6RF<@y{hl_Qyl2 zRdh>gWyr&rj-TmBVa~l0g-EWuk#WqPgx0ure2V|klh;4=KQV%yBZ<&=`Hd`3vbOwb zM`EK7C~{MW#PqMwf&TJ@9#J1^mA=^L?)=LLp?z4} zz^fRs$dnB19)LxSBwkz09b)2&L~W|Jf5_!{@4+(syl>;jtxMRO)@!;>_C* zf|Li*srkh>E${4jGP6<;xw<_rokHRO<7G2pVd?P#keF5p9sPK4xZ#+U7-rMwnLkG= zQp}}lGrZ!*cZq-z186@_t{%;RgXMksAD(?aQ)6-CqZ=`L_M!Oh1Io|y@hP=8=Z;nE6WMYM!8hA-?f{1$b8cd%+$!rUIY(C?#tyd?@}8%cbPu%fuV zHmJ?qK(RGCn^1^sz0*lppm$UUzNT_2bypgib!{*TbgoE-8kMliGrE|*OR;L`nD~#8B-YU(wWNs_(+5Un**Ep zff5*To$NlVS%x59R8Luue(S12jXGt_L*fDL?dgaseG8>+IdO-~L@F|zkWY>U^Dh1x z0rk7Qi)kd!8?2c~1Fy)kWslqI^)fQSdt)j@1z`Z2M)M41OCzTRx}ZKg!ot(XDZH5;arI>LD3nB^1q++cv|OT~`i z8ZoAX%GydeBvt!>ee56IT-VRx%(otrPQUJ(00XuH?IE}$Y?tClldCSub+=SuqEB+D zkt!~vrgb*u#_nbS1i$a3D{OkQhQ9C*_ovEATl&}ISmP<2KAlQ_-Grxw;okhm`w5qK z$_!LEkAFQ2I`dNsF(z*}iya2}T2Gyy!JHg6a?(VNYQ-;G6|4Wf_7F}vyw!Qmqj_bZ z4>QdG;vN z=^|&NU-I7b*sajdJc@(!q=!6FXSTadlX49Q)nc-2%~l9^p=1bvHRosomH4qXkdb@k zwK%z;z?zgB&4?-P8#|sLzsT z%{Y;tU%0KwHCb3~$ktLakPPO$8i3d~dkjW@-}c&{roA_Xy008E#BLYgH~|6E5d|T5 z1-=~Mav%F2rjId+NmKW#&3}4tNTnvK&2WU!&Nh^Zcj&P(k)yJceJO~@ zoS%KO6uItbmOcCzhD!{lYhWV4@#fZO*oy7o-8*q#kz1lxvw;y#OF@^7UpH9N5Gr9D zYX;BMkr2>|+2vZuzwSUhgC&IIbE^sZG9UEj@$y~S&z<4_c`&!!@pbI=$YmMMAVTzP z!hhUsnCf~c_FROUC;_J{ehp==1oXfm^pPqb?6%TBxJWN{YB}-$xNgnc47!yy?)4~9 zW6^M%8DbP(-}y*_8Fcpo(^}Ga9~-mB)pA8)~?JOV4olI{h0(@B+Q$xC5d~le-8b& zY#`>{j%RNi=Y+3Q8JeK8lqc~AWDpn6ABE0bo)xBW^l5+iByDp*_AG z{a+ch7yxnh2-*Dy0ou!wH}(i)Tdy_C+LlrjNC}H6oR&W~t|{>)!iqZ@y6F z{Z9uEMXfon-58Px??G!D5oo{xn_qE58U8r<{UL@3iFJ7md=6aaM45`lyZE<6eG8P0 zM+Mung>esC$yKLmsfO4+x7~jV3cjMTb@*iwBQd_KiT~bVMD7G_Fp-i#3Ag3VvwvgJ zeDa^SDwA}O33bLZdDOqk{PT2>}^ZuiwC z;D=h{g{AxG60UoTEx_=y8X}RY`67bD=rAHwZ~`vs`Cl9+)W^D#c=^|MK^l0IzPS41 z>RH|V-K#!>g^OjYfWDh6G?-KFP~=n8*#jfad4nU}&x-_VP)ifu|NZ2NXLv%`xe)Rm zaN2*^Is&#*_a^vh`05^UOnY*g&NH5O**!7oW}4H9xfyUZnHgZ~0K+~v_b!(td%2#s zA|rICEg_#ru(Op_*H7m-p+vt=$fN zl0Qxne}1|j#4)x@(su-^ZXsUZ&0`U>#&wsB4sdxCkP>pfg9q8I)PzY^z-%`J?NJ5B#wAUF*E2Sh8%o4VuZNg zhn+rNdZLtMTj=$|uiVd*tJpT=#8*~vliD`09q3=`vI~SPiE2whwhMl##D7H+MK?>c z9qx91xPZQD#cTSpLwZk5pbp&Wau1%yZ&}IM+_TuhJ}t1BDZ>aUr;y5D*_dLM_>Nhu zW{83uG!i$muzqsesr7=fVVV|SlyYf&jCFxqiSH+5-I=A@KglOh93TnIQ06WWwkHLi z`0(;_E#OI;>y-BS` zRm|I);;aH=hTh%rn;-wey*2XFe+YF-UJX&cX5d(H!3o{=vw*t1xcbYe_}x`48RXm( z2qznisI9=Rd#nlMm0S%6sVZoNE5d{J7WmoU2tT+%aICh?!;F{08 zghazF>D0pG24#JQ)Ma6K)cNP>Qr8}e3zM4XO&dkAwC6^+Tqz0GK((Yks9PR52Y)ee zaK?{9Fh z1OzF{6Z6zi=_B4F_4tM&(p6ufcX59*0K|pS-EFRos`0#BxB7L5LxZ5_UPTdAX^u+4 zk$9hZ+`{9j{Wzi@62z>L9lE~Nu3YmmKinE@mFXWlux76q1Ml#$2J zy~IT%@vm!(DmvUe<1z?0uks9UEt46=ExfsnMMi5nUL=8;h@pbhLh_fZRqa!_-VAAd zZ4kcH@p+K$r|y5suWeCLiF|VN$gz@cGdn9NDaOHVBs;=*wIW}drsdk;6KY3lo`2{AI5+U$BDWJUFm)aqj6;(x(Lbi7|Yf6yphgBoS@~ z@&3jP+jYo3-s7Jh6Ll86nw__T=~6!L{6`!G;#on#%J<>gaa>pc!8nirBEEOvD83b2DkFGe}n&vL_Vt7~BYWb7J?oTY5-bIK) zp$Wj)JV^Tv$30cGG-B}zio@Xc`g9iODv@tv5F<*T9f*EXNsILj(&5p#`)vj&LmKE@ zJYK=(vAM@6xoIfSeNoq*%i(xKmjsrk_OgAueO~k`*L~Z7e zG3nQs*XWS(`E4m7!$u$_u$@tYTjlC(IjL@S==w_alVmiyuJ(^(Bk{5D*_u!pd?>(} z^uz1f=n5YEtRF!919q7GvVTZ946bY&zn`pou#&sWCoFn+UqEnf?{`r&uIVIm^~=t0jOnZog6W`^$>?)m1L z2WWq_QHkKRuh>q}4<3bzfY;F?HpDLG%OYwa7>9-nN+Ul$mb z)}d>ObXR{(Il?cG)(n0iFAyZ)9h^xvS4GnJ9BiMuw#9}|PnZ4``H#`sEItn+NY_H$ zMv-g$J)?uqt%56~B=5pwGp^d|uO2)V^?gePPWIHo$*p{ z6+>TaHo3+CrpMqvE_U%n%+Vyhm-mR_ATK2a?1MwQ%*mg=@YteVRT%l&W=yGK4z;hMYLiI-d7jH45`uo~Q7q7}y zfK7gF5dWbfX3pw)gOG;zXTO37mt-de`NkO^)!O{6<{4L)>i%1|53+~T9A(i`akJ^c zVFDALp43U8v>D_o9SpxwQi_`DP?%B&Ku-1){GRrlX=HAikQD)Me2ovR&?D%ca(EBy zc=&6#_LtuIsY!%%sA6fY@p~ziWhoQ=OCt;>AmG}gWuKyRHw+T%Zbbhx{2bgE2x;5! zB)Z951iOh|T-)vNQ3|j7e*I<$-p-u(XT(}{B8#*cX%1cNXeg+HS=?>T`tI0~hTw>N zhzHIt z-wJuuWFu!DV+jd3l5|wjKaQ|98RQ;JOz;H4ncj#z+^U` zrh{^b3RJ;17r6k%*gQr2UScJ8CD{Z1z(^5DtkdW}FR`S0=iBIWdp-)hfq8OYqaLfU z1j)d>Q8r|9uSww}e2xa&1zfFBm|-k`-&=jWhFe5At#mxI%{ zxjnzZQw#Kz8CyxCor{W>(GN?%*p)0Xv_PMTs$O2ZtL9|Ug4sOdsva*IZz%yyz6G$* z;-;YwJo=@9yjDSv?qfC`PdR~rF{7Wd);QPDwHYZ!7!Y7Gm~U! zPTv^s34I*{I?#&xv?sFNk?XNy@n%dg#LZ~za)Xn18G{%qTRd_Op)?D{3rivId@I6w zWO>o~SO{H*=eR5;{Z(3$xo3UK!SZcP9P99=JicQ3&^^Dw^?L%;Fj+G>Xe>|_dx)<~~ZxS{*H1P97@Za9mlfgC*wjU)~yV?`)M#>TrI1Q(tWCw*OwNV6^i5qdA5vX?j-LrqYfo7yX$8s?i zB&WcgzHzMi`pM*atDU{M*6tg4=^GUi0(f9>GJ;sxPN-fqYe^WAM3x@MzT=A*ViVp~YzR!-_9svJmMlBU;YuI& zB7T*I{Ix8mee5wL*+JO8dUtdMBbwX!t(~x2fO~qFx(8f*9Neeg4#bHB=YUKSmdzEziS6~iVSC^u(*farDs5R(tY^Xw6_y%; z^E>>!^z6x7;=2R?S(xHg#>*bjZ>y12AMNW>=vUWb> z{bfD^cEU>vj`kl$t;6MidWc4%E?U$wc+7wgbwC7g>^gFH1o2o@d(9PE>al6T6J;pAt)TKLm zG5w}$NZ@v)%JyIY?_6iiObOg2t$}0#g|R3~p0~x^h4LjU-918XT5Vz;XmRa@&Ycu3 z)(0M;zK)$F*|@oUcs1eSgQp#Fq&9Ykc^C_x)1XTA82F*U+S-Oo?Gl)RDsMpc70trd zg3{VgqdG=0Xlem!%O1q5_Fj|y<8stHbqkYdB(dUj%{tB8qLLJj^v^mPDp^~H?Yw_~ zkM}I-*RTA&g+nbnt+uww4yo;%)&wz0L)F6@1q$e>4xDKg-+Bjx9RRI7H`SOGIGhxG zD$V_3JanT!yi%WTyM-NfD8m|uru{+MME}-aT@wny`_(~~bd+yN1DR4@833DS?Yqm-|<5+gF7u)C>4f?f}&Xc{@vbRpcB?YG2!*^m1M)UieMh zw~N)&APr53HF6MxBukt?E$KQC zB6A}^=jseIY#R|bC#fB9q)U-tfj;U+X^&&GiiY3hT${ym`!k$>pSFA(8+*`kFHK2q zAzFTtdV4^C+7<0JROnyM>u0C_Dqx*`=y-KKDM-PGzwiTFX!XdJu=tEBfkT!=(Tl@2 zz!_e0q8m8?nYo!t_k9D{N*svv7bn9Y-9Y^K|9x=S6m#G$rc(wM0aXw+(%A(J6C`6S z+jY@&Q3v8v$9>(}aL&d)Mz+jc8?^qi8FJ|+3TS_^d-=vx zKFR8FKAp!#ex_PL&W?_3Fw~_S;9jSiqaVR=65uVF2ImC3+dre!&uGe7NGn>-_jI%g zj1)1_#*OVA*!_CK(Ido zaR)cL>XJ5VK%w3MpW!cuVY9{^!l)JzJDwr6Wt#I@(nF-1rw-P0a_b2_`=<8rYuS%R zn@fUwb*pJhgylPNKPBuoI=lT3=wNYD@S8PXU>Ng(7z5dny=~6v-k$-tPIftYNyJ>U z?xgCCsQddaz=^zurlg+=_-(qqp4(*B$J19*IALzYuZaQ`@11i_r(kQ$$XLPN?V5ul ztIh)9K-#Qb2YiJJQQ=e?GR;ixB86K%-GlKjt=0`kRqn(XMeM=VLhc}^&#Nrh!uS!Z z%=x8p;9w~NqLaz$`v-5wrJWwMoZfd%!M#ExN&m;a5sYxy|6BkR&5lBpR{mTh@@O&V_ar;XKeAZ*~?F4PEGzjal z(F_R1QT?90Le7%LUCR^%S*B;lk?&Xf}{r(5{mwO-Y zdtT=}pA~+SSKH!J@e;dPI{T-7&!;Mo) zhWCtZ*wr{k8#RuE|LSgxnf`TL;vhKSL}Fe|-fQT_#Hv^@r}wor1OAm;t{17?V|QkK!+JqCehFni7@_sOh_S3HiwgNHRV6>J%EwIQdXB>rIBo^_yCT zUx(?^>NTtUQtkCi*6#=vlTx4KDH0{p%lDMb9ehT3K$6PS-39q>{<>NR zm;Q?W6vAX|ck2|BQDgYMp<*klK(QoAYGrbq4=m$~a^5f-DqP;d0LZwv)>vdBEqUwF z?B35U0^_!80O1I<#q$a!MkU*&>y`J=Xe70qdF45 zLGzB#Blk3N57~M-L{F*;N60obdO(5`~06DL?qHL$^kx= zZ&>@B(*8Qimsl>B)(;P+#*q84%;u=Ek}`aI!aucI3mFLhzspI#YoT0@i0}~-nO3_E zDiu&ZT^j5Nw_7~R0Uc8X{;+!2{NSTvIC|ETwaxem?A9u;`||VXmc*7E#)F&*ATbHv zj?(kR-LL>|!!}D=?QFPEMFY&xYl<>o-kl9bfhoN-f55_9j3*M>KMa%&U+A6Q==?T8*J;%dbIRf-;pYA&M@X;-D*1i z7wouNogBnKFJa&IvY1vA|Np5K0%Y}@FW<8GM&%{p(haA776W?f?_Mv${1}+&Q zwqiY{_>6{XZd(sSnX*69BnIb?zu+cD?|-WnbeUiUiP=Cb7RpQ7%e7+5?s6eMIPGjU zMc(O&B1N##BW-b~)1~Ec+1X2sfFAAk)10mHJw|})SYZD6SK$eyt{$9OJ5RosaMzLJ z@qN0pgrW5!b4zH;U{o#0Oxkph2JD)ao%=C$+BD)s}q-aJI zRv_?_7i8^a!G8}&9D*%hrhKzbbt~5$gZ}tty!?XPp?@Ohg+sdgud6Z$evIBSgEkXT zFr1qTb2_M+kCX*=cE4qSxQO0Am%3QRI=FZmSq1WSmxnWwXg9UZ0pewPh_EQq!vT$B zr>S6+p;SF961n^rFJk%>Kj-21{K4c)iIG$o^~lR*fyyIkfmj4G*VJ3y?UlA;T)-*a zp=(PXBLDCBos+S9)o-U49|Q;`3cK>Etz7xJ!nSU!y1itzR) zcpaG+%B%9lU;Vz;WQ^FyHr(GW*FsyJg463D9G~_TC+so+tAqkWkS-!KHj40C#{`l* z@5g&wi85gFTWcxhtDn3UdjRJ}c5X`dE&Yc1j-vS8=yex>-1SUo&?YGzuD55o#H zqu;vsdRpMw`G`-_89A+FfdAZcJ#8dhXy?z`q?WOEW2f^zGR>T^p?i$2tA|TIzp;O|ZwINSoEoHpO z^E$(+rz@ycjUiyXPQaOd?C_wNPj;M@oP$EzWCn~|6`|sxu74>Hp}A~W7KefshCT8b zZY3YJ-}z8ieFhH&N5sk1=sqV?ZB@rFo&V9j>vNdAyGs^Q74Y-L^v3&7USa)(Vqo1c z*5zUw$Za=yStsg^)izn$fK4x%YT71W=E>mxKY;sf4vwrkY(SY|Fjp_e{IVOMcoOc4 zBYBhHpj_^?LjFoa*>utBiIsMyQ@V}ACt~Wz&p*Z=u2;$4=%K9uhU=K}T6fqD3qnt6 z_Ex4S8z@F5T&vv?+}y$Pn2+97bMc2P!)8rU9w8Cxm-=O^ca2HiO^SPZ^kHQ^N3RZ3 zn+W1i7W+E(TVr>>r?uQoQ+&+)4>A`&%0+8##oi0TZ_aEC^L|Y{j6LF*@&GQ_?5jab zrX%chQIWK&3O!ckoBz6*12;xW2*!MMe)utN14?lyz_flV^mn2PeyuvTZ{Pz~mkkIT zr1h;iH3P;wql4n|Ul-NJdh5LF(CquRW$szN&1zH7&!q73bRHo4>4p z_O*+feaIKIZv$l?2Gf&nBNkyB^&~l@1^Q3dG@yj|SgBE~sQi*olYapT+1;qP(E>bwc?=sSAhQrrN8%ey; zNyxa1bNH2;zzrQCM0=>y?ZDv?KUsMKm%@$IezQbo_@!-LrzN8t3G=a3T@0a zB$-^g`m+gnEBCoI_3mL7Ge;chmf}$BJqKzRDc}&e3`-1tvp#zpbex7`E>-kQ&?V5D zkWlr)w}l|sG0r8O`?1v#OT6>NiuRwlNoE}v9m?EtsD539S1<-JyAHOvGW(MOqtivR zUB4Q;sFYMLIFAKT=UC1#c(OsEMdN4}N(^Zq&Z8jZFUuikG9>Ico@N`*let@10Tl(Y zbC$~O7v0(M5vm4Z+oCkt{#_J(M)qFM`u(zL!U213*Zz$$hVRCbb0cVg#W#mI6)wKqz$W>3pn>%45liDw^ETFqD7 z546xl)PqV8>K3nyXIzRANr|LDRv#!*t^i_!J?iea6g7O!@%edv&-;)sX=PAuebbj` zqEpWYQty;ciJrz*|Kr#seFjl)C~TS#4Ih^8k$!_A#CeVY@@!>jZ)W&*(%Tsr zj}x5JkSy%X3G|Zv3HdEXj6+p>{_qyd{MmjZ&}@cJp*ncyy`D~b>q7W5c~WvGCw9fM zNaFDRu#5~pGjbzF*2{1>A|n}^zn6s)%u+y$fIS8t{yUziuPEmB=+Wsbg3aB z7EG(0D^^&jBrb;}6|ftWg^pzVYVDc%nzm8BlQE}zQ|mCG>KU!47Otu}X*KH-1R`I= z)4z;tRejDuKHRN1*B1fL1VwgZ1>nmmpSO?Uj~`49|M#bIj)$#W9C*c>`Gehk?07k3 z(78ie-MDA#y(o2*M|;+BX}7$By<(i*_Xa##+seuG+HG=eH~@&fcYSN5-FIlu17Y*E z2_$t8*(BR_X4rhuvp+MTs9+YP{dyvo@iNGa-Mj0JtCoB-U%~-nIqt-xB?*}=> z!Q#P-xyS<}D9beLe4L>Zi=$P4<WAFo; z1Ik5R)Fjxf^$CpT&ueiU_YIUm`pf}vDZx(8A?rVxK4=Z%cKEL`0Jb!>PqtJYjIaDU zKhpWjZNCpjXWg}=86)5t8vLDqA>N$7%Sv93V{7^s47ba;MVFoI!dtYzOY4lLLHraP z{Y=_C2O5OG>}6~fQ);n(y!*!8gOq}HM&!ixtpb$Ui+17W2$zX+P@)YbqD7#Z7Uli@ zrBaXv_3QPT8-_iLxvgY&SSEYQfAa%5S=n{6$~%?4+)tzrzwZw zT9oli5B}_tx8nw}EAYME$%7l6^~*guhP7_*+|&J@9zd?Oovw*1$7qxG=RtGV6y%}b6qBb!V$-MA|P^@|a`8a$7bdCBCyi!vY_bmgYLMRl- zC%-38_HuR~B;;GTrED8rcYHy6*lTVa5=s}rBqW=k4$G%54}G`g`D$(!UGVeLts>`b zX&YhX&u!-8X@r_$1o}hKG^WKrW+{s6UTu_zk{_)}+9&ZZBNJcpnF>HJ+NF+zPVTLe zC`gtFHJvxE2sR`!ej2t$xyiSg@JRH|BE{jX_t8Q(xkFmFyo|;i9QMH#1m1AM)~i*d zTIk_OMO#hM`sjLjqTltyON}R#ZZvArA>`cua+RDPrn%e+5=P(<;Ah-3Vz4Lp4N&LH zxFthC3Pd#R>3@5}O64(uVZdIEBcGWk?Am*;&Z*F>usHRkvBd0*jQpX1?*)E^vjYY= zYkft|Zv{4_FmNj5&HkCEYsu$5J_r{A>k~PO_(1dJ=7$%DC%FOgM1$sU>8Zo<+Fu~p z*Q=UeemyYo&W}*W8z@1xM?C8KxauaW<-h`Pe60YT8g1atirF9wY4CVa97`{%{wv=; z+1u@n&6OWdOYmOgoto`9nd0RuKd&>1RD4LX^hNVT`OKcfM`ZyXMh-4fLu=X}QIxi>8fhws)z>zwT2V&}Dp=ov zjwy#+!j2DK(OvKeb9YW=MOyD` zHn>&8`!8^(u#|n@{FCd6DQuAQf@-&t->L#BaUzQUxV@5`cr*+w1yMhf)*=x zoV}dHfw3C!V@7Bp$F7vZWsJ)HjZfH!C*S(Kb*aS}>Lp!YXOK!kJ0i_y`faDq(0{xD z2nKPgCy!f>tS;~fHvM>m#5OGT3{UYbx{Fk>IQ7+)$Du0qsu}JQUG(tfXy{piOu5-Z zkz?7d-zLm-Kx4tYk?-DXIZ15C5PGD`+vJw90ZrWZxLXgDeIEVWy`@oi_L45W?ta$< zBh=UUHB$jU0?W}v{okg+(3ZlKg*x%X zHC`?fE9u5v?B)a`JCmh5_IysX;t>_gig{wKP81wYO9{SBx$nUv9T}2xaDa9k!ka?4 z&DbUi4gv@;bRiJWVL>8jdxUYU;8Pfn1~cVN`R_?Xi*sJGfqsoCbiK(uHypUK1>z!A zzcac|az+3kG3G|YIh~iHUwuMQs#il7Q@XDR(`(c~9Ou#QwU7A)c>#D{mj$BI^UsQB z7xL;e-g|u2fw^<$3=5!k}S?Xg7AhdpF^JUM^F zOR=@eQ?P3G^fD@hAATp$c>}y|;(kFo=|N_TZQM!K*wUvt|5;ABU))UOa{#8T8=p!D_~U8%ME>V2Irm^m$HnxvYMmNC$e1*MOmbXBYvJt*bW`1 zZl%R~Z_QFf%3Y7re)wrsQgiulGeY6N<00;VjPvB;e+PpC|KLiUb1}b z`5L?bC0VV^IW?ALoblV0#V?F57jW(KJ=;y%-;bb&k6> z!0N^Gqu>83e#7WZ`$k6l-^*%8ft&a@uz!c;G_D;OsdUPuZW_44LXBQ__Q(5^QL|z` zWp=nMwRRArI5a*G1PRzqnKU?jGy=MOA_knp2fEImd2qC8-M1(B+qU9O?5FO@g~`q@ ziUEPRl!rvLu5hd`=J|ojU?xJ=48cAEcC|Hf09TKV^Gf?R((Vw{{i)&#Swe1@dF_ z8bF7y|FPH!Ep$bKrghtD#m02`dBkvBzdsx(W*XooPL!RJ!_^jDZTs&a*I7Gb9M)hs z+C!(PgGdydXSb=V;dd#1YTSeYb~XavtesuF`G()j_UAli_Q-qbh5glUxc|&{6hQ3r ziu39m5)Z6t@7`?stYxs<7WY~pqtLi#@IPZcv(q0}=kfO9b4hyKeyJRERpi3jWuj3Nkcbl$TzOQTl|+a_wH&*%phVtk^V1ad--#iLN77V8e-0e?YT^! zf-HP+q75i=@h@uR7aS)VE_}KBaxahk+X!O%uYwB^P94otejug)@7Z3Smk0BMn*B6v zpMV354hSh?c~e8_r?@Ejo{6}9f-5|!J>mlv-R*u)`J4n;0UmEd++l+HQ;B>mZ~mNFY%`>JuCWKvbnPFLrOAxRE)+Xt}yt4YA&DG`lK z`7y57u`AO?yx_);#vn&)v1!MO&1;9o=l0aOqYy5ZZ z1?$>YqV;%#ds``o!_hVxyXpE4JEWHC@kz#hhZ=;tt3%0+z@_d?|A=NJD&79wGWo%P z(%wYTgS3r(0p#bZS{*x`8XR_0`thirMoGNqs4H`L`5)xT!q;>7s9dL4xF;iAC0TT1 zfP|s#-gv}OAEIj?N;S^BZe_oQ_h$_6gddG{ndaFJ z{3p4o5Z?DIu-fPK8|mU4dE{&pq&$9x}{~okfwzMlJ+Tjnua5nC<(Ge85&_ z`64SI==z}c8cueu@#f|oSyG^N3$Z*1>-~;V3o7|LKNe0MKe6>STsPbFOuZRb!R}zz zcFz@_i*lB(^B|J6rrT@Ya8V-vq)2Z8opKVK%SxV@4qOB$aU7e~1|>Mrq)Wa2dn^4Y zm8tFab)!=tG_x3jYhEmbe+(G`QT}dF#Ib_W=%M`wM5y2}$XWzOR+r=3xSscSDy1VS zDMimsiD~n%qigf;X+yE6@gt_V4=(f55_A4Rmnnmf8;gu<3acYF1ky+6-Zngk4|cA2 zgyChD{@&=f@4)6atG(O8+w0Nk_yQW>Y0+t2cJu`UT%6RxzSLN`UK+No{D8}$MLe%5Z7xd$z7+H zq_va|EGiLjYcUH9xi5511H5|1&kfa(>s0t#1^eMm5GKyaD+bCw4xax^0m9a%1R|Dx zEd1+sv_CkVrIy+^Txtd5L(1wNn=$)c>tu4w8r|#J3dQK0&F{aK#t1+sat2(mH(;1Q z=zOg*e?=Bf-e6@4YPMFKD-$^Q3b89UL9_R&L9YmcuLzdv53gQJm9)qglViHSw&l#z+UO)(6kwwhneyUv$=c z4&H zwY{VMxu?@_;7*V#@Hh=vZCQaooPCl(v||t{?w>40S2k&S{SArw1YqczbymV#lKXp8 zO;TC^Am-wvjQs0`V5sUl1pWa6(N9_h5cXaCl0X|bH7VOGLpBu|aOXcb^mQZ7+-+O+ zWwZi4gZ&cX_w_olH|F?d*Hb|E#Gy?T0);5%b}ajZwBJS>ncnpO_Q~0L=a0qLSy%}6 zKkc>Y?byWMqTL(ATr`x@r>T2un1M1cX%EEnEFjYmBdkmmS(^Cx>j7!31XiitqVsOB znK0ILnxm(VD?VS(^6KJ7L{&UuPOlF8B2Xc6>l@8>FfMw~Uvb2lCe{AqC!Ooh5t5rw z?6#CBZdJhUx)B7p}ImJCvuH2<%YgQ3N zo3;Os4HJxYYtnS|nqq`9$%vK@+m|f!u`nE@_!nRDk6{iE<4Lln_nH_&dUJLNe^ zL;DS3P(xnN@w+W))Rb{=^V2_Wgn*P`Oc{ynf1NPseSdg(lk&Cq$u16Z{C6B}4U>3=a)uaH0tg_D4~#r!ql5;4_VtN_)sb_o6B0(t)Ip)X7Ov6~Dq6e|Fw zpYm&PP(C)k9UHm7pwz`QsMse}gOYyTPDS!=-)-zNft-h!2S@euiZm86!15SCeRqgi zAkLdX*>8Wb!fFq$uU!IE!FYLRwmBJy)UGoQI=ueX`R!K!#1H?To*UY^Ik_oELCR`bWUXv9zn_v)e@D^=;u0Ms9Y|P7MD&>*TsBrGq4f5OL)4i# za<~Qos`b*53M0X?HI$NQ_)#qByNegESw(?*Z%Redvh~ZU7g0#cDI!|kO^U&R=LX*= zTG+}T_B%aW@NOrL+x2`Bh@`rX5OjKM>X*evOD7%q`z6eZQ`95xMZO+mvc%^?7s2=+ z!->Ust<%q(IyNmoj7YCjk~I&ry+cA|ZVL@7r9>(`^UeL`qbxT7^y2LSD}RQfMNO`c z#C=y1FC}eK%I}%m?JBhm3KObP#m0}uF*F}I1WFWN=XPH!e-FF!W+ep-7Dv!#0PjVC zT><#uJsSup`*_0S$2BCogeM{au9gl!9Zx)o1ml%hpa0lQN{4Ix+Vz0K0`Mz6?3avC z>ly^H6DRA1-NqUA$~IB@9Y~D1zN!^nS|QBkxz*K$P5IuM>yqotF(dxh8LY3k$P~GC zJNQa~_+Jv;ALsBCMv{41_o~bJr1kzKu<+UsY#7$3PuDaIX$ljg1TP?&c8dun`b6f+fPmOfc3*voorAuD8!)ALz z9zmE=$M(#ucTl0&f)2S$r7i%;8K-AK7e{pAhX6C}_7JKR!Q>=*E zI>zmtr1{dOf&z64lKZJ(FOABJ;)6a+3FP~I1>%;DVV~|x*b@YHBXHT8xY8#0=_2|4#`FMq=gy>8??~k+8Sri<=(^<)lp~ z(x7CwP&6=LW~EkW(uA;#Ip)W4GFVCdNL+Q3??o6xP~>Ize#cgUbMRg&d~VEgZ>@8D zV(L#8Bhc`&8jhMSpM1rQNcvVm<^fNn(c$ZFC-Z^v6>d@A48ne63-!K&@ezQI0NjcM zIm4fR4GVL52{XdHDj*+Mi0hq&PoJWMUGxj7HFZVAh2mzd*24onvm)(=CwVs;vtHb! z8(Nivy(f5J`3QNSY_l+kQvB7(G}iQ}XWJw{Rh!dbV;UeCP(eyS67`9(AOJmjvm&>$ zlAFXdqog{#Zg&OlxK}*-bZC9|lgrsqFXM(dbfl$&EaITOcg2A1wRA9|>s;nH7B-A;3h7$0;GOCM$ke znTned0rm$g0EK;N zDLIeIf4j~~dU|lsmuP;r(3G|gn)sT}*`Ie{1`H*kkBYZo{Da0SjiJl}@#nQ4HCTB1 z*ev>vS@?e*4;J6$pUL4-F`U>sXSMh%;F!^83$qK*nu*H!Spn#m2K?M`f4VidAc z964PLdw}u+G{J)IihQ#->zC5Cz&0Sm4}6}{*YPi3uh?S!^rTi>QJdLk4=~-7{QmA} z4usypjbj8c)}WgdJTLz({aR44rW)!b=(}?l55%NpA?+XY-4xE%MgFjYyi~y_UIw_H z5f;U*%QgQZ#-w8p;=|WtO{BNd)`}++rUNwaSKbG&Uq?iAq6rm37QfK3Hf8u1>9F_H zlYwaAtw6VV1n%)D_54O9xasz%W13G#^IPnDh4W)$^XK&(Ev6=yoqx86hIr{(YcPjqnS0dIglTK*jWdpr!eLkr;J&p5gns&Hb zc`F#s{4_L?{o>36d(v#65)*xDXY-LoHT7<3=vBza)TTL!wa1d^=By(Cz%w;b;g1@kCc95U9Rn zzI~K%GFGB(eMqj~a2Qcv3U@wx$6heU2BCF-EJyNxnruGA;cvtJbL!tlfVM=#lN{#) z4NK}~@~oVa?IvH+2w=%!tB7+bc0Ee*R-HnwFCL5!!f)jKj##!_aB*J>ygA}LGXF%f zm=XTk={<~2?$JeLLi3HD@^Wr|%hso?!~gVcGA7=`l1|sItgZ>L3yXP8Nc+#4J6iXJ zsWA!cj3s*FHLRd{5VSdvK@CW8t@5YDi$txkKc5|{c6a>2`X01E~3MgRA3_ws31vt+DENJiEr8BW+} zv%`C)s0`sD&%b}}b6{5l48Ko^Zh%fS(lKeqLBrgy2^mt-T+2y*@(<3}+>2{?xG5DM zl;?E3zf_IlZYqD41VTr(;C)6-CQ6#s=#KRpn;D{z{zg3BuOx4NyF|>LU?^S$VXN>- zdX?KJMwNO6QJuj&m!|{tYVcod>XJWAmk%Qd<1UH3e z3yX0ru`B%}3b)_}wFbrGL}5hZ($ThKeV%>Ausf!PTlF-bto&kBN>u&Fn+@jK8Q`Bi zh>v(+Z<>M%m*Z3Mea=a?vKn_$s@RqKUf<~$?;eKRnQ9HnZ0sFa!>-JBuk4G?m90Ps zmS#h0s9c7=;?ab+m&LOS*PfgHK)>ZZrKfM|tgJ*70C&1t$SWOFxaPeaQZiW4^Ka8M zTEJtc2DL{C(F|^j5%Iss5ZM?>WSS1XfMRl7_RwT)BF8rWuaxl8t_;SO<7o*N-Q3X} zfEytr(d6EQpers`Lna?0+fgJ!GyPDmUu?q7{{@3EzvX(I)H{W9kwO+fW++hAtP7$`Y@-OyKm|JCJij8#Te4JE&w3oa+S1`XXN4^!2|7Wsq?~-;?vr=a7N|`_E-FE zEPE&={pK8g?mQ4v2GXJ{W&?+FOUA$Vj_rBh=H_%mg{v8p6!%D*2z3>!G*rJqni7A8z;wiCOhVZt;3!|9xfM-^RWFyi{)#7W_zr{q67dT1+DxI{BvNk%ok zo@Dd!DU`@dQZ}=Lr0kY3d;f{0EX&*+^g&uWFP%PCZJ1PlQ@G**JQmp`#Wh3Tu>ZwN zsXigqr9eOo7g?vBcP8B|Z22-m{hIlvsc-6xW4$@6{Fs z=eX>H3uwH*eUQjtLAm1cgY83?^BG#+@(*~RibD}UXfAp4(F4PvNukrBruIW22l-~v zd>6Bg56qE?YpbrcT%KPP%7Xz%WWjA;2O_ zzy0!a)Wkby1BaVnMdzVNz(TRWN9GO2E%WjB_8W|TxL|G(fjY<^1qm;4#Ci9(1a7}F z$qz(1QUUpOICJ_7R52-pMh6<93VAyj89U9(pc}4&nT?H~c#cy@ECDB_5||$G_#1L` z`{>zqRgXjx2+a!sQehS<8!*+oyt-=ESJU)=Xv_l{H-662Zj_NQfAV`Kmg?J*xPjXB z6ga{9RaE#UMt=Upy$J%3zq4<&r))&V=vd268jsvXDONCeRcq6{4k%0v>&7}vVvY8G zrvWEdqe^V9rEqzoiG%Z|1Rx}OsCtJL^u5-b8f}V4!P8EjDSpd-3-D_i`C4;P4pR7p zt4KrKxV^f#xB5dO!e>_%~x1xshps8f^f6`A1 zTP$J76FV&k@?A=>+lptg7~$S$;Mrzq?RJ+=nzCZ3rZwAtv>S7GQWA2m?tIcvk>WT_{TrDw+JD;PtZ$m!g7EYLiyx-oe z=3)h5oijW@*_^?OEaK!N=h~;WDdL9rviT=0aeU0oy-&fDO_Ol-!vOWFDpK-4KFHR6 z#Z;%K5Gn9ablk@?hF=p6Y7>TYFT~+}PG80Xu(hE6>)zt_H-B~&Q+&dPbeu=0McUr} z$ukJY2TB!Y+&+Ngh*a8R=j(J!rBt=cGIHTVi}xyHn9Iy#=yQj4-)8NxnMl?pP*%%| zCnc?1o9QvN`z4`zQ^r)`jb>JMRUX5=4y=zpl*Uq|TGZ17gu7oSa4_ql=LyWZB&{%i zV0|rDaygdKrEc*zDj6o8^W_nDyQ$uDBgKFd0SXY#{ZTDJ6M9loK!q~=z7T=Hx?dzh zm_#@H2s=}R>?8pu?3l+Ru5X&tVo<_0$cK>>7y$n|x=*F`Dr3SzeP0ZZ z(@N7Pw6(s}73u7Bz4l9;AC5kvUueD~vDG4!vZ5c9r^O)KN zAn0{r2(q$0=p2>DdGg_mOv-IT13Ev9cFsJx*$*fFb%#aw)XnVQbO#S=zy~*MhwY)jvcFvf|jPcZ%$FHf|o0N5lk7(0qZrGNHD?@@na2O-F zV>$x}+&H0tgn%LGbn4O&Iek@S^><|WIsoyx?#{11JnqKlIOm{_w_bl+G$A9IrUsiWgU3vh@d+TIWa}S(L+8$>>$^$Frv*N4q^1ZC^ zTY}4;1P?jawj$Z$KYzu&lub|2mcQ*gAz%sf5FWbJik5d^cI>>!ocPMp->1T>6PXZWh<7+ z%lLTajSwXwY5XvA+tCL28YY&^W7y~kWI-vjbHMYf(i zQ{4-7L=Wk$pbzGoefNMPmn2F+7QS6!lAID!LXO=$+YD6Z#G#1{Aid<-D_a9`xXMx4QI$7Q$r6eMcVaGxt!(Uv8QJcVl(dBX#_m%**6G=*M4z9ptE3%c=4X~fj?BfrFRI7fQ zXC2rX^LVjAySbJh!Ogh|z`L{ky^lH73F*n(7a4ot@Gq$z?+T_d!*d!u0<6YO$dawkN;1(go^0Fo2ffdmob*hx#)5N$(+N_T9 zKm`A&y^7Y+Mr|QqKG?I>KlaGw^6!7jCLx>aKWTfTMZ36kpq6p9jgGvsELP!AB#BF!)?Z6 ziHwYt!-vz0%dgb$6zDmHY>2`K`Y2sLjrfoDlSGkoVWq18JP^@X@DqX4?%`N@)bL*)5)V`W5u-@Ws6>w8h~w@iDAk~=Y&Dj+al}|F=3<~6 zf5izR$#$rhj`sE5YMGAnZt0Qg$#72BOt&JVl(LXYk@G&`kEZussaRJS3pms3_^lua zk}O7D5EdQN=0z1Vsu`En&P$sVZ&Z~ zuik`VN|eO&Db7)6YtB{?Ouh_2NaXCku*)j)jev!p7~a3(Z>g5I~{f4I?|d7 zWt>u6pM}H+J{Mc+8R=B~J%i?J(msew+X@XuD>f-qNv@B;`t{?upw5a#2Q_3xRbIo3 zL&y+sPi#q++PvA&MX2dwTX%6o>s$A%O-J@s&I+TIKDcwY-Si#JpyMnyE+d;ImUVjf z7oV~-0eXpPrfEzl}FPi=k8FEdXH|ARpw5J_+V_9vTtP#b35y z-F`r>nXm_b8S!_)(Z4xgP0`q3MV8oLJ%FFZNS#<$E#k3D%SIzeG&J5gk%ZZ4tbBcc z{S3a+vP(i!LVda6u=R2hX;_g`RLg5w6VX;eBB2!JyhFMNhj+7P^L>PcTAzebQG`=E zIGl~XzW5!1sf_+_>yi_%0bITNZ4#FlEbvKZsM~aq;m+o@z*@iM(bJdOdH0yZ>(|HW z{O{iqMm~`4u4hZ^5zxr>g<)URP_!;*&2~`4QPBNIG!5y~4Y@KHkOxO0^{TyqSZ&ri zh+m`#w!eUO*k2Nl6L4vpAP&X!U^Wf}(}Kz%>@{ge!}^~(-@!m_;;lID43G(S zmMc7-3+4RkO_d4+Gx5f#R-6^Sgg?BWo+#}z_!hmUY6y}~Bb|gE?`~)Ncj*lF zxm~F{8QZkI#ynizt0&GOr3J(}{8!NjeJFxG+nTDl{j&V%&?{!Y}a4 z-k=?%dL%~3X|3!Ujizd0W49PgiW@dx&<&#sMhU;gwznSSmAL~oaagI^4iJ_vZf^ZZ zsR0fNiWz>Db3GTbD&9y4I5pbR11{945~N_e8*j5t?oZva8-QS^LzL=H(f5#6=K}I2 ztzfJQ5;F7qR&6kT+_XISl_s1wWe`W!56|(zm_*%I@9z`)h5E=Nkn#DVYOdSj>~#@xg1do>VbZ3I&YPiX=G zsF3stE0q~1#!aADQwS@(`{X?%sFXa~U?8wU)0t)5N)?%+FT3YI9uz<^C?oak4+>pK zta-`Z!I7VJ6sgs_`A%m877UL*aw2|-BgADd8Ie@6qVTI&um?2X=y#4@YlUDj zNdUPKY@qT<86Qy2H?f){XVWtPDqj4Mk2STiQn>SRX5NzXpVV`uOR2Mv(A9vXiL9gKK&|P}GAM=|0^Aas_|a1xvpUdfwD!d|-FEB;lV|Fpu7>qR}qU$cKyILbUUp>{m5#j-_t zX!@`9!3)7e?1)FmT>xHZZ1KO560#`|moyt<&P5o}n_P8n=y)8xj+z&~H6iw$M+fzA zd(4!_%^U~?;a1v`KQX)tRl2PipwR<5lp}Rh*S7BtkZ4Hwp`uPKg^p9sdqtj zL(-LK9GOj7v+8(m3c*Kv`eXHq{Pw%}K6nY2SLxk3=<2rn;toGa&HB?Xqy0yveNuMd z`0^}zC`rQ*sAA`mNlEUT`BV8wF?3=$Ofh2<1@J--CF9(bjP4w8-39tdO=lK6;Zhtr zc+$o-)Nbzq&C^Or!x( z8A*)EpHX`0UDyRat$#0i{`QqD`Zv;4ix4$&O_J3OxABRpnF~06X=-K{Wc;)(bbR^K zzl}s1h+jIw9~_r}u_}l4+IBC)hNh;9V~$%S)6F;~iUV=&{M4g>9+@bf!G?uf*(^w0 zhGN=>#};(&jw>mE;1q$5z-7^^DCpeZ+tMPPDy!4&pMTmERlA_#U~|M#0S#tZPD$qz z6BrvLt@%(Y1&05;su^M?G7)l&p|KS?6w&Etwkz7{N^7Ti>3scv6`hGc6aF8^UBx#_ zCCa&!tCF))WGh1CsN99g8Oa>EXH#TuIYx+8lB-C`S(|(A$z6`wm}_E(W7Ce`exJYL z^LTtd@AvC?uC}?z!xkmbYed%L7^70p18+^m_q(UM#nKW%-OT>n+Bb+l zSqH8|`QAur+(M-);uX>tGc|kis&JCVLCiFTcIM*wLY%(W#b3b1A(PkVD65)K756nZ zU!1QDD_T(#ojel4xaZ=|lnA2wdcIZqO_-UrL~QZFOjIuJ=a4CWL+<4QMr#Lb=G>r} za}UK&8?CNGz1K^f!ekRokg5?WhAa*EQLe@kU$}BRBle zl~PIZkT17oV7f;I@M%24qOn&T#%ZhjPw0jl$xH3&1x5sALWow&=#7V%$|iVNEQO5p z4LqBiwQ&839J^6njLC@)M&JB)*hQr1dF<4ckKyN~1foa7T)D+A&o$9&94Y+h*=~x@ z%Hks#N{-F*wd0&ON;QE|2u(KiE8yby>4YE5&N$D|BXF_KlYo55o*(+2bx2|I4LB~^ z?5FKhc*p7S1e)v6Uy3V~x&nX&>BuW0ARwK5fJL9vPRPjbRbE|Ra*&*Ts-Ylh8sI^X zr9a8Sjk^6c^+DjZt=6CSeiMAPb}$oR6K{YWK2Q-qOU-;B4YhktnZHXPgXvpBeN^)^5%}xrU_rdc%d33*q;Y20HZM&X0bm zJO(=|)FlC&4kyHGrYO&qQ%GkcSR^c`9UIE@a&8g&rXT?Mm70nBFOpIC4Ila78t!Lrq{E!Q#_v*6R__?`ZP-ZeUz8`VfE{dGtsw#QMg;-0?0H%LxEK6Nt`L@w4?%v%Y=A~fpKd# zF@^&oS2_Jc#&&4l{aSvq-Yq({;}!Vx^8NV;pkgF#kiD8YREuKq*yTFv_#>$uRW=pU zjs6ku^j~5Z2{|^MN+M$%cg{<&9V`Gw60eyyf>9JT0q{M?J44f}8|zzX2BOWQU#jjZ zB|5_0pjSU-kG*~F#e#VC+6^e^FkE`V45_yi3TkvcnDI|#e4*6e*=pr$npT26OV;; zGS?{NSCyn1Zh!e;`expBc6$a~E;o63zh|YEaX{ixwL5FU_#t}BhAE>7bSv29=Dj6t z#O$Y|?9BgL2aqJR{Z~TWnY*W5sv;Rr4=TSMHuwnM;ST5jsN-2%ddJWIu+8{Bk$6S^ z5_Y#~rQQcf)|MCnZ{8HVUtRBU*uDLrdr@Skvl<@YL9;w=DwlVJ#;CqnPrzc2NtsoP zH=GQacFI{CS`dc6i8?w`Z2B3h_r=R=Z7eD8Umwa?I^W0M(72{;AX9NroIOx$J-avr z3D}0M39HmE%>&R&Mc|d$V{B3QMxV$WQPtcb`ZMSJ7MmfF18xNsRAHPfp3b*p7&*Ro zMN}7QMXfURQxwV$TNL>GLRc?+i3~Smjo99t80Ffn=MMKZ?9VnWTd&dYhy66ayIFY) z+=%5P4WG-Q<=}k^1N;BAtI|${GL#rSkb4uTFedDTJp78JN;b}Xy?!$ z_8rsf9Kt?ghHm#EMGY=|eHL8EIYn*925V#!w_+K(KezLZrq>}Svl%M|e_ z+2yZ3ak4Z&d?KjQzauYB0|ef0?|ty<4moc5Tf|7N(zpN9SdDl8@N!qF90VGQ8|yzK zd5hPFE@AOHJZ|{*q-aV$)O3-j2}|31_uf75-w$4bQpzvzCbi4iMtC^7Cn=>Gy!^#G z4^aK8RPL=auT;#@St{gdl%cUWXl^4!VG*@5_VMXn?=@RJ$zl=xNH4wcovlDccc#*8 zb=#*nMKzMh(w=y?!DqN7uR^Wp8S7;63ZEIv+S6(ZO{IQ8DV^D}jwueTTtE$N;LufxV^OO+#+psO~ocX-5I93%G6mctSgcFPGgxBzwLYI5NM1w_~nX{A%- zQ~=hgA4ezp@&>B)N8%dXPMo`!EA+VX8YxrY?LyLm5k|R7Q;J&c%a8+He}}Y*d+7ot z3jm=ZNO5QRf+MK_3&U9h!ZqQu;(&A7wl}{Fe^n91bm|caHnK^A4akvWjmIw- zR>sehuo(GwESIH_SFPuRA`b^K7W5VJZ6cUi4e!X-WiK9hBCHFF|Gk=*bQOK?{Dr{p#W(XqZOk*8qrS>u z=a;5ZQ9DH_5r&de032c*a?-p7T6f`b9elxdonok5a6mu#RJd4)vgSlZ`Td=nHyxP6 z*_#KuQqrJ9kiH}ES)RHw@yeYEJ7g!A+;4LN%5mv9^=Z?Qv+d7V7Q-ABzB_zFrRR$XL;n*&xnB?%ty0QwqX8=6`=H97Add5 zgEhoA+cZXOo_Rr4E#}}EZGF>C2PRo{4Zu~+J1M_6 z+B|+8Jhpp248{tsGq3Y>pI)@V>; zn&kyfS7nZdJPeDd1v%9~SaTIr=2<`o!O@uM!(F0RBCM#=>0R=5Nm;rzvuj5^YidNF zR``BOU+00>{Eb!e!mcB5>#Gp68Od{|L5Z^aqVUT<8SabV_M>tJuJE)WP7dbDL1ONc zVrhMivCHag8PMlW$Tz(z4(CqBszunvuvkSD?%TVrM2XFYhbQI!`?&Yd(^WH7>d)!< z{nN-d#(qJd$V1mT9cFja#ZgNe&LIl$?+Nu#BM8v!;>SfU5iv=uhBI!-aZ>>^(A&U$ zHh&XKymV0>zYo?0R)&CSuY~j#cxv) zI9T@!Jw=tz?c=Szwvt53?o_uPjImq+t2~L48}ewuEXCV%0ZgRBE|^l}vZI2)d7pXt z9%rO;7gnwd%f3oGaOd1+fcc5Zrpv-tC#><20gn{Or+$3Vv9rF|j1_?Aeg#6WO!RUd z>+nUWHMda35L=2@S%G)_nl!mh|FWTrHisA%6RK}J9SMXYVkR`s?l1D*oumUChlgSr z87&u&&8+F6UA5d9`kmOKK4Fxd^77`nwmOcJN2~vKy6J}4bbl4Q!#8;XVdJMp1;!H= zlbbX&P^%=tQ4^8*7-?N+G<}NRJyp>=+Yxm8r}NQ1cdRf-kaajIMtE*W9u%mj1bZCV58=2k zE_ORNGYs`vC#>wgbSV_ZlOPO&UMj~%5e<1LsXu|*=|qfOymXIPRHu7kQn?H?J*Fo6 zmF2{h2I}8NlEo4;4THSQ}dFv3UkI?<)NqdlxK@_#9ti2PrKLi%2 zaO*zEQiWN>(O=fO{uF#=(YIAyJrwNVslH3hQFi<*pKE7?MU1TBV%)U$E=R=V#n_m; z$i7*Vo}QqVOJ&#Mqk0TY7cUxfzg6OyLa*}UQc+A{e2C*w$h}KiFY)>QB#VSZ0wrgG z;>i+3J!SO(9#C%Qsi1E0A@JdR1W^P17T2A|*;3Fq=H1s52*~M|OZ(}ydlZ}ZUZn!` z5F5&xsid-4*m*Dz*lieL8WJg{6>kIlYlr4|@DMluPQzK2;5~`H8=nWtH&5}3OYWSj zXc4BFp+z&`D-p&{s;a*Z=rnB`IFBnk*MjD0FDg4@aQrdWGAYjj9$1Xu#pNiawx%+) z72r+Tv>&Yk$i)z9x(hlQ#QY&iLNk$Yy8Sn(l3m!Q(sqC6`s=g>beQXeXvB+Hbrdoc zyhm8{^D5Oj=PN^d=DrcE*LJDq&uc=fKJI(oYW`r{fJ=>s2MR9uZlp^l4#0C(w0qF<3R$nCK;ldd{ zlP=_V)gQ@d$EF&IRls|+6<}&70V>5YYmGBL32tu#`!&IjD+D-&05g~7bGQ$KOJfDc zz8}HR6%D6Wr-G<6Uwokb@(9NkYE%+;wik0!TSQdQ#MhSg8)WcVvb-kZgMR+EvtTx1 z=rU{5g=y$Us(m=sX>%UkT1^6TY(_HB6u~&HRp5ma;R4gfg9}kWj_h{A;>E+bznO;% z#LOz0{rRc%?ug%?91W~E6kU59#om^aM_;y)&mEXhS=KEZn{TaP?0=ZA`9y2flXk#B zWqmjV&|1>$Z?#XbEEF{V#h&B~BzQm0J!{M5PC!fX(0X_6UZ^IDa#t}F;4Zx5N;GQ` z-sXCBVR*&*N}_rZ$^}e|GWszC51zdRwJF`z9yDVT=^BEni%HT(76@%nv`2lO>kn=a z$tBk=3=Xx|XfnSCEK?Q*b+x^=j#{i?E|>c6NQhvHwRZ`)%&WcK{l0~<6CZL_ zBDeE#$JH3kt2Tpk;HpLYj%ui78J$s@f|>wxB; zV!n?%v@;e4kNmEKwod3BDn)&KN^wls}WE98?}`ogG~W7%*AbR-Xt7jhfh z#SZhfOyVPYs*AqSg?BQvajV2uHQmw_{XMbau*^&<$fJ#GM&Gowk*KWJdT3@}`F$qY zcOShO9^A252-M?~mBO|gXFI1FPtUyP5C={U zr9)lL_vbJvs)8-94qU%-fy3#QN2&nm3n$?cc0y&!gBLDfXy(T+|FG1R`FXi%WAxnH z-aknn@`?cS^&nt4KM}uRBU7;Fgr;uyJwXAIKY9HzOt^lVi;7`_E{&aB;uZgUdwm>}*NAV4eKUxa}N8$*BzCE}DS3MX>>eMm>eeYEy}#QXlt zX#Y-;I-odap3l4-13llvCJ6FP44l!i>s?B~Xxth_72%pV(}+y!p$8nGsyIz>sXE`2 zsbL=P%ssO1GLXRL!nVO7BZ;|V{eENNehua4>#T#1Y}!^B29^U%9z1yvkl#LhMGTZa z&rz0ARdx~F6zstom)bLkc4{6DbXh85}FxVEdkLi z$&Z_E!$W6Nxa})i>;>^%qF}fFbfT6#5720~gTxR{yR|%7m?!hX+T4Sf1Kb1Lvzc>& zfKX6;q)Bgq!#E9#{s2!dhkM7NyedKEh~fb~Y;y2Jx5a?)h*+zb_a6hV*c)x`;Q1#w z3xJ56(Thc9qEygNA%C!{`z+OlzSo;v0G3r3-5A8zt)@26_A}r>sl1)8n1%x_X+x?CwjqDxeM_(>kwQ?t zckV}7=1c^~J^588R}Yp}4M4jApk6l1qYv;FWwW93p6V})%ixtad8WyhYqet~1Gze~ z-tyxnHlIp#r#^oN1g}D_%%=DS%RY)@-3r~NPw+$kWIO+!f&R0I?>bH;3d468s({1B zXr@3jzvZZlCd}va-txmQ#mS?*+%=J;8yQy+ODkHXNTM4f38%IZ)hKKzkGPv^6r~^`$$~7=Cv38mE@XnbOb-2psK<3!<4&L|O{_KdwXGc%4-3eqSPFI>e zbKSrNYy76<*wnj%8JhrK%_RWj$LnccB>%+M*IQ(rY37Dw&lvoZNQ}~|Fkps(^Ouy- zc0*+%G#^z<8yYAdf?f6s@t#^S=KAKrhoZQ5GEN}DC%iOuZX*XDXp}u@u0xsYxW_ouBxwM}`0H_=wyA| zE8)_i>OKbmw$;eho9to8`su9p#>P@i{m>v!HYrMx`by5{s2fgqV%IN2u``G2{;S#} z7(C_JHL#g4!TVKzH-;cqyTWYUbYJYD51;o&OW{neeF^8u{&=>3MOrA~?FdpJV zSYd`@e7yIF=r>t}q62JMgr{OifCEZ+OqL@U0qnPCM~vzAVAWSinbTGsoAj%8aAv*o zuWD3^SdZJGJp`)nD#ZmjSqj)I^?gr($f>AJ$#J))lJ(;mu}!}FFX04CDff;uyZT$@ z44yzaWcc(;REg2B-keS7+|){0hao1Ky6u~P!(lZL$EGcIp3i^I>#mUn%_C6l5a^P! z>!#Rsp#cEt6KG$x)xQV)s9bQ9Udl5Q!j2ysPa78L&HdLqdHuyUL@dr}NJnn_or0#u z)ho3h3FLS-gf8mRizhfvtzM0;@IyPk-^a6h9oP}I+0o=6~N{Rb6BX3y4 z5iV4cW^ZW|en}IQMT+TnetP+OC=>YD9ENf2e>0Cg{8J!oHPOl6dW}=^aM*Unss)1+rbRF+Sba7% zS^dsY{r8^f?G9m8-(u)oUlX_hU>wvBfuHDZcJ$scFzxx_sGe>&>$_MnNuJCsS&yi* z?S#{Ys<=ZKzX4zFL(&!$TFy;eGq<}lHtC1pKHZ{AsJ|Suh|q}G&Hj5`YQ6kg>-TLH z@Kyi8(;^duC=6+%3mPF4l)6`@ir!|39??Zz7I ztV%vhgYW=#7VO2Wemv>Gq}*g@;q;+w3>`V;kYxK;6FPKtq`3YYe^ONz(}&E_>Aq4d zi=*$Z4@FD3K~IDg#yC21E&p50#uK=4t=!6S^zF}6jtF|OY2C#@@z}oC8anXk#M0LC zd+<`)JID$k59QE^GI&PGf^LN=Mk)-?G zAp#plve>m9P|9#iZEcyjfDFB2Y_A!F^9a*j3Pm!I-(LKYNI0 A4*&oF literal 0 HcmV?d00001 diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/foreground.png b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/foreground.png new file mode 100644 index 0000000000000000000000000000000000000000..4483ddad1f079e1089d685bd204ee1cfe1d01902 GIT binary patch literal 12430 zcmeHuS6EX)+pUO#NL3(IK|}&d7YKwF5CM@UBE5tjTBw4Q5KwvxB2pw25vBJIB27p@ zOaSQt5eZd#CxmkF|4+F-=Q)?(#XNgvmzlk1)~tDFz3+~Fs;5bRo%8yoOPA=i9zS|^ z=@P~5f9V?4rAwDs!Yjfq4p(5Rx~i8hRVUG&*j~LT%Q>2AIqB+Nx_^yhg70E+c&i!%2~zqE0}mxIX= zz1$7|sWj&3yL#7D|4uLjQqV+x(Rz4WC{A9|^m@1A6`BNi38Cf3B^aJyqxF{TjS&2q=3$BC zB1Fu04C;%o9V_Yg;Ed;xpmge>%b<|5q52W_pTd9o;Qty2mQ+-Peu)^(K)RH^d5byH z>AGB-I7$|~9l)J0H_LPDsUUL#brIHpjO1>dJ9@_5&W zLV)s!AVn7*Hy{o<1zLA_Ky-TWzJ_^1=W=Gfyc#1ssqeY_2ww>;ANX%JT)(9uNHOtU zeqU2_{Wu6pLvCMBLgy+dx=13ZG-+cMrBf;#8KezD^}_F2x>_Nob0^iXEv>aML;8RQ@@sN(#bq~VsOa>) zW9RDe#_!zLkj)PyQ<05AjbPk5yJ^|B6q=sMX2L0JE|(P%=v2$6+4QL)cu$c*yt`EC z?)p#@xE12zK?QF2u^(xb0>KieYWS%DH`?=eOiFd!6)WRmCo6Joq6}7e=Nl_;oNJ{1 zu&szm^c0s*wAxfHSlk^+hb)aB<&B?9+_YvxC1LEy$(dDJ8J)d!>rwz?q zGTpJ5&uVwR#t4%B`T{*~RAd_Unnf&`*9c^zbZfsVc;v*@=BHOCX7VbyhnS5G*Pik} z@`U!W&dq$A-&GCYAWg@rG3W6ANL_2a)|;&HJSig{zyfyO87W{;ej&@-)yx~eu|G6S zO)U5U?QD)!ey@XcxEKX?m{R4VZN!*V9gT}6_lv@YD^}}y4OM(*#%kMMBij<9x4*by zCkGRQ3vqoZ)HvQ4oY~=kh{c09u`@Lzqk8)3R+$+hcYuhqajQqgq8qWy8X_QMy@1+T z0&yU)D$XzuW+GZpAB%%|^3*{x!r`8nOWhu6>t(2mvERH# zwD(@F(UyHL)A@d0q#?|SOaIrK7`~^_KhtD69y6E{G70hSpvkOuvhEmR1(|2efAmi@Xw9*}m%vZb>kVqe?t6*aL%179k2-;CD<(T2&{-rQ;%g&4b= zStwf@&UH8&T6lBt>jybuLy}~>HTF7(kmQuR6(8*l&xSQq79o~y=t@1Z0aSiA&-LWp z0NQ{@*q$n1m#1Z}?sFj0=6jxX!@eHh_D<=qD}vOG`kCQ^44In=iDu`srXYt8{4c&) z7G9;S9(*ydG({X#u#N%3l}&Yaq*lzrY-E%htNRQTrjCrX1NMi~a!soU$|=0*dXokbDxSFnm6OHLV@%5(K&ZQB%e+ZFne-TrP|veCOrVj;0pG zdbMMl{Z%MBfVA6b>SKLi zXyRQXFc}Krl(owbvDh?Um&9l0#P)rbdiZxK)8=RY8XvSG1@0=@vGxtW|3E{`T&9Zk zC0==A6=d?8`t>?}z3d12SZ$YU4KZHQPf~|w zJD7n^6bjSS+&0Kq6nxhj*9}9qDZC~A`nzEz{<+9lxx)v#qaCsGWko<{ahFVncU-R|715> z33|Jp;8Iq?Z)NXe;h$K{z8#lRB#JC*XUod!9+#hCfkg#-^FD5Jq@>Dt!SzYr@q0(& z;I!1>qg(PU*HMX7>G-#T5V;IOw~4L@XQ&5le>B4Va!sx0P1pm1PMa!%L##WB{CukUKwQLR#mw_r{d1DneIIJT(j#O#-det^FD zbdwZ-8R%84+Bo+g5iyd(a6x;*5F0xuclibP*ff{7PNPESiBNJu^Q2?h!4}38?XKcb z1cb%?RlBpM10D9~`7(D`#uzQxY}K)shcU_}%#WJZ`~FU)C1j&^b5i=Wc7uJW8^-NB z(rs3^Wms@#S~)+us~_(~uocjV^vU^euJHB^upc~CY%6gqBXHR3{FJ}D^V0uB8xrdo z%j>^}CvVUV6jaGJf5i$e;gXng&>{)uK?nWhEUaVrv+x8njtfCz>cqP8uUTn1`McQ;CD+jm zGle#Cefq~0!!v@W2XnNsA~8j@Gaaj+fT)QzP<&gR$L=bGEJ8^z*tHxS)sZ=vZPV!4 zw*)4rK3To_7<;de8PvEPu4Q5d;D=g00$bPnaG|sEP6(kDsxwc2+y=l@=8Gy3^DW?X z$=3@Y|B6^8mUadWxX-6z(Oh@9|3%Nv*Hz=bA3)}AiK3MrA@eOvp)YSd(Nf|v;6dz-v zI5xYnKImXz)PTM}jxK=GJh_OrE2HXqKgh*KB!U~;4W!DpXN6A98^kNt%~i7+I+`g5 zW}~Qod0A;Lw*Q@m73+!Rfuir!WXqcTd5mXE^DWV3AUSVk>5EA&b6Svd&!yh*!z+6( zh^>CvoV~2?y`UJ#Jho<+PlUEw=Y?Hyd8C#Oj$c!5d!Du*w4OQ9G&OxhDmQ=)tzD()srM-?#=f>aw-$x}3Z?qLOIJ{gnZu zd`Y3Pu@-6CD7)$*a6189&`vfy%c7^DmCj90Mw>5FgU_yh15-*dsMPOLpn%G&Gbq@c z)NN;i4jF!g3-}@w-}i(YUbp4WY;xYi8`sa3ep2V_UXf_!7A{;Fhp25CGF=6{xLd&d z!Mvrklt74KI=0hsCRMYBXM0Z?v1sDfN=Y&W2dW!hUyqiiU@A}R-XCxbIudes32?<&DQ!Hr>qn`aYQ?jSq?4X|x(CCDAB;b=wcWVCH1CfwqU1di z!|LlwpE@R5*{9XlM;`OM$(VZBN$c{`%$ZT3S3aYJwVO}kw)@4_EyP4SXgXkd)Q z7PtWeexnE98(N{TMKt-aG+YpQs`a~e_Y;}upm;CRXlTWI->sMI?cj%D`$7K@mQ<-e z6c3=23v>}kQ!+Z{G2&KQ99s+el!e053~lQJc`8%`$;xt_RQ&16M-jjl$HK)VZG-0esPL)%m(*xgTxhvj>YKkE?dOv3G%g-W9;dgR&pG1FoW|wrm7v|b_Y-VU zKV&S7NcSkHSjm4nrPIy#Wvwp8(lbN>^x7o60ICQ5m?QwOuUY9q(q~<6`0+a7 z_`Zhdli4>YUiT%XT1&z74m|S7pZ;||I*2@$Zd5=|9{V~xFLGS|sAE`ZQ=toXwPUzSz%(Ar!@#M}4%I2r*Ca<9 ze?7@cjo0^QC6zocYls~PXjm{I-w|^|?Hpmvl_!6;&?vERiS^(A2e-)2qxQ#IfuJ_M zgEhyUo8K;fE}w8OE$6nq26w$M-YgMyeYnhwguXF-@5ca=0xYn%I)Rl=_lZaUn5tgl zq{GPw`_E=ilA8s)Jy=%ks{*^ijmr0SqHYg5D%zYfzlqy~#fp6GHI7wm_SN!mo*B=(4jED535Cy$0WQgpMk_!VjQ zhjwgVnse1csNUVP_rkF)3q*bk`=D| zRm=kyT3qxBA7a}d4b433h)JR1r_zBVy6)DMRyM?5%=@^}YMnjurETi?w8)8Y2lox+B2Mc9(WcW709kmg&QO^PydT;QZ_K7tmYO8aA8M?Y);N zSn^>S4^jpy!tF}ZAn_;hcCNY$eyakky`&>*Nh{Yf8H17GR#{9&%f^ps6IAlo`0a7| z-5WT~hwWze!uONxb4D$Was0UyM#f|Al`@rMWg(+oyWOL{(2>P6$`ht&d;q3uD6W+D zQQKN!nzWpx$Ya8CUKa3dgn={(ad!Lm7qDcu`SB#dKHvAM#GW}Z>EZmS6yG22dWcVi zef}3H%>*xQE6XidovM|h{PD;~31ijm0ia9g=-tnlFk!0PDn12luSSt7gWP{nbUK-G z_;*xp66cFpR2OkYg+1wGZF$3SCHuNOh~T{QxmE}&DI?a%s+Q&BqRkJ^37TgbKmAKA z-lXW9)FAv@J#Z=C2lSk4@W5q7S0~BpAs>m(p{^)b2MCFka=_0~yTtPvSKJEH%6&GW zKv;f{iTBYXA0^wmTAmssRXI(3556s-FYRfgXSs2F7D?)Muw3X(n96>Fe~#_y!;5dQ zdOQ?Kp<{m8r8ee4PPIETr3Sr=L{BgNp=Hl~>nSiYS!vY-rs7>zJE&K9>k00!&bs>P zD`CMT*(GNFuh#^fdZE?R`V};&3K^rq3z5UT^^KE~V+Yq@nxU<{+Ug^t(FEIk@f~5* zgnEN(6_Zcdmg55!i|T1Xn2NBcinnnFghvgYxT5oG<#r&$ky|k5SaFs(+Vr@W6W!wc zhr8=;xACvw0kVQ6m+uK@w0M_|3*`l1D1SbQ1B%k-HMIa!=~kGkCfuQ8^C^ZQ&7xn%?zUs@ zJv~f?$}gE-(aEgrt|vKx z;}Q@0S-w8jTszP4_+Em>MvCg@+IT%eNk_MIr)gA`;*lhuP%vm}{=>pIah-$r^3{Da zp;l8BZIY#N3v`sN%POMh>Q=e-o^BM2OK_7-ztamrbZ{m49XWXIgg1Gqa+C!XfX?gxVvl@Yc z?lm`jKKariU3($HdVP4LPtp4+4mV=+tw*rjI~_q%R6DfIW|6`<`}My)W_VK!6c^i* zIvi5RI=c%+#{fOc1^%pnKBkmGk{n2 zC<)woa7^dmGd|$2v77jNVg{v9cP;?R<5Hz&w)i1YTrbpNc6%p0{Khx8hi!J94klTx zC9LuDS+2u)()U%ug}~voR<>Cq}#OQfXF2)TCm)4nk4dkJK<{Ji<% zcP30SBMi`eN&Lves%5zi8b`z0j<83Tc~cBqc7F%;N9zZcNAe!JR3!n;@j1h z1lCS;R&Xw6EFbwYNCw_`r4_DiPb}ogRDYy^watxfz7Xy(zQ=RKaRMV#RY}`WgLrrF zVY?S>T2T_0_gmfEc1P>euBpQk$h-TAw(GijhS$+YK=Tg$zQ6?>D}F1vFkHMoukc{a zEy_ED8Uf0r#&yr0HH7|2|B-{vV9-6x6%+AEp3Hd}4fvb`f5|t#1a^r!L``xWv0pYp zK_sWYo?M7Ka~?Ti?_2#VSWzD;+NOTq_0`+=>-+<27aH>r;wtxc2mAJdsVzr(62hGT z)&mW2D1I;#ot)2O9iIWid6J}Na=-qm<@K(sk9ppYVwcO*IkP(P8P9ER7!PsMfNBn& za^K3zdtRPHN^c^l9lmBs5m>rjxgOV7Io|5p!v}X)j;Ax&u7K?;q%XjX_~o%@lPr_8 z*9Uqq$6~D2?gL>l^=mP&+~8z3yT!99Io|+z9QCQwYR2S? z(t}t86UG(B`86l3E&Y`O1p($K!sj_~Szh|(peg0h(+?ymZ?)sk6C*iUD89q@SVAIS z4_&>H|FtF3pZ<_*-;w|rv%!y93`xISUXVWp-T~!8n*#@16?Q}v>{P^~9I69_ z%n*6qXY%Yy!%fWkW5OADjlkEKjP5d$8>`wRrhp=ra6@iEL)prjHQ=o3@+N$WN7maZarII1Zz-rqUrBVRY znukG8!4Q$))$$`IcgoPA;izr~)m2%Wl&%&EHeRmOXUJsiSwge{CQ5;l6K*f{(Y$dK zr+Ms$jZr918R?`Rysv0Z+#6wT~L%t0b;+Q^{rT$Y_J%=|3^Wd zt6$*epNax{<>cRLLyEm2t&MjM8j1U)pYxwc-MDWDwN~$V|G#;ney}e?-YB~f0-n-M zw?G0{JBvufZPvKoY*5O85X8y3)1IFwLkMFr+5G1knQdDje8Y{BGoelP12*9EUN%KY zxk|^L1xHs)rNCp_@p0*`=#9{%r)_7IsX3T&x{b&X;mgnjUOMtgKs#ylC}%kSdtkjl z8!FE;zg-elNMzzYzDjZ0)^Ieq?HW_G)|Sg=4mBA1EloCGZTG(+tr)OPwRZ{J7OY5O z-u^rg$|QACu3Cq*Al+><3gPrW!35XM#YAriTfXw+!m_NkpMN$HY+wKfNr4L9PYUX6 zzlS_jplR*TFaNt8ide7lbsipOGdSE!+zhi$@D8y%FCwjQ$r9L{z>FOk9`c^?Kjmj` zMuYzJ3lU=4n6Q;tr@a$L?%8~af{fraE2*s=hn>Cp;YCQ#>re~C6xoCO7}(mj#Xh*k zba*^&l5yo%qnHQd!W*<-IXZ+8vnMb>c^cM={07F5{v1ulw!aVecf>C42Ir44Vz);s zT-%=b<-{YEZ*nD{U;m4uIi#wyf4G^ggB0@5%#DRIbN7hz&!Bb!hl?A6#(~|dZ%%iN z%o^Sc0oq?wn5_;1HQ*s%km5+`HK!Bq9^dL$ZL7!o2j@&piKs-)bi>dGD9BCC4PSIk zrGJIk0P-Fv?{`4G0`eU>*i`V_XN2xXw%*xTUlVENh%_|iZDkl5p@Y866#=@Xg{cbE zjZtS75AB(^xEogv2B)1x^m!0XZdCqOZ~=~2%7kuI!6E74!u_j2iau*{do^aD^2Vk^O2eW~KSv(BzRD>xw` z&*Gb6ksujl^_Fg<9{Nxn%B8jSv6jcmU+Kw5-Q&psk7EU|G|_)%rogKwNzemwy6QX^ z@ujX`ZkT$alQ%3oWJ2VOJGz{G(ukN|LF&Ga)nKml$M>IY@1F)}2mL&m6~?A)CN|YS zLi^lZj;aN$DQnmlc~AgqcDB7)?<<0=D*JMD zM3%;`BX_AsO%3+;YjwAbOnkT+m^;*q5X>@S2hO@Aa1J zJCCx~6B|ewT}HQECVls)>JqY95!(x8tJTl^D9t}c_G8p6;&167Z{2*+*qbjZdPBKR zwYTwFdQwnL?Q_fZ1S5+O2`Bi&@(s_P_cQY7?>NOU&FL}U5YmlM6yw@TASK}~;pon& z&{?aE)kw+rf)rVR1R!KIA&R@6^&5tt+oJ8h+P)7GWpbZ0xhG1hCCSz8pFjdYT5mJUum4y`e6ST z&@%+@8U+Bx-^#X6vpu~G2`=~;;97zryltTvX_;q&`r%A)oV7(xhxX1-Obw!r%_aBq zXumue@LLi`iFY=9t~-zHYJC&!zW;W6TKK3YgAe-4E5@wu_HwjtlH4Ep5vqLS-2C5$ zSxHdkc#a7g$_vSgCJ_dxxPL&~SeaPflc=j>z18KsBxhHfhSRvim6wzyuJBI@*m2g@ zc2$Hh#1|Nide`x;s zFEY{lfS)AO1(&M2`md$eil6mNBxu2_M(#la)vUt>ub2uO+!3=jb#6Ic2xq$*jBF`n z%L9sP{NK&^17myQl!*yca`I%e*{%{^D5ld#5&5Dbmw2He%xl{Z?Bv@+UmIbjXEHB5 zH5Sh@UPidw19)2ZMmXkn`O@)IsF`Fbj+RLtb$qTJ#B-vXrZ?7??}cA6N56t|TzFj4 z=rAukcL+Zk?vE$J3_QP=HeaZiJ>sPUrar&8Ao}%X-FpDz+o?UsRbtr6!(ES)@vCo94^P>R%u%q(-9wy%Duenrn)jXuW z+2hV;WWLbrH-awRI4^BBwkb{USY=a|U+=L6IJbHc+!%aSb|KB}H$ z?;wmaMfCf`2o^LLsVRHayM++C2aVlLWRbMjawRSh!|`u4I8tjLx>H>?ZR&ba(LJXj z?DRP5gyUNUnznwc)C%qsQ!aTlw6i(@viQ+~|0fLN?FR=&Mz z!m?8%ms9Zm`@?A{S+a>p-JQ}TICnZa{gktp_;s>#3Wv_=7#GC;f$M! z&TRADKS2F7Grq42P=N2(^g3PHSv9Sr5khe~OZap~yE3UUWM-{Fh{H-BGK9MOV3L#y zw*TZQX^enrYRj7iXkEaCLTZF5z%T)MU*{_RxA-*;G{sl{7ry_e1h+X~HM>NyBnnV6 zzcFEEZvv5PId&nY^VG0nqu!l%4Ln9L8OVmkfQi1}=-j_u=t%I1_~|`SZ_zv+SV@2>e1;w+Y$vY75F((`NKQU2vax&tTw!~HE>c2M3z3d>g zk@W;ee$-qtx3IgJ&cQ;-5AmGPIIdtV0YQvcV7G)N!(PWkx#qq=;AiOzb$C@x+Z zu##CR=Q`hVF-LGTr?w9-umq+&6PrkTr)T1CJ!@XV9i+em9sS#E=UO}BNMwuBrCayH zAub{V#`%5ecrycz1$eSV8<2Ikv6CQ5E=h^K%3m6h74APzqFYP{oejD^Y7o_E2b3p| zeA*LbkS?zNs8`f>wX`CuZF=Vcnc?D9l|P;QF8KedIQiHkm!f>Y3}# zl9AL|w=FC#e&CG1Vj1SX@K&6z&wEdwI}i+9}=0 zD)hP8t2qSqGq-zz1>nRbHpsOX+Ou&rc&B>1K5Z`l|60?OVRG!%y@dyXhC`Y)1x&pBnbuTa%|7f^nM;OIHu%(W6&Ci`84e(2e5z z*ThM)rgG_sjP#cQ+Xs8;_5jS%p3?)1Cd0epUI+qH6)RAoaWyIr#O{wWN#wI+_de=e zPHAv`+(8DcYwZezvF?o<#{{xGw05-!dGx*J-i6B-YsG?>W6ke;g4Hg#P+$=@?s0UEI-*Bw6RE<{1I7> zjBlz61z%K{w(Fbs@*+5i`|zyRlh@qP_iu#(*1Wcpz$is&$q|YHc+dRFT7N)#@B@znBGn$2wXOi+ggc5BJ<+2( zlI3ksg*I$2(gaUp4h9pJY${1?hgh6#mU-3e=N{4cTb2V_4R`HbSASd)X&1AJD{hd8 z^}36_R=S?hhh>k{b|Q{V4g^$!<)__{4ZCIAOzE}*nn%8FpA_Bmaub%88)q94qdSj& zU&K}EwoAH(N;V`V{ZfKgP}7P8xX{2STb>)D)y3#SF&&=+6Jz=_o8pqGbBI1lUdL(1 zD2L567hm`YXfrYLV3fz4yv?7yE!3uaicqZ7ufRny<0U&B6qh8bcqsL`r9)-JOxkXy z+l@a1(ptpJ`{M2l$g!g@DX;KZcoPP93JT=vi}|dQ!tn5*k@U)brT5a*!NEAJ2Apj0 z3jNsKvYjiiy-sUG06+A3T)f+N_X|`ZAX$1+M8W1ZaK3Nm6Dd}Xw#CnL+A?Xi*n>}B z+g^J-yeBCQ;(6yjA1~5bLwIzXXp>6syw2d^&DXBrf$G@}~y*QOne;u_UdZD^Cl zXxza$QKpgXzp22W4GZI|8N{0M2?78Z`$wi+S>waN@uSr9`u5+ghvrjfhcjQNuoDp; zk9szfi0j_VBAd2M+55}LBoF!BASF5?QV6q5zf94lQ$2goh8#I@&N4tiMK&5WOgt0H zRiGPL-7G)N zj%2#teK$kweDwBL1+DK?B#>r?tjR02JIr zUq=)|zME?3CA9?-DRGfqM+;h7w&xgGmLjhTAOdy`b%#?iM;>=l7v)^GADOA64 zy}x#1eDIpJ^iQ-mHzp5#R2_{6(~wo;npi>z4tuCy@Z6Ovw1EGFOaCWi{Qog*{?+*F cSLciz6 + + + Public/ic_public_list_add_light + Created with Sketch. + + + + + \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/icon_home.svg b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/icon_home.svg new file mode 100644 index 000000000..504af3400 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/icon_home.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/icon_info.svg b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/icon_info.svg new file mode 100644 index 000000000..2210223f4 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/icon_info.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/icon_view.svg b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/icon_view.svg new file mode 100644 index 000000000..b7958b5ed --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/icon_view.svg @@ -0,0 +1,13 @@ + + + Public/ic_public_view_list_filled + + + + + + + + + + \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/layered_image.json b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/layered_image.json new file mode 100644 index 000000000..fb4992044 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/layered_image.json @@ -0,0 +1,7 @@ +{ + "layered-image": + { + "background" : "$media:background", + "foreground" : "$media:foreground" + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/startIcon.png b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/media/startIcon.png new file mode 100644 index 0000000000000000000000000000000000000000..205ad8b5a8a42e8762fbe4899b8e5e31ce822b8b GIT binary patch literal 20093 zcmV)JK)b(*P)AsI{U0tD9;7S&f z3`9H(<`G*WCN>bN493AFOi{!!!L|afI7%o`6&6lXK&2`L1YumJiZTQ+5doQ^Fu|gz zI6Nvw1cME>!8`;4iI*N+z3;u_gZtzG5&vyF~^*1 z?S1yyXYbweAFzGO*PdLxe&gE9j&{c{J=rY}9i1#6cCzdq+ASx~UzXhiC(H6orN{Ar zj;qq$yDTU7NWP@ws1J2_*G}Ykx7%{iE$G@-7-eF^Y3#}`(v#ySiIZdTj}`y+a>=Im9Vq=f1W5yxR*!@kj+Rxz&v=+4_?qb>2v z^P8^zTt$BB=j8B|JpIS7`QY>Jz4z#w<>ZT>lB09T6nS2-t-LNa`Yg!ixr}^gvZsB` z{B;rQ@uVEqwOt7oA8%Sn=e2VBs;^`dNc~|xx$^LKH+*6BuO8<1`K9&UDuw8t_%!FY zoV0NZ!^eH~qhBH?uakr4K4~ZC5VHnAA|L9#J5r^|-)7;Y zUl$mM>pDMqeipwr+7#N+YO&F-3t!twD#tH9_S*S{wQ+C`@f*(uNuw}s=xXMh&DI;Q z;_u$0c(3`5*FEq(O?pz@6#ee_pZMDAFS)(D{hdnlGw+UhHaZ&vMC3y~_HorR=oT!) zD&Jv0*w5!@vBS?MX~$>r(d*!xjZ=9%U3__Gl0?W|%cDAF&TIVSk@)+3cqc!3boGhhYzil=`)k_5%wL2pqQz`Ju@50G)sNfVj zoXGZ|Q(f3+@xx0`O2~K<`L6lJ-SXStp$#*Nk@$Du%RKJ9@n>4_fX zCq4RXG{SB86?4nquk-Hy-E#B;AN86?zpBs|J16`d(I5ZXNB^!~KL7eV0uKN-_1L$Q zfhXMkzP+y=*8|%=cJL*vJ8JS$i*h!V@e z?gp)OZL3q^qPRQ$mTS*l z!1Lo9sgwA)pzOQd7ry0nSAP)8dF^z>J#;@|{wb*sK5UU+HV4!!`0VEJLKou6^E1;q z{-F(t{g8gMTs+F%4CL8B(dE++Be1u} zQa1d_@^?2B{4?(K#G2gBZ2YKxYj^wS1vv8wb2h-K`rtLS+C4j5oS5zZQT6pjk(( zJ4B5)x)C<~DS-Jn#3lX27u>p0yp_M+jn)mGYaUy>+T%Nnb1#0!>tbyAQ%)nklRSgJ z&7=Ic?ks-hoA@5fJ^x~JiY`PYkDmW0C(plGd!Q$Ex;t|N@d~qieC9rdJUa(Jbmg%% zxJoLcUW^RY7oUugb$iXkOVyLI8AJG+ zNchYly!4G7Y^6~5nrXo&e$8p}lUVB0m<1UOEOBY-ht5+)-??6hPx|GZjRV(b``>-$ zM|{PjUt-09)0*964ZWy4qG3A!iZuCL5J4vSq$?ol?wO2=1e&!;9t z{HK#&d2T{`aKZSSV$8nw`5IF+b?d?_&_RB2Nn@S=KEJHRZ&{wfFD-HANt+d!8=g@V${FeVy<@Q=p|RCl}k1iW;RIY+rXYw+ro1J ztScYrS3bq4R+FlcH(!!*-yB2t`NcV#59x0CP?FiqC-VdG1vMIuAg3o=Td=#P|3Z0B%|-@17rLGk-6p<6~!$6~POh1kU3(XXZO`=|>$d z!lw$=5_RyEi#Jr~RP#^%iC^4A^2m;K+VClBHe2;z6Z14*Mk&|$%X0f<_lmdugY8>E zPThfcKaZ0b)2b2Pn1`Dkmvb_pUZ*zC08jjo)ep|hccB`;;R{6kL;Ts-DL%Zk@M}Ec zYe??S-~5VIlRb~$9A!25WQb$>P5#6re$4=RZ7!m^$ICJHQwLq8^3qO zSIW*0ziJfhY2#Np#+5qaD29V6USiSHHu0r%dVQte1>d!Te30L9h<8T(gM1~;2HMmK zAIaG=K2h~u$+A`Ao#yL~^C@rnmi3*Dn>*0%_Q|VFij#Is9D-CUfq|-t52LPSO>Mf;|h8QzG9r>i*kxj)D&%wf12-@hxpQE(boL;`OLW% z&4ra*97R9KXL{m{MVR>LH~jeO-Z?hkb&`yq#K-O6lT$@0DD?-g)^Uzc7T&5n8gw__ z0DpXP`45D@vQE5>CYLA9MXJba02$ioVhjTWVS5bZ6(4zN`ENe`p5>!H^k})NKh(Lb zKhik@lUA-Xx~smjY)TJqEB4J>%kshNC(AGX&hhfC|NQ3id+))>f~iYr%eBS5L6diS z0c(T7VNUk2yzB*+mM{H`dzO#=6GzJf`m=$1G@nblG}%hD(09V$W~@UCQLSS;5BqEV zWae*vfSYo>EH@?Gc;aOFp#GTWmw)f}@_j#ZYkBJ*Le`;RxE%9>G%3oHFxKHSfF_;E zFF&fw_1jO}dg1SWTfI@g(_fZ9_1ee&mj2x4J1a|pX>wLqgaW;Whu>GnNZR9Y^4s;%W zx4i1NzvUU8TZ6Uq$a?oX>%J5^9jAU9em|0;-_C;e(1}uEYG}e zr$t+qTP`-spu!U-M~AgevS79|o^g>`wAc>y@e7Vk`?z91a^qxq>GOBXzxbc8ET8gX z-7Xxv6CigTGJZUUv*`9=vmA1gzg4h49N+Y^ODZ8#@KI9`q-_X zaPu5;fuSS!*@le$mhP;#HK&jK(B1NbUvXvmPhY0_kiYDk{5AHRoIkT@vw@Z8z;F1q z7l7fCCi(MA@@nf@5q}|i{jv8-IsM&M6%o3LI{BfEQREKp4HG$@wUJ1eYx}Q!%BAIh z`K$LWk8838tEq&7|H$p$UeKq__MwZg*U!9Rnw3=(J#1>imzU))z3%$*uKvrZuZ{Wd>ES!5dgNmrfBPTZ zSl;rks&UNFhD?$g9J)KT33%MPXFTyAfBeSP=e+&fch`Iedi2_(FPHhgB&G`tFhZFY^iGZTPO8%A6S;JedWE&6Z7VgKJMLTtbV@Au;oe}a$|fo@8QFpeTE;~ z=(!{4cwATZ_x+vv)3p?oK6COMai}`b-FNw9`G;R}pRW2^Ajgt*_)SjojgA<};ZV-D zH)q&q4iEL*eWU|BFmM=S?>NY;&)5I;`<6?(5sl{jyXGx}^8>dxQX%Vtv5PEo8w6JK zToHH6efQkYp6Q3Mqvhz+s$i(tXF7XpLn?CV%Z6Oqu_p_+nw!5{zT;K*3%heMNzF;f zzun5oTzGVll(CU?9of+U+nP1y(OpU zvv~w9Sr;nLG5?3p<|70ueyyDbUY}Yd!E0=`V+1F2S@%7DUU z!+3G5v_Yp@FhhD(9o{OXys6YM@?dLP0LotS!( zZ~o{ThY!62s*m!Sg&e-XdU0#<$S=0*Pb|w{eYqaXoLkS+K6Rp~Y^EN+{G*Qi6P;tq z8XuKI#YV0>%Nz^2?6yhv9fh2b=evx?JV#`6&=bQOMZM+dz(~P{OOO4g=JV%2_LA3t zIWdLGe~6_L*6U?ZoidN$t=;E~mp$XEY0L*5)a)#9%C_**_ejXj1}SaGL~lF&7ro-L z5_Il{V)fCw*fu?YZqYMj%cgB7z3S~eAahn{_@cQMlFic3)%3UY#Noj!JH4cEvRr#S z^9EDCiHH1&FTSjo9Q4r{^K&2ha-QnFK^=vKuFYqvdxW=7K2uz)M)&XO4}*2S)oU;32*?s`tzhPoNdy zMK~{~T*=4;PVlC()T`0MfB8pTs;kbv+GgKHr(Rq!;3+S|5(B&y+n5*@z^5dLrcGjDVs3` zF=w9B8T=Q$;LA>~9`X4+qVFJ-liI=f8qb5;adlP9$i*t%;M>z~dBL;M7jh(|v1O@a za}jzx7Y{1+b#a=fVe#WfJ$C)~F&^GD!hg8&3xD97hwY{wLOxnA2;wJqo|?br07>n| zdc9}P-SQkmio~mhtX%z&MJycY7!O^|^}~~L*w+vLY!DscBm0>6jPaAr#6u#lPtl}a zn^g8A4RF_SY<9BpclX?P?PZtsH(oFGD^X@u>A2cxb^Xba#{f#>E7Bp? ztFxkR`P@dmpq)Vyx9`@uFnA8e#&tpr-DGb_G^IYIlqLQGW*i-bW1&6e29O6Y4AR#5 zvw3QcRQo|aIrZklmvExE$M4X$oUyA07_9mhM=sXuWE_~5;nT=?xmN7c}VZTZ(}?rL~jVuDCHDd zW0I>4RkJL)P{rpZ{mdS{51lA{3Pf+T`jPlbs|k>vbZN6ZbRkPI+fmPp0DeI6t7Nc~ z$NhZ%nT)>k;6(Zz50&~yf1iG^fs4sKviK#}-Dl{r>Bu~hY2DR;F}T*pmL9|4wUTbw z@xnlPQdFhr&E%R&<~6QfTI+#VgCJrYF+`(acGqTfD_@rASLH)IiT<#`a<+xCqjpL` z>#D>_%Q%UnL=``~nBcrnhfBLfp$0UGM~}`pY-%%xL2Su?1!0>O+=jhV^Q|SHHsi~S zD~0ov1zlYjfNIlt^GFNNb-;qpg1EPAM(ME^ps)?4i@M~QXic5q&!wGA8~zyJ#}kr& z^`4JJ%2R4dCKVL9!V%6$c5)Gv^*q_xt7|K06))bGDUPP7^FtSfX;?h<0|XKb062A zIY|b0!pj0C)Y$7;i^P=d-~9Mh&zQKh^`h&1%>hsw!5hUsnpx4t z<}nU3;cAnu{B7X&Vn5^sgN95?k&<*Nw-dMSz$p_Pc^$xvIFk*X^*T}DEO_*uml7(B z&nEcAJ#m?Xu}#P#5u(vuOElFSM`G;J(?_?d0s0skGYz4+p=0BMwY@=f?C04B`6n16 z7Y+?9wH$J zAxS-==YiY@80*`{n1+s)KEk056AV77g?$%2H0xq(Q))9XS&VWbRL_G=l_J9>UJl0D zL}N3`NDj2QCw^L+J)AKpGPZ04N*&EdoH2o<_uVvg5ExqK?h8cD!pAn(v{$fP*#~QU zh>wrmGmlPAjvv4qPUcCCWLhX|Ka2&~1>W*WY1;yK(tBoXnGCEf#s(&kaR8=O7&`Rb z4)NokexjR!kF~8MOFmU5aQ$lW3aOlWOo#8pn)8ot^lQLVQZO5XoZ}x``u%x;$Cmjs zwt{}jE1RV@QuzczTVvNF(%{QMY#aX3$pievr_W(l1ZA{3C6z9Llh!WOKW`#3*AYhq z-tucRhL5MYjUq^yq;P4yz(j=;Uhu<*6tg}0;12PFp$~4~hxPm_+Zg8Ct>f7*BneZNsSb8?%&Jh@KlZTTrOg zc*d4a&)A=--&QSt^&=aCKtMfi2RM(tjY0_3lN)$zC%(pMOo(G{xaW#VQD)ml*8}*( zn%f398D{+~2NGYgRbLr0gOY-ta%{uQ8}bVGoMs=E!xb*`2zR1d+}H1qgGY~B`-@YJ z>*a;j$od&444i_t&M>U#WibY2>CmtI+6%Qc>JFq&fKMxFac!J|LFhSyp@oAfvh|$Q!ky#K zhS(4BtuuI=bE{5uez>A2b4!3M+hm`g$1$&w|CB6iS~rUj(~}eO8bJK3dJ?_67ebx{ zSHS|R%y8%`=YQMnAR>?_}JgGOix59Mum~lwBBOj7l{Dr%(^B9~CeuB#Ukb0`^qvuU*Y(62BICR)&Tg!A&&-M+!2eTcS zQp|kcb?_I5@TRuW`$zm0SeN?*o>tHfJx!tLIT3p}glz!EcCx$YvH;wLhF24aiOPLh zoyM4vMhXD7pn%KA%I|SJ3pjFVbc&HshPKa%R-zM#w$p3fhA+q*C$x=DN^`o8SMD%{ zlYy6XyKVf(AvWYbX0=U|B7A&%L$qy^lSpgCbq?mNVK#inCYah3&VIO?=1DXw=#`qC zbt3TAho;;JwjNhLV1kW_T;f+5&f5zw$zb{>8{!V`+%h~%KVy-DqlO+=H=VZ=FkY%TPJGOKbO-eUMZb@k`Qw5*kXQI4 zNn-VY-V}k{dvi=NgDj)aFv2b;9&Lhj62jH0Xgt5%4NV`a$nS9VFeZ8jwL3ZT-35mn zvUwAUQ9a=cgBJ%U^%9B`*>UXEt~NPJ9a#K=jILPgIq5_LF4);`bivL2J}%hVmz_pI z&(zfWn4ASNsVrtA?CTky6@SLgnCP>dnQ&s$k2bCduV@v=0M<$2v&?X_w&f?0 zdVL4q!ob4O|06wo;ixOrj>l#y;~Gg=-=WAx*pV-hTSqte=+)3!U&FCJJ(R7IGj_tH zSk_m_@)csRD}7KQl3@|As*N?`C_c!U@vo=O(oUUM9HYTXr$fev>%5uanu%NzjR zCb4pse%58Ff_FbT99ZTs=22SCWBp8Il>D>{j4u>gKeWxhWg0&$HJ{gkdPXCf61P@& ztiI#OvjYd~D)hvhL4pdPanYqKH?T(AS0xsJjcpoa4(T1TJw`VIoTCqRpI?P*;>dsN z5f0BOf=znyxkaZ2tJWn8N$N>lK}c;lWS?W5vOBR=JKko}KC|$3Z%PH$J5|jKJ-NqE z_ZknrZ7W~D$^f(y8P~onU3Oty2J4NY*@llDx%i|JpU9&wHDK(xtG@VU#^kYat*h>i zdSLC^jL7(-#cz$a=M=p%&kPDtW4)wR`B-^()-G4{E(m^LY+5LRq%6%7l<6vOPNhVCyvY=4yUI zIx&MxLE28(nmXlm7viLOLSs$b4|GCD7I{^>sJ)bo<7qB^r=YAS^^JFY6;xwEh zZpDM~;ZEeb0~BvkTQTEG0U3VZL5j9H_mXvxdHwoPMGk8H%GZ$DSUoG};o!Bp*+kXX z`qy7&0LlzDGC5UnIv&!hC5g%LKEG*AaEI$`J|`zF9*~_UC6v2ef%Yt=w?iGS=`x{m`*tc1v}Pz zf~slY{K=p-7He#u7L@_cNMwKhd*f^(-Vaneam*r{gTf>LelwEqaEL>^IXTI3UTi}^ zZkltHCYX)!fRgkGlZFWF0F?CZ*bebcbNh5(fov2_4=P{4lkUMPb=`l~2uhFxu>7&DseW}mFpI(L7m<98w3m<&s^gYwzKLS`@ ziH2UU5yjHI=Sa0E5;z6n)mm>R$Iaaa0HpF2H=cyKrST)6aY5j>Y2EFa4KyaOJpi`Y z0cR0NFVNX;eH&s&2RLs_Wk`!X1Ktl5EXMuVY^M5^Na4ay{PgzMr(hU*GqwVm<`|tx zHqpMHc}$IYj}CnPhO8RSa9ryZ-xY7p0CWe2u`wOua|f#J0CPySsjO015zUoj^|=$R z&P!8a>m2?Q`plg2TfXWox!mch;lqB)b!%4}(i&%-8hjt^C)?8v8krgXwGp&JSbXUmUuKNKj;seLQ@+i{*gD4%I@RALNg?5Nv zHQN3d?-dcg{ZuEQo!};N-E}JHlr|#Z=D+=Y^?ah~?(8cL)5{VsbD?G)a@Zyct*NHxP>~FNNVt39Nz-u{udkt;$vC~g<^Q~(o z@!$ErW946qkAsrqYR=YH5b{$F!kam>41*1>C($G?Qu;QuA8=!KcHIVdWNDr-8-7uK zNuNiULdrZEx{d!~v71dXW?a|C=vhDe#uyuYWb4hW)6k0ypF8ER{BAwTAx;YE-wb!) zU;16Was^(;$OUp5dXvkJY0hDAS|8fn=gyP6&xSuan8cZ0vW)z(=x@DiJPDG%HphC= z- zpYdSh-(EFF=R=BYI@>x#_%jYWdLEjhM|USaBzVpNLG3+y_(R$BD_RmMas$MWs~oG^0ClV~+&9ED$w?cD|Yz+=nu2k$xd2U}uu6PP0V zCo+iBf#`{lqWxs#{-;()(J&9)cV& z*MIxg+j{>(@hd`~jcXbH;1z zth?n%0u(-3tD58KJI#tQPuPp_{T#@NnLsv#(utmIWON>=r)G}FN{F5lNBD@6U;Bn9 z>MqnKn+0+&Jbe!0Sg#XY1|IL>WT_VXUT;oA+Kv6ir{@DlMjpC8`1rDX*N^ifn3Oa- zP>v=r{|3wSjsMrp<+?rvZ1#&IQ%o*?Q%fUy9{OfIvd7w82leqs-`IVe19y5!^8?p+ z%lE(O);9mymq@O`lr{MH-Gap%a!lvK(+9_5!wv_d}s`<0wzR2F;-6sG^f)1 zfAhBE<$Hhn)^a}|--)B-fGBwkg|A}DfUPxB;ADB-k7x(+!4Wu(Z^V|l+qB6&n>1q*9dcD_jHBlT z*vR|+hTp{?KmT(AyX9Nn__#hpI{B~9Yw%ik6(uW2wP}cuI}>`1H0k-6=fBTqX`C$v zyXpzH+GeRX%|8xjW>_S<&=S+Pnr``~H$Jia)W5&2PruNUE@20Cie;tIvIjt59r&b0 zjV=c|+__#ALk??qI+k=+1B_gv^QeSsUl&j? z;p|tZ|KgJ`FMscq_bfcG=0&dhz{tYj7c4!e`8Av9+C(?nNM0J_+A`~hL2+5Y%lGV- zcj`{^cVGXwo}+cX;<;dQvT7u2?0R+qYFq{XM198e*L=}E%d_>lL3~zo=0om&Voy%^ z%h9>f^lD0ytPpr zg~{1jZAiO~^T97J@yeh09w`1xwSh24F`NSEhCjRLSXJn`%mH@4#+$x@;up2ebwIl&_3snm%EJ(YEoj{-clclgY{Q#$UL- z{G^^VuQM1Gu)n(U2vif97a;}2J2D&cm4Ei0<mZtf?9#n|`tkjxXn6KX&EI1=R@*$+Kyw>;|^ zN6TfsKa#H^pu#R*_}$O*#n-X_6q!ggu8IzGT!q@a0d4&GoYsxW{s08 zxcb6`!zl91*VjDiv#}r4pKJ1goci!UFDRc`2%OJ$tT_0@2dCnL<$j-qr9L&M`lL5D z(Jg%h*(2AFmk(S^Onhux>cB?H;>YJE=cKZwR~3}pmJcYob}zo~KupBx=(Nh~M4*nz zFreXsw&7fy?>G)Rb7uLh_>fd0az4fHf;q3Jlg~yVw=Ucr;=5V{Uqw2b-#L3OowL9U z9j+Ix`1q<;8v}WtQ-xXig+I)9(3;nXc|pGNB1^pvR0~0A$kl-?YrweTR}h1GVi

c)ijgxDm}8EsRXFt3h@+Ufr7@DN z^55r2UpdZvo*$)c`MJ_3zXBARbH%T}ifygzYy6g*WBtspGU<*Ccb`wpyW!Ui$gZ}y zo>MwK`K>f-62KfvO2{S zXF|ni6T=gB=C>=mF~5ojWS?I%DBt!ouB^&}v*S8G>5&(6>bM<0W9)PIeSXbv;v2lq zgZx&0)nJZqzUPEz=3RZouldy~VSciFe9|fxrs_KoD#u$hYz3BTu8Twxs@yt>*lp{< zm_XbpVEfL5#v}%x;+@AY<0*cV$ZF-248A&7CXCUG-9e@z7Va=V8J*&{q4I$n{~M-~K{qUmg-Y{N~tC__Y!6wZ`uS zAN=8SKnb`wARia}P{>}4q*mFJ2rt$xz9z}40>2@prKgMpJ4y?1MK zsu;8LLY(s8tNKp-L`??i35r}^567PuI=u8S&*EdFoy9Nf;48%{S#m8d=h|q*N!*Hw zE&QzCc2jn4u4(uar*pTPKCQ7DC)&Cs49?>3$7+X~)XJA`!=HT>p7`~r%@S~FvIWT% zL)t28t$h|BY!xpHnSQNXihG*>p${(0U;hi2mrwZcOUrZh0ee^UiT1oYO{3$5Hop*u zLXEN0l1qM=vD`rN)XOLJdon_5oHz3`AzpsrE1f=|*Mk1={U^)6{EcJ3kodUYZmX=p z&l4~2a)h&L*mG4|<3d+3_?Prr)`vgu$Y1U7EWIl2?@iUEd5K>;n9zxxlFNU^0vTLl zH@o9AcfQkuuVr{d?>6N1tv`70$?|*eKGqA1!uC8^rS(s+P1LOQ9lYFac+7nk_^^=}_9|LQHrRm;gm z#jgtmwd-2xd;fSm;rGSZd-@wbDeXS|)%sP&lv@b1qs`Sf43!0V?3qvsHeeF4^Q(*h z^}o7zxuRcU@`@_U0N4FIMxo}rPTLvJc{K#}XhYWmowJJ2$Yjbl`u)zkPnNIv?#GvR zeQ>x@oZ)FOm|m&l>_ivC(ek;URCk@4f5BINBIPcJedSknv#$7sL09O4r%@qb_M zz2et2d?)PSD|vhJv?jf^coe^7;*5D_(i{GoNjc@GFgNZjMJ5=HK91L-#6s_k5ZsDS zGS%RQ&sF+5eNE*3{W~3);ByDsjH9O)4$S@$?yR>?gy?){V`EPI$n>{$7kZJt&E|jq z@9tl&>KhB0wjiX?fvux_ph<@^P`xU#l~@YcVmvoP|52 zFCDST=db-|m-UT`(xE24+%n&4gZ%FnLi&Yo)!)!<`8*?XqEn@~PlG4oI{hPQc|SBA-3UqQo@Ok7n} zIAZ21l@78Rn`X^sw|ukiJP&AnypS?sjm)BYgRrvd_2vm*-zj>cKd@`Ab&91Yp=>6{)F%4)7auKu@lUJhnvWozKNZb^uG+`E@Y3=U zeK~|@uUf1nf;jWRpXQgYuqA_|MTZQJmcB;TNR^GlS{T8}iC6rO{IH|tWqO{uY5h}C zK^05FmfvX7IMk$1hE*ehH{+tKyHIa1DdB;;rJvHi z@XysN8q8vy7k-&z&tLr~zqICPT-#vO+|kk)bI{UP%}!$rHS^6TDD1uXt~a|@W*~+c z8vo^wJW;Rw34f4ZJkG`2_D~Yj%WRNd2O^Mwn=s<$0*s{9@EYCPT5v)bA~e(n|~6M0EUxGtnrcN&$s(s zzN8S(XWAcol9+ za@NCPqQw`HsBTqo#8>DWj&U^~+CTP~&69^IHqX$ty#E|%_>m7|XO7~asM|V+|Xy_l(fh&fm#RNST>VcoN?=6S_DPi%0~BG=sQt4-78)-@|b)lahBHa~PL<9jHj zNE~dl9PG02qUPM@QPu+cEDu-Af8%z}zB%Ihfge*{9Wd$&G+)E(=&9+o!^CjO`cwNdjVRH+WU`h_MXAOitJp5x3ifW{$igPf9iBj$(b=HI#x==`-hy-E&gI#->XR(BW&pMdcoR19-nNcPkY4s2bR7uK27u z;T-wi{Jv$d3tg^Khr|3zu!D-f$3GV1rd-BjB{h8+psmB&uHFO}3e<>-KnIym}P_oSC zslstp61Dm&1NiV|^pEbaNt}ZX!rh1GA<@OoA~K`yhAgd{@foOROsg!`F}gM(u1!jB zP-&PeM7Vk8W1#d^)-p1e`o(13g|c~w?dj`;4_bZu^_E|g3d=E{cLES;rdxmDH283uG=7WUKG<2~ea{IxU4q0( zBCeM((XD0e;O571>R|^u&Ev*jpsQGwzvm-2(K$^ICifY)?_e`E(umG-isbY(H;sFS z_TV{-u;uIR9OWMt?$V=eCxZbQ9k$3lC>2^A@xz~@XvD&(_uWN31AO=Zpf(=jB!lHh zOT3|j8)NsuFr00(J`~5*Aa@-yCcZDeY#2MK^7+byjE?yuYo4B|14zoWZPTeh8BIOF zi#LZ9-0pPpQq1&2arSg`YF@vQoGhb26RLwnlb*1L_^M-Vlx>giHItHpV-y+pt6ZEK z556G7lZ4?GS?qbNp_S;OAM&IlDs9+mIL@;^vinA)D6z3H9OHAVWxzHP_n^luSJ#<< zbsIty2lS^g(Tp%sL>_Jx%DMrbLPR&IRuN*2au@Mv3b3wQaDyVnmOp4Ma3Q*l1@}l- z7!@6xqcC>X;&3#^WC@2>d~Pt-WCFI;DSS*he8-yHfN>hl!&k7gZRoJWX*}IU_<3Dv zFh%O=_d;$wPTu#$88_QzeaYlJH`gOD^~u}%0AtVi0{v!P<5awgzdH2uJ`V|wUL*2lawezA2~fq&{P;mfB?8T6HUC*4h6A&Uoa8O-j$RT~z$aZBVg6 zzF?cyl6N zdHw?sJ7Tp$XXHMr#>SS7hWS(q4Vv|F6FxR`qoAKa__u1W&%AQI4T^VKan^IyU>zfs zE|$R$NQPNwnbWKcmi{dLjG5%b9r@2i8f!K??SvY4H+*lPY@EblJRiC1P#E;CqroIW z@amJ2xy(A56v{9|GuaTpMMj+DK>H#%Xah4-!k=}#^ zneQH-ALI49-brtya+(0Rs?MoH;W4xa=7q~HKFb7Z1nBuy5&@vrkTKXDY=saRII;oP z3R%&P2^nF-NYearIVR*J3O2Ys934KH3%!qF8Ezacu`vg0S*Oab^yt!p+xLq-xy5gM z#Kw5jI=`XA!CkZ&zAqE&VEj1=NFmPhl*4MSO=PEas`~e2-T71-1sApc|fu*Q}= zsYFnC_DZcy+zSDb@&j)&>t^-n;oK7;%>Y=GI zf;q6^#lf=W>#ky4S#ll)lVVQT_DO*_|C(c%5cIB9nT$1w zdZdwu#x~{=-+@S!Al?*`YqRX_$W)w|mL<42l`iKk-%cwYqIN?eH8`i)kL=}d1?JZx ztLCs2KGwvGug#(X==ud4yo;s5T!B+uNNV9YMyc!;d~C+efEeaJa{IVw7aDzJFOkR6 zSlJt<<>?A3vyx@)YW!;#RD~3cJ<+yt$FWi*K*_8K6|i@y5t3Ja zJ+H|ads>I+vjj95MRGK=^x>=qv2joEMXBp_IFN4`AdHaye#ZCSN+T3ki zEEWhGJ-%>&Q^eAnKgqhuJba{|Jl+AxddOr{Cxi+(@50!IbHi4?hjyY5LQ=XVPTEpb zyqVjwx1@vOf~d3GC@cCi=V6PSGqd|Ua>`SZ|JP5mkUUL?=|EPi{@-nlH?JLkAw z*sMbLgtgvL+o_1?*wJfZjcXpC5>GR~M4yu?y`l7N54Pg1hB01ME2+8Z!14qfU-Yz@ zpP&@C_lf&Q^@(4j;1EbkPV$`KhCay2t@XoalE&DO(HG;)bGsV$(1$|8a365@r{WKw zNW$FkEp^Sm<|7b9uV3Ad{N#D~L@0goVuYqx6L^T_<{Zg#=0otZT7J0Sg93< zJ_mX2IquB#Bm6s#^rsweb>du#$y5q2icb}=oNpi;{UA7T{^iK)*yGw5d6=pq_?*D>mRC&iQRDaItw;A9 zUwyN}YMcO55)^&3H9%p>YklyFuHBgRqrZ5o{^}Fg-RyE2Q&BkPr4P7!;2dsBBY5kZ z6MOo=-HSke#!JD&S`O^!e_!8v^T8YV)+p1?{L!gB{K1puy1vT%sWe=-JBLXqC(&~o zh8QdS8g_rYT88wPo<6+$(H>5CKO8#&q^#c>*j4hprAvR9e{%Kyt8YGf`?u>?8Tz14 zS1k!Et{sV(!ehcu#U^0M9yMmukRS`=W<1D5*Xuj%0?f#3B#i1AuV%Dk0a#p(np`Z z@Ny<>{{ZDV5+@v)mOs>&&;9Vv>-)pHaOkS3YygE%;ePHnZ!h`bKx(H9HZuLnZ`piM z2ii=ClLN3rsu>=c{+jNjKd(=0rLpid^!u4*y(mWJPG6kjm0Yv8i=0jt@0q$c?3SO6 zo`T_+i0(Myt98b;JQvD(PJ8@c_^spR4R6xbATVp;gA^fWJoolt6Viy=aHkR(bL6>a z0*u#QIOR-CHs#1eI_@gp{LgMJH~1i?ZcMM{ufkCb2He+@V%l*Br$@ccN`(OGk)9u)8Cl^IS$70>cnNtJOD;^adIv1mfzOH@{j*A zpUGT+)Iu&-&YD8$81J|E-`Afpo?Sod(=~-f1KG?W4N<>A4H|trX(W)6k{Oa&+m(#9NV~FpO<-jgq5FpLo=R80h%`t-tc094&kfl2?<-(g>J|r?=r^r}OA> zmp&f(`pX~wSI3@L@|*kMoPV!t)up3lQ3afNHGkNJ?ukAA%&S+P!*d|=aQo0Nz5YfK zKR4s_UId|>uzYyqbjJt5=GTt(Ez-yS$U9G{Cqm(9+ajN> zgT~ide(a0*RMefm>R_qQXttNTKUJiWa#G(o>gibbxL(-&eO>l^>-4Yw{;}#f=Ndog zTpjgwLr5GKkp=Bm^VjU9%39U~*@|iCk3RCfSN<|`f4G7d?}tSDTy`AIwQL?;#$97+ ztSvnwvYK=4p}Io0?fv>@g@5oyeJpBc$rtZF^xS26hCWZ4#Yok->p2VeHu^YSPUGG2k^A|XtmgmW>+a9E=9)4OCk5TSW^(Rd;pI_JfySLre zQLOv*sbCN46V?6wuS}=FN|eBT_p(bFq*`MXpIA`Vg(EMp(umI{;a4t?=!xmyYV?&H2P7PMKv=d+vjRBWh(As6Lj0Qcn$#3?!%y6`&&<3aj!!;n$@xk0 z*`QFf2~yb7*ZgYBR84)J;s=KZ&x_vE!tWtII60`G5(@|IFyHPr=5zVG<@(X_<1hTc z_kGCwAo)o&!Uw+XL*A!{f;S*LxN;y5=0e-ZrK)pdNED2liw(!iVbw-%n7!XMpG8kA zGUJMmr0RBj5-MyJddQOpL{O*s7%s{`6u+WXrgQwlI?smCIg$&Q{AYgqCt0wKb7$_% zm%{TugWsEv_{Fa|uJO;}cZ_9uLpG0)>jq*Vhu`WPlbLjiH(IU~Fm-o{X+n|rIebs+ zBK*FBMohVN%r4@=_@qH>4)KXqe5CL#cK)Tu;+Dei@z-rsKEYOe;uO{W-~*^lGv{e} zg4af91r84J?WZul<4pXy&Q9bMAD7uEiayKu@j6WtFdw~+#;%<5b$dDfR;X#?4us;} z-~EhV6zs>~=Rof`?o~=VM~9%M_?8J+n!&AcCV)?AP=;fE71{~UeEA>#S{QucDki=r zzHybu$j{hvT>Nr&n2+r=zY;+&dlw*cHh$KbFJ$UN=-6jIG7AR2vDH_c$iN1FmhpRt z?{%2s!?BZglURd~-k|DP8~&9Flv)o?mLI$Jz3h>-Z8i{UeJRS<(K9vL#!-~$F*1Sp z9>4-|wb7EC2gB>kF9$2`EI#_O(HBeOdGZy+=Ze2BPH_+Mi?qgP47=j(>kB=mJ%oMS z9r<0iE@an9F`Z)KGra&4x%#2EIrCiSSMf=2pI?~4w>$UPbpC{gT;8zlrl=Bb2 zc!MuoiVfHWSDf^|NDlF(^ZW;&*`LSHX6X1EeyW$cIeN{P*pA<}=H;OUB#~>P2l%!Y z!u69#KlsSz*U2UJ{M*;+{q-Mwz4pdlJGFtZ-+TGiS1Ql<#B&y|xO2F8BP#-G95X!= zS3AtF&0v5*jT?Lk8~!j1%0_T}otooBko6is#Sgz&6@Aj7$ONp`$^7Ks*zOGN$=Vl+ z!3WfQyRB%BY(65Ff(S*v1=yWtyJ{I0gB$4W-~OP!g>&~BlI$ss{JeWJ0Y~lvE4La}LgwmJ{B^=-^LrxrR*K+!NY34Y z%M z<9FfUS32e(gAJbEtbl5ub8iasSIo+HYW6cI2(;PPCVrX9hj6>)HIID%gYPzH@6^%v zv^{*@-@5)2n!;y#NN$bBu|)+fn^0}89(_q=8AGE|lG!A3qm}-*G$sPd@g2 zSN`*ry_F8$fdaX8yu3>5_^=Mm3a>SxDq|(W496V3gthog+!l-+gI^0x3>K~U0B9_I z@g1v9#%%cbQY(J<)|7{e%NhR$c6@0R)3;{wt|Y5hT-qAn?23((Ie*Is_;P_4Gx3j1 z3^!RMCcZ=O#~*wM_}}BBm6H6+W|(D1K9`SA_)O&v{7zZehxLm7tBQH}eC`H%|3AL+ zwv$WC=ZSiwBbOHn*aasRMW->jDp-wcQfvqt$sDPv&GGOq`KuGkd^o;c>O`@?JJE_` zdU788%6;TNa;;()znFK!uf=i(n|UXb!}$}T5F5S&N6!Fu`(`Au^2Zij=Z|V?HNBZ# z{Jg_J&>P3Qlh3>HhAVHIXs5)?*?J{TB9TPPY-Gp32p`^F3!lv=`TY2MT!#Dn_EX5YDwXjm4@%zo zyA%j0dpPZ8aUi>rp!dHqyG~d+l6Q>+x9T-*oC&4dQmFv;TYcH~Spj>DJ0esIt zzWNO+#A`{>E5i(Xk;Z0`sjgNLsQM^ePYfMu`tZTDpWqGSgiZetwnduxeT7P8ynTsi zel~9SC}kpn5&t6m<~Z?*-@e9Xw_7%@1cxGiwOUv!*ZAgV{^YpI;WyoHSsAi`#H6j9 zt$aSe;%xY&tQ7Q@%CCLw|GfH*c7B0V=63;TLHuy07aBFXpK@e@kz6>#YSGcv3{ghz zzVXF3=^Q@()T&z5KP7&Q>i!XZTNu&$kfkNQnO!8-_aDL+?R~C8sjF4t! z6x@c9tB)3F@nK85F<=By?G&Gi4}X@LiXJ2XmM&tvDMDVeZJcH{s6W+y1bgFn`9~ZXTFjEjziZ(}(o3vn z`%X>ZGshK%2W48h%Jnqix>9=bSGbGC-{Va~Hp{r_k-l2)R5e=9GXJFTue#GuTPtHLO_kpoE;{;<|N8ou=yCIP zN<{A~WY5T@7mLhsKlK)EER*b9LF?v{dT-&+=Hpvd_~PVB{13->Hs|DD_AU++MKR^? zVbs#s_)ceV^X6!`7vaB08NBAP@4xarcZzYI{jMLv_MN@||G4r!x9+?3(b^}k&qm0m zIJo%3!Mf<)XVROminu6NX7e>E)#+h2O$}L)eu$)~=3}XaGUgyZ_V8KMnK#)7zjPHp z_Ts=j%wK(OAJ%4maf|Pa51wLAKZDR6(r+-k<@J}An;-pDHxE9y+0Rj)g#6$aUwirP zX!kYxQ0mVy-QN2yL-92;)+QS*i|kvrv|fAPK+-?Jmin%y1ZS6N0LGw(w2!|y(vgZ*y#F}>^b>-1db)Nj=f;xC|Ft8@YI zMIq1nn~#0+?)d1{!hey9e+8a5izk@{Oplez2GHqrSUlSN&@^wrvVyP!giSlmuO%9r zW`jOGD83?gYTjdlCEZT%G_f_YKb`yp!)N?Qcc8y6-5c~LFW-9YpKRX@b^v?Vs?#fW z*DlT`JnOH$|Jl3C_q|fP=kqnu&(d`7^YSrkS5(VraZMu&zIv_2t3qXyto_-1d=_pk z^vbJk!~$p|XLVszAW2V_Pv+Y=r{jaEb~--#@C&o@YkYyT{(x!uak=@SdyXFer}KN5 zFTlMk$hvZOMZ0@2f4q3@#*LTjFKs?eK|fUioJEMtmjUO-<02&yOE|p|V-%X=6Xv@X(oCxjr1jf2;npdQ$tQM<2QW z=azp~pZ|S`@O0`r&8O4l#eLPLy7n@?{`u15<>(>(HP?sj)ax^gp0C0^Q@=iWK*f2c zD)fL#sXs~F-K&MVM;neWi6M8@tERwteOT%%cv{JMqtu2a&-F?ld~arKwAH@y=LKKw z#h-2EA?L&VSjQ(K-_mq$Dl8u&b4}hKRXUGo8jtD{dqj15STlZy(C<7sI)2CQ_~fnE k9@EG3{4s5ok?kb>|H;3ubeVRY^#A|>07*qoM6N<$f~C=$asU7T literal 0 HcmV?d00001 diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/profile/backup_config.json b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/profile/backup_config.json new file mode 100644 index 000000000..78f40ae7c --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/profile/backup_config.json @@ -0,0 +1,3 @@ +{ + "allowToBackupRestore": true +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/profile/main_pages.json b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/profile/main_pages.json new file mode 100644 index 000000000..1898d94f5 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/base/profile/main_pages.json @@ -0,0 +1,5 @@ +{ + "src": [ + "pages/Index" + ] +} diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/en_US/element/string.json b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/en_US/element/string.json new file mode 100644 index 000000000..d2ba01212 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/en_US/element/string.json @@ -0,0 +1,20 @@ +{ + "string": [ + { + "name": "module_desc", + "value": "On-device speaker identification with Next-gen Kaldi" + }, + { + "name": "EntryAbility_desc", + "value": "On-device speaker identification with Next-gen Kaldi" + }, + { + "name": "EntryAbility_label", + "value": "Speaker identification" + }, + { + "name": "mic_reason", + "value": "access the microphone for on-device speaker identification with Next-gen Kaldi" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/rawfile/.gitkeep b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/rawfile/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/zh_CN/element/string.json b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/zh_CN/element/string.json new file mode 100644 index 000000000..0a580accd --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/zh_CN/element/string.json @@ -0,0 +1,20 @@ +{ + "string": [ + { + "name": "module_desc", + "value": "新一代Kaldi: 本地说话人识别" + }, + { + "name": "EntryAbility_desc", + "value": "新一代Kaldi: 本地说话人识别" + }, + { + "name": "EntryAbility_label", + "value": "说话人识别" + }, + { + "name": "mic_reason", + "value": "使用新一代Kaldi, 访问麦克风进行本地说话人识别 (不需要联网)" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/ohosTest/ets/test/Ability.test.ets b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/ohosTest/ets/test/Ability.test.ets new file mode 100644 index 000000000..8aa374977 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/ohosTest/ets/test/Ability.test.ets @@ -0,0 +1,35 @@ +import hilog from '@ohos.hilog'; +import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from '@ohos/hypium'; + +export default function abilityTest() { + describe('ActsAbilityTest', () => { + // Defines a test suite. Two parameters are supported: test suite name and test suite function. + beforeAll(() => { + // Presets an action, which is performed only once before all test cases of the test suite start. + // This API supports only one parameter: preset action function. + }) + beforeEach(() => { + // Presets an action, which is performed before each unit test case starts. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: preset action function. + }) + afterEach(() => { + // Presets a clear action, which is performed after each unit test case ends. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: clear action function. + }) + afterAll(() => { + // Presets a clear action, which is performed after all test cases of the test suite end. + // This API supports only one parameter: clear action function. + }) + it('assertContain', 0, () => { + // Defines a test case. This API supports three parameters: test case name, filter parameter, and test case function. + hilog.info(0x0000, 'testTag', '%{public}s', 'it begin'); + let a = 'abc'; + let b = 'b'; + // Defines a variety of assertion methods, which are used to declare expected boolean conditions. + expect(a).assertContain(b); + expect(a).assertEqual(a); + }) + }) +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/ohosTest/ets/test/List.test.ets b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/ohosTest/ets/test/List.test.ets new file mode 100644 index 000000000..794c7dc4e --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/ohosTest/ets/test/List.test.ets @@ -0,0 +1,5 @@ +import abilityTest from './Ability.test'; + +export default function testsuite() { + abilityTest(); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/ohosTest/module.json5 b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/ohosTest/module.json5 new file mode 100644 index 000000000..55725a929 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/ohosTest/module.json5 @@ -0,0 +1,13 @@ +{ + "module": { + "name": "entry_test", + "type": "feature", + "deviceTypes": [ + "phone", + "tablet", + "2in1" + ], + "deliveryWithInstall": true, + "installationFree": false + } +} diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/test/List.test.ets b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/test/List.test.ets new file mode 100644 index 000000000..bb5b5c373 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/test/List.test.ets @@ -0,0 +1,5 @@ +import localUnitTest from './LocalUnit.test'; + +export default function testsuite() { + localUnitTest(); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/test/LocalUnit.test.ets b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/test/LocalUnit.test.ets new file mode 100644 index 000000000..165fc1615 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/test/LocalUnit.test.ets @@ -0,0 +1,33 @@ +import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from '@ohos/hypium'; + +export default function localUnitTest() { + describe('localUnitTest', () => { + // Defines a test suite. Two parameters are supported: test suite name and test suite function. + beforeAll(() => { + // Presets an action, which is performed only once before all test cases of the test suite start. + // This API supports only one parameter: preset action function. + }); + beforeEach(() => { + // Presets an action, which is performed before each unit test case starts. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: preset action function. + }); + afterEach(() => { + // Presets a clear action, which is performed after each unit test case ends. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: clear action function. + }); + afterAll(() => { + // Presets a clear action, which is performed after all test cases of the test suite end. + // This API supports only one parameter: clear action function. + }); + it('assertContain', 0, () => { + // Defines a test case. This API supports three parameters: test case name, filter parameter, and test case function. + let a = 'abc'; + let b = 'b'; + // Defines a variety of assertion methods, which are used to declare expected boolean conditions. + expect(a).assertContain(b); + expect(a).assertEqual(a); + }); + }); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/hvigor/hvigor-config.json5 b/harmony-os/SherpaOnnxSpeakerIdentification/hvigor/hvigor-config.json5 new file mode 100644 index 000000000..06b278367 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/hvigor/hvigor-config.json5 @@ -0,0 +1,22 @@ +{ + "modelVersion": "5.0.0", + "dependencies": { + }, + "execution": { + // "analyze": "normal", /* Define the build analyze mode. Value: [ "normal" | "advanced" | false ]. Default: "normal" */ + // "daemon": true, /* Enable daemon compilation. Value: [ true | false ]. Default: true */ + // "incremental": true, /* Enable incremental compilation. Value: [ true | false ]. Default: true */ + // "parallel": true, /* Enable parallel compilation. Value: [ true | false ]. Default: true */ + // "typeCheck": false, /* Enable typeCheck. Value: [ true | false ]. Default: false */ + }, + "logging": { + // "level": "info" /* Define the log level. Value: [ "debug" | "info" | "warn" | "error" ]. Default: "info" */ + }, + "debugging": { + // "stacktrace": false /* Disable stacktrace compilation. Value: [ true | false ]. Default: false */ + }, + "nodeOptions": { + // "maxOldSpaceSize": 8192 /* Enable nodeOptions maxOldSpaceSize compilation. Unit M. Used for the daemon process. Default: 8192*/ + // "exposeGC": true /* Enable to trigger garbage collection explicitly. Default: true*/ + } +} diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/hvigorfile.ts b/harmony-os/SherpaOnnxSpeakerIdentification/hvigorfile.ts new file mode 100644 index 000000000..f3cb9f1a8 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/hvigorfile.ts @@ -0,0 +1,6 @@ +import { appTasks } from '@ohos/hvigor-ohos-plugin'; + +export default { + system: appTasks, /* Built-in plugin of Hvigor. It cannot be modified. */ + plugins:[] /* Custom plugin to extend the functionality of Hvigor. */ +} diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/oh-package-lock.json5 b/harmony-os/SherpaOnnxSpeakerIdentification/oh-package-lock.json5 new file mode 100644 index 000000000..f538ae290 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/oh-package-lock.json5 @@ -0,0 +1,19 @@ +{ + "meta": { + "stableOrder": true + }, + "lockfileVersion": 3, + "ATTENTION": "THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.", + "specifiers": { + "@ohos/hypium@1.0.19": "@ohos/hypium@1.0.19" + }, + "packages": { + "@ohos/hypium@1.0.19": { + "name": "@ohos/hypium", + "version": "1.0.19", + "integrity": "sha512-cEjDgLFCm3cWZDeRXk7agBUkPqjWxUo6AQeiu0gEkb3J8ESqlduQLSIXeo3cCsm8U/asL7iKjF85ZyOuufAGSQ==", + "resolved": "https://ohpm.openharmony.cn/ohpm/@ohos/hypium/-/hypium-1.0.19.har", + "registryType": "ohpm" + } + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/oh-package.json5 b/harmony-os/SherpaOnnxSpeakerIdentification/oh-package.json5 new file mode 100644 index 000000000..a79d5300e --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/oh-package.json5 @@ -0,0 +1,9 @@ +{ + "modelVersion": "5.0.0", + "description": "Please describe the basic information.", + "dependencies": { + }, + "devDependencies": { + "@ohos/hypium": "1.0.19" + } +} diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/element/string.json b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/element/string.json index 8679134da..207a99982 100644 --- a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/element/string.json +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/element/string.json @@ -14,7 +14,7 @@ }, { "name": "mic_reason", - "value": "access the microhone for on-device real-time speech recognition with Next-gen Kaldi" + "value": "access the microphone for on-device real-time speech recognition with Next-gen Kaldi" } ] -} \ No newline at end of file +} diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/en_US/element/string.json b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/en_US/element/string.json index 8679134da..207a99982 100644 --- a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/en_US/element/string.json +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/en_US/element/string.json @@ -14,7 +14,7 @@ }, { "name": "mic_reason", - "value": "access the microhone for on-device real-time speech recognition with Next-gen Kaldi" + "value": "access the microphone for on-device real-time speech recognition with Next-gen Kaldi" } ] -} \ No newline at end of file +} diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/element/string.json b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/element/string.json index 652fac4cc..a47ca31cc 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/element/string.json +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/element/string.json @@ -14,7 +14,7 @@ }, { "name": "mic_reason", - "value": "access the microhone for on-device speech recognition with Next-gen Kaldi" + "value": "access the microphone for on-device speech recognition with Next-gen Kaldi" } ] -} \ No newline at end of file +} diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/en_US/element/string.json b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/en_US/element/string.json index 652fac4cc..a47ca31cc 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/en_US/element/string.json +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/en_US/element/string.json @@ -14,7 +14,7 @@ }, { "name": "mic_reason", - "value": "access the microhone for on-device speech recognition with Next-gen Kaldi" + "value": "access the microphone for on-device speech recognition with Next-gen Kaldi" } ] -} \ No newline at end of file +} From 1bae4085caa61012af123554858f14f8649091ea Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 10 Dec 2024 16:03:03 +0800 Subject: [PATCH 098/183] Add speaker diarization API for HarmonyOS. (#1609) --- .../SherpaOnnxHar/sherpa_onnx/Index.ets | 29 ++++---- .../cpp/non-streaming-speaker-diarization.cc | 30 ++++++++ .../main/cpp/types/libsherpa_onnx/Index.d.ts | 5 ++ .../sherpa_onnx/src/main/cpp/wave-writer.cc | 7 +- .../NonStreamingSpeakerDiarization.ets | 73 +++++++++++++++++++ .../ets/components/SpeakerIdentification.ets | 10 +-- .../src/main/ets/components/StreamingAsr.ets | 3 +- .../src/main/ets/components/Vad.ets | 6 +- .../lib/non-streaming-speaker-diarization.js | 2 +- sherpa-onnx/c-api/c-api.cc | 51 +++++++++++-- sherpa-onnx/c-api/c-api.h | 5 ++ .../csrc/offline-speaker-diarization-impl.cc | 24 +++++- .../csrc/offline-speaker-diarization-impl.h | 10 +-- ...ffline-speaker-diarization-pyannote-impl.h | 22 ++++-- .../csrc/offline-speaker-diarization.cc | 24 +++++- .../csrc/offline-speaker-diarization.h | 10 +-- ...ine-speaker-segmentation-pyannote-model.cc | 37 ++++++++-- ...line-speaker-segmentation-pyannote-model.h | 10 +-- 18 files changed, 279 insertions(+), 79 deletions(-) create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingSpeakerDiarization.ets diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets index 5132df5f1..16c6279e1 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets @@ -1,11 +1,6 @@ -export { - listRawfileDir, - readWave, - readWaveFromBinary, -} from "libsherpa_onnx.so"; +export { listRawfileDir, readWave, readWaveFromBinary, } from "libsherpa_onnx.so"; -export { - CircularBuffer, +export { CircularBuffer, SileroVadConfig, SpeechSegment, Vad, @@ -13,8 +8,7 @@ export { } from './src/main/ets/components/Vad'; -export { - Samples, +export { Samples, OfflineStream, FeatureConfig, OfflineTransducerModelConfig, @@ -31,8 +25,7 @@ export { OfflineRecognizer, } from './src/main/ets/components/NonStreamingAsr'; -export { - OnlineStream, +export { OnlineStream, OnlineTransducerModelConfig, OnlineParaformerModelConfig, OnlineZipformer2CtcModelConfig, @@ -43,8 +36,7 @@ export { OnlineRecognizer, } from './src/main/ets/components/StreamingAsr'; -export { - OfflineTtsVitsModelConfig, +export { OfflineTtsVitsModelConfig, OfflineTtsModelConfig, OfflineTtsConfig, OfflineTts, @@ -52,8 +44,15 @@ export { TtsInput, } from './src/main/ets/components/NonStreamingTts'; -export { - SpeakerEmbeddingExtractorConfig, +export { SpeakerEmbeddingExtractorConfig, SpeakerEmbeddingExtractor, SpeakerEmbeddingManager, } from './src/main/ets/components/SpeakerIdentification'; + +export { OfflineSpeakerSegmentationPyannoteModelConfig, + OfflineSpeakerSegmentationModelConfig, + OfflineSpeakerDiarizationConfig, + OfflineSpeakerDiarizationSegment, + OfflineSpeakerDiarization, + FastClusteringConfig, +} from './src/main/ets/components/NonStreamingSpeakerDiarization'; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-speaker-diarization.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-speaker-diarization.cc index a35f7924a..2cda40e76 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-speaker-diarization.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-speaker-diarization.cc @@ -101,6 +101,17 @@ static SherpaOnnxFastClusteringConfig GetFastClusteringConfig( static Napi::External CreateOfflineSpeakerDiarizationWrapper(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); + +#if __OHOS__ + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } +#else if (info.Length() != 1) { std::ostringstream os; os << "Expect only 1 argument. Given: " << info.Length(); @@ -109,6 +120,7 @@ CreateOfflineSpeakerDiarizationWrapper(const Napi::CallbackInfo &info) { return {}; } +#endif if (!info[0].IsObject()) { Napi::TypeError::New(env, "Expect an object as the argument") @@ -129,8 +141,18 @@ CreateOfflineSpeakerDiarizationWrapper(const Napi::CallbackInfo &info) { SHERPA_ONNX_ASSIGN_ATTR_FLOAT(min_duration_on, minDurationOn); SHERPA_ONNX_ASSIGN_ATTR_FLOAT(min_duration_off, minDurationOff); +#if __OHOS__ + std::unique_ptr + mgr(OH_ResourceManager_InitNativeResourceManager(env, info[1]), + &OH_ResourceManager_ReleaseNativeResourceManager); + + const SherpaOnnxOfflineSpeakerDiarization *sd = + SherpaOnnxCreateOfflineSpeakerDiarizationOHOS(&c, mgr.get()); +#else const SherpaOnnxOfflineSpeakerDiarization *sd = SherpaOnnxCreateOfflineSpeakerDiarization(&c); +#endif if (c.segmentation.pyannote.model) { delete[] c.segmentation.pyannote.model; @@ -224,9 +246,17 @@ static Napi::Array OfflineSpeakerDiarizationProcessWrapper( Napi::Float32Array samples = info[1].As(); +#if __OHOS__ + // Note(fangjun): For unknown reasons on HarmonyOS, we need to divide it by + // sizeof(float) here + const SherpaOnnxOfflineSpeakerDiarizationResult *r = + SherpaOnnxOfflineSpeakerDiarizationProcess( + sd, samples.Data(), samples.ElementLength() / sizeof(float)); +#else const SherpaOnnxOfflineSpeakerDiarizationResult *r = SherpaOnnxOfflineSpeakerDiarizationProcess(sd, samples.Data(), samples.ElementLength()); +#endif int32_t num_segments = SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments(r); diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts index d2b6d6ea4..f71e2f6ee 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts @@ -62,3 +62,8 @@ export const speakerEmbeddingManagerVerify: (handle: object, obj: {name: string, export const speakerEmbeddingManagerContains: (handle: object, name: string) => boolean; export const speakerEmbeddingManagerNumSpeakers: (handle: object) => number; export const speakerEmbeddingManagerGetAllSpeakers: (handle: object) => Array; + +export const createOfflineSpeakerDiarization: (config: object, mgr?: object) => object; +export const getOfflineSpeakerDiarizationSampleRate: (handle: object) => number; +export const offlineSpeakerDiarizationProcess: (handle: object, samples: Float32Array) => object; +export const offlineSpeakerDiarizationSetConfig: (handle: object, config: object) => void; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-writer.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-writer.cc index 3ade695a0..8f6d7bcab 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-writer.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-writer.cc @@ -67,10 +67,15 @@ static Napi::Boolean WriteWaveWrapper(const Napi::CallbackInfo &info) { Napi::Float32Array samples = obj.Get("samples").As(); int32_t sample_rate = obj.Get("sampleRate").As().Int32Value(); - +#if __OHOS__ + int32_t ok = SherpaOnnxWriteWave( + samples.Data(), samples.ElementLength() / sizeof(float), sample_rate, + info[0].As().Utf8Value().c_str()); +#else int32_t ok = SherpaOnnxWriteWave(samples.Data(), samples.ElementLength(), sample_rate, info[0].As().Utf8Value().c_str()); +#endif return Napi::Boolean::New(env, ok); } diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingSpeakerDiarization.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingSpeakerDiarization.ets new file mode 100644 index 000000000..c0dcd3af1 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingSpeakerDiarization.ets @@ -0,0 +1,73 @@ +import { + createOfflineSpeakerDiarization, + getOfflineSpeakerDiarizationSampleRate, + offlineSpeakerDiarizationProcess, + offlineSpeakerDiarizationSetConfig, +} from 'libsherpa_onnx.so'; + +import { SpeakerEmbeddingExtractorConfig } from './SpeakerIdentification'; + +export class OfflineSpeakerSegmentationPyannoteModelConfig { + public model: string = ''; +} + +export class OfflineSpeakerSegmentationModelConfig { + public pyannote: OfflineSpeakerSegmentationPyannoteModelConfig = new OfflineSpeakerSegmentationPyannoteModelConfig(); + public numThreads: number = 1; + public debug: boolean = false; + public provider: string = 'cpu'; +} + +export class FastClusteringConfig { + public numClusters: number = -1; + public threshold: number = 0.5; +} + +export class OfflineSpeakerDiarizationConfig { + public segmentation: OfflineSpeakerSegmentationModelConfig = new OfflineSpeakerSegmentationModelConfig(); + public embedding: SpeakerEmbeddingExtractorConfig = new SpeakerEmbeddingExtractorConfig(); + public clustering: FastClusteringConfig = new FastClusteringConfig(); + public minDurationOn: number = 0.2; + public minDurationOff: number = 0.5; +} + +export class OfflineSpeakerDiarizationSegment { + public start: number = 0; // in secondspublic end: number = 0; // in secondspublic speaker: number = + 0; // ID of the speaker; count from 0 +} + +export class OfflineSpeakerDiarization { + public config: OfflineSpeakerDiarizationConfig; + public sampleRate: number; + private handle: object; + + constructor(config: OfflineSpeakerDiarizationConfig, mgr?: object) { + this.handle = createOfflineSpeakerDiarization(config, mgr); + this.config = config; + + this.sampleRate = getOfflineSpeakerDiarizationSampleRate(this.handle); + } + + /** + * samples is a 1-d float32 array. Each element of the array should be + * in the range [-1, 1]. + * + * We assume its sample rate equals to this.sampleRate. + * + * Returns an array of object, where an object is + * + * { + * "start": start_time_in_seconds, + * "end": end_time_in_seconds, + * "speaker": an_integer, + * } + */ + process(samples: Float32Array): OfflineSpeakerDiarizationSegment { + return offlineSpeakerDiarizationProcess(this.handle, samples) as OfflineSpeakerDiarizationSegment; + } + + setConfig(config: OfflineSpeakerDiarizationConfig) { + offlineSpeakerDiarizationSetConfig(this.handle, config); + this.config.clustering = config.clustering; + } +} diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/SpeakerIdentification.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/SpeakerIdentification.ets index e490ab15f..50868dfb0 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/SpeakerIdentification.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/SpeakerIdentification.ets @@ -35,8 +35,7 @@ export class SpeakerEmbeddingExtractor { } createStream(): OnlineStream { - return new OnlineStream( - speakerEmbeddingExtractorCreateStream(this.handle)); + return new OnlineStream(speakerEmbeddingExtractorCreateStream(this.handle)); } isReady(stream: OnlineStream): boolean { @@ -44,8 +43,7 @@ export class SpeakerEmbeddingExtractor { } compute(stream: OnlineStream, enableExternalBuffer: boolean = true): Float32Array { - return speakerEmbeddingExtractorComputeEmbedding( - this.handle, stream.handle, enableExternalBuffer); + return speakerEmbeddingExtractorComputeEmbedding(this.handle, stream.handle, enableExternalBuffer); } } @@ -106,9 +104,7 @@ export class SpeakerEmbeddingManager { addMulti(speaker: SpeakerNameWithEmbeddingList): boolean { const c: SpeakerNameWithEmbeddingN = { - name: speaker.name, - vv: flatten(speaker.v), - n: speaker.v.length, + name: speaker.name, vv: flatten(speaker.v), n: speaker.v.length, }; return speakerEmbeddingManagerAddListFlattened(this.handle, c); } diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/StreamingAsr.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/StreamingAsr.ets index 3b2985771..f8b3c61e4 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/StreamingAsr.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/StreamingAsr.ets @@ -125,8 +125,7 @@ export class OnlineRecognizer { } getResult(stream: OnlineStream): OnlineRecognizerResult { - const jsonStr: string = - getOnlineStreamResultAsJson(this.handle, stream.handle); + const jsonStr: string = getOnlineStreamResultAsJson(this.handle, stream.handle); let o = JSON.parse(jsonStr) as OnlineRecognizerResultJson; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/Vad.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/Vad.ets index 8f1bf18d6..cae2cbf13 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/Vad.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/Vad.ets @@ -62,8 +62,7 @@ export class CircularBuffer { // return a float32 array get(startIndex: number, n: number, enableExternalBuffer: boolean = true): Float32Array { - return circularBufferGet( - this.handle, startIndex, n, enableExternalBuffer); + return circularBufferGet(this.handle, startIndex, n, enableExternalBuffer); } pop(n: number) { @@ -93,8 +92,7 @@ export class Vad { private handle: object; constructor(config: VadConfig, bufferSizeInSeconds?: number, mgr?: object) { - this.handle = - createVoiceActivityDetector(config, bufferSizeInSeconds, mgr); + this.handle = createVoiceActivityDetector(config, bufferSizeInSeconds, mgr); this.config = config; } diff --git a/scripts/node-addon-api/lib/non-streaming-speaker-diarization.js b/scripts/node-addon-api/lib/non-streaming-speaker-diarization.js index 8ec31ee10..37c4a7493 100644 --- a/scripts/node-addon-api/lib/non-streaming-speaker-diarization.js +++ b/scripts/node-addon-api/lib/non-streaming-speaker-diarization.js @@ -27,7 +27,7 @@ class OfflineSpeakerDiarization { } setConfig(config) { - addon.offlineSpeakerDiarizationSetConfig(config); + addon.offlineSpeakerDiarizationSetConfig(this.handle, config); this.config.clustering = config.clustering; } } diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index 7748b9fee..84b33eb0d 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -1784,8 +1784,8 @@ struct SherpaOnnxOfflineSpeakerDiarizationResult { sherpa_onnx::OfflineSpeakerDiarizationResult impl; }; -const SherpaOnnxOfflineSpeakerDiarization * -SherpaOnnxCreateOfflineSpeakerDiarization( +static sherpa_onnx::OfflineSpeakerDiarizationConfig +GetOfflineSpeakerDiarizationConfig( const SherpaOnnxOfflineSpeakerDiarizationConfig *config) { sherpa_onnx::OfflineSpeakerDiarizationConfig sd_config; @@ -1820,6 +1820,22 @@ SherpaOnnxCreateOfflineSpeakerDiarization( sd_config.min_duration_off = SHERPA_ONNX_OR(config->min_duration_off, 0.5); + if (sd_config.segmentation.debug || sd_config.embedding.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s\n", sd_config.ToString().c_str()); +#else + SHERPA_ONNX_LOGE("%s\n", sd_config.ToString().c_str()); +#endif + } + + return sd_config; +} + +const SherpaOnnxOfflineSpeakerDiarization * +SherpaOnnxCreateOfflineSpeakerDiarization( + const SherpaOnnxOfflineSpeakerDiarizationConfig *config) { + auto sd_config = GetOfflineSpeakerDiarizationConfig(config); + if (!sd_config.Validate()) { SHERPA_ONNX_LOGE("Errors in config"); return nullptr; @@ -1831,10 +1847,6 @@ SherpaOnnxCreateOfflineSpeakerDiarization( sd->impl = std::make_unique(sd_config); - if (sd_config.segmentation.debug || sd_config.embedding.debug) { - SHERPA_ONNX_LOGE("%s\n", sd_config.ToString().c_str()); - } - return sd; } @@ -2029,5 +2041,32 @@ SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTtsOHOS( } #endif // #if SHERPA_ONNX_ENABLE_TTS == 1 + // +#if SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION == 1 +const SherpaOnnxOfflineSpeakerDiarization * +SherpaOnnxCreateOfflineSpeakerDiarizationOHOS( + const SherpaOnnxOfflineSpeakerDiarizationConfig *config, + NativeResourceManager *mgr) { + if (!mgr) { + return SherpaOnnxCreateOfflineSpeakerDiarization(config); + } + + auto sd_config = GetOfflineSpeakerDiarizationConfig(config); + + if (!sd_config.Validate()) { + SHERPA_ONNX_LOGE("Errors in config"); + return nullptr; + } + + SherpaOnnxOfflineSpeakerDiarization *sd = + new SherpaOnnxOfflineSpeakerDiarization; + + sd->impl = + std::make_unique(mgr, sd_config); + + return sd; +} + +#endif // #if SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION == 1 #endif // #ifdef __OHOS__ diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index 111aae779..a781520ff 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -1577,6 +1577,11 @@ SHERPA_ONNX_API const SherpaOnnxSpeakerEmbeddingExtractor * SherpaOnnxCreateSpeakerEmbeddingExtractorOHOS( const SherpaOnnxSpeakerEmbeddingExtractorConfig *config, NativeResourceManager *mgr); + +SHERPA_ONNX_API const SherpaOnnxOfflineSpeakerDiarization * +SherpaOnnxCreateOfflineSpeakerDiarizationOHOS( + const SherpaOnnxOfflineSpeakerDiarizationConfig *config, + NativeResourceManager *mgr); #endif #if defined(__GNUC__) diff --git a/sherpa-onnx/csrc/offline-speaker-diarization-impl.cc b/sherpa-onnx/csrc/offline-speaker-diarization-impl.cc index 15c3a2eb4..c0af5c7c4 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization-impl.cc +++ b/sherpa-onnx/csrc/offline-speaker-diarization-impl.cc @@ -6,6 +6,15 @@ #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h" @@ -23,10 +32,10 @@ OfflineSpeakerDiarizationImpl::Create( return nullptr; } -#if __ANDROID_API__ >= 9 +template std::unique_ptr OfflineSpeakerDiarizationImpl::Create( - AAssetManager *mgr, const OfflineSpeakerDiarizationConfig &config) { + Manager *mgr, const OfflineSpeakerDiarizationConfig &config) { if (!config.segmentation.pyannote.model.empty()) { return std::make_unique(mgr, config); } @@ -35,6 +44,17 @@ OfflineSpeakerDiarizationImpl::Create( return nullptr; } + +#if __ANDROID_API__ >= 9 +template std::unique_ptr +OfflineSpeakerDiarizationImpl::Create( + AAssetManager *mgr, const OfflineSpeakerDiarizationConfig &config); +#endif + +#if __OHOS__ +template std::unique_ptr +OfflineSpeakerDiarizationImpl::Create( + NativeResourceManager *mgr, const OfflineSpeakerDiarizationConfig &config); #endif } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-speaker-diarization-impl.h b/sherpa-onnx/csrc/offline-speaker-diarization-impl.h index 41f0e1e2f..d2cbdebd2 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization-impl.h +++ b/sherpa-onnx/csrc/offline-speaker-diarization-impl.h @@ -8,11 +8,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/offline-speaker-diarization.h" namespace sherpa_onnx { @@ -21,10 +16,9 @@ class OfflineSpeakerDiarizationImpl { static std::unique_ptr Create( const OfflineSpeakerDiarizationConfig &config); -#if __ANDROID_API__ >= 9 + template static std::unique_ptr Create( - AAssetManager *mgr, const OfflineSpeakerDiarizationConfig &config); -#endif + Manager *mgr, const OfflineSpeakerDiarizationConfig &config); virtual ~OfflineSpeakerDiarizationImpl() = default; diff --git a/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h b/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h index 51d712eb8..e8228d47b 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h +++ b/sherpa-onnx/csrc/offline-speaker-diarization-pyannote-impl.h @@ -11,11 +11,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "Eigen/Dense" #include "sherpa-onnx/csrc/fast-clustering.h" #include "sherpa-onnx/csrc/math.h" @@ -71,16 +66,15 @@ class OfflineSpeakerDiarizationPyannoteImpl Init(); } -#if __ANDROID_API__ >= 9 + template OfflineSpeakerDiarizationPyannoteImpl( - AAssetManager *mgr, const OfflineSpeakerDiarizationConfig &config) + Manager *mgr, const OfflineSpeakerDiarizationConfig &config) : config_(config), segmentation_model_(mgr, config_.segmentation), embedding_extractor_(mgr, config_.embedding), clustering_(std::make_unique(config_.clustering)) { Init(); } -#endif int32_t SampleRate() const override { const auto &meta_data = segmentation_model_.GetModelMetaData(); @@ -213,8 +207,13 @@ class OfflineSpeakerDiarizationPyannoteImpl } } } else { +#if __OHOS__ + SHERPA_ONNX_LOGE( + "powerset_max_classes = %{public}d is currently not supported!", i); +#else SHERPA_ONNX_LOGE( "powerset_max_classes = %d is currently not supported!", i); +#endif SHERPA_ONNX_EXIT(-1); } } @@ -229,10 +228,17 @@ class OfflineSpeakerDiarizationPyannoteImpl int32_t window_shift = meta_data.window_shift; if (n <= 0) { +#if __OHOS__ + SHERPA_ONNX_LOGE( + "number of audio samples is %{public}d (<= 0). Please provide a " + "positive number", + n); +#else SHERPA_ONNX_LOGE( "number of audio samples is %d (<= 0). Please provide a positive " "number", n); +#endif return {}; } diff --git a/sherpa-onnx/csrc/offline-speaker-diarization.cc b/sherpa-onnx/csrc/offline-speaker-diarization.cc index a4b021b73..1e861ab88 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization.cc +++ b/sherpa-onnx/csrc/offline-speaker-diarization.cc @@ -7,6 +7,15 @@ #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/offline-speaker-diarization-impl.h" namespace sherpa_onnx { @@ -74,11 +83,10 @@ OfflineSpeakerDiarization::OfflineSpeakerDiarization( const OfflineSpeakerDiarizationConfig &config) : impl_(OfflineSpeakerDiarizationImpl::Create(config)) {} -#if __ANDROID_API__ >= 9 +template OfflineSpeakerDiarization::OfflineSpeakerDiarization( - AAssetManager *mgr, const OfflineSpeakerDiarizationConfig &config) + Manager *mgr, const OfflineSpeakerDiarizationConfig &config) : impl_(OfflineSpeakerDiarizationImpl::Create(mgr, config)) {} -#endif OfflineSpeakerDiarization::~OfflineSpeakerDiarization() = default; @@ -98,4 +106,14 @@ OfflineSpeakerDiarizationResult OfflineSpeakerDiarization::Process( return impl_->Process(audio, n, std::move(callback), callback_arg); } +#if __ANDROID_API__ >= 9 +template OfflineSpeakerDiarization::OfflineSpeakerDiarization( + AAssetManager *mgr, const OfflineSpeakerDiarizationConfig &config); +#endif + +#if __OHOS__ +template OfflineSpeakerDiarization::OfflineSpeakerDiarization( + NativeResourceManager *mgr, const OfflineSpeakerDiarizationConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-speaker-diarization.h b/sherpa-onnx/csrc/offline-speaker-diarization.h index 4a517fbb2..acbb6f524 100644 --- a/sherpa-onnx/csrc/offline-speaker-diarization.h +++ b/sherpa-onnx/csrc/offline-speaker-diarization.h @@ -9,11 +9,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/fast-clustering-config.h" #include "sherpa-onnx/csrc/offline-speaker-diarization-result.h" #include "sherpa-onnx/csrc/offline-speaker-segmentation-model-config.h" @@ -62,10 +57,9 @@ class OfflineSpeakerDiarization { explicit OfflineSpeakerDiarization( const OfflineSpeakerDiarizationConfig &config); -#if __ANDROID_API__ >= 9 - OfflineSpeakerDiarization(AAssetManager *mgr, + template + OfflineSpeakerDiarization(Manager *mgr, const OfflineSpeakerDiarizationConfig &config); -#endif ~OfflineSpeakerDiarization(); diff --git a/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.cc b/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.cc index e3768dcf4..093e871b4 100644 --- a/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.cc +++ b/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.cc @@ -8,6 +8,15 @@ #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/onnx-utils.h" #include "sherpa-onnx/csrc/session.h" @@ -24,8 +33,8 @@ class OfflineSpeakerSegmentationPyannoteModel::Impl { Init(buf.data(), buf.size()); } -#if __ANDROID_API__ >= 9 - Impl(AAssetManager *mgr, const OfflineSpeakerSegmentationModelConfig &config) + template + Impl(Manager *mgr, const OfflineSpeakerSegmentationModelConfig &config) : config_(config), env_(ORT_LOGGING_LEVEL_ERROR), sess_opts_(GetSessionOptions(config)), @@ -33,7 +42,6 @@ class OfflineSpeakerSegmentationPyannoteModel::Impl { auto buf = ReadFile(mgr, config_.pyannote.model); Init(buf.data(), buf.size()); } -#endif const OfflineSpeakerSegmentationPyannoteModelMetaData &GetModelMetaData() const { @@ -61,7 +69,11 @@ class OfflineSpeakerSegmentationPyannoteModel::Impl { if (config_.debug) { std::ostringstream os; PrintModelMetadata(os, meta_data); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s\n", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s\n", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; // used in the macro below @@ -103,12 +115,11 @@ OfflineSpeakerSegmentationPyannoteModel:: const OfflineSpeakerSegmentationModelConfig &config) : impl_(std::make_unique(config)) {} -#if __ANDROID_API__ >= 9 +template OfflineSpeakerSegmentationPyannoteModel:: OfflineSpeakerSegmentationPyannoteModel( - AAssetManager *mgr, const OfflineSpeakerSegmentationModelConfig &config) + Manager *mgr, const OfflineSpeakerSegmentationModelConfig &config) : impl_(std::make_unique(mgr, config)) {} -#endif OfflineSpeakerSegmentationPyannoteModel:: ~OfflineSpeakerSegmentationPyannoteModel() = default; @@ -123,4 +134,18 @@ Ort::Value OfflineSpeakerSegmentationPyannoteModel::Forward( return impl_->Forward(std::move(x)); } +#if __ANDROID_API__ >= 9 +template OfflineSpeakerSegmentationPyannoteModel:: + OfflineSpeakerSegmentationPyannoteModel( + AAssetManager *mgr, + const OfflineSpeakerSegmentationModelConfig &config); +#endif + +#if __OHOS__ +template OfflineSpeakerSegmentationPyannoteModel:: + OfflineSpeakerSegmentationPyannoteModel( + NativeResourceManager *mgr, + const OfflineSpeakerSegmentationModelConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.h b/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.h index 6b835763b..a3cc7ed3f 100644 --- a/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.h +++ b/sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.h @@ -6,11 +6,6 @@ #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/offline-speaker-segmentation-model-config.h" #include "sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model-meta-data.h" @@ -22,10 +17,9 @@ class OfflineSpeakerSegmentationPyannoteModel { explicit OfflineSpeakerSegmentationPyannoteModel( const OfflineSpeakerSegmentationModelConfig &config); -#if __ANDROID_API__ >= 9 + template OfflineSpeakerSegmentationPyannoteModel( - AAssetManager *mgr, const OfflineSpeakerSegmentationModelConfig &config); -#endif + Manager *mgr, const OfflineSpeakerSegmentationModelConfig &config); ~OfflineSpeakerSegmentationPyannoteModel(); From 914cbad6a3604fb1b585b083f409c6bddb40dc2a Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 10 Dec 2024 20:11:44 +0800 Subject: [PATCH 099/183] Add speaker diarization demo for HarmonyOS (#1610) --- .../cpp/non-streaming-speaker-diarization.cc | 170 ++++++++++++++- .../src/main/cpp/non-streaming-tts.cc | 7 +- .../main/cpp/types/libsherpa_onnx/Index.d.ts | 3 +- .../NonStreamingSpeakerDiarization.ets | 19 +- .../SherpaOnnxSpeakerDiarization/.gitignore | 12 ++ .../AppScope/app.json5 | 10 + .../resources/base/element/string.json | 8 + .../resources/base/media/app_icon.png | Bin 0 -> 2777 bytes .../build-profile.json5 | 40 ++++ .../code-linter.json5 | 20 ++ .../entry/.gitignore | 6 + .../entry/build-profile.json5 | 33 +++ .../entry/hvigorfile.ts | 6 + .../entry/obfuscation-rules.txt | 23 +++ .../entry/oh-package.json5 | 12 ++ .../main/ets/entryability/EntryAbility.ets | 43 ++++ .../entrybackupability/EntryBackupAbility.ets | 12 ++ .../entry/src/main/ets/pages/Index.ets | 194 ++++++++++++++++++ .../ets/workers/SpeakerDiarizationWorker.ets | 189 +++++++++++++++++ .../entry/src/main/module.json5 | 52 +++++ .../main/resources/base/element/color.json | 8 + .../main/resources/base/element/string.json | 16 ++ .../main/resources/base/media/background.png | Bin 0 -> 57364 bytes .../main/resources/base/media/foreground.png | Bin 0 -> 12430 bytes .../main/resources/base/media/icon_doc.svg | 2 + .../main/resources/base/media/icon_mic.svg | 2 + .../src/main/resources/base/media/info.svg | 1 + .../resources/base/media/layered_image.json | 7 + .../main/resources/base/media/startIcon.png | Bin 0 -> 20093 bytes .../resources/base/profile/backup_config.json | 3 + .../resources/base/profile/main_pages.json | 5 + .../main/resources/en_US/element/string.json | 16 ++ .../entry/src/main/resources/rawfile/.gitkeep | 0 .../main/resources/zh_CN/element/string.json | 16 ++ .../src/ohosTest/ets/test/Ability.test.ets | 35 ++++ .../entry/src/ohosTest/ets/test/List.test.ets | 5 + .../entry/src/ohosTest/module.json5 | 13 ++ .../entry/src/test/List.test.ets | 5 + .../entry/src/test/LocalUnit.test.ets | 33 +++ .../hvigor/hvigor-config.json5 | 22 ++ .../hvigorfile.ts | 6 + .../oh-package-lock.json5 | 19 ++ .../oh-package.json5 | 9 + sherpa-onnx/c-api/c-api.cc | 5 - sherpa-onnx/c-api/c-api.h | 4 +- 45 files changed, 1074 insertions(+), 17 deletions(-) create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/.gitignore create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/AppScope/app.json5 create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/AppScope/resources/base/element/string.json create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/AppScope/resources/base/media/app_icon.png create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/build-profile.json5 create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/code-linter.json5 create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/.gitignore create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/build-profile.json5 create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/hvigorfile.ts create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/obfuscation-rules.txt create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/ets/entryability/EntryAbility.ets create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/ets/pages/Index.ets create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/ets/workers/SpeakerDiarizationWorker.ets create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/module.json5 create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/element/color.json create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/element/string.json create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/media/background.png create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/media/foreground.png create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/media/icon_doc.svg create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/media/icon_mic.svg create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/media/info.svg create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/media/layered_image.json create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/media/startIcon.png create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/profile/backup_config.json create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/profile/main_pages.json create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/en_US/element/string.json create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/rawfile/.gitkeep create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/zh_CN/element/string.json create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/ohosTest/ets/test/Ability.test.ets create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/ohosTest/ets/test/List.test.ets create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/ohosTest/module.json5 create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/test/List.test.ets create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/entry/src/test/LocalUnit.test.ets create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/hvigor/hvigor-config.json5 create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/hvigorfile.ts create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/oh-package-lock.json5 create mode 100644 harmony-os/SherpaOnnxSpeakerDiarization/oh-package.json5 diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-speaker-diarization.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-speaker-diarization.cc index 2cda40e76..a6b1f302e 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-speaker-diarization.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-speaker-diarization.cc @@ -282,6 +282,170 @@ static Napi::Array OfflineSpeakerDiarizationProcessWrapper( return ans; } +struct SpeakerDiarizationCallbackData { + int32_t num_processed_chunks; + int32_t num_total_chunks; +}; + +// see +// https://github.com/nodejs/node-addon-examples/blob/main/src/6-threadsafe-function/typed_threadsafe_function/node-addon-api/clock.cc +static void InvokeJsCallback(Napi::Env env, Napi::Function callback, + Napi::Reference *context, + SpeakerDiarizationCallbackData *data) { + if (env != nullptr) { + if (callback != nullptr) { + Napi::Number num_processed_chunks = + Napi::Number::New(env, data->num_processed_chunks); + Napi::Number num_total_chunks = + Napi::Number::New(env, data->num_total_chunks); + + callback.Call(context->Value(), {num_processed_chunks, num_total_chunks}); + } + } + delete data; +} + +using TSFN = Napi::TypedThreadSafeFunction, + SpeakerDiarizationCallbackData, + InvokeJsCallback>; + +class SpeakerDiarizationProcessWorker : public Napi::AsyncWorker { + public: + SpeakerDiarizationProcessWorker(const Napi::Env &env, TSFN tsfn, + const SherpaOnnxOfflineSpeakerDiarization *sd, + std::vector samples) + : tsfn_(tsfn), + Napi::AsyncWorker{env, "SpeakerDiarizationProcessAsyncWorker"}, + deferred_(env), + sd_(sd), + samples_(std::move(samples)) {} + + Napi::Promise Promise() { return deferred_.Promise(); } + + protected: + void Execute() override { + auto callback = [](int32_t num_processed_chunks, int32_t num_total_chunks, + void *arg) -> int32_t { + auto _this = reinterpret_cast(arg); + + auto data = new SpeakerDiarizationCallbackData; + data->num_processed_chunks = num_processed_chunks; + data->num_total_chunks = num_total_chunks; + + _this->tsfn_.NonBlockingCall(data); + + return 0; + }; + + r_ = SherpaOnnxOfflineSpeakerDiarizationProcessWithCallback( + sd_, samples_.data(), samples_.size(), callback, this); + + tsfn_.Release(); + } + + void OnOK() override { + Napi::Env env = deferred_.Env(); + + int32_t num_segments = + SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments(r_); + + const SherpaOnnxOfflineSpeakerDiarizationSegment *segments = + SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime(r_); + + Napi::Array ans = Napi::Array::New(env, num_segments); + + for (int32_t i = 0; i != num_segments; ++i) { + Napi::Object obj = Napi::Object::New(env); + + obj.Set(Napi::String::New(env, "start"), segments[i].start); + obj.Set(Napi::String::New(env, "end"), segments[i].end); + obj.Set(Napi::String::New(env, "speaker"), segments[i].speaker); + + ans.Set(i, obj); + } + + SherpaOnnxOfflineSpeakerDiarizationDestroySegment(segments); + SherpaOnnxOfflineSpeakerDiarizationDestroyResult(r_); + + deferred_.Resolve(ans); + } + + private: + TSFN tsfn_; + Napi::Promise::Deferred deferred_; + const SherpaOnnxOfflineSpeakerDiarization *sd_; + std::vector samples_; + const SherpaOnnxOfflineSpeakerDiarizationResult *r_; +}; + +static Napi::Object OfflineSpeakerDiarizationProcessAsyncWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 3) { + std::ostringstream os; + os << "Expect only 3 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New( + env, "Argument 0 should be an offline speaker diarization pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + const SherpaOnnxOfflineSpeakerDiarization *sd = + info[0].As>().Data(); + + if (!info[1].IsTypedArray()) { + Napi::TypeError::New(env, "Argument 1 should be a typed array") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[2].IsFunction()) { + Napi::TypeError::New(env, "Argument 2 should be a function") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Function cb = info[2].As(); + + auto context = + new Napi::Reference(Napi::Persistent(info.This())); + + TSFN tsfn = TSFN::New( + env, + cb, // JavaScript function called asynchronously + "SpeakerDiarizationProcessAsyncFunc", // Name + 0, // Unlimited queue + 1, // Only one thread will use this initially + context, + [](Napi::Env, void *, Napi::Reference *ctx) { delete ctx; }); + + Napi::Float32Array samples = info[1].As(); + +#if __OHOS__ + int32_t num_samples = samples.ElementLength() / sizeof(float); +#else + int32_t num_samples = samples.ElementLength(); +#endif + std::vector v(num_samples); + std::copy(samples.Data(), samples.Data() + num_samples, v.begin()); + + SpeakerDiarizationProcessWorker *worker = + new SpeakerDiarizationProcessWorker(env, tsfn, sd, v); + worker->Queue(); + return worker->Promise(); +} + static void OfflineSpeakerDiarizationSetConfigWrapper( const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); @@ -313,7 +477,7 @@ static void OfflineSpeakerDiarizationSetConfigWrapper( return; } - Napi::Object o = info[0].As(); + Napi::Object o = info[1].As(); SherpaOnnxOfflineSpeakerDiarizationConfig c; memset(&c, 0, sizeof(c)); @@ -334,6 +498,10 @@ void InitNonStreamingSpeakerDiarization(Napi::Env env, Napi::Object exports) { Napi::String::New(env, "offlineSpeakerDiarizationProcess"), Napi::Function::New(env, OfflineSpeakerDiarizationProcessWrapper)); + exports.Set( + Napi::String::New(env, "offlineSpeakerDiarizationProcessAsync"), + Napi::Function::New(env, OfflineSpeakerDiarizationProcessAsyncWrapper)); + exports.Set( Napi::String::New(env, "offlineSpeakerDiarizationSetConfig"), Napi::Function::New(env, OfflineSpeakerDiarizationSetConfigWrapper)); diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc index ed1f3afb3..05e27846d 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc @@ -344,9 +344,9 @@ struct TtsCallbackData { // see // https://github.com/nodejs/node-addon-examples/blob/main/src/6-threadsafe-function/typed_threadsafe_function/node-addon-api/clock.cc -void InvokeJsCallback(Napi::Env env, Napi::Function callback, - Napi::Reference *context, - TtsCallbackData *data) { +static void InvokeJsCallback(Napi::Env env, Napi::Function callback, + Napi::Reference *context, + TtsCallbackData *data) { if (env != nullptr) { if (callback != nullptr) { Napi::ArrayBuffer arrayBuffer = @@ -580,7 +580,6 @@ static Napi::Object OfflineTtsGenerateAsyncWrapper( context, [](Napi::Env, void *, Napi::Reference *ctx) { delete ctx; }); - const SherpaOnnxGeneratedAudio *audio; TtsGenerateWorker *worker = new TtsGenerateWorker( env, tsfn, tts, text, speed, sid, enable_external_buffer); worker->Queue(); diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts index f71e2f6ee..7db410a4c 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts @@ -65,5 +65,6 @@ export const speakerEmbeddingManagerGetAllSpeakers: (handle: object) => Array object; export const getOfflineSpeakerDiarizationSampleRate: (handle: object) => number; -export const offlineSpeakerDiarizationProcess: (handle: object, samples: Float32Array) => object; +export const offlineSpeakerDiarizationProcess: (handle: object, input: object) => object; +export const offlineSpeakerDiarizationProcessAsync: (handle: object, input: object, callback: object) => object; export const offlineSpeakerDiarizationSetConfig: (handle: object, config: object) => void; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingSpeakerDiarization.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingSpeakerDiarization.ets index c0dcd3af1..176da87a5 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingSpeakerDiarization.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingSpeakerDiarization.ets @@ -2,6 +2,7 @@ import { createOfflineSpeakerDiarization, getOfflineSpeakerDiarizationSampleRate, offlineSpeakerDiarizationProcess, + offlineSpeakerDiarizationProcessAsync, offlineSpeakerDiarizationSetConfig, } from 'libsherpa_onnx.so'; @@ -32,8 +33,12 @@ export class OfflineSpeakerDiarizationConfig { } export class OfflineSpeakerDiarizationSegment { - public start: number = 0; // in secondspublic end: number = 0; // in secondspublic speaker: number = - 0; // ID of the speaker; count from 0 + // in seconds + public start: number = 0; + // in seconds + public end: number = 0; + // ID of the speaker; count from 0 + public speaker: number = 0; } export class OfflineSpeakerDiarization { @@ -62,8 +67,14 @@ export class OfflineSpeakerDiarization { * "speaker": an_integer, * } */ - process(samples: Float32Array): OfflineSpeakerDiarizationSegment { - return offlineSpeakerDiarizationProcess(this.handle, samples) as OfflineSpeakerDiarizationSegment; + process(samples: Float32Array): OfflineSpeakerDiarizationSegment[] { + return offlineSpeakerDiarizationProcess(this.handle, samples) as OfflineSpeakerDiarizationSegment[]; + } + + processAsync(samples: Float32Array, callback: (numProcessedChunks: number, + numTotalChunks: number) => void): Promise { + return offlineSpeakerDiarizationProcessAsync(this.handle, samples, + callback) as Promise; } setConfig(config: OfflineSpeakerDiarizationConfig) { diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/.gitignore b/harmony-os/SherpaOnnxSpeakerDiarization/.gitignore new file mode 100644 index 000000000..d2ff20141 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/.gitignore @@ -0,0 +1,12 @@ +/node_modules +/oh_modules +/local.properties +/.idea +**/build +/.hvigor +.cxx +/.clangd +/.clang-format +/.clang-tidy +**/.test +/.appanalyzer \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/AppScope/app.json5 b/harmony-os/SherpaOnnxSpeakerDiarization/AppScope/app.json5 new file mode 100644 index 000000000..93816310b --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/AppScope/app.json5 @@ -0,0 +1,10 @@ +{ + "app": { + "bundleName": "com.k2fsa.sherpa.onnx.speaker.diarization", + "vendor": "example", + "versionCode": 1000000, + "versionName": "1.0.0", + "icon": "$media:app_icon", + "label": "$string:app_name" + } +} diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/AppScope/resources/base/element/string.json b/harmony-os/SherpaOnnxSpeakerDiarization/AppScope/resources/base/element/string.json new file mode 100644 index 000000000..a9d8f0625 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/AppScope/resources/base/element/string.json @@ -0,0 +1,8 @@ +{ + "string": [ + { + "name": "app_name", + "value": "SherpaOnnxSpeakerDiarization" + } + ] +} diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/AppScope/resources/base/media/app_icon.png b/harmony-os/SherpaOnnxSpeakerDiarization/AppScope/resources/base/media/app_icon.png new file mode 100644 index 0000000000000000000000000000000000000000..a39445dc87828b76fed6d2ec470dd455c45319e3 GIT binary patch literal 2777 zcmV;~3MTc5P)9*YHQQH znh@I(s7WDIN`nJ+5@|<)iZcg=qN74U#DNnD1Se7u4fs(|1ivr?9ayP|B3iYCD$mfQ zCQ{S1n2)}^yxe#1J=_0pt-a1UPwQ^Z*?X_`Uu*sM+8<}X+baE^a`3seUF}?bEaiMO zrD`Qrd5@qw^epHZ>Df|p-qKBUEB%*?!m0{PHC6j|RplEgR~PkM5a^}N)Sfwi>W;Uz zdhwo_4HXBU%kRl^w@&7iKPx$e-n9%#IU!&oMI~iNsw0n19qSX;dS>I`G_G=WdcN9r z;_Rtv9XC<7kbL+HHxJ782T~pg05t)tf^>2vNJqfYt{YmqQDoBxkv+ra*BxxhcuK2v zm5%@Y)biQz)R8O%e=o%n${;ojY;EUP>`Qj6Cq)7GHm)C%2%^+hI;Z4T#a|oKIvshv z5H%!I+|I4PEXaXj04%ybsVolr%vhKnW7AEhC?eP!o1{y;8m2R#;}{6VZPc!+)ou0C zVWz$|1#2(|L5z%EYRxOzP+uLB>qYGuajX-<#^u;Kw&2uh&93)h>nHaFA%{&2PW=Nn zr?*a;gk3xvRhQIRa1de-!r(ss&?tRmZ=L2FMkhxI3lK6Jn<>5c*ID|@KU#^MCIo6> zpFA{|R(4fsBwHIW z9v!7G|7enadv4}~*8q_h%tD^j$7=PCnn0=dR0GKA(fgb9`2IRg6ksBIo+Gdw#|-3eSe=3tmDe zIqVN)tScM`0W#Z>2wc>~2Uv=3L)~D4gXqZtPQ8rifbYJqwkG>bv}95G7+};9Br?hF zWSa3b)X}z#79W9kukM%6-b_54WDJm~Ub=gsrJ0lz-8&lrQ7zfK1qzuZQkZvcE3|~S zZWmk0ETaNIHnMALn>akuvHLf5c4`y%!f+u>ZGp%@q_;T!`76_snc_?K;Wx%YpF;5K zw^F+BCYUPy`fpRif@5O@Im5cf?evD$>KlAgX;D0*HiO0`Yg3j;R4jT(9h(L_TsY6yxk*@ZBe%+dMqY=cB5oGs{D$QwOFbH)G$iVf<3Olcd7^#fr- zM{!ILWt#coT)s9ySkwDCPHv0oww8g8K%Yr{aR}msELVX(}JQr%F4Q8=KKn*OjSO*uSp;JK%GwhRF_K??vGC$ZqmJX z@+}8sQ)9Z}3*DiWl+L_7OXn_^{SW~2&C*b^;%IP!j$lkre7H&bMR1}7aTT*G8P}|G zHM1)hZDe{r_E3{{Y=d}}_PxJO_w4MaE4)$<<3JwzPdwPzfNemK(-X;{UCzmVr0zu5 zEnT}fzx)oVd!*W77`1Ig`DFcZ6TkPaI$hO1+`cGb$({ukz&{p4Ic-Xnwrg-KEkDqW zW3l$7Q`V$!1T(=QL1jgjIachdr75>-8>1A^h+;rTrD^nnwf?bw(Rang!*16Odj$Pn z@)JN5&5w~}ae6d};oa|&G>sT!)ixE#5;QW(u(=bqYHXcOflE%@t4A?n5fTUm0F~8_ zwpoz9rrU`@G=vsNjDRY(CrF(jIjqg8bd|CP02>eFag7T?u;C^ir+Z7YKmBYw;%%XdT2T}a$X4yR7EI;zaof3a)5Z;`OwVi%D?gbkBj!{;z2tOBSFk&E1DeiZXD**uvNqL}+|pO{ ztO$}2NMRit2ddU?)7Prq&*&H3X>&=E{-+j4iUz zrvL;?0$^@lyl=LHz9G^$SJV6ID__@7z->Bh>Vm=6AK&5bP%@heveHja5F@agGgUsY z@L@W2+^*NVoId0!kS~4XkWb%y;f}XBf>S+NIw9aHK;vN+4mJ|em)_QjIVfb2$;bwv zDKmoq6AThgKydS6Hs+UpKPWq|UA}s=UOEBZNM3oNT5qTAabY)X>L6jxfGDuu7&GD_ z=@@m?sJ-o2GS}&hNRW}-zHkr>o4&138@a8IC-FjSBxzjx?(*3@YmdmWGAd%0QvXzS zJ53JpX%Fp!=>v&`Hd7F@+Atw2vx9%^2M-APg0Jd|ePsRn3*B$#9Z5hCou4fo7W#SN z#}-@-N=##yQDh26pNzr9f*Q88krhI5@DHcf{dU-~PLSs}MvI4s1i|<=qxD~9`7>*~ znlw5lr$_6mTG4XbBNF_79BzvZ!TeIP)exdk3)kSHjYdW1P10ZJ_NCJSlrCuIU#gqw f88(SSw!Z%ZUzhC#9QlKF00000NkvXXu0mjfG$}gK literal 0 HcmV?d00001 diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/build-profile.json5 b/harmony-os/SherpaOnnxSpeakerDiarization/build-profile.json5 new file mode 100644 index 000000000..8e63d9768 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/build-profile.json5 @@ -0,0 +1,40 @@ +{ + "app": { + "signingConfigs": [], + "products": [ + { + "name": "default", + "signingConfig": "default", + "compatibleSdkVersion": "4.0.0(10)", + "runtimeOS": "HarmonyOS", + "buildOption": { + "strictMode": { + "caseSensitiveCheck": true, + } + } + } + ], + "buildModeSet": [ + { + "name": "debug", + }, + { + "name": "release" + } + ] + }, + "modules": [ + { + "name": "entry", + "srcPath": "./entry", + "targets": [ + { + "name": "default", + "applyToProducts": [ + "default" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/code-linter.json5 b/harmony-os/SherpaOnnxSpeakerDiarization/code-linter.json5 new file mode 100644 index 000000000..77b31b517 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/code-linter.json5 @@ -0,0 +1,20 @@ +{ + "files": [ + "**/*.ets" + ], + "ignore": [ + "**/src/ohosTest/**/*", + "**/src/test/**/*", + "**/src/mock/**/*", + "**/node_modules/**/*", + "**/oh_modules/**/*", + "**/build/**/*", + "**/.preview/**/*" + ], + "ruleSet": [ + "plugin:@performance/recommended", + "plugin:@typescript-eslint/recommended" + ], + "rules": { + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/.gitignore b/harmony-os/SherpaOnnxSpeakerDiarization/entry/.gitignore new file mode 100644 index 000000000..e2713a277 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/.gitignore @@ -0,0 +1,6 @@ +/node_modules +/oh_modules +/.preview +/build +/.cxx +/.test \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/build-profile.json5 b/harmony-os/SherpaOnnxSpeakerDiarization/entry/build-profile.json5 new file mode 100644 index 000000000..cb3e674c9 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/build-profile.json5 @@ -0,0 +1,33 @@ +{ + "apiType": "stageMode", + "buildOption": { + "sourceOption": { + "workers": [ + './src/main/ets/workers/SpeakerDiarizationWorker.ets' + ] + } + }, + "buildOptionSet": [ + { + "name": "release", + "arkOptions": { + "obfuscation": { + "ruleOptions": { + "enable": false, + "files": [ + "./obfuscation-rules.txt" + ] + } + } + } + }, + ], + "targets": [ + { + "name": "default" + }, + { + "name": "ohosTest", + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/hvigorfile.ts b/harmony-os/SherpaOnnxSpeakerDiarization/entry/hvigorfile.ts new file mode 100644 index 000000000..c6edcd904 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/hvigorfile.ts @@ -0,0 +1,6 @@ +import { hapTasks } from '@ohos/hvigor-ohos-plugin'; + +export default { + system: hapTasks, /* Built-in plugin of Hvigor. It cannot be modified. */ + plugins:[] /* Custom plugin to extend the functionality of Hvigor. */ +} diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/obfuscation-rules.txt b/harmony-os/SherpaOnnxSpeakerDiarization/entry/obfuscation-rules.txt new file mode 100644 index 000000000..272efb6ca --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/obfuscation-rules.txt @@ -0,0 +1,23 @@ +# Define project specific obfuscation rules here. +# You can include the obfuscation configuration files in the current module's build-profile.json5. +# +# For more details, see +# https://developer.huawei.com/consumer/cn/doc/harmonyos-guides-V5/source-obfuscation-V5 + +# Obfuscation options: +# -disable-obfuscation: disable all obfuscations +# -enable-property-obfuscation: obfuscate the property names +# -enable-toplevel-obfuscation: obfuscate the names in the global scope +# -compact: remove unnecessary blank spaces and all line feeds +# -remove-log: remove all console.* statements +# -print-namecache: print the name cache that contains the mapping from the old names to new names +# -apply-namecache: reuse the given cache file + +# Keep options: +# -keep-property-name: specifies property names that you want to keep +# -keep-global-name: specifies names that you want to keep in the global scope + +-enable-property-obfuscation +-enable-toplevel-obfuscation +-enable-filename-obfuscation +-enable-export-obfuscation \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 b/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 new file mode 100644 index 000000000..97448bbf3 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 @@ -0,0 +1,12 @@ +{ + "name": "entry", + "version": "1.0.0", + "description": "Please describe the basic information.", + "main": "", + "author": "", + "license": "", + "dependencies": { + "sherpa_onnx": "1.10.33" + } +} + diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/ets/entryability/EntryAbility.ets b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/ets/entryability/EntryAbility.ets new file mode 100644 index 000000000..679d91453 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/ets/entryability/EntryAbility.ets @@ -0,0 +1,43 @@ +import AbilityConstant from '@ohos.app.ability.AbilityConstant'; +import hilog from '@ohos.hilog'; +import UIAbility from '@ohos.app.ability.UIAbility'; +import Want from '@ohos.app.ability.Want'; +import window from '@ohos.window'; + +export default class EntryAbility extends UIAbility { + onCreate(want: Want, launchParam: AbilityConstant.LaunchParam): void { + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onCreate'); + } + + onDestroy(): void { + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onDestroy'); + } + + onWindowStageCreate(windowStage: window.WindowStage): void { + // Main window is created, set main page for this ability + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onWindowStageCreate'); + + windowStage.loadContent('pages/Index', (err) => { + if (err.code) { + hilog.error(0x0000, 'testTag', 'Failed to load the content. Cause: %{public}s', JSON.stringify(err) ?? ''); + return; + } + hilog.info(0x0000, 'testTag', 'Succeeded in loading the content.'); + }); + } + + onWindowStageDestroy(): void { + // Main window is destroyed, release UI related resources + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onWindowStageDestroy'); + } + + onForeground(): void { + // Ability has brought to foreground + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onForeground'); + } + + onBackground(): void { + // Ability has back to background + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onBackground'); + } +} diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets new file mode 100644 index 000000000..d2c48b421 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets @@ -0,0 +1,12 @@ +import hilog from '@ohos.hilog'; +import BackupExtensionAbility, { BundleVersion } from '@ohos.application.BackupExtensionAbility'; + +export default class EntryBackupAbility extends BackupExtensionAbility { + async onBackup() { + hilog.info(0x0000, 'testTag', 'onBackup ok'); + } + + async onRestore(bundleVersion: BundleVersion) { + hilog.info(0x0000, 'testTag', 'onRestore ok %{public}s', JSON.stringify(bundleVersion)); + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/ets/pages/Index.ets b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/ets/pages/Index.ets new file mode 100644 index 000000000..f6fc537b4 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/ets/pages/Index.ets @@ -0,0 +1,194 @@ +import { LengthUnit, promptAction } from '@kit.ArkUI'; +import worker, { MessageEvents } from '@ohos.worker'; +import { BusinessError, pasteboard } from '@kit.BasicServicesKit'; +import { picker } from '@kit.CoreFileKit'; + + +@Entry +@Component +struct Index { + @State title: string = 'Next-gen Kaldi: Speaker Diarization'; + @State titleFontSize: number = 15; + @State currentIndex: number = 0; + @State resultForFile: string = ''; + @State resultForMic: string = ''; + @State progressForFile: number = 0; + @State selectFileBtnEnabled: boolean = false; + @State copyBtnForFileEnabled: boolean = false; + private controller: TabsController = new TabsController(); + private workerInstance?: worker.ThreadWorker + private readonly scriptURL: string = 'entry/ets/workers/SpeakerDiarizationWorker.ets' + private numSpeakers: string = '-1'; + + @Builder + TabBuilder(title: string, targetIndex: number, selectedImg: Resource, normalImg: Resource) { + Column() { + Image(this.currentIndex == targetIndex ? selectedImg : normalImg).size({ width: 25, height: 25 }) + Text(title).fontColor(this.currentIndex == targetIndex ? '#28bff1' : '#8a8a8a') + }.width('100%').height(50).justifyContent(FlexAlign.Center).onClick(() => { + this.currentIndex = targetIndex; + this.controller.changeIndex(this.currentIndex); + }) + } + + aboutToAppear(): void { + this.workerInstance = new worker.ThreadWorker(this.scriptURL, { + name: 'Streaming ASR worker' + }); + + this.workerInstance.onmessage = (e: MessageEvents) => { + const msgType = e.data['msgType'] as string; + + if (msgType != 'speaker-diarization-file-progress') { + console.log(`received msg from worker: ${msgType}`); + } + + if (msgType == 'init-speaker-diarization-done') { + console.log('Speaker diarization initialized successfully'); + + this.resultForFile = 'Initialization finished.\nPlease select a .wav file.'; + this.resultForMic = 'Initialization finished.\nPlease click the button Start recording.'; + + this.selectFileBtnEnabled = true; + } + + if (msgType == 'speaker-diarization-file-progress') { + this.progressForFile = e.data['progress'] as number; + } + + if (msgType == 'speaker-diarization-file-done') { + const result = e.data['result'] as string; + this.resultForFile = result; + + this.selectFileBtnEnabled = true; + this.copyBtnForFileEnabled = true; + } + }; + + const context = getContext(); + this.workerInstance.postMessage({ msgType: 'init-speaker-diarization', context }); + console.log('initializing'); + this.resultForFile = 'Initializing models. Please wait'; + this.resultForMic = this.resultForFile; + } + + build() { + Column() { + Tabs({ barPosition: BarPosition.End, controller: this.controller }) { + TabContent() { + Column({ space: 10 }) { + Text(this.title).fontSize(this.titleFontSize).fontWeight(FontWeight.Bold); + Row({ space: 10 }) { + Text(`Number of speakers`).width('60%') + + TextInput({ text: this.numSpeakers }).onChange((text) => { + this.numSpeakers = text.trim(); + }).width('20%') + }.justifyContent(FlexAlign.Center) + + Row({ space: 10 }) { + Button('Select .wav file (16kHz) ').enabled(this.selectFileBtnEnabled).onClick(() => { + this.resultForFile = ''; + this.progressForFile = 0; + this.copyBtnForFileEnabled = false; + + let numSpeakers = parseInt(this.numSpeakers); + if (numSpeakers.toString() != this.numSpeakers) { + this.resultForFile = + 'Please input a valid value for the number of speakers in the .wav file you are going to select'; + return; + } + + if (numSpeakers < 1) { + this.resultForFile = + 'Please input a positive value for the number of speakers in the .wav file you are going to select'; + return; + } + + this.selectFileBtnEnabled = false; + + const documentSelectOptions = new picker.DocumentSelectOptions(); + documentSelectOptions.maxSelectNumber = 1; + documentSelectOptions.fileSuffixFilters = ['.wav']; + const documentViewPicker = new picker.DocumentViewPicker(); + + documentViewPicker.select(documentSelectOptions).then((result: Array) => { + console.log(`select file result: ${result}`); + + if (!result[0]) { + this.resultForFile = 'Please select a file to decode'; + this.selectFileBtnEnabled = true; + return; + } + + if (this.workerInstance) { + this.workerInstance.postMessage({ + msgType: 'speaker-diarization-file', filename: result[0], numSpeakers, + }); + this.resultForFile = `Decoding ${result[0]} ... ...`; + } else { + console.log(`this worker instance is undefined ${this.workerInstance}`); + } + }).catch((err: BusinessError) => { + console.error(`Failed to select file, code is ${err.code}, message is ${err.message}`); + this.selectFileBtnEnabled = true; + }) + }) + Button('Copy results') + .enabled(this.copyBtnForFileEnabled) + .onClick(() => { // See https://developer.huawei.com/consumer/cn/doc/harmonyos-faqs/faqs-arkui-308-V5 + const pasteboardData = pasteboard.createData(pasteboard.MIMETYPE_TEXT_PLAIN, this.resultForFile); + const systemPasteboard = pasteboard.getSystemPasteboard(); + systemPasteboard.setData(pasteboardData); + systemPasteboard.getData().then((data) => { + if (data) { + promptAction.showToast({ message: 'Result copied.' }); + } else { + promptAction.showToast({ message: 'Failed to copy' }); + } + }) + }) + } + + if (this.progressForFile > 0) { + Row() { + Progress({ value: 0, total: 100, type: ProgressType.Capsule }) + .width('80%') + .height(20) + .value(this.progressForFile); + + Text(`${this.progressForFile.toFixed(2)}%`).width('15%') + }.width('100%').justifyContent(FlexAlign.Center) + } + + TextArea({ text: this.resultForFile }) + .lineSpacing({ value: 10, unit: LengthUnit.VP }) + .width('100%') + .height('100%') + } + }.tabBar(this.TabBuilder('From file', 0, $r('app.media.icon_doc'), $r('app.media.icon_doc'))) + + TabContent() { + Column({ space: 10 }) { + Text(this.title).fontSize(this.titleFontSize).fontWeight(FontWeight.Bold); + TextArea({ + text: ` +Everyting is open-sourced. + +It runs locally, without accessing the network + +See also https://github.com/k2-fsa/sherpa-onnx + +新一代 Kaldi QQ 和微信交流群: 请看 + +https://k2-fsa.github.io/sherpa/social-groups.html + +微信公众号: 新一代 Kaldi + ` + }).width('100%').height('100%').focusable(false) + }.justifyContent(FlexAlign.Start) + }.tabBar(this.TabBuilder('Help', 1, $r('app.media.info'), $r('app.media.info'))) + }.scrollable(false) + } + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/ets/workers/SpeakerDiarizationWorker.ets b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/ets/workers/SpeakerDiarizationWorker.ets new file mode 100644 index 000000000..4a297ec87 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/ets/workers/SpeakerDiarizationWorker.ets @@ -0,0 +1,189 @@ +import worker, { ErrorEvent, MessageEvents, ThreadWorkerGlobalScope } from '@ohos.worker'; +import { + OfflineSpeakerDiarization, + OfflineSpeakerDiarizationConfig, + OfflineSpeakerDiarizationSegment, + readWaveFromBinary, + Samples +} from 'sherpa_onnx'; +import { fileIo } from '@kit.CoreFileKit'; + +const workerPort: ThreadWorkerGlobalScope = worker.workerPort; + +let sd: OfflineSpeakerDiarization; +let useAsync: boolean = true; + +function readWave(filename: string): Samples { + const fp = fileIo.openSync(filename); + const stat = fileIo.statSync(fp.fd); + const arrayBuffer = new ArrayBuffer(stat.size); + fileIo.readSync(fp.fd, arrayBuffer); + const data: Uint8Array = new Uint8Array(arrayBuffer); + return readWaveFromBinary(data) as Samples; +} + +function initOfflineSpeakerDiarization(context: Context): OfflineSpeakerDiarization { + const config: OfflineSpeakerDiarizationConfig = new OfflineSpeakerDiarizationConfig(); + + // Please refer to https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-segmentation-models + // to download models. + // Make sure you have placed it inside the directory + // harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/rawfile + // + // Also, please delete unused files to reduce the size of the app + config.segmentation.pyannote.model = 'sherpa-onnx-pyannote-segmentation-3-0/model.int8.onnx'; + config.segmentation.numThreads = 2; + config.segmentation.debug = true; + + // Please refer to https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models + // to download models. + // Make sure you have placed it inside the directory + // harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/rawfile + config.embedding.model = '3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx'; + config.embedding.numThreads = 2; + config.embedding.debug = true; + + config.minDurationOn = 0.2; + config.minDurationOff = 0.5; + return new OfflineSpeakerDiarization(config, context.resourceManager); + + // For the above two models files, you should have the following directory structure + /* + (py38) fangjuns-MacBook-Pro:rawfile fangjun$ pwd + /Users/fangjun/open-source/sherpa-onnx/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/rawfile + (py38) fangjuns-MacBook-Pro:rawfile fangjun$ ls -lh + total 77336 + -rw-r--r-- 1 fangjun staff 38M Dec 10 16:28 3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx + drwxr-xr-x 3 fangjun staff 96B Dec 10 19:36 sherpa-onnx-pyannote-segmentation-3-0 + (py38) fangjuns-MacBook-Pro:rawfile fangjun$ tree . + . + ├── 3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx + └── sherpa-onnx-pyannote-segmentation-3-0 + └── model.int8.onnx + + 1 directory, 2 files + + (Note that we have kept only model.int8.onnx and removed all other files + from sherpa-onnx-pyannote-segmentation-3-0 + ) + */ +} + +/** + * Defines the event handler to be called when the worker thread receives a message sent by the host thread. + * The event handler is executed in the worker thread. + * + * @param e message data + */ +workerPort.onmessage = (e: MessageEvents) => { + const msgType = e.data['msgType'] as string; + + console.log(`from the main thread, msg-type: ${msgType}`); + if (msgType == 'init-speaker-diarization' && !sd) { + const context: Context = e.data['context'] as Context; + sd = initOfflineSpeakerDiarization(context); + workerPort.postMessage({ msgType: 'init-speaker-diarization-done' }); + console.log('Init sd done'); + } + + if (msgType == 'speaker-diarization-file') { + const filename = e.data['filename'] as string; + const numSpeakers = e.data['numSpeakers'] as number; + const wave = readWave(filename); + let result = ''; + if (wave == undefined || wave == null) { + result = `Failed to read ${filename}`; + + workerPort.postMessage({ + msgType: 'speaker-diarization-file-done', result + }); + return; + } + + if (wave.sampleRate != sd.sampleRate) { + result = `Expected sample rate: ${sd.sampleRate}`; + result += '\n'; + result += `Sample rate in file ${filename} is ${wave.sampleRate}`; + + workerPort.postMessage({ + msgType: 'speaker-diarization-file-done', result + }); + + return; + } + + const duration = wave.samples.length / wave.sampleRate; + console.log(`Processing ${filename} of ${duration} seconds`); + + // You can remove this if statement if you want + if (duration < 0.3) { + result = `${filename} has only ${duration} seconds. Please use a longer file`; + + workerPort.postMessage({ + msgType: 'speaker-diarization-file-done', result + }); + return; + } + sd.config.clustering.numClusters = numSpeakers; + sd.setConfig(sd.config); + + if (useAsync) { + sd.processAsync(wave.samples, (numProcessedChunks: number, numTotalChunks: number) => { + const progress = numProcessedChunks / numTotalChunks * 100; + workerPort.postMessage({ + msgType: 'speaker-diarization-file-progress', progress + }); + }).then((r: OfflineSpeakerDiarizationSegment[]) => { + console.log(`r is ${r.length}, ${r}`); + + for (const s of r) { + const start: string = s.start.toFixed(3); + const end: string = s.end.toFixed(3); + result += `${start}\t--\t${end}\tspeaker_${s.speaker}\n`; + console.log(`result: ${result}`); + } + + if (r.length == 0) { + result = 'The result is empty'; + } + + workerPort.postMessage({ + msgType: 'speaker-diarization-file-done', result + }); + }); + } else { + const r: OfflineSpeakerDiarizationSegment[] = sd.process(wave.samples) + console.log(`r is ${r.length}, ${r}`); + for (const s of r) { + const start: string = s.start.toFixed(3); + const end: string = s.end.toFixed(3); + result += `${start}\t--\t${end}\tspeaker_${s.speaker}\n`; + console.log(`result: ${result}`); + } + + if (r.length == 0) { + result = 'The result is empty'; + } + + workerPort.postMessage({ + msgType: 'speaker-diarization-file-done', result + }); + } + } +} /** + * Defines the event handler to be called when the worker receives a message that cannot be deserialized. + * The event handler is executed in the worker thread. + * + * @param e message data + */ +workerPort.onmessageerror = (e: MessageEvents) => { +} + +/** + * Defines the event handler to be called when an exception occurs during worker execution. + * The event handler is executed in the worker thread. + * + * @param e error message + */ +workerPort.onerror = (e: ErrorEvent) => { +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/module.json5 b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/module.json5 new file mode 100644 index 000000000..a1cea8b6a --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/module.json5 @@ -0,0 +1,52 @@ +{ + "module": { + "name": "entry", + "type": "entry", + "description": "$string:module_desc", + "mainElement": "EntryAbility", + "deviceTypes": [ + "phone", + "tablet", + "2in1" + ], + "deliveryWithInstall": true, + "installationFree": false, + "pages": "$profile:main_pages", + "abilities": [ + { + "name": "EntryAbility", + "srcEntry": "./ets/entryability/EntryAbility.ets", + "description": "$string:EntryAbility_desc", + "icon": "$media:layered_image", + "label": "$string:EntryAbility_label", + "startWindowIcon": "$media:startIcon", + "startWindowBackground": "$color:start_window_background", + "exported": true, + "skills": [ + { + "entities": [ + "entity.system.home" + ], + "actions": [ + "action.system.home" + ] + } + ] + } + ], + "extensionAbilities": [ + { + "name": "EntryBackupAbility", + "srcEntry": "./ets/entrybackupability/EntryBackupAbility.ets", + "type": "backup", + "exported": false, + "metadata": [ + { + "name": "ohos.extension.backup", + "resource": "$profile:backup_config" + } + ], + } + ] + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/element/color.json b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/element/color.json new file mode 100644 index 000000000..3c712962d --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/element/color.json @@ -0,0 +1,8 @@ +{ + "color": [ + { + "name": "start_window_background", + "value": "#FFFFFF" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/element/string.json b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/element/string.json new file mode 100644 index 000000000..55c8939f3 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/element/string.json @@ -0,0 +1,16 @@ +{ + "string": [ + { + "name": "module_desc", + "value": "On-device speaker diarization with Next-gen Kaldi" + }, + { + "name": "EntryAbility_desc", + "value": "On-device speaker diarization with Next-gen Kaldi" + }, + { + "name": "EntryAbility_label", + "value": "Speaker diarization" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/media/background.png b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/media/background.png new file mode 100644 index 0000000000000000000000000000000000000000..f939c9fa8cc8914832e602198745f592a0dfa34d GIT binary patch literal 57364 zcmYIuc|6qL_rIk#Su&MMQlYU)cz{|$Qc0x~A^BEf( z`{n=HaSk>%wsfNM*uUkN^8dI{qxxW z*@b_`#>VlLWSG9 z0>QdPQ-&i_RCVdp2s$-u%S362^SHV0`EO6;@n(xK));G>#qwhPWrDXGk@OBMV}H!J za!48&`xhWJKj{_+f3ir<>Jg6Ax<&Xgn;)U7UJyAw{(u?zlf{oLsJTS-_o1?+lSg-j z8fcZj1*Ad(!X>WuuxM!H5t@V3*8vLL6`QnC!q!BwQjI{yk*;~@|3;B)`p$WYcDmnZ zt`R zr=oS6o-D$WZsYKh1PiOdhhX&YWGOzpc<6ITKzr^zi-#>z){t;yz3tu_a!>)(tTU9d zd}COuy~Tb}UIRNX@aVGJqEKUa)1#E-u}pl!sY)z4cu+Hu9==`6=0Ob#x-%q}t@jBp zmoiZDcfF1WL{PB0ZO**8yZ+%;LF6K*JDUoHrJkl0Wzak+Y%E( znUmuA^p@Jv6{%Y;MsiZ4O?#ID2b2ssEq6_KGL z8T%zdA3YhMnkBu19bNsa_$$_1^16jadx`0ZzPx`M%T>qZpYyNYOeDdmqLTNWpR5T% zOlRrW_xNCD+*3_WSxvt4P-@qQ9g_$aedDk-hcV~t>Oxw;UaAk1V?9m5<2k4%VrM$- z?{KH{)m_U~yJcBbX+vqVfq&4)Vf+FvAHd|s{V34=f#uJM!Tp?b32THmfzNn1unwY& zPNtaE{ZZ=OkZFh*xW2FT&fDF?64Q%l>dwdZ#Bg;^v;dAbU*QLEQG@_|ucNXFyx~H( z#h?kJKeI3jD^U~`e`*^zcm?PlIWj|tL_a8NC?HVl*gX%;5PW5Y%ZZ*G=jPn5#o+Sh zhnE>D@Wb!f*O>cZ0}ZT=HlEdoWVWk}5H1S;$vxe#Rv~;l5rJ=w--wPl621jCW}B|gxECKzT9 z3FKlD{=OfN5$J3?Ag0g4F5t8_D(RvO8W!*~?#q{Dhx(Sj=)^9ZlE|LyI?p1NXMWr| zGGbzFN^3)5?P^vfnD7XZo*8yf&O&>7XULUUvhJT@rHcF>PmjodH~u4RSmX4TH?v`IKg2cy7 z(T@e4&pPRHRczikEvwvO?jbblSVp z2qpyT+LHUFhHwcunP(^h)G#uA95vF`Gd&1k%F@wuCk3DnjNjw;b}*;dY{F5{7tNsg zLf4y|)RTV`PjQ^!NoWB3YA@S@Cw zUAr?mUcn7g)n!3J`D7*H`y{%TuT$wNY;))rP(y@kdFdPH#h|rjcW2#oRybxTchXlQ zwMW{bVcqRRc_2r^tI)Zav_+qLwdd|Bw=*pM!|pflbT%K&Eof^{6+|k{2_;HcrWd3? z#z;>@Y3dp#B^R5c9RhH8lT5MRr*;>xd<%C3sV2Y}>{On@a*oump`g#H<6V&DKeZ-?Zic$S$>ulEiZvJG8kHMeSzVE(R|E-<}cEG^n2E*Cp z-25-DQv_Mf+&WhT3r?23Phid$q`Z3HE($RgC{EJA0Yc1SP6(a(oZ4RU2L1~H6k0Q< zHY1Mj{)b(ll3Wr=HakbiEk13zYKN&f#9*}tMZiQ7h@Us+N(Jk`aWQHf)r!ObZAT>_STJuzjuO{qHMlTjN9^hPZ8sZBMl zl&MX}xk{d5VUEInRK9r^Tnx#HE2;hFoa7?NDufAxZV6Mj9B^NaAt4;oStAtWfVg8< zjQAfLPj#u>Xp*sALAi;M(f1>la|_-k(E*-1Sa_Vdt$KsCNAwAbm8CmvpDbwL$`Cx8 zkBC0&3#@q@7E3LVtGQcrGS=s-uh6FHuC)WTtU_@t5c_T~`Wv+F0Jd$a9s(?ucd&l{ zWThjQ*u4YqU6Wq{+^0sC%S;vXx~qO|+s%Am1m-N}zkd84>GT;5u}a1*p9&!g%3wk2 zl=rj+H9g>!z4_zdU1iItL}Zox?lwK^ykQ+_#Ym~p>s8CgcLQYV4wezL^K-_HzM$r! z1m$U&G13HqDckgHschNcoe73o=)$P$j46Y)SnaZK(U|F7d#{AGb%>@b+jX#5*Rf5x zq}@ejPTyyn&&@n|dDGl-o-=XF%6dndW+}@7JDd?6b}Mt-SX_GV^3{!3Yz5a~X@$Fw zyDIkaWq*rtn{8knumG6=yF(6lzQnq)&M@%8RzdC%{%-0Ey{v&0pp-aIPP$bTrF|=~!MvLftx2pd=0-86i#@A;;(b^r-TzBJn~W4d42|-V*)} zt}h95!TwDQ%xWD9TFS{BwGO@d9P>kia=+LQ@r>0>5VvEV8(&tEuw%+YP*Qm6KzZs9 z#qL6SPwl9DtPZ{0`)Vph`^ryNV|=I7r2Vf@LrX3<=$f6zv1^z*!<6j{f$|6Jw=%s2 zb)|d{?()1m_Xoab$B5r9#&taTI^E@0yTQ$UB1_f0nc<oQhFOi;b@!o=y6w&Tsrw|K5XXEJEA>@Eb?8hi( zlT-*bXZd6g*C+W9V6B5iF$2f(_-ek(ko^JW%$@}`#GJVV0S8A~FwzM(JdY)c1B&ls(qJ=bvy&S10cqD8@1Clbooq|3kmbD_she z@O#tu^ibgYfM#HD%WIF%%uf7+)sc&Dejs@WRQE+Q1jXlN2z>9dB;X9e>Y3a-&-A;T z>||D+o$j^$E>F`4y02DTELRMYH*biv(5+ed(cQq&82Gu z2~UNnOcNc&MwT3lD@S}nPJMsvOT%0L{`dN}DU&^Z#6?2^aE!5ulUV_Zct}2~K6R!_ z4ReuaX$@AP?M!XMpi&ZJwsY2up5F-xe0{ym`9#@pr%63v->d&@UoFthcC1`k$L=ze zYX1{xl49Q=z953h>NzyMc3UuH96t7)-k|lRw-P=T%Q`;dC7@r`uCOq8Eqi7gKC)~7 zb(*Q>H|T2(e>5DVf9nswM~C%V2G2 z#B|VOitZm{FlV>EydvsFF|Ue~ium0%0KOaFiMOLk(X}jHq@dI@*AM2G6XzCU zSpFR?#U4MPz~VZR>RA@a!CZu45#f<)^f#kJ+ULtRLJKzSj=cf+NxQ}Kw)Yme6wJz; zu3W=Jz<}rEm$g7sNy>yr-Z|OiI>qQ4m37~);`_~Xgr~N4wOAssk(HTh5er1XtFm+! zb`5FT&FoKA{ADaUP!Y#o^sGPb?mT2wBY9ZfQ}ujLk`C_dyTvT&)34sj!RXJcZ%lCzF?kE~i-xCSGh{ zy%iUR0+S_JP(#%W9!Npk=RL(8tFB7(up1ms-Q#8 z$-{dva97!EQB<5#@0KgW&2S|ddKN*<(?}37-=O@1bF668sG)3(D61=Ech&sJ;j|An zqu1a;`}bcMj;#tF3l~&Ue9ES7GRw~kIPKK&q&^No_3M#yjp?ygI;To&wcXbe%ju*z zpMI!gbi8@{AJVkgXR+py{dMSfko}H`^q^elZQ-5<2bG-K8tYq8Q@*4t)`Blvz!#v> zE;3kk_e^|Kew4?}eU;3n)q48yWgAm)d+F(;W(>jPB_glQLiH|IE=EDVFI*j_FBebS0vXyh5@x9LS?RNi7vXf?RckfXjvy^Pifki$9G zzwp&k7S+aNOI8%DUON~#xxv+a5rJDE+^6;@RcjnwKZ|%#%Ukq~@&vL#Pts;`f?jwYL)Y zDOROB^T8hlFfA@(=$bFYKWy{F^5$#{h*A1FG5GZZ1?>Y+!}UULap(oEekfHZCJkpC zppRS@+Uvrs>_Df!YT#HWpuaEwRq)V49)TgZ7Jf{A6@tpv&>tG)c9F&eZWo)(tDPDB z4Fkl6@ov*S4!gboeokhZ>My7@q%!Z93-zy>Y(_9axnH2W2Ie&#X2Z->o1A6ZoV(OgY z@PpdL`E%U}QN-vzdLCdkVX)Vp-z|CGg)^e06LvMfbj%1)ZdXNB>r>{Jk&ApwTkkLr z-2C5e31{3c{*xsm?)EItQ%pSW(%723B}AHgke#M{7KJW6TT*>9^+`FIe4;VHRwSF$ z9rBta7_>vwCuV;vFY=|NZ2KlX$A`EUk*phH=Pd~I8Kkr|v!j3sBAD^fPD!FoPpnHf zqP&jc&^s{jm0M&oBNXjUol2${7|G^u7UtOd2kxA0b?japS#xlwo_TaY+jh-`+$sfO zFLgfqb~kaemX{ErUn7}?_tb>g?G@UyT99HoY^;BG(5|gh>F3J!9J* zvrz6TP+;XdE$<41%Vony^Y}i*aCz@+4v^38p)5?Nhw`m%Cbg5Lpz%VOxaBnlA9P;N z9D=#{(>`$N_!?&CKf9eJGzIc>dhWes8XtpX`{gOhP;HMklZ8~@Yu~YT1bZZ{VwrAffDNiZ6Mh5vEzpq z=5A;0ff@>1MG@vbwRU!?7ZFD-SYng>JN(=>uwrkrl@4u6M^n6jl1shsk;DM`t#|F? z(H9W(@&~b(mmUR)30H=vAZdIrX%9iR7rMruZ_I4$Eq7YnBI4Z8T zj5;RTUu8?(ZsW>30%Hk#$^zfAtgZ&y!|p@5%e_4oe7)3{Y6c^x>zv=o_XPiF*wI1y zNe5L3p=L;8_D7-+5I+LfNgDYrOIUD_Iu_VJQD^=4v=Gd z_u%h$8{Lobyu6%VkeZI%T_vssgc#J4yD+&6pVkdLYl@3@NdcQbwl!J%4{RC80oF1z z`ksIXyrZT=Apq3kOR#m795+y}-8NizKBNESZCmBS#jqG`n4kCydp-4DZ^BF-zWD2# z1@F?p*^9m)EPrkd^E&cimk<1mN+iwSCVTHpqz^#`_Dj;-5xURqxK*!kp5asE##*=< zc{bFC-`m;q4VL3=| zKN6@)%XIu=yS*-K-9Bw`jN+-lWBttd77x>|g)~$UgPB_qH0h&bm}j3#sdLfV&xcR^ zQFk=d3;U8~YLqm@^61C zmaLbHw=dJ0oLP?>eyJ&=wgtZm!2mS9V!i~62x+n`%jyesf0bKruxRDH-)c2uF;&qT z4Z0drBbHg-G#ueH1vVaEJFTw$U))8mlUjFz?!PDqNpcIqZ%B6$Ju$CzrK@_na@?na5LpJODS}`)`8j7i#>C z0RNEb>nnQ8v$qXrgh)-(=VVRFwj4 zZKH}5T4rlZ$PiI2z3_*{`av5A0jPJY!Y*RQ?XbKPZmNdwp6ufAH4m~1%r{gYeOJBR zai+gl7I{Z35P0Q7EoGmkkLGHe5rR^{bdxWyMiC1~&kI@I-bYJrdGv{=O7!p&kKxN3 ztOoyzWj_asX!zA>`fa~&>#$n@3{c@VVcl3(1m5=dCI-~1uR+4s;@87ozKCU|Z(EhE z7~Csbr}e|&-zPK~*W}WcKqB+rv-rNRzvqfY299AvP zA5u^Rs->xN6b@MzP_f(M+}|~RxUHs#zO%D772V@B$F;5<%Jx|0#Oh_?#%yrHfV>}I z!Lfe59_VCjJ!pEQOWyUr;CdyL z-RzERMQjU_j%}N!Av?++44uVMc#r_KCTZxxSZL>4`xbm)#)*?4I#nFDOZLv10s^{6 zAyo6zfA)w8n^jk|KBb4J;|Gbx9)grFflY-Nyl_v8_@}gizDNn(Y2l6TqM&aN(+9Qg zTBo#J4N$h%f!;K&2NqBlT~J6aqHGy6HI`Xn*)UV$w2>iLk~P=l)VTdah9Ab`z%}dg zxIvG$xPG=H0NRw|6_-~Bzh+BPv9&C;z)58?`7t~$HupdHcF!F5dirrGrn3d}wAHr! z^@&!aoW@3sENjl#i@LzRYOZ4b#v|Jk_Mo$-VYlgbE3LQVKniS1mH)uO`90X{bc~{1 z*%Wm4$E_2-W__`4`mDu;Ld(wv8e147=mMu!AKSC=mw*4n^8S>~fm9mJgf4~8t(bb> z^_3WSK>aAZ6lK3OZ#_7g@)?z1#pZ zoR2>rm%_enbG!+Y34#Jmal)V9@-s8li+_Le^~z8cxHeF5vR%p~{93TJv%YmeTB|@^ zc=}q4Gofbju_Z#%Iv9|44|pawNvh^mFGBA_KZ5C^rx-l~Ytqf4;%SxezE8%O)aJh& z>2it7b`epB=P&=s^y`mJMjMq&9Jvpdhn}6sFHk)q%d zE_RV6%-}?H)w7yAW9TA)&7XbMyu=N}tRA-JTl2iG6u8;@?;!BW;ykyof{i+alo zJu1v~ITow6y^)5crWdi)&;yNs0d)3*vN+aSszJ%`1`(%9X-Hi}3gH#iRg@{Svm?cP zM}T*)U{A8FTQ7b@oc$7vr_EeTIj6N%Cr}VI5VcfZk+@1UFc>zpJkm3S%cb<~=~`BV ztbyjzOPJuDkTJJ!hL^nLk}*=2EXd?->%+3NWrq&5a$%1G{r2~cLQT2W>8!pd$9G;K ziQIDUErsVk$XQPRm)pDFYVuLFlx&eiFlnoixT|jvAoB)ryM_}euaYFXrdKLqi|4AL zG`rnvWi4Qa>Wvo=;Y+t@ecMjl{#37K;?VkYdoSbT(2m}8!k~RT{yv0l8cPp{jtiXr z$7KAJAvM_g4ak}0Yo*q!sO%PN_CK)Pv>lC7xoB~vG1hs?Wv>^kpOBU0WV@$|oL!cE z1FV3%^4Pjr5Fqc)|Sv+upxx8BCM z9*cYQYi3jY(^pUL8`I|3rHf+5>sq98e!hkPsfNMQ1@y7Tnf4{F2p zx9AO&@zYO;WpCQUf4G@!d<{t43@&RHh2Ukg^D-8_;De`dc{hz?yPS_7BzU!x^P-tj zBWt_uk{g94M1uo_&0l?m1qh!Q>=dKy5cx zRa7mv(}`xYKJOm)h3s8goQ*XK1OT<#&Ozf35uTB^VD8m)Z6Bnlal5r-bkso}J^TcM zo)ZSc#2@`h0Si}lrnCFt67JFa*e&}2avKCL|IIk<$R2*5sILkv4P( zesTX_tP#NqXN#>Q{4oe!N=G{SZ_I#~%^kq5ilGc=Q63_5uRt!D^j$k=&$`Ha&bGlAjZ2&hWa=M};Cw|5onME2e;8le z)-hK+mgNbGw-4puLN6g_q5p6T?0XM^dMo810rSBSw7Rrl(jt2JNVBwhB0o3``lZ1y zBr`Dy8LdVilxv`X5b0N8#{#(y<2vQrLj;qv`XA#RZ+@Q~*aYa^UY~;#F>6BL>75+E zeH2(L#HhLeI=Mz1#%^96zY$Se;@N)biYOvM6H1p6-4LcvA=&GP()#?u=_WXgAoZl* z+bR{6BA52?12Rex)v?(LMRsKvf9{KzP<^4&NISV{2!a;wEhr&E)EloHqSR9%ezb)? zl9X;qQSTg@es%UevGs9-KQk6RqJ;Ui(v@S0=JpkXQVYgXlRKQcfFLT2A%*#c?7(b} zjki==Q^Y#Qf}ZVpFtF6<4SbGKkkU>I6wY*Ps*EAzemS5Z0r!-oD>~r!<<+c~fHK+{ z`u4nWcW&4!()0%2>r>@zr$F6$;5*IAuq5bc>cn-IEZ+B|hkO&NPeBi&47YiU-<$w0 zq-j9aGH~K;Y%0{D&e90RZ(J_@o*`(e0TgqWM zz>V1_2|7MMg_6zbeK`A2oW6>`dUuDIll*?4hKaK{^>2t!B*N9o7_!iC51?A=hss#S zTOD48mGM}}JkMLeB>f0zNw|zPj8Efyx1Qh?QyT7Bp*PsC1%+$kgboSqDR=rTEs%8X z-t2|68n3XC`A-sBYO9tXuQqE7{}pE3mRASQTvScN7(%JH0{M|k4t%rE7xh`qUf4A- zgEE3f#zcuMyMYyiu;w=#PFC-_W0rb;u#{l@E}K0uMy~Ec1MBz-KglT}I_AG%m9nb!XAkpoW-`_85Umy)5g0j(3(>`;o1;w;CKp zLKdGc@@LrE*Y6B#H>jMeTcD6nZx;FZw zZ?8nd;T;sv#~t>9Stu`V2=$pLBHrDq3VNw9{KZU-50LlNLK@?o*hLF?1Kjl3op`;u z=nFLXc(CuUKp%gcxwwBm08`iDki>51cyobB9Eypc5@0Uv%$x+m$P}vtzJ@yXv2Y(6 z%G|Dfw#*GyPhoZ)9Obc;u$h*k0~W zv)EW8ChYvHNP~Ws5(MQk4JSGnG!l*4I-odrw$8E;u9uTN)1sDTSK-9%H|jqRi1XpO z_RLbdR5?V7FZiM9a@_RLzrIa?o8u(&ct}&dJFEmRO#py=1J(LW)$S@B$xLi6T)SOw|;fa7Myzv z?MOZ*b$o!rCg?J9&v6SsP#m&goHWvlC%0`IUKT~X&=s1cU$O`0Ea`_f|aU@(<=bXW{`6+7W#cu@H9t zagx-Usc&&vez&!Mjqpdk+Ol(}Uo_B;A&JhUaOe-iG9|*Z<)SYRZ;!m{$5X=V;9Cl+ zs(#H}WR`823f+9`wmRKF;(;wyt*?b3@Y`H^;&@1GipUF_{Gb_RzIV!3$qMq++{iyr8Th+msVi*eA69cY1K|TmaXNA-rCXT%k z%$21aDiQY_-+BI`52BI$rv}FI)tg7-CaaD7_O`l9ngVYH9#Xu44ly2flHy-xuzEyCWC^6c-^K*QrZW zNG1PL`B#xfh_CD57q**Q+=Ty9EEolHUwT`)Z`SWJPvsxa-f8_iHO;AQOj^^?v$Pd6 zy~3pjahT&?UwB@2zW1)s8+UfK$SFAL~tHHx3whuvPyW4mh3w z`_Q5~nHOsoDT0sx@+N~J<-Y&TvqV4MCkgXgo^ntecjdoSopR%@?wkEfAuHDOIVHQe z|K0}d$IAWT3jC{=QJCD$*L3=%k#f)T)tT7R=nTHqn)i5$Q)sm)53ZV1w&{swK_X#n zpD3;2Eb$r)$CDg__L8tv=0-5U5hB))B~SI2(6`QM95phAkktAVs0hU305vOGT{|^t zH`?>)3!24y5TBnjRfAJG|J9jjj_JYwB?gujfD3QwPf@~K(A2Z4KynC|m! zMt!}`yx4=~u?@-#ab5-T?In;dGAUlGajcN(yFF%ypy(av6(B6-=d(A}}k7wcgUJ%c_TA&p~<@ZA~EU-mvx5S_ykM?O8{R|mH|RE75BR5QQ#CTpy{;f{(N zFpFjUOJ}EEwov(%eX6wm&~H5dD|PO&*VQvG&6Br6eo1I>i7L)sk`T?{8}`lQfCB2R z@nDF(51Rl?^;uv9K%Wz-qpmyIoZjoO+tGhY)P>lU7U1Rpv;b{^)mu_I7=1e%POI7M zneWYe`!E(sG!D4Pm@9XD2!jhItDw15w=Vl)ioN}tjFK(3~fxy=!h!`6@!cQuCP6#aH;{{dyV2@O1#ZX{Zl4pLmD z7*{Ip)`V*gV-QVaE+>|4R`><5Z1*;n%pfkb3AiZ1s39)5f5khONJ{XZ5dEj{AwE^i zj6G1{WVlyMNlC3!_Nyk^Z0DjKo$ha)xbx}7UO*rnNj8he_fyO?v!so#$d4^uhxAXf zZNG(a)^5wM7^{-xB|`JITdre*!q^0$>^GMLKm@oauH?5G^;l>0Hp)xxzomAmYTE02 z+c%CPd*0$Be%v~(u%mMywt>EgIlKPOZH{Q%Y5c6=;F0usNLUPph9Xez1H1>s1YOPG zz|s4D9}W5qUuupaM_2#&;|@Jl=mK~Bc0i~OYb643=Gzqz>i%czm6IJ}e-nj~`8ZFe zGWf#c?5=VP0hlqMCIlRJj0p>6ob8O5e(*AYuP~QI>C$d^Yi`)_a|r1LwH(~NZ9P?Y ze?ts^N2upq=Br??YX8%HZ%xopU$9Z$(sjX zPFNIynnhW{IRi^L#G9#+Ew!gHJ%T1dibisJk2~6dM4r$&WR1@Yh3+PZbrp7G519Z>UKXw(mZMT+M-ozzkggshV_x`b zthj%~?f*E&m2-P{17aTUsk&fyuduoa3w}G`Ii-fByRE*XlORaY&Ax;2q^Y}9DeUiq zyMK?>G}eX;GkTjbS%GZr z5T&~;Y#yW|>Ep#W|B^P_r=X1$4uFNPGyw?zjr2lT?F6>ZQaaY;=%~?w4R^35Z=yWu z?(pW}`Hbg{7^L5u3abb48R>Wz-8&e~ld& zG34mkg*Nsz8LkANRe$e1~y0OAYcFkLVXfFw#0X3 z=EB)RkCjS-zhk?;_Eww$ZWCeYf2AIt@_v0+O&5H%+nUcKQQZ*-D#Mj9~nh zx&c!=`cApy)!}O~mTV6{@dbum`*7{`e8wKXQ$qf(L_&%pEl%&9Hz4Ua`%w=5(|{Fe zG=KtAxRHvVR%isJiC+qS)RMDX`xiqORyFg!x&NkABWs5}rYfi3W6r|&5P*I>{#$0n zSspPdl-FAPCWDVqU+`hp5SJ)}U4;QxQ*A|gM$`7-D_HnBBw1Px+%y8Fr*ZBkK&P(5 zLO)g}sM)3#vqJr|zOLiUYMzC)Ip0^+BMHE(YMU_d9|WolPeKCgmx*JYTE6;S>Wa~2 z4x7~9yMFQiL85QHvJtCUi;sWX->6#j?bP;4-B$$B=t*-7v~dwa7d_l5=?cxUgm6Cd zaZr_|B^X5;{k6{%BEZY5G{tgIXaw~PMvhi$_PDnHbyno3v;_gj5-=Qm12)lz+O@kglm5{q;M_RZxMCq-* znMrLfk)rYkS^lo@-6`Sd+^FUeRw9NYH^+}naYE(H+Zh38KI`SA9vUIYM`w7n(({Fc z<0<5oW06nE*}@UB$5AV7a^dI2srSJRcWrClmn7EQdBmJ6?_NrBl@wo_%pe-;K3ph= zN1j@y%^Bw-|7I#-OsQL<1zRV2i1N8h%Jz zJwR0GxN$z5cL7T2`h@=Nn-d!(GsG9!?+6zh=pQ$E{l5S3TiBHQ1&Bvy(*8{} z3j>EOJw+p*2|#VfcRh@u)N+@NMx-@QrQhRg>Tr5cY}aHl3CA*moGLkK0}rdRVR=E^ z{#;gyR7l*RccCrEo*H}%3X|@5YPQ+FM>u|=k#sp-M{J+EGRGl7LH4Z8UIUZqJ%O1S$-a-TXZC__K^ zV}HQ($I)a#fHDGwtEVN4+}*Rszq5|ewZGZEuA5Iw2OpA6%g^thr!`g2lSe?v{V!Zs zZR|Oezz_e)(WIs7nejBn3Q;m~{el(T15QaA3slu+pDiHa->pWfN1C6rVtf%}cuYmO zgKLKj2iNqdxC5nzUkN5bWkY7QyW{~Jm`(yqq=456x~COUo&to>DhmwrE0T1u8eLBX zmGKaO;crc6pm6&VjM@?bZCAXTbba*pRUvkbglVZYwEkF8YfO`T(Y8Hj5McaI z|C{H>yx3qKlRMuy-lc?Sc1!2)CVr8jr{HCfqbxH-_?m>w0h)fl`U3oh{a{=<4u=GG zzB1dSG{rJNtgG}nPU<2q1UPrW{mUkc8)_`L7OAnol7dZB_a(SX@-|yK8Wwm(0F1NEm_aN1wVsURw>% zPcJ-K`1h9E5@B%#7Tn`q0}2)m8v1N;72R}2#~JeoV=z!u6nMx5Hh%7WcQf@>B}s}j zpX2a$CtQcsC3W?=6QyG8m#bS^7MwKolNJR0blaxwZnvS?S;Zd`$Td4sdlY4B=DpVj z;GB--4WcwwL>bZgwia+-FoH)nTd?9m$)`kWfURntsPevI9OkDUq}At_Fhr2*m>J<7 z|K^#22*1UDq{{(|XIx*ulqtAAdQ3OrRygED^IBKe*@i}bZ9_@AZve0qu;T?J2LZ}j zw%cP}y=TD%H^Z>GUW2*063o&E!US9==;FnvZpXFNHRbelmmD_~T)}7{w z&e;xBEsak%$=pypJ3t9=dtnbS!6w40@X`hEdjEiR%*$gfB`8X5t54B?{Y@k+{O-C( zyWn|kD&H^1e6{Z}+mjH!-{_d1n-62-&sj0eAIe`j`?O4m+Khn*F7;(ko`grc}wJs-Gcu{X=-q9>JtlE}duQ+wL-kpryH@ zy?9QcUQwlU%a{$3@vO{6uEg-;vQ6$i3UQK;nO(8qR*T1<;wvvr-5aev6Kzq@WY?yI z8CkJ-_v2o5#Cy<>1tkp7W+umyd18ce*OX=Fs(i}ooB^lb_(Z+B(#0c+peWSQ7vamb z`z_V8WZ6ITb0VsHVCX3uI!$aMYq+2H_VJv|H+xOae}8%g0Ho5T!|3N(fPIQlqqpY} zehINqo%!U~bwZHmWWWQHbG6yOu;gWGMqLHRHz7_bwPG8clq4AvuJY+yO|fZb!!O?8 zu}-gsTJ7>_YGOwb9ZP{7Y~E_-54t0uZ3t;;kkys%#n||9@a5P2V=teS?-R*n9l4LU zX`b4bjK#bVZd&U8y01tpmu%od$DMxAMMv9l&MoL=#mqz@UrVGR_l0_DR1(?*60e1Gde-2*c+IsqkdsUBQplCu zbAh}kVEU~E+wWc#ljwacB1;-}=6;qO#+T9U6+R*7gTqwax52TW8BT?9baXZbe&3!{KI_6)y4?e%W{LkWI2jCl?{Trz8L**uH#O^Q>E0F; zvZVDQPmj+y3P_#pP5&8F;btP7L{R3-N@^b&z}P6C*IselB-bHG;@9&O))tmx7<0R@ zq~8V%kqZ)eZcoE~O~sQ8B8+i&1Ue*r4H|9dY8S&zqWooS;5LT2)V0emG9SEr9t7AM z08Kh_ER&MkZz||l>!~yU@mi`?QQ4AitwkZp6F1DCU$U*G8x922-bf6%3pYrD#i2*< zwpz(G$kV;(&?c|bI?kVkWtK(xu`&B#;UTMoJn+{-FXYMJH&~sfC%3D^A2%%pYB~Fx zYFb@KR!L)a;xpqnrzd^@O_;-5c!|es9)R%NkQ;Y{;h&+Ck8^jTn&jZ}P=M)n>!7A9 zbI=`ms%#Cn4 zcD|SP<@REH*!8~greM*drUAx|97aK~i?nk84xe+fW zZ{VZUt^WcR{^_IyCA?BgZ6gdxVu5?G1|-aEz1&EUsaWP+cJ~=7?fk17Km5W&X3{&= zr6*juZl+Xa>izM!qk7^<2X1*30KepqIdjyV2i+e+zNXSxbK7Tpa}Fm~tK0+5Cmz|g zd=qVePKdNVx^>DVw^plZ?2M6Lxb`!8Ti#RkyDG;^w5l=4mTJ7GuF?>G>j?|lQi82< zNSi&Ar21!4wJGm%haIm3(&qHRaalgKQ+Zo*VUmdvO3d*r$tQiZdevGg?sUI{@hBMB z#c4dG%$ziRt^bWNf~3wy9fsIN_Xz#^hwnqZ)3n%{%nU9mIShVGJbF@_aV%R@{2`Bd zRRV1z;iLf8vnhQhV!*)}h_XFiU+=HG5zruPk-I(^EEW2+SP43iUg88Ktt+fn{a3`C zxH5^rzt^)}NibifBptLnWW>O$q<;o81Ytp^|JHO2c^)R9nQizz@=pua-L?WcDwzsk zqLYg1NS}l0EoS1SEwfU_n>3wtIkq4r(>>1vzP9Z)u* z7!cFZk(y94Ta9;@KGI}VuVTz%OclFRP84+NBUYBAN9)j18h-Dk(N_YxRc+#$@;E!G zk3>;{dx`$+A4-y+OCDz=U?O~&oq10pF2=@SEP`h*hn*uC*BdqRBV;NUWL%7GQwvf+ zy^@Jg8oV=aF&&>FIZfBUhPx!`mVdKBuW_kcOjuX6o{4h~GUS(Oc#=*IhjnUUK6V>q z3|r^NJ1i%lyLPs-RMaW{5i$=F!>FC4M0Pj0<<@G%muXC?eGi&&ai*KS|^#9Ba>V z1r&49PJmi&clkkAhrws5!q)&@Ng2>63rG~VPQPfM6P3_7JQhw!k2;x7`97!rb;o&f zj*N+5e^fk>D^vzYxcBT!!vc`_!+5f!_>XV3z@oz}r2l;7v?ybOOoLg1yQEm1p==et z8!M{V&DaVz@Xg1^2sOzN<|B~4p!Qqom;IvMJuhY^iq(pcg1vcJBD)9j$F|MVwyRM%Pf=l_jD+NyPHL%YE6 z$(-O5y>IX=Oj2(?JA*YBgFzC#Ok z9`8k0Tqim&9(eUu$uOl3X@wSOFmmcm0q`1mIA64Ve_<%3$nNID@10j(FXICMN0-)z_1h!Y(wFt@%rzn&KWkzAN|(aV{DA=J;-G z#?ZdfVo{uhmv0)tmnXPt7NlYVPN%)+Ps(HATs zB#a;EeCAVi=f9W$o`(OvXpJzf;CLh}-04ibR;6BeF3%HSpb7P|@BS;Ns&;?bSOo4F z4DlH!B~h1(AX80$!u6fC-}OET`Dlw`(|?}OMDd~ z>qFr8tnPYIjcmoZtVUn^-ei%&OQA5Tc=Z`Iz9m6b8v)SNDYgGI z&ufpuaQSeQ_2BtH5K)eKXd4pr>O-P(?zf3-LUZVNwLsusL-~7SqM_*WS%%V#M4_TG z{P&M5x)q1sQS4zgx}C=u@Q?t@YU*P&n!}ih@#Hx{2kRN*I*QhP*keYtJ=k?c?y9!B$5bcgrQql3d(MDOE& z$&4)a62X+@f)63w)4wmU=x5`h3F6ai?c0HhJ~iZLYXK!aa#)hyA>2 z|mZaulq=2%a+*w}~-#`f<0;rmBC$8kUReVyk83I8Vz z9h*!SORnHE+X=(t1767g6#NDfz8iGC>whkQKj)G}l@4r;Kv22N_b&h+TX2u|j7#Oj z(K3uiNL1XY*yk@SMq0V^nF^C4tY7F%Xkl1!XVbIhi9k&fR@zT?lM-aSH@RdqE*fzT z0x=nU5YhN`oe2_Me7X&Slwrh-emZTam}o^KV=~utowP0%qBQVdeF^BBD(JrsnqT=g z0Kw~8J^_6p*PaLgV@w0$mjgf4%j*&bCxW;?u04g`wLQC{3<iiFFlUUNQ@-0`3U0PTr^* zMu`6+{ji*^jscj}HzT-Ix^mFBSE+}Zet434IpXr-z;GbHM|<6Z$ud>QLOHm$q>Yj? zi=X^?XVKh5dmh63E6q?c-(MkM>f(9y>kJ)X*W=($$*zh%V_IowxHcM_Px=q^tBS~D z^CNokYN*qIzqTFLw@*J|W1E6Y93dEjFM7bVH;omm!&C=Z%kF zDZ!5^rmEV)HlD6O6Tr*st_e4;^fb1cMxb2+e*K7{dMXd+lY~LT*&%qoG(^LQ;xu2U zlX&3i8OG86!Vntf_USh9iF4*U|J`}Z=mVM)PeAs{D4WZ*4$7P zB%t)P&$2Kr04o8Xy;J`g@KPzWe`1T}m6IZ9MOy`GPfato?=$ik(>JsouPv<{^B1k$GpotiH# zAFc}^jX-(p!24l8(M&7@pUe|Pfm=;J8d^`&7M`y}lC2ikiklLO3&7s(v`TZM_wLvp z)BGvu*V#(5myOg0-#f?hZM~gOm)pbI4r6l2`c;x+BoKN zlf8pTUa5LIE_z>y*IP(5Wwu|3hR`D}LJe2Z{OO%LwF75itx_bm2;*V*L_d!<^U`113iZ?AUR2fo{~@G!O7S z8ry*a+L@ya1s~1tUwKIw=9Y$~W4(^vWXYd@p8Pzd41rg5Et!ZFn)0i|BZzsFQS{Ma z45FpX$A2OpdxJDya+vhWuRX!EMr)~=G60EB#(9=Cm{yUH#1~9tH^>Jf<0R6m#c}G< zi(K*ezx7%l*|KrLE}7Nbi?ghND_o~9`pZ1q-*}Q*Q`{_{6rWZ;i3So3-$FI8e&&NC zWaY{pZS>)b>-mE2`c_1^jB#|!C|63e+q*hQFKyk1RQ#UTkJI!M6}>*G=VmpY(8bq{tn;^1f`?7^Zc-BLmxn4n zI7ms3JW&2@wCq%Iun#b{=0FF4fUU|6)~D`fAdrMDf-%qb7}(_}O-Q%nk`;V~i0&E` znTDr*@a5IOZ9_&vz`~lLmNpX8``JG1kxEJD;}0!4K|3<0TVqBa%r23*zlrBZWH4U0 z5PQ(DoTHN$fb7YEFYgjdU<)3`W~2TCFZR=#A)q&Z+nJ$iP35--s`>pS@B(Z1_+$t{8(iqnGXFSA(Eez$U z(rAcMIv(%#M&j7W?q4q*k#Rn$E zuip+NtT*wwH#{;4u5GD8u}hZ<6@&20Q`j4GxWAW}!MyTY;KIYKaj~9lLj|ADb-{w> zXQV5^!qH%Z=(nxMKm85L9tLs3cFQNel6fR6KmK|2x@yy>gzqqyx%l2?3(eDsLCocG zdslQ2dcLqbO%Nc`$|v^)KCTKql8YQ&?l90WQGtlNjj$*dWc`kau){M=;cMhq|fFjQ_6$TE)+((=L zN}9jU#9gO~MwryIRsj`Atd^e}?`()lD^;B%s>2xr9u$3Ux0maqBQ-M>|74?_%Xg7K z!Rj9hvpde``3walaYgh+!5Q07qw5!{qQ@py4<7ToKiaHbesEVf#mwc)!Ha{sUwaYR zYil{4w$X?jszTm52%aZddax+>6ZVji-I*L2fukc8YS$2F;Fp7qW|#QMx9#UKh&WC@ z@b|j|WKkGzxI%6W_|)$N(vBy^<2S&=M}T&+nZ~}8nxXRO<)lH7nb=UnCA)@o7GYXG zo3mta!~WY5Dh@By(QrLSG!7x6di% zS9=>}2G(da?F-j0X5}QM<)9<2P^&l*D$0iYCMgnRBFhgP;FHiQ{{xc#7njIn&F46G z?iOCDCSZ+j2-Bt2p^J`aBdnQ2?1U{L4m?WeF)8Z<2czjUtR`T$m;{Z_29g z>0R-hEnP?RcHD}C;UCvlJW`!Q#=eH%5m;&(#~y)~Xxx)!XmTP*e;VXL8x+aO(;`p| z^Y7W=lRA)%A&Qg4Ci82P=5l54I9(e#7KD~f&prgcc-_0=Y$*(6kGR#%a+Hj=nMsHH z{nStbI?Mq~mcO0m3g4GMOW%!sg=~(F zHo*;$bSAPDVg*dJd-V~f&<4;QrUGPQ6G10(WzW(3hbT`A_0#Y>R2$q%MZMcYywII% z>aI2%Lsu?S5d6~Z&+thwjJ}cHCua1T#4KIVsE)J)J~nf3t4Di|CU2=n)FGexBvJ*U zcqjy-l@EC24Xf1KX1_uW^(#D5hrp2oIs)xY*_=Xl}7sic0DaxuVQ;Vj(H8jl6{ ztl@;=7&sO8d1Gy79NJS|g5yuZoY}H4{hxfL0oDiPGb?VB&s?rXwe~sbb+Sdvx96Mi zf7XvCdY<~>#8qEs6=adRIh)T#cly&iVqloGZYgq2DE$sBY(0R;w#HyO5m{Xi|j`ryzeJhFvObXi}zQ$^dkUa z8-=*j7t{_XJ~$Hv+WXY=obm2O&HfejylNDi~KEqaO>WLW#z~4D&S_4?L?|I7O zd9bOA>y97h8sWz}k$zJxC8agx00PU z=&q>}m9ckFl0H+8hHU7@QXQTDL?Q5QW~dH6U!?M-P2yvDhHyR=*S$jlFb&0tEg}In&YcQjdt18>ST2pa1*s+G_eQ z$i_(cvP~<#>q^Bp?-6%4Xz=QHw?E&1dQfBsGqE1{N7)PW@SLg91&af=IdJ<2o23%I z=B3MHDwg?zEY+b7?2pWuog5RCD;Ts$p6L=wk|sWaAE$aA+6Z*uB?%5v$opCbw9)s| zLe|cu36WL79#gea+kAOY86xuP@wbA8`P>mQkI<_463)vU;mhz}ev%wYe9GJV8DG zsI*WsdD7gNyjS4W75N&vocg7{z5xOXo$IkwyV2@+8uJ0z_5FJ|yr3t0HolQ8DNX*! z@UtBrYSwpRoJm))>Ui-&I|GfHtg}9}+AglmSHBzP+5p0(>?gKNG`pAQ!o9wA#@CUV?kk=n|xk;NAC7^On%cCA6GUg(8h74Mx zmW0D{fTc@BUs1k3M=8z#svN%Ei)~)D$!SRh)g|_VkdkQiW;lkt?N}oDiND=P-Idjx zkXC>GUNXXJwB{;*6!`ng08u+T37|1I=G#2R0wvra0A!Sc!<9r=?}l{$d_EW{5PB5< zwUrHoXWjP(om^Xc&*V*LNj~HwO;dHpPQq`eu13BY+nHVMI=pjOlsk;VH~8AK#p3E# z1Ayw~&8+%!P<)FVQz)NqdGfTyNTcPU!_)~5lQhDRYkp zC_%1KG3Srg*YlBCiN@6Rz58(IAeQR&A_FooBDOZM83P*b{nB%0neKaT#g$Y7rGmbH zHMCz_Yq+w?u72_rRDz6F4}2GfvaFfx80_zu;fIdvk1$FYLSXCbPQ#V%gzb)_Nq(}y zU3ZOC)Aq>!)bT44i|W`IwFgrG;@_%k*I%D4G6?l|eYRk%UGdM|8h^+cnFz~LymyV5 z5h^5j|4ieG`CvT0^v)hdx>x$4e6v^czfVQlAfgj#Fy_(pxneG?yXsOU8$@^>PX-We zw`wab$am3g+C&Uz4)|>7a*fvwKsEZ&?Ybqt9)qDXf}-cC5E22Loax}F)rj@7O7$(2 z?!By3nfztcBnGSUa1VZ)041(8iYs;m!`C^1Tiyg?|0l^IwgFc*BSY;i+Ru*Uh}%B( zpGlO&;XTgsH^=xdf>7^jmsz*4(_pfM?Wj~cXnBx z$yXh{O^XBq{@qVmy!3{Fe;!W@={=aK2j2UzP5%pMBJj0CeFX*AMz0*|e5> z0wrQ0n97T;j_W9N+s3LX;fTC8`{qy)IZ0K9riL!D!5uE5b9WPVf&!-Q=RVOjTSwBi z;k8~2s=sRnuy~C3mJ|d`StNjPSpD|gN1T; zzn|xTg~NK#smNy7NR@gBtcTMt3~%0kdbzV9%NPq6P)tbZzz0`C{C#mdv%>;Ao>|XF z9T!uW%f{;V^q70#wi`Y&^GyCG4UkW@$`FG>2r$|+R>cng%Ay@aip@1NWmZ1+gcN$V zGh=iq+^Iy7a|>y}@#KfqSDsgM>yr($WF&@~n1*KGhMF{vmm|Fakd5mo!~zM$Gew zn{T}s^aD5dq_;fJQ%))f`$5s3r1`G7tNu9Cv_YzL=G)n86=SkQN(esj_>Q{^f$Q0l zj$sILcM@Rv$kp*t$s4ktEp{iiV&b;eWR+O7^3?$9y^dc_N(V^%wbpl*ZmZW}s~61t zC)3`KlBcpmunVa)|J8NwWr3e`izfB^AQkzeKpWXQY){k@)2p5_!R@8GcPFT#3p_sS zU2P7<-pWbsgYLk%M&LUO#ycYKV59bKe8nkHyyH-9+I^Gtsekp|x9$Vh6x$K2JW4MH z?B97keW}HJL>CBgaJvcIuqZwH&v0t{zp6rmOjcJdt=5#U0gz%O;r5BPbli`~bn-B~x)jPcuX;Qa4p=fVKCY!AcXB)_9R@svcMQ3a+3Qf#anpAW6c zy`hp8b*Np5O#tA*6rhnIK0?8wYULw21)NewAS@DQyw=aryfmQb0zC~6F(8jHAmH%yD&YeYF3g2R$mBpYO8RPkdMs{f+{XJILUCPEi(lE9^uM}al?6z}`_pj_)mbUDDEc^i26 z^#|94ClCxrF#PNB6U=hBSP%DQzhg!rc^sg`bNY4$x@IgCJ_Sk>1Ce0sp47kZzXIY9 z|7!cT`@e6#M>bl%n(^E0X@sPdj`Wk)&2m9A|eG&Uv*S&;NUT2*W&tD|}H=7Wpy5$Op4C z;lrxxFPj050yU58a@~5snJrO;gF|XTcxBFwrycmk?zoNvu6Cu}Gr@DrqBwXLlharC zl1vBO)RIe=mBUAV+QtI_*stF9v3zwjExdyrp!b|Em z^Qi{xZ+SxKi*%CxJR`=belBN2@N*NRaj@ydsNK{UIK2gkP!gwG=z;sfD^oQzTA#La zO5vBp_e3}q=cE4-Kbqa{n-PV-zF=n@csZ2&dJ< zfPr0T)65}Y8PR7?#2yb`jv;P)6TsvSoOqenNdzgKy#1i7h!>dojt|V;PIc}Z;55sXdP=l9(^p|759HpLCBthH#}Aa`oZ`9GAO=*n{lX#bRAm^gh`ld{8~~gycM6iYEUB7zn&$9I}i%`)4W;V0V(Jht>^f zV!k8yO{{Cv1jw`yBk8d85UqHM5mK#FpJ3fnn2WQtrDy9`CEQO68Kxw??(_}4`m&iQ zn>(Hh5S=F6y#FT24V9j|Trq(4`!-UVkr>`Hu!LD=3vz0ks3PQsHSoStgeYXiK=vGzZpKaR8a6rQN!4etGo|kBLTOdJzt8YADqF*68=L zY+4i#i9+9$xs`EF*s$V5G6!#;J-EZDvfDh2F4xfkUa^ny{IpzpCqRC?vPY5~C+HEo zw2A<6CfR4qiAr<&J`>#S`=sNLi@g%rg=i@z|;p+JN}{J+d~3!bwR|1_p_WZ*zFg8JdY2H&$(=>qm|h~`0d88 zWfyZh%%J_j4Dq6hl=rxTCAnU4frH$_ytGsCU*D1mn`Z+sw9>F*#!002LkOF@J|RgG z&VYXmonzYG{uD{CvS4 z2zvgHZG^kGrEZme_YMX^>Jp5Ekly?SG)UqM2$JF;2kQZuO3HlZJBAWt5XB?QAtk6p z;PZBUYmLv}O4#vA`t8Ta9W!j|LYfuO*R{kX~Gkj&k=x{OR zgyuxc7eyW4QKwM~Y;XaJ4k9|Rj;;=@E%@FF)P+@9Wx#6|HcbPs9Er>v%et4vJrx)Y z3O+mlAgaHtAg>Nf|0Z2za?+B6+hfpony5lDAE$d(o?L1}N0%V|tJR#e1J<;%&1W}W z4sdoDCj#!=VGrjHHMfK~!Aastb2s_g)o|qjTPwpxh%bS!912Ze_R1@tsT?0hUX>l= z0g~f3qq>IyyT|fEsc3UU%%e9f@6tYuSbu!PUgly3^o}%#>ptxjwWfP1pM1AwR0`_Q z%ul*q5UsD$nLPe0@(4Nfp56?GD!KCH8Cq7Ut-*bUr}KB^_liJCg=aP&2w@$IA|4wz z09gyWU?8N!5TMlMU;(rK)zk;6jObF@{cH>4aH;$*7AvDf@#!;Um?R*(8&!b z5TAj!VC4&7_>dCm<;$(+T{TeoPk0>2{Bi?uVfbTXN!yb(S#~8f2){1p713Ty*{jc_ zRf2HseOZT8+!fPXa&@%N3i994vCh!EtP(;}!4)kKE%-$Ir&(6wqjxugE|6~v?;rNi z^h=ZRn^;Nzm0U~}M7eO*=BYA-tWFv8ZnP1qe?Ete!mwVw)ZOGc|2qNyR1{vBFqdt9 zt8xG7xKiWPD||`~g42zB1A?)^}Kb zHZN&k&5<=QopZ~J#!ma`OZ1?J|EfUB-SQyjl4>N4fd(x7L!Tv?k{Xl|Zi zj!2NPdK#Lr$aN7wpAeRyx5Er=tJ$^W!M|(Z|tTlIzdC>lf3BIlUt5Nq<^Tm~-|%FF_W;5qeHfl!yrS z9V6$z>|&Do^kuvZw?FH)k}b0zXk(QJeS<=)fX#LP&{-( zR1mXZ<8?!2fYl{@0Ezi8RS2-g=bTa3d*Q&5p}B_RA`OEM>K{D%u@0Na==gQGyV{eE z-kFU(OR^Kv7pt2ORs?Lq@qv7IXi2vKqKf33 zR~4e`{tcY0mG_o&UQI&*yPiUi5dRcXr0|&)XZQi&;?5gVlgjsGONiCF!slVgk!>pJ ztZJM|yhmK~(d5AOK36q1cB9m~^hW}b?T;y(@{Wy2Pli96zt0DS-1xLeo%g87+w+(p z>nEs|=n}0MPb;Eh_?gkGvf)rv3^I(x!*_Q~yK^$LoJi7p0jnH_?F3AMe?u6qKfACz zxBXJe>2EQe*q$tu`?_BD9)1(HV@WigmKpH)8qa8vN?apP0c^wh78>C_RjVEiq^C_M ziLc~F=qyRnDrNWFk00VNCHidqC;&lO-YJo^ilZH&&-2-nnG7s%+mw0h_s~!K*O8R3 zdXceMp|+2$u<*a4dybOy{rsWgc1HcLhxIs2qQ3&MoFc#~p7=ka}> zSXC^xPkO?8?qUqhJM_C!S!&(m8G3Jwc`Rc0Lv(=16$e0NUMq zg&0AcMq)4ca){?MH15c7r++038WzbRm^di@BInT7Q-|RVTyl#F$ zN#cH-@iNC$)^ouQ!q6}$)J3U?09q+e;jv%7R-)S-Tg~Fv-s)g$Za{wkkBTK+0U;hs zJXGJte6PM&iTX!8$oZr`sB{db{2cefDoJ1AZ*D#m-oYZdmG{q?_rL4IK4v0^_kBK= z-j#xDpZt3e8`$7C&CK}3T!m8lU>~eN6kQ*41SgS%V5hKZw=j)Y0#FP)dY2(Th|uUH z*sKv>v8vZVEx?Sto1+TzzFaFnv5g#17WrL9fQ9+6OXt`vpdPYF5qWs`#godJitEns zqdqueW_c6LUNyQ!6e)bV(zIh${I@c-qB98Qqq!2VR${EvJCyR!=6RF<@y{hl_Qyl2 zRdh>gWyr&rj-TmBVa~l0g-EWuk#WqPgx0ure2V|klh;4=KQV%yBZ<&=`Hd`3vbOwb zM`EK7C~{MW#PqMwf&TJ@9#J1^mA=^L?)=LLp?z4} zz^fRs$dnB19)LxSBwkz09b)2&L~W|Jf5_!{@4+(syl>;jtxMRO)@!;>_C* zf|Li*srkh>E${4jGP6<;xw<_rokHRO<7G2pVd?P#keF5p9sPK4xZ#+U7-rMwnLkG= zQp}}lGrZ!*cZq-z186@_t{%;RgXMksAD(?aQ)6-CqZ=`L_M!Oh1Io|y@hP=8=Z;nE6WMYM!8hA-?f{1$b8cd%+$!rUIY(C?#tyd?@}8%cbPu%fuV zHmJ?qK(RGCn^1^sz0*lppm$UUzNT_2bypgib!{*TbgoE-8kMliGrE|*OR;L`nD~#8B-YU(wWNs_(+5Un**Ep zff5*To$NlVS%x59R8Luue(S12jXGt_L*fDL?dgaseG8>+IdO-~L@F|zkWY>U^Dh1x z0rk7Qi)kd!8?2c~1Fy)kWslqI^)fQSdt)j@1z`Z2M)M41OCzTRx}ZKg!ot(XDZH5;arI>LD3nB^1q++cv|OT~`i z8ZoAX%GydeBvt!>ee56IT-VRx%(otrPQUJ(00XuH?IE}$Y?tClldCSub+=SuqEB+D zkt!~vrgb*u#_nbS1i$a3D{OkQhQ9C*_ovEATl&}ISmP<2KAlQ_-Grxw;okhm`w5qK z$_!LEkAFQ2I`dNsF(z*}iya2}T2Gyy!JHg6a?(VNYQ-;G6|4Wf_7F}vyw!Qmqj_bZ z4>QdG;vN z=^|&NU-I7b*sajdJc@(!q=!6FXSTadlX49Q)nc-2%~l9^p=1bvHRosomH4qXkdb@k zwK%z;z?zgB&4?-P8#|sLzsT z%{Y;tU%0KwHCb3~$ktLakPPO$8i3d~dkjW@-}c&{roA_Xy008E#BLYgH~|6E5d|T5 z1-=~Mav%F2rjId+NmKW#&3}4tNTnvK&2WU!&Nh^Zcj&P(k)yJceJO~@ zoS%KO6uItbmOcCzhD!{lYhWV4@#fZO*oy7o-8*q#kz1lxvw;y#OF@^7UpH9N5Gr9D zYX;BMkr2>|+2vZuzwSUhgC&IIbE^sZG9UEj@$y~S&z<4_c`&!!@pbI=$YmMMAVTzP z!hhUsnCf~c_FROUC;_J{ehp==1oXfm^pPqb?6%TBxJWN{YB}-$xNgnc47!yy?)4~9 zW6^M%8DbP(-}y*_8Fcpo(^}Ga9~-mB)pA8)~?JOV4olI{h0(@B+Q$xC5d~le-8b& zY#`>{j%RNi=Y+3Q8JeK8lqc~AWDpn6ABE0bo)xBW^l5+iByDp*_AG z{a+ch7yxnh2-*Dy0ou!wH}(i)Tdy_C+LlrjNC}H6oR&W~t|{>)!iqZ@y6F z{Z9uEMXfon-58Px??G!D5oo{xn_qE58U8r<{UL@3iFJ7md=6aaM45`lyZE<6eG8P0 zM+Mung>esC$yKLmsfO4+x7~jV3cjMTb@*iwBQd_KiT~bVMD7G_Fp-i#3Ag3VvwvgJ zeDa^SDwA}O33bLZdDOqk{PT2>}^ZuiwC z;D=h{g{AxG60UoTEx_=y8X}RY`67bD=rAHwZ~`vs`Cl9+)W^D#c=^|MK^l0IzPS41 z>RH|V-K#!>g^OjYfWDh6G?-KFP~=n8*#jfad4nU}&x-_VP)ifu|NZ2NXLv%`xe)Rm zaN2*^Is&#*_a^vh`05^UOnY*g&NH5O**!7oW}4H9xfyUZnHgZ~0K+~v_b!(td%2#s zA|rICEg_#ru(Op_*H7m-p+vt=$fN zl0Qxne}1|j#4)x@(su-^ZXsUZ&0`U>#&wsB4sdxCkP>pfg9q8I)PzY^z-%`J?NJ5B#wAUF*E2Sh8%o4VuZNg zhn+rNdZLtMTj=$|uiVd*tJpT=#8*~vliD`09q3=`vI~SPiE2whwhMl##D7H+MK?>c z9qx91xPZQD#cTSpLwZk5pbp&Wau1%yZ&}IM+_TuhJ}t1BDZ>aUr;y5D*_dLM_>Nhu zW{83uG!i$muzqsesr7=fVVV|SlyYf&jCFxqiSH+5-I=A@KglOh93TnIQ06WWwkHLi z`0(;_E#OI;>y-BS` zRm|I);;aH=hTh%rn;-wey*2XFe+YF-UJX&cX5d(H!3o{=vw*t1xcbYe_}x`48RXm( z2qznisI9=Rd#nlMm0S%6sVZoNE5d{J7WmoU2tT+%aICh?!;F{08 zghazF>D0pG24#JQ)Ma6K)cNP>Qr8}e3zM4XO&dkAwC6^+Tqz0GK((Yks9PR52Y)ee zaK?{9Fh z1OzF{6Z6zi=_B4F_4tM&(p6ufcX59*0K|pS-EFRos`0#BxB7L5LxZ5_UPTdAX^u+4 zk$9hZ+`{9j{Wzi@62z>L9lE~Nu3YmmKinE@mFXWlux76q1Ml#$2J zy~IT%@vm!(DmvUe<1z?0uks9UEt46=ExfsnMMi5nUL=8;h@pbhLh_fZRqa!_-VAAd zZ4kcH@p+K$r|y5suWeCLiF|VN$gz@cGdn9NDaOHVBs;=*wIW}drsdk;6KY3lo`2{AI5+U$BDWJUFm)aqj6;(x(Lbi7|Yf6yphgBoS@~ z@&3jP+jYo3-s7Jh6Ll86nw__T=~6!L{6`!G;#on#%J<>gaa>pc!8nirBEEOvD83b2DkFGe}n&vL_Vt7~BYWb7J?oTY5-bIK) zp$Wj)JV^Tv$30cGG-B}zio@Xc`g9iODv@tv5F<*T9f*EXNsILj(&5p#`)vj&LmKE@ zJYK=(vAM@6xoIfSeNoq*%i(xKmjsrk_OgAueO~k`*L~Z7e zG3nQs*XWS(`E4m7!$u$_u$@tYTjlC(IjL@S==w_alVmiyuJ(^(Bk{5D*_u!pd?>(} z^uz1f=n5YEtRF!919q7GvVTZ946bY&zn`pou#&sWCoFn+UqEnf?{`r&uIVIm^~=t0jOnZog6W`^$>?)m1L z2WWq_QHkKRuh>q}4<3bzfY;F?HpDLG%OYwa7>9-nN+Ul$mb z)}d>ObXR{(Il?cG)(n0iFAyZ)9h^xvS4GnJ9BiMuw#9}|PnZ4``H#`sEItn+NY_H$ zMv-g$J)?uqt%56~B=5pwGp^d|uO2)V^?gePPWIHo$*p{ z6+>TaHo3+CrpMqvE_U%n%+Vyhm-mR_ATK2a?1MwQ%*mg=@YteVRT%l&W=yGK4z;hMYLiI-d7jH45`uo~Q7q7}y zfK7gF5dWbfX3pw)gOG;zXTO37mt-de`NkO^)!O{6<{4L)>i%1|53+~T9A(i`akJ^c zVFDALp43U8v>D_o9SpxwQi_`DP?%B&Ku-1){GRrlX=HAikQD)Me2ovR&?D%ca(EBy zc=&6#_LtuIsY!%%sA6fY@p~ziWhoQ=OCt;>AmG}gWuKyRHw+T%Zbbhx{2bgE2x;5! zB)Z951iOh|T-)vNQ3|j7e*I<$-p-u(XT(}{B8#*cX%1cNXeg+HS=?>T`tI0~hTw>N zhzHIt z-wJuuWFu!DV+jd3l5|wjKaQ|98RQ;JOz;H4ncj#z+^U` zrh{^b3RJ;17r6k%*gQr2UScJ8CD{Z1z(^5DtkdW}FR`S0=iBIWdp-)hfq8OYqaLfU z1j)d>Q8r|9uSww}e2xa&1zfFBm|-k`-&=jWhFe5At#mxI%{ zxjnzZQw#Kz8CyxCor{W>(GN?%*p)0Xv_PMTs$O2ZtL9|Ug4sOdsva*IZz%yyz6G$* z;-;YwJo=@9yjDSv?qfC`PdR~rF{7Wd);QPDwHYZ!7!Y7Gm~U! zPTv^s34I*{I?#&xv?sFNk?XNy@n%dg#LZ~za)Xn18G{%qTRd_Op)?D{3rivId@I6w zWO>o~SO{H*=eR5;{Z(3$xo3UK!SZcP9P99=JicQ3&^^Dw^?L%;Fj+G>Xe>|_dx)<~~ZxS{*H1P97@Za9mlfgC*wjU)~yV?`)M#>TrI1Q(tWCw*OwNV6^i5qdA5vX?j-LrqYfo7yX$8s?i zB&WcgzHzMi`pM*atDU{M*6tg4=^GUi0(f9>GJ;sxPN-fqYe^WAM3x@MzT=A*ViVp~YzR!-_9svJmMlBU;YuI& zB7T*I{Ix8mee5wL*+JO8dUtdMBbwX!t(~x2fO~qFx(8f*9Neeg4#bHB=YUKSmdzEziS6~iVSC^u(*farDs5R(tY^Xw6_y%; z^E>>!^z6x7;=2R?S(xHg#>*bjZ>y12AMNW>=vUWb> z{bfD^cEU>vj`kl$t;6MidWc4%E?U$wc+7wgbwC7g>^gFH1o2o@d(9PE>al6T6J;pAt)TKLm zG5w}$NZ@v)%JyIY?_6iiObOg2t$}0#g|R3~p0~x^h4LjU-918XT5Vz;XmRa@&Ycu3 z)(0M;zK)$F*|@oUcs1eSgQp#Fq&9Ykc^C_x)1XTA82F*U+S-Oo?Gl)RDsMpc70trd zg3{VgqdG=0Xlem!%O1q5_Fj|y<8stHbqkYdB(dUj%{tB8qLLJj^v^mPDp^~H?Yw_~ zkM}I-*RTA&g+nbnt+uww4yo;%)&wz0L)F6@1q$e>4xDKg-+Bjx9RRI7H`SOGIGhxG zD$V_3JanT!yi%WTyM-NfD8m|uru{+MME}-aT@wny`_(~~bd+yN1DR4@833DS?Yqm-|<5+gF7u)C>4f?f}&Xc{@vbRpcB?YG2!*^m1M)UieMh zw~N)&APr53HF6MxBukt?E$KQC zB6A}^=jseIY#R|bC#fB9q)U-tfj;U+X^&&GiiY3hT${ym`!k$>pSFA(8+*`kFHK2q zAzFTtdV4^C+7<0JROnyM>u0C_Dqx*`=y-KKDM-PGzwiTFX!XdJu=tEBfkT!=(Tl@2 zz!_e0q8m8?nYo!t_k9D{N*svv7bn9Y-9Y^K|9x=S6m#G$rc(wM0aXw+(%A(J6C`6S z+jY@&Q3v8v$9>(}aL&d)Mz+jc8?^qi8FJ|+3TS_^d-=vx zKFR8FKAp!#ex_PL&W?_3Fw~_S;9jSiqaVR=65uVF2ImC3+dre!&uGe7NGn>-_jI%g zj1)1_#*OVA*!_CK(Ido zaR)cL>XJ5VK%w3MpW!cuVY9{^!l)JzJDwr6Wt#I@(nF-1rw-P0a_b2_`=<8rYuS%R zn@fUwb*pJhgylPNKPBuoI=lT3=wNYD@S8PXU>Ng(7z5dny=~6v-k$-tPIftYNyJ>U z?xgCCsQddaz=^zurlg+=_-(qqp4(*B$J19*IALzYuZaQ`@11i_r(kQ$$XLPN?V5ul ztIh)9K-#Qb2YiJJQQ=e?GR;ixB86K%-GlKjt=0`kRqn(XMeM=VLhc}^&#Nrh!uS!Z z%=x8p;9w~NqLaz$`v-5wrJWwMoZfd%!M#ExN&m;a5sYxy|6BkR&5lBpR{mTh@@O&V_ar;XKeAZ*~?F4PEGzjal z(F_R1QT?90Le7%LUCR^%S*B;lk?&Xf}{r(5{mwO-Y zdtT=}pA~+SSKH!J@e;dPI{T-7&!;Mo) zhWCtZ*wr{k8#RuE|LSgxnf`TL;vhKSL}Fe|-fQT_#Hv^@r}wor1OAm;t{17?V|QkK!+JqCehFni7@_sOh_S3HiwgNHRV6>J%EwIQdXB>rIBo^_yCT zUx(?^>NTtUQtkCi*6#=vlTx4KDH0{p%lDMb9ehT3K$6PS-39q>{<>NR zm;Q?W6vAX|ck2|BQDgYMp<*klK(QoAYGrbq4=m$~a^5f-DqP;d0LZwv)>vdBEqUwF z?B35U0^_!80O1I<#q$a!MkU*&>y`J=Xe70qdF45 zLGzB#Blk3N57~M-L{F*;N60obdO(5`~06DL?qHL$^kx= zZ&>@B(*8Qimsl>B)(;P+#*q84%;u=Ek}`aI!aucI3mFLhzspI#YoT0@i0}~-nO3_E zDiu&ZT^j5Nw_7~R0Uc8X{;+!2{NSTvIC|ETwaxem?A9u;`||VXmc*7E#)F&*ATbHv zj?(kR-LL>|!!}D=?QFPEMFY&xYl<>o-kl9bfhoN-f55_9j3*M>KMa%&U+A6Q==?T8*J;%dbIRf-;pYA&M@X;-D*1i z7wouNogBnKFJa&IvY1vA|Np5K0%Y}@FW<8GM&%{p(haA776W?f?_Mv${1}+&Q zwqiY{_>6{XZd(sSnX*69BnIb?zu+cD?|-WnbeUiUiP=Cb7RpQ7%e7+5?s6eMIPGjU zMc(O&B1N##BW-b~)1~Ec+1X2sfFAAk)10mHJw|})SYZD6SK$eyt{$9OJ5RosaMzLJ z@qN0pgrW5!b4zH;U{o#0Oxkph2JD)ao%=C$+BD)s}q-aJI zRv_?_7i8^a!G8}&9D*%hrhKzbbt~5$gZ}tty!?XPp?@Ohg+sdgud6Z$evIBSgEkXT zFr1qTb2_M+kCX*=cE4qSxQO0Am%3QRI=FZmSq1WSmxnWwXg9UZ0pewPh_EQq!vT$B zr>S6+p;SF961n^rFJk%>Kj-21{K4c)iIG$o^~lR*fyyIkfmj4G*VJ3y?UlA;T)-*a zp=(PXBLDCBos+S9)o-U49|Q;`3cK>Etz7xJ!nSU!y1itzR) zcpaG+%B%9lU;Vz;WQ^FyHr(GW*FsyJg463D9G~_TC+so+tAqkWkS-!KHj40C#{`l* z@5g&wi85gFTWcxhtDn3UdjRJ}c5X`dE&Yc1j-vS8=yex>-1SUo&?YGzuD55o#H zqu;vsdRpMw`G`-_89A+FfdAZcJ#8dhXy?z`q?WOEW2f^zGR>T^p?i$2tA|TIzp;O|ZwINSoEoHpO z^E$(+rz@ycjUiyXPQaOd?C_wNPj;M@oP$EzWCn~|6`|sxu74>Hp}A~W7KefshCT8b zZY3YJ-}z8ieFhH&N5sk1=sqV?ZB@rFo&V9j>vNdAyGs^Q74Y-L^v3&7USa)(Vqo1c z*5zUw$Za=yStsg^)izn$fK4x%YT71W=E>mxKY;sf4vwrkY(SY|Fjp_e{IVOMcoOc4 zBYBhHpj_^?LjFoa*>utBiIsMyQ@V}ACt~Wz&p*Z=u2;$4=%K9uhU=K}T6fqD3qnt6 z_Ex4S8z@F5T&vv?+}y$Pn2+97bMc2P!)8rU9w8Cxm-=O^ca2HiO^SPZ^kHQ^N3RZ3 zn+W1i7W+E(TVr>>r?uQoQ+&+)4>A`&%0+8##oi0TZ_aEC^L|Y{j6LF*@&GQ_?5jab zrX%chQIWK&3O!ckoBz6*12;xW2*!MMe)utN14?lyz_flV^mn2PeyuvTZ{Pz~mkkIT zr1h;iH3P;wql4n|Ul-NJdh5LF(CquRW$szN&1zH7&!q73bRHo4>4p z_O*+feaIKIZv$l?2Gf&nBNkyB^&~l@1^Q3dG@yj|SgBE~sQi*olYapT+1;qP(E>bwc?=sSAhQrrN8%ey; zNyxa1bNH2;zzrQCM0=>y?ZDv?KUsMKm%@$IezQbo_@!-LrzN8t3G=a3T@0a zB$-^g`m+gnEBCoI_3mL7Ge;chmf}$BJqKzRDc}&e3`-1tvp#zpbex7`E>-kQ&?V5D zkWlr)w}l|sG0r8O`?1v#OT6>NiuRwlNoE}v9m?EtsD539S1<-JyAHOvGW(MOqtivR zUB4Q;sFYMLIFAKT=UC1#c(OsEMdN4}N(^Zq&Z8jZFUuikG9>Ico@N`*let@10Tl(Y zbC$~O7v0(M5vm4Z+oCkt{#_J(M)qFM`u(zL!U213*Zz$$hVRCbb0cVg#W#mI6)wKqz$W>3pn>%45liDw^ETFqD7 z546xl)PqV8>K3nyXIzRANr|LDRv#!*t^i_!J?iea6g7O!@%edv&-;)sX=PAuebbj` zqEpWYQty;ciJrz*|Kr#seFjl)C~TS#4Ih^8k$!_A#CeVY@@!>jZ)W&*(%Tsr zj}x5JkSy%X3G|Zv3HdEXj6+p>{_qyd{MmjZ&}@cJp*ncyy`D~b>q7W5c~WvGCw9fM zNaFDRu#5~pGjbzF*2{1>A|n}^zn6s)%u+y$fIS8t{yUziuPEmB=+Wsbg3aB z7EG(0D^^&jBrb;}6|ftWg^pzVYVDc%nzm8BlQE}zQ|mCG>KU!47Otu}X*KH-1R`I= z)4z;tRejDuKHRN1*B1fL1VwgZ1>nmmpSO?Uj~`49|M#bIj)$#W9C*c>`Gehk?07k3 z(78ie-MDA#y(o2*M|;+BX}7$By<(i*_Xa##+seuG+HG=eH~@&fcYSN5-FIlu17Y*E z2_$t8*(BR_X4rhuvp+MTs9+YP{dyvo@iNGa-Mj0JtCoB-U%~-nIqt-xB?*}=> z!Q#P-xyS<}D9beLe4L>Zi=$P4<WAFo; z1Ik5R)Fjxf^$CpT&ueiU_YIUm`pf}vDZx(8A?rVxK4=Z%cKEL`0Jb!>PqtJYjIaDU zKhpWjZNCpjXWg}=86)5t8vLDqA>N$7%Sv93V{7^s47ba;MVFoI!dtYzOY4lLLHraP z{Y=_C2O5OG>}6~fQ);n(y!*!8gOq}HM&!ixtpb$Ui+17W2$zX+P@)YbqD7#Z7Uli@ zrBaXv_3QPT8-_iLxvgY&SSEYQfAa%5S=n{6$~%?4+)tzrzwZw zT9oli5B}_tx8nw}EAYME$%7l6^~*guhP7_*+|&J@9zd?Oovw*1$7qxG=RtGV6y%}b6qBb!V$-MA|P^@|a`8a$7bdCBCyi!vY_bmgYLMRl- zC%-38_HuR~B;;GTrED8rcYHy6*lTVa5=s}rBqW=k4$G%54}G`g`D$(!UGVeLts>`b zX&YhX&u!-8X@r_$1o}hKG^WKrW+{s6UTu_zk{_)}+9&ZZBNJcpnF>HJ+NF+zPVTLe zC`gtFHJvxE2sR`!ej2t$xyiSg@JRH|BE{jX_t8Q(xkFmFyo|;i9QMH#1m1AM)~i*d zTIk_OMO#hM`sjLjqTltyON}R#ZZvArA>`cua+RDPrn%e+5=P(<;Ah-3Vz4Lp4N&LH zxFthC3Pd#R>3@5}O64(uVZdIEBcGWk?Am*;&Z*F>usHRkvBd0*jQpX1?*)E^vjYY= zYkft|Zv{4_FmNj5&HkCEYsu$5J_r{A>k~PO_(1dJ=7$%DC%FOgM1$sU>8Zo<+Fu~p z*Q=UeemyYo&W}*W8z@1xM?C8KxauaW<-h`Pe60YT8g1atirF9wY4CVa97`{%{wv=; z+1u@n&6OWdOYmOgoto`9nd0RuKd&>1RD4LX^hNVT`OKcfM`ZyXMh-4fLu=X}QIxi>8fhws)z>zwT2V&}Dp=ov zjwy#+!j2DK(OvKeb9YW=MOyD` zHn>&8`!8^(u#|n@{FCd6DQuAQf@-&t->L#BaUzQUxV@5`cr*+w1yMhf)*=x zoV}dHfw3C!V@7Bp$F7vZWsJ)HjZfH!C*S(Kb*aS}>Lp!YXOK!kJ0i_y`faDq(0{xD z2nKPgCy!f>tS;~fHvM>m#5OGT3{UYbx{Fk>IQ7+)$Du0qsu}JQUG(tfXy{piOu5-Z zkz?7d-zLm-Kx4tYk?-DXIZ15C5PGD`+vJw90ZrWZxLXgDeIEVWy`@oi_L45W?ta$< zBh=UUHB$jU0?W}v{okg+(3ZlKg*x%X zHC`?fE9u5v?B)a`JCmh5_IysX;t>_gig{wKP81wYO9{SBx$nUv9T}2xaDa9k!ka?4 z&DbUi4gv@;bRiJWVL>8jdxUYU;8Pfn1~cVN`R_?Xi*sJGfqsoCbiK(uHypUK1>z!A zzcac|az+3kG3G|YIh~iHUwuMQs#il7Q@XDR(`(c~9Ou#QwU7A)c>#D{mj$BI^UsQB z7xL;e-g|u2fw^<$3=5!k}S?Xg7AhdpF^JUM^F zOR=@eQ?P3G^fD@hAATp$c>}y|;(kFo=|N_TZQM!K*wUvt|5;ABU))UOa{#8T8=p!D_~U8%ME>V2Irm^m$HnxvYMmNC$e1*MOmbXBYvJt*bW`1 zZl%R~Z_QFf%3Y7re)wrsQgiulGeY6N<00;VjPvB;e+PpC|KLiUb1}b z`5L?bC0VV^IW?ALoblV0#V?F57jW(KJ=;y%-;bb&k6> z!0N^Gqu>83e#7WZ`$k6l-^*%8ft&a@uz!c;G_D;OsdUPuZW_44LXBQ__Q(5^QL|z` zWp=nMwRRArI5a*G1PRzqnKU?jGy=MOA_knp2fEImd2qC8-M1(B+qU9O?5FO@g~`q@ ziUEPRl!rvLu5hd`=J|ojU?xJ=48cAEcC|Hf09TKV^Gf?R((Vw{{i)&#Swe1@dF_ z8bF7y|FPH!Ep$bKrghtD#m02`dBkvBzdsx(W*XooPL!RJ!_^jDZTs&a*I7Gb9M)hs z+C!(PgGdydXSb=V;dd#1YTSeYb~XavtesuF`G()j_UAli_Q-qbh5glUxc|&{6hQ3r ziu39m5)Z6t@7`?stYxs<7WY~pqtLi#@IPZcv(q0}=kfO9b4hyKeyJRERpi3jWuj3Nkcbl$TzOQTl|+a_wH&*%phVtk^V1ad--#iLN77V8e-0e?YT^! zf-HP+q75i=@h@uR7aS)VE_}KBaxahk+X!O%uYwB^P94otejug)@7Z3Smk0BMn*B6v zpMV354hSh?c~e8_r?@Ejo{6}9f-5|!J>mlv-R*u)`J4n;0UmEd++l+HQ;B>mZ~mNFY%`>JuCWKvbnPFLrOAxRE)+Xt}yt4YA&DG`lK z`7y57u`AO?yx_);#vn&)v1!MO&1;9o=l0aOqYy5ZZ z1?$>YqV;%#ds``o!_hVxyXpE4JEWHC@kz#hhZ=;tt3%0+z@_d?|A=NJD&79wGWo%P z(%wYTgS3r(0p#bZS{*x`8XR_0`thirMoGNqs4H`L`5)xT!q;>7s9dL4xF;iAC0TT1 zfP|s#-gv}OAEIj?N;S^BZe_oQ_h$_6gddG{ndaFJ z{3p4o5Z?DIu-fPK8|mU4dE{&pq&$9x}{~okfwzMlJ+Tjnua5nC<(Ge85&_ z`64SI==z}c8cueu@#f|oSyG^N3$Z*1>-~;V3o7|LKNe0MKe6>STsPbFOuZRb!R}zz zcFz@_i*lB(^B|J6rrT@Ya8V-vq)2Z8opKVK%SxV@4qOB$aU7e~1|>Mrq)Wa2dn^4Y zm8tFab)!=tG_x3jYhEmbe+(G`QT}dF#Ib_W=%M`wM5y2}$XWzOR+r=3xSscSDy1VS zDMimsiD~n%qigf;X+yE6@gt_V4=(f55_A4Rmnnmf8;gu<3acYF1ky+6-Zngk4|cA2 zgyChD{@&=f@4)6atG(O8+w0Nk_yQW>Y0+t2cJu`UT%6RxzSLN`UK+No{D8}$MLe%5Z7xd$z7+H zq_va|EGiLjYcUH9xi5511H5|1&kfa(>s0t#1^eMm5GKyaD+bCw4xax^0m9a%1R|Dx zEd1+sv_CkVrIy+^Txtd5L(1wNn=$)c>tu4w8r|#J3dQK0&F{aK#t1+sat2(mH(;1Q z=zOg*e?=Bf-e6@4YPMFKD-$^Q3b89UL9_R&L9YmcuLzdv53gQJm9)qglViHSw&l#z+UO)(6kwwhneyUv$=c z4&H zwY{VMxu?@_;7*V#@Hh=vZCQaooPCl(v||t{?w>40S2k&S{SArw1YqczbymV#lKXp8 zO;TC^Am-wvjQs0`V5sUl1pWa6(N9_h5cXaCl0X|bH7VOGLpBu|aOXcb^mQZ7+-+O+ zWwZi4gZ&cX_w_olH|F?d*Hb|E#Gy?T0);5%b}ajZwBJS>ncnpO_Q~0L=a0qLSy%}6 zKkc>Y?byWMqTL(ATr`x@r>T2un1M1cX%EEnEFjYmBdkmmS(^Cx>j7!31XiitqVsOB znK0ILnxm(VD?VS(^6KJ7L{&UuPOlF8B2Xc6>l@8>FfMw~Uvb2lCe{AqC!Ooh5t5rw z?6#CBZdJhUx)B7p}ImJCvuH2<%YgQ3N zo3;Os4HJxYYtnS|nqq`9$%vK@+m|f!u`nE@_!nRDk6{iE<4Lln_nH_&dUJLNe^ zL;DS3P(xnN@w+W))Rb{=^V2_Wgn*P`Oc{ynf1NPseSdg(lk&Cq$u16Z{C6B}4U>3=a)uaH0tg_D4~#r!ql5;4_VtN_)sb_o6B0(t)Ip)X7Ov6~Dq6e|Fw zpYm&PP(C)k9UHm7pwz`QsMse}gOYyTPDS!=-)-zNft-h!2S@euiZm86!15SCeRqgi zAkLdX*>8Wb!fFq$uU!IE!FYLRwmBJy)UGoQI=ueX`R!K!#1H?To*UY^Ik_oELCR`bWUXv9zn_v)e@D^=;u0Ms9Y|P7MD&>*TsBrGq4f5OL)4i# za<~Qos`b*53M0X?HI$NQ_)#qByNegESw(?*Z%Redvh~ZU7g0#cDI!|kO^U&R=LX*= zTG+}T_B%aW@NOrL+x2`Bh@`rX5OjKM>X*evOD7%q`z6eZQ`95xMZO+mvc%^?7s2=+ z!->Ust<%q(IyNmoj7YCjk~I&ry+cA|ZVL@7r9>(`^UeL`qbxT7^y2LSD}RQfMNO`c z#C=y1FC}eK%I}%m?JBhm3KObP#m0}uF*F}I1WFWN=XPH!e-FF!W+ep-7Dv!#0PjVC zT><#uJsSup`*_0S$2BCogeM{au9gl!9Zx)o1ml%hpa0lQN{4Ix+Vz0K0`Mz6?3avC z>ly^H6DRA1-NqUA$~IB@9Y~D1zN!^nS|QBkxz*K$P5IuM>yqotF(dxh8LY3k$P~GC zJNQa~_+Jv;ALsBCMv{41_o~bJr1kzKu<+UsY#7$3PuDaIX$ljg1TP?&c8dun`b6f+fPmOfc3*voorAuD8!)ALz z9zmE=$M(#ucTl0&f)2S$r7i%;8K-AK7e{pAhX6C}_7JKR!Q>=*E zI>zmtr1{dOf&z64lKZJ(FOABJ;)6a+3FP~I1>%;DVV~|x*b@YHBXHT8xY8#0=_2|4#`FMq=gy>8??~k+8Sri<=(^<)lp~ z(x7CwP&6=LW~EkW(uA;#Ip)W4GFVCdNL+Q3??o6xP~>Ize#cgUbMRg&d~VEgZ>@8D zV(L#8Bhc`&8jhMSpM1rQNcvVm<^fNn(c$ZFC-Z^v6>d@A48ne63-!K&@ezQI0NjcM zIm4fR4GVL52{XdHDj*+Mi0hq&PoJWMUGxj7HFZVAh2mzd*24onvm)(=CwVs;vtHb! z8(Nivy(f5J`3QNSY_l+kQvB7(G}iQ}XWJw{Rh!dbV;UeCP(eyS67`9(AOJmjvm&>$ zlAFXdqog{#Zg&OlxK}*-bZC9|lgrsqFXM(dbfl$&EaITOcg2A1wRA9|>s;nH7B-A;3h7$0;GOCM$ke znTned0rm$g0EK;N zDLIeIf4j~~dU|lsmuP;r(3G|gn)sT}*`Ie{1`H*kkBYZo{Da0SjiJl}@#nQ4HCTB1 z*ev>vS@?e*4;J6$pUL4-F`U>sXSMh%;F!^83$qK*nu*H!Spn#m2K?M`f4VidAc z964PLdw}u+G{J)IihQ#->zC5Cz&0Sm4}6}{*YPi3uh?S!^rTi>QJdLk4=~-7{QmA} z4usypjbj8c)}WgdJTLz({aR44rW)!b=(}?l55%NpA?+XY-4xE%MgFjYyi~y_UIw_H z5f;U*%QgQZ#-w8p;=|WtO{BNd)`}++rUNwaSKbG&Uq?iAq6rm37QfK3Hf8u1>9F_H zlYwaAtw6VV1n%)D_54O9xasz%W13G#^IPnDh4W)$^XK&(Ev6=yoqx86hIr{(YcPjqnS0dIglTK*jWdpr!eLkr;J&p5gns&Hb zc`F#s{4_L?{o>36d(v#65)*xDXY-LoHT7<3=vBza)TTL!wa1d^=By(Cz%w;b;g1@kCc95U9Rn zzI~K%GFGB(eMqj~a2Qcv3U@wx$6heU2BCF-EJyNxnruGA;cvtJbL!tlfVM=#lN{#) z4NK}~@~oVa?IvH+2w=%!tB7+bc0Ee*R-HnwFCL5!!f)jKj##!_aB*J>ygA}LGXF%f zm=XTk={<~2?$JeLLi3HD@^Wr|%hso?!~gVcGA7=`l1|sItgZ>L3yXP8Nc+#4J6iXJ zsWA!cj3s*FHLRd{5VSdvK@CW8t@5YDi$txkKc5|{c6a>2`X01E~3MgRA3_ws31vt+DENJiEr8BW+} zv%`C)s0`sD&%b}}b6{5l48Ko^Zh%fS(lKeqLBrgy2^mt-T+2y*@(<3}+>2{?xG5DM zl;?E3zf_IlZYqD41VTr(;C)6-CQ6#s=#KRpn;D{z{zg3BuOx4NyF|>LU?^S$VXN>- zdX?KJMwNO6QJuj&m!|{tYVcod>XJWAmk%Qd<1UH3e z3yX0ru`B%}3b)_}wFbrGL}5hZ($ThKeV%>Ausf!PTlF-bto&kBN>u&Fn+@jK8Q`Bi zh>v(+Z<>M%m*Z3Mea=a?vKn_$s@RqKUf<~$?;eKRnQ9HnZ0sFa!>-JBuk4G?m90Ps zmS#h0s9c7=;?ab+m&LOS*PfgHK)>ZZrKfM|tgJ*70C&1t$SWOFxaPeaQZiW4^Ka8M zTEJtc2DL{C(F|^j5%Iss5ZM?>WSS1XfMRl7_RwT)BF8rWuaxl8t_;SO<7o*N-Q3X} zfEytr(d6EQpers`Lna?0+fgJ!GyPDmUu?q7{{@3EzvX(I)H{W9kwO+fW++hAtP7$`Y@-OyKm|JCJij8#Te4JE&w3oa+S1`XXN4^!2|7Wsq?~-;?vr=a7N|`_E-FE zEPE&={pK8g?mQ4v2GXJ{W&?+FOUA$Vj_rBh=H_%mg{v8p6!%D*2z3>!G*rJqni7A8z;wiCOhVZt;3!|9xfM-^RWFyi{)#7W_zr{q67dT1+DxI{BvNk%ok zo@Dd!DU`@dQZ}=Lr0kY3d;f{0EX&*+^g&uWFP%PCZJ1PlQ@G**JQmp`#Wh3Tu>ZwN zsXigqr9eOo7g?vBcP8B|Z22-m{hIlvsc-6xW4$@6{Fs z=eX>H3uwH*eUQjtLAm1cgY83?^BG#+@(*~RibD}UXfAp4(F4PvNukrBruIW22l-~v zd>6Bg56qE?YpbrcT%KPP%7Xz%WWjA;2O_ zzy0!a)Wkby1BaVnMdzVNz(TRWN9GO2E%WjB_8W|TxL|G(fjY<^1qm;4#Ci9(1a7}F z$qz(1QUUpOICJ_7R52-pMh6<93VAyj89U9(pc}4&nT?H~c#cy@ECDB_5||$G_#1L` z`{>zqRgXjx2+a!sQehS<8!*+oyt-=ESJU)=Xv_l{H-662Zj_NQfAV`Kmg?J*xPjXB z6ga{9RaE#UMt=Upy$J%3zq4<&r))&V=vd268jsvXDONCeRcq6{4k%0v>&7}vVvY8G zrvWEdqe^V9rEqzoiG%Z|1Rx}OsCtJL^u5-b8f}V4!P8EjDSpd-3-D_i`C4;P4pR7p zt4KrKxV^f#xB5dO!e>_%~x1xshps8f^f6`A1 zTP$J76FV&k@?A=>+lptg7~$S$;Mrzq?RJ+=nzCZ3rZwAtv>S7GQWA2m?tIcvk>WT_{TrDw+JD;PtZ$m!g7EYLiyx-oe z=3)h5oijW@*_^?OEaK!N=h~;WDdL9rviT=0aeU0oy-&fDO_Ol-!vOWFDpK-4KFHR6 z#Z;%K5Gn9ablk@?hF=p6Y7>TYFT~+}PG80Xu(hE6>)zt_H-B~&Q+&dPbeu=0McUr} z$ukJY2TB!Y+&+Ngh*a8R=j(J!rBt=cGIHTVi}xyHn9Iy#=yQj4-)8NxnMl?pP*%%| zCnc?1o9QvN`z4`zQ^r)`jb>JMRUX5=4y=zpl*Uq|TGZ17gu7oSa4_ql=LyWZB&{%i zV0|rDaygdKrEc*zDj6o8^W_nDyQ$uDBgKFd0SXY#{ZTDJ6M9loK!q~=z7T=Hx?dzh zm_#@H2s=}R>?8pu?3l+Ru5X&tVo<_0$cK>>7y$n|x=*F`Dr3SzeP0ZZ z(@N7Pw6(s}73u7Bz4l9;AC5kvUueD~vDG4!vZ5c9r^O)KN zAn0{r2(q$0=p2>DdGg_mOv-IT13Ev9cFsJx*$*fFb%#aw)XnVQbO#S=zy~*MhwY)jvcFvf|jPcZ%$FHf|o0N5lk7(0qZrGNHD?@@na2O-F zV>$x}+&H0tgn%LGbn4O&Iek@S^><|WIsoyx?#{11JnqKlIOm{_w_bl+G$A9IrUsiWgU3vh@d+TIWa}S(L+8$>>$^$Frv*N4q^1ZC^ zTY}4;1P?jawj$Z$KYzu&lub|2mcQ*gAz%sf5FWbJik5d^cI>>!ocPMp->1T>6PXZWh<7+ z%lLTajSwXwY5XvA+tCL28YY&^W7y~kWI-vjbHMYf(i zQ{4-7L=Wk$pbzGoefNMPmn2F+7QS6!lAID!LXO=$+YD6Z#G#1{Aid<-D_a9`xXMx4QI$7Q$r6eMcVaGxt!(Uv8QJcVl(dBX#_m%**6G=*M4z9ptE3%c=4X~fj?BfrFRI7fQ zXC2rX^LVjAySbJh!Ogh|z`L{ky^lH73F*n(7a4ot@Gq$z?+T_d!*d!u0<6YO$dawkN;1(go^0Fo2ffdmob*hx#)5N$(+N_T9 zKm`A&y^7Y+Mr|QqKG?I>KlaGw^6!7jCLx>aKWTfTMZ36kpq6p9jgGvsELP!AB#BF!)?Z6 ziHwYt!-vz0%dgb$6zDmHY>2`K`Y2sLjrfoDlSGkoVWq18JP^@X@DqX4?%`N@)bL*)5)V`W5u-@Ws6>w8h~w@iDAk~=Y&Dj+al}|F=3<~6 zf5izR$#$rhj`sE5YMGAnZt0Qg$#72BOt&JVl(LXYk@G&`kEZussaRJS3pms3_^lua zk}O7D5EdQN=0z1Vsu`En&P$sVZ&Z~ zuik`VN|eO&Db7)6YtB{?Ouh_2NaXCku*)j)jev!p7~a3(Z>g5I~{f4I?|d7 zWt>u6pM}H+J{Mc+8R=B~J%i?J(msew+X@XuD>f-qNv@B;`t{?upw5a#2Q_3xRbIo3 zL&y+sPi#q++PvA&MX2dwTX%6o>s$A%O-J@s&I+TIKDcwY-Si#JpyMnyE+d;ImUVjf z7oV~-0eXpPrfEzl}FPi=k8FEdXH|ARpw5J_+V_9vTtP#b35y z-F`r>nXm_b8S!_)(Z4xgP0`q3MV8oLJ%FFZNS#<$E#k3D%SIzeG&J5gk%ZZ4tbBcc z{S3a+vP(i!LVda6u=R2hX;_g`RLg5w6VX;eBB2!JyhFMNhj+7P^L>PcTAzebQG`=E zIGl~XzW5!1sf_+_>yi_%0bITNZ4#FlEbvKZsM~aq;m+o@z*@iM(bJdOdH0yZ>(|HW z{O{iqMm~`4u4hZ^5zxr>g<)URP_!;*&2~`4QPBNIG!5y~4Y@KHkOxO0^{TyqSZ&ri zh+m`#w!eUO*k2Nl6L4vpAP&X!U^Wf}(}Kz%>@{ge!}^~(-@!m_;;lID43G(S zmMc7-3+4RkO_d4+Gx5f#R-6^Sgg?BWo+#}z_!hmUY6y}~Bb|gE?`~)Ncj*lF zxm~F{8QZkI#ynizt0&GOr3J(}{8!NjeJFxG+nTDl{j&V%&?{!Y}a4 z-k=?%dL%~3X|3!Ujizd0W49PgiW@dx&<&#sMhU;gwznSSmAL~oaagI^4iJ_vZf^ZZ zsR0fNiWz>Db3GTbD&9y4I5pbR11{945~N_e8*j5t?oZva8-QS^LzL=H(f5#6=K}I2 ztzfJQ5;F7qR&6kT+_XISl_s1wWe`W!56|(zm_*%I@9z`)h5E=Nkn#DVYOdSj>~#@xg1do>VbZ3I&YPiX=G zsF3stE0q~1#!aADQwS@(`{X?%sFXa~U?8wU)0t)5N)?%+FT3YI9uz<^C?oak4+>pK zta-`Z!I7VJ6sgs_`A%m877UL*aw2|-BgADd8Ie@6qVTI&um?2X=y#4@YlUDj zNdUPKY@qT<86Qy2H?f){XVWtPDqj4Mk2STiQn>SRX5NzXpVV`uOR2Mv(A9vXiL9gKK&|P}GAM=|0^Aas_|a1xvpUdfwD!d|-FEB;lV|Fpu7>qR}qU$cKyILbUUp>{m5#j-_t zX!@`9!3)7e?1)FmT>xHZZ1KO560#`|moyt<&P5o}n_P8n=y)8xj+z&~H6iw$M+fzA zd(4!_%^U~?;a1v`KQX)tRl2PipwR<5lp}Rh*S7BtkZ4Hwp`uPKg^p9sdqtj zL(-LK9GOj7v+8(m3c*Kv`eXHq{Pw%}K6nY2SLxk3=<2rn;toGa&HB?Xqy0yveNuMd z`0^}zC`rQ*sAA`mNlEUT`BV8wF?3=$Ofh2<1@J--CF9(bjP4w8-39tdO=lK6;Zhtr zc+$o-)Nbzq&C^Or!x( z8A*)EpHX`0UDyRat$#0i{`QqD`Zv;4ix4$&O_J3OxABRpnF~06X=-K{Wc;)(bbR^K zzl}s1h+jIw9~_r}u_}l4+IBC)hNh;9V~$%S)6F;~iUV=&{M4g>9+@bf!G?uf*(^w0 zhGN=>#};(&jw>mE;1q$5z-7^^DCpeZ+tMPPDy!4&pMTmERlA_#U~|M#0S#tZPD$qz z6BrvLt@%(Y1&05;su^M?G7)l&p|KS?6w&Etwkz7{N^7Ti>3scv6`hGc6aF8^UBx#_ zCCa&!tCF))WGh1CsN99g8Oa>EXH#TuIYx+8lB-C`S(|(A$z6`wm}_E(W7Ce`exJYL z^LTtd@AvC?uC}?z!xkmbYed%L7^70p18+^m_q(UM#nKW%-OT>n+Bb+l zSqH8|`QAur+(M-);uX>tGc|kis&JCVLCiFTcIM*wLY%(W#b3b1A(PkVD65)K756nZ zU!1QDD_T(#ojel4xaZ=|lnA2wdcIZqO_-UrL~QZFOjIuJ=a4CWL+<4QMr#Lb=G>r} za}UK&8?CNGz1K^f!ekRokg5?WhAa*EQLe@kU$}BRBle zl~PIZkT17oV7f;I@M%24qOn&T#%ZhjPw0jl$xH3&1x5sALWow&=#7V%$|iVNEQO5p z4LqBiwQ&839J^6njLC@)M&JB)*hQr1dF<4ckKyN~1foa7T)D+A&o$9&94Y+h*=~x@ z%Hks#N{-F*wd0&ON;QE|2u(KiE8yby>4YE5&N$D|BXF_KlYo55o*(+2bx2|I4LB~^ z?5FKhc*p7S1e)v6Uy3V~x&nX&>BuW0ARwK5fJL9vPRPjbRbE|Ra*&*Ts-Ylh8sI^X zr9a8Sjk^6c^+DjZt=6CSeiMAPb}$oR6K{YWK2Q-qOU-;B4YhktnZHXPgXvpBeN^)^5%}xrU_rdc%d33*q;Y20HZM&X0bm zJO(=|)FlC&4kyHGrYO&qQ%GkcSR^c`9UIE@a&8g&rXT?Mm70nBFOpIC4Ila78t!Lrq{E!Q#_v*6R__?`ZP-ZeUz8`VfE{dGtsw#QMg;-0?0H%LxEK6Nt`L@w4?%v%Y=A~fpKd# zF@^&oS2_Jc#&&4l{aSvq-Yq({;}!Vx^8NV;pkgF#kiD8YREuKq*yTFv_#>$uRW=pU zjs6ku^j~5Z2{|^MN+M$%cg{<&9V`Gw60eyyf>9JT0q{M?J44f}8|zzX2BOWQU#jjZ zB|5_0pjSU-kG*~F#e#VC+6^e^FkE`V45_yi3TkvcnDI|#e4*6e*=pr$npT26OV;; zGS?{NSCyn1Zh!e;`expBc6$a~E;o63zh|YEaX{ixwL5FU_#t}BhAE>7bSv29=Dj6t z#O$Y|?9BgL2aqJR{Z~TWnY*W5sv;Rr4=TSMHuwnM;ST5jsN-2%ddJWIu+8{Bk$6S^ z5_Y#~rQQcf)|MCnZ{8HVUtRBU*uDLrdr@Skvl<@YL9;w=DwlVJ#;CqnPrzc2NtsoP zH=GQacFI{CS`dc6i8?w`Z2B3h_r=R=Z7eD8Umwa?I^W0M(72{;AX9NroIOx$J-avr z3D}0M39HmE%>&R&Mc|d$V{B3QMxV$WQPtcb`ZMSJ7MmfF18xNsRAHPfp3b*p7&*Ro zMN}7QMXfURQxwV$TNL>GLRc?+i3~Smjo99t80Ffn=MMKZ?9VnWTd&dYhy66ayIFY) z+=%5P4WG-Q<=}k^1N;BAtI|${GL#rSkb4uTFedDTJp78JN;b}Xy?!$ z_8rsf9Kt?ghHm#EMGY=|eHL8EIYn*925V#!w_+K(KezLZrq>}Svl%M|e_ z+2yZ3ak4Z&d?KjQzauYB0|ef0?|ty<4moc5Tf|7N(zpN9SdDl8@N!qF90VGQ8|yzK zd5hPFE@AOHJZ|{*q-aV$)O3-j2}|31_uf75-w$4bQpzvzCbi4iMtC^7Cn=>Gy!^#G z4^aK8RPL=auT;#@St{gdl%cUWXl^4!VG*@5_VMXn?=@RJ$zl=xNH4wcovlDccc#*8 zb=#*nMKzMh(w=y?!DqN7uR^Wp8S7;63ZEIv+S6(ZO{IQ8DV^D}jwueTTtE$N;LufxV^OO+#+psO~ocX-5I93%G6mctSgcFPGgxBzwLYI5NM1w_~nX{A%- zQ~=hgA4ezp@&>B)N8%dXPMo`!EA+VX8YxrY?LyLm5k|R7Q;J&c%a8+He}}Y*d+7ot z3jm=ZNO5QRf+MK_3&U9h!ZqQu;(&A7wl}{Fe^n91bm|caHnK^A4akvWjmIw- zR>sehuo(GwESIH_SFPuRA`b^K7W5VJZ6cUi4e!X-WiK9hBCHFF|Gk=*bQOK?{Dr{p#W(XqZOk*8qrS>u z=a;5ZQ9DH_5r&de032c*a?-p7T6f`b9elxdonok5a6mu#RJd4)vgSlZ`Td=nHyxP6 z*_#KuQqrJ9kiH}ES)RHw@yeYEJ7g!A+;4LN%5mv9^=Z?Qv+d7V7Q-ABzB_zFrRR$XL;n*&xnB?%ty0QwqX8=6`=H97Add5 zgEhoA+cZXOo_Rr4E#}}EZGF>C2PRo{4Zu~+J1M_6 z+B|+8Jhpp248{tsGq3Y>pI)@V>; zn&kyfS7nZdJPeDd1v%9~SaTIr=2<`o!O@uM!(F0RBCM#=>0R=5Nm;rzvuj5^YidNF zR``BOU+00>{Eb!e!mcB5>#Gp68Od{|L5Z^aqVUT<8SabV_M>tJuJE)WP7dbDL1ONc zVrhMivCHag8PMlW$Tz(z4(CqBszunvuvkSD?%TVrM2XFYhbQI!`?&Yd(^WH7>d)!< z{nN-d#(qJd$V1mT9cFja#ZgNe&LIl$?+Nu#BM8v!;>SfU5iv=uhBI!-aZ>>^(A&U$ zHh&XKymV0>zYo?0R)&CSuY~j#cxv) zI9T@!Jw=tz?c=Szwvt53?o_uPjImq+t2~L48}ewuEXCV%0ZgRBE|^l}vZI2)d7pXt z9%rO;7gnwd%f3oGaOd1+fcc5Zrpv-tC#><20gn{Or+$3Vv9rF|j1_?Aeg#6WO!RUd z>+nUWHMda35L=2@S%G)_nl!mh|FWTrHisA%6RK}J9SMXYVkR`s?l1D*oumUChlgSr z87&u&&8+F6UA5d9`kmOKK4Fxd^77`nwmOcJN2~vKy6J}4bbl4Q!#8;XVdJMp1;!H= zlbbX&P^%=tQ4^8*7-?N+G<}NRJyp>=+Yxm8r}NQ1cdRf-kaajIMtE*W9u%mj1bZCV58=2k zE_ORNGYs`vC#>wgbSV_ZlOPO&UMj~%5e<1LsXu|*=|qfOymXIPRHu7kQn?H?J*Fo6 zmF2{h2I}8NlEo4;4THSQ}dFv3UkI?<)NqdlxK@_#9ti2PrKLi%2 zaO*zEQiWN>(O=fO{uF#=(YIAyJrwNVslH3hQFi<*pKE7?MU1TBV%)U$E=R=V#n_m; z$i7*Vo}QqVOJ&#Mqk0TY7cUxfzg6OyLa*}UQc+A{e2C*w$h}KiFY)>QB#VSZ0wrgG z;>i+3J!SO(9#C%Qsi1E0A@JdR1W^P17T2A|*;3Fq=H1s52*~M|OZ(}ydlZ}ZUZn!` z5F5&xsid-4*m*Dz*lieL8WJg{6>kIlYlr4|@DMluPQzK2;5~`H8=nWtH&5}3OYWSj zXc4BFp+z&`D-p&{s;a*Z=rnB`IFBnk*MjD0FDg4@aQrdWGAYjj9$1Xu#pNiawx%+) z72r+Tv>&Yk$i)z9x(hlQ#QY&iLNk$Yy8Sn(l3m!Q(sqC6`s=g>beQXeXvB+Hbrdoc zyhm8{^D5Oj=PN^d=DrcE*LJDq&uc=fKJI(oYW`r{fJ=>s2MR9uZlp^l4#0C(w0qF<3R$nCK;ldd{ zlP=_V)gQ@d$EF&IRls|+6<}&70V>5YYmGBL32tu#`!&IjD+D-&05g~7bGQ$KOJfDc zz8}HR6%D6Wr-G<6Uwokb@(9NkYE%+;wik0!TSQdQ#MhSg8)WcVvb-kZgMR+EvtTx1 z=rU{5g=y$Us(m=sX>%UkT1^6TY(_HB6u~&HRp5ma;R4gfg9}kWj_h{A;>E+bznO;% z#LOz0{rRc%?ug%?91W~E6kU59#om^aM_;y)&mEXhS=KEZn{TaP?0=ZA`9y2flXk#B zWqmjV&|1>$Z?#XbEEF{V#h&B~BzQm0J!{M5PC!fX(0X_6UZ^IDa#t}F;4Zx5N;GQ` z-sXCBVR*&*N}_rZ$^}e|GWszC51zdRwJF`z9yDVT=^BEni%HT(76@%nv`2lO>kn=a z$tBk=3=Xx|XfnSCEK?Q*b+x^=j#{i?E|>c6NQhvHwRZ`)%&WcK{l0~<6CZL_ zBDeE#$JH3kt2Tpk;HpLYj%ui78J$s@f|>wxB; zV!n?%v@;e4kNmEKwod3BDn)&KN^wls}WE98?}`ogG~W7%*AbR-Xt7jhfh z#SZhfOyVPYs*AqSg?BQvajV2uHQmw_{XMbau*^&<$fJ#GM&Gowk*KWJdT3@}`F$qY zcOShO9^A252-M?~mBO|gXFI1FPtUyP5C={U zr9)lL_vbJvs)8-94qU%-fy3#QN2&nm3n$?cc0y&!gBLDfXy(T+|FG1R`FXi%WAxnH z-aknn@`?cS^&nt4KM}uRBU7;Fgr;uyJwXAIKY9HzOt^lVi;7`_E{&aB;uZgUdwm>}*NAV4eKUxa}N8$*BzCE}DS3MX>>eMm>eeYEy}#QXlt zX#Y-;I-odap3l4-13llvCJ6FP44l!i>s?B~Xxth_72%pV(}+y!p$8nGsyIz>sXE`2 zsbL=P%ssO1GLXRL!nVO7BZ;|V{eENNehua4>#T#1Y}!^B29^U%9z1yvkl#LhMGTZa z&rz0ARdx~F6zstom)bLkc4{6DbXh85}FxVEdkLi z$&Z_E!$W6Nxa})i>;>^%qF}fFbfT6#5720~gTxR{yR|%7m?!hX+T4Sf1Kb1Lvzc>& zfKX6;q)Bgq!#E9#{s2!dhkM7NyedKEh~fb~Y;y2Jx5a?)h*+zb_a6hV*c)x`;Q1#w z3xJ56(Thc9qEygNA%C!{`z+OlzSo;v0G3r3-5A8zt)@26_A}r>sl1)8n1%x_X+x?CwjqDxeM_(>kwQ?t zckV}7=1c^~J^588R}Yp}4M4jApk6l1qYv;FWwW93p6V})%ixtad8WyhYqet~1Gze~ z-tyxnHlIp#r#^oN1g}D_%%=DS%RY)@-3r~NPw+$kWIO+!f&R0I?>bH;3d468s({1B zXr@3jzvZZlCd}va-txmQ#mS?*+%=J;8yQy+ODkHXNTM4f38%IZ)hKKzkGPv^6r~^`$$~7=Cv38mE@XnbOb-2psK<3!<4&L|O{_KdwXGc%4-3eqSPFI>e zbKSrNYy76<*wnj%8JhrK%_RWj$LnccB>%+M*IQ(rY37Dw&lvoZNQ}~|Fkps(^Ouy- zc0*+%G#^z<8yYAdf?f6s@t#^S=KAKrhoZQ5GEN}DC%iOuZX*XDXp}u@u0xsYxW_ouBxwM}`0H_=wyA| zE8)_i>OKbmw$;eho9to8`su9p#>P@i{m>v!HYrMx`by5{s2fgqV%IN2u``G2{;S#} z7(C_JHL#g4!TVKzH-;cqyTWYUbYJYD51;o&OW{neeF^8u{&=>3MOrA~?FdpJV zSYd`@e7yIF=r>t}q62JMgr{OifCEZ+OqL@U0qnPCM~vzAVAWSinbTGsoAj%8aAv*o zuWD3^SdZJGJp`)nD#ZmjSqj)I^?gr($f>AJ$#J))lJ(;mu}!}FFX04CDff;uyZT$@ z44yzaWcc(;REg2B-keS7+|){0hao1Ky6u~P!(lZL$EGcIp3i^I>#mUn%_C6l5a^P! z>!#Rsp#cEt6KG$x)xQV)s9bQ9Udl5Q!j2ysPa78L&HdLqdHuyUL@dr}NJnn_or0#u z)ho3h3FLS-gf8mRizhfvtzM0;@IyPk-^a6h9oP}I+0o=6~N{Rb6BX3y4 z5iV4cW^ZW|en}IQMT+TnetP+OC=>YD9ENf2e>0Cg{8J!oHPOl6dW}=^aM*Unss)1+rbRF+Sba7% zS^dsY{r8^f?G9m8-(u)oUlX_hU>wvBfuHDZcJ$scFzxx_sGe>&>$_MnNuJCsS&yi* z?S#{Ys<=ZKzX4zFL(&!$TFy;eGq<}lHtC1pKHZ{AsJ|Suh|q}G&Hj5`YQ6kg>-TLH z@Kyi8(;^duC=6+%3mPF4l)6`@ir!|39??Zz7I ztV%vhgYW=#7VO2Wemv>Gq}*g@;q;+w3>`V;kYxK;6FPKtq`3YYe^ONz(}&E_>Aq4d zi=*$Z4@FD3K~IDg#yC21E&p50#uK=4t=!6S^zF}6jtF|OY2C#@@z}oC8anXk#M0LC zd+<`)JID$k59QE^GI&PGf^LN=Mk)-?G zAp#plve>m9P|9#iZEcyjfDFB2Y_A!F^9a*j3Pm!I-(LKYNI0 A4*&oF literal 0 HcmV?d00001 diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/media/foreground.png b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/media/foreground.png new file mode 100644 index 0000000000000000000000000000000000000000..4483ddad1f079e1089d685bd204ee1cfe1d01902 GIT binary patch literal 12430 zcmeHuS6EX)+pUO#NL3(IK|}&d7YKwF5CM@UBE5tjTBw4Q5KwvxB2pw25vBJIB27p@ zOaSQt5eZd#CxmkF|4+F-=Q)?(#XNgvmzlk1)~tDFz3+~Fs;5bRo%8yoOPA=i9zS|^ z=@P~5f9V?4rAwDs!Yjfq4p(5Rx~i8hRVUG&*j~LT%Q>2AIqB+Nx_^yhg70E+c&i!%2~zqE0}mxIX= zz1$7|sWj&3yL#7D|4uLjQqV+x(Rz4WC{A9|^m@1A6`BNi38Cf3B^aJyqxF{TjS&2q=3$BC zB1Fu04C;%o9V_Yg;Ed;xpmge>%b<|5q52W_pTd9o;Qty2mQ+-Peu)^(K)RH^d5byH z>AGB-I7$|~9l)J0H_LPDsUUL#brIHpjO1>dJ9@_5&W zLV)s!AVn7*Hy{o<1zLA_Ky-TWzJ_^1=W=Gfyc#1ssqeY_2ww>;ANX%JT)(9uNHOtU zeqU2_{Wu6pLvCMBLgy+dx=13ZG-+cMrBf;#8KezD^}_F2x>_Nob0^iXEv>aML;8RQ@@sN(#bq~VsOa>) zW9RDe#_!zLkj)PyQ<05AjbPk5yJ^|B6q=sMX2L0JE|(P%=v2$6+4QL)cu$c*yt`EC z?)p#@xE12zK?QF2u^(xb0>KieYWS%DH`?=eOiFd!6)WRmCo6Joq6}7e=Nl_;oNJ{1 zu&szm^c0s*wAxfHSlk^+hb)aB<&B?9+_YvxC1LEy$(dDJ8J)d!>rwz?q zGTpJ5&uVwR#t4%B`T{*~RAd_Unnf&`*9c^zbZfsVc;v*@=BHOCX7VbyhnS5G*Pik} z@`U!W&dq$A-&GCYAWg@rG3W6ANL_2a)|;&HJSig{zyfyO87W{;ej&@-)yx~eu|G6S zO)U5U?QD)!ey@XcxEKX?m{R4VZN!*V9gT}6_lv@YD^}}y4OM(*#%kMMBij<9x4*by zCkGRQ3vqoZ)HvQ4oY~=kh{c09u`@Lzqk8)3R+$+hcYuhqajQqgq8qWy8X_QMy@1+T z0&yU)D$XzuW+GZpAB%%|^3*{x!r`8nOWhu6>t(2mvERH# zwD(@F(UyHL)A@d0q#?|SOaIrK7`~^_KhtD69y6E{G70hSpvkOuvhEmR1(|2efAmi@Xw9*}m%vZb>kVqe?t6*aL%179k2-;CD<(T2&{-rQ;%g&4b= zStwf@&UH8&T6lBt>jybuLy}~>HTF7(kmQuR6(8*l&xSQq79o~y=t@1Z0aSiA&-LWp z0NQ{@*q$n1m#1Z}?sFj0=6jxX!@eHh_D<=qD}vOG`kCQ^44In=iDu`srXYt8{4c&) z7G9;S9(*ydG({X#u#N%3l}&Yaq*lzrY-E%htNRQTrjCrX1NMi~a!soU$|=0*dXokbDxSFnm6OHLV@%5(K&ZQB%e+ZFne-TrP|veCOrVj;0pG zdbMMl{Z%MBfVA6b>SKLi zXyRQXFc}Krl(owbvDh?Um&9l0#P)rbdiZxK)8=RY8XvSG1@0=@vGxtW|3E{`T&9Zk zC0==A6=d?8`t>?}z3d12SZ$YU4KZHQPf~|w zJD7n^6bjSS+&0Kq6nxhj*9}9qDZC~A`nzEz{<+9lxx)v#qaCsGWko<{ahFVncU-R|715> z33|Jp;8Iq?Z)NXe;h$K{z8#lRB#JC*XUod!9+#hCfkg#-^FD5Jq@>Dt!SzYr@q0(& z;I!1>qg(PU*HMX7>G-#T5V;IOw~4L@XQ&5le>B4Va!sx0P1pm1PMa!%L##WB{CukUKwQLR#mw_r{d1DneIIJT(j#O#-det^FD zbdwZ-8R%84+Bo+g5iyd(a6x;*5F0xuclibP*ff{7PNPESiBNJu^Q2?h!4}38?XKcb z1cb%?RlBpM10D9~`7(D`#uzQxY}K)shcU_}%#WJZ`~FU)C1j&^b5i=Wc7uJW8^-NB z(rs3^Wms@#S~)+us~_(~uocjV^vU^euJHB^upc~CY%6gqBXHR3{FJ}D^V0uB8xrdo z%j>^}CvVUV6jaGJf5i$e;gXng&>{)uK?nWhEUaVrv+x8njtfCz>cqP8uUTn1`McQ;CD+jm zGle#Cefq~0!!v@W2XnNsA~8j@Gaaj+fT)QzP<&gR$L=bGEJ8^z*tHxS)sZ=vZPV!4 zw*)4rK3To_7<;de8PvEPu4Q5d;D=g00$bPnaG|sEP6(kDsxwc2+y=l@=8Gy3^DW?X z$=3@Y|B6^8mUadWxX-6z(Oh@9|3%Nv*Hz=bA3)}AiK3MrA@eOvp)YSd(Nf|v;6dz-v zI5xYnKImXz)PTM}jxK=GJh_OrE2HXqKgh*KB!U~;4W!DpXN6A98^kNt%~i7+I+`g5 zW}~Qod0A;Lw*Q@m73+!Rfuir!WXqcTd5mXE^DWV3AUSVk>5EA&b6Svd&!yh*!z+6( zh^>CvoV~2?y`UJ#Jho<+PlUEw=Y?Hyd8C#Oj$c!5d!Du*w4OQ9G&OxhDmQ=)tzD()srM-?#=f>aw-$x}3Z?qLOIJ{gnZu zd`Y3Pu@-6CD7)$*a6189&`vfy%c7^DmCj90Mw>5FgU_yh15-*dsMPOLpn%G&Gbq@c z)NN;i4jF!g3-}@w-}i(YUbp4WY;xYi8`sa3ep2V_UXf_!7A{;Fhp25CGF=6{xLd&d z!Mvrklt74KI=0hsCRMYBXM0Z?v1sDfN=Y&W2dW!hUyqiiU@A}R-XCxbIudes32?<&DQ!Hr>qn`aYQ?jSq?4X|x(CCDAB;b=wcWVCH1CfwqU1di z!|LlwpE@R5*{9XlM;`OM$(VZBN$c{`%$ZT3S3aYJwVO}kw)@4_EyP4SXgXkd)Q z7PtWeexnE98(N{TMKt-aG+YpQs`a~e_Y;}upm;CRXlTWI->sMI?cj%D`$7K@mQ<-e z6c3=23v>}kQ!+Z{G2&KQ99s+el!e053~lQJc`8%`$;xt_RQ&16M-jjl$HK)VZG-0esPL)%m(*xgTxhvj>YKkE?dOv3G%g-W9;dgR&pG1FoW|wrm7v|b_Y-VU zKV&S7NcSkHSjm4nrPIy#Wvwp8(lbN>^x7o60ICQ5m?QwOuUY9q(q~<6`0+a7 z_`Zhdli4>YUiT%XT1&z74m|S7pZ;||I*2@$Zd5=|9{V~xFLGS|sAE`ZQ=toXwPUzSz%(Ar!@#M}4%I2r*Ca<9 ze?7@cjo0^QC6zocYls~PXjm{I-w|^|?Hpmvl_!6;&?vERiS^(A2e-)2qxQ#IfuJ_M zgEhyUo8K;fE}w8OE$6nq26w$M-YgMyeYnhwguXF-@5ca=0xYn%I)Rl=_lZaUn5tgl zq{GPw`_E=ilA8s)Jy=%ks{*^ijmr0SqHYg5D%zYfzlqy~#fp6GHI7wm_SN!mo*B=(4jED535Cy$0WQgpMk_!VjQ zhjwgVnse1csNUVP_rkF)3q*bk`=D| zRm=kyT3qxBA7a}d4b433h)JR1r_zBVy6)DMRyM?5%=@^}YMnjurETi?w8)8Y2lox+B2Mc9(WcW709kmg&QO^PydT;QZ_K7tmYO8aA8M?Y);N zSn^>S4^jpy!tF}ZAn_;hcCNY$eyakky`&>*Nh{Yf8H17GR#{9&%f^ps6IAlo`0a7| z-5WT~hwWze!uONxb4D$Was0UyM#f|Al`@rMWg(+oyWOL{(2>P6$`ht&d;q3uD6W+D zQQKN!nzWpx$Ya8CUKa3dgn={(ad!Lm7qDcu`SB#dKHvAM#GW}Z>EZmS6yG22dWcVi zef}3H%>*xQE6XidovM|h{PD;~31ijm0ia9g=-tnlFk!0PDn12luSSt7gWP{nbUK-G z_;*xp66cFpR2OkYg+1wGZF$3SCHuNOh~T{QxmE}&DI?a%s+Q&BqRkJ^37TgbKmAKA z-lXW9)FAv@J#Z=C2lSk4@W5q7S0~BpAs>m(p{^)b2MCFka=_0~yTtPvSKJEH%6&GW zKv;f{iTBYXA0^wmTAmssRXI(3556s-FYRfgXSs2F7D?)Muw3X(n96>Fe~#_y!;5dQ zdOQ?Kp<{m8r8ee4PPIETr3Sr=L{BgNp=Hl~>nSiYS!vY-rs7>zJE&K9>k00!&bs>P zD`CMT*(GNFuh#^fdZE?R`V};&3K^rq3z5UT^^KE~V+Yq@nxU<{+Ug^t(FEIk@f~5* zgnEN(6_Zcdmg55!i|T1Xn2NBcinnnFghvgYxT5oG<#r&$ky|k5SaFs(+Vr@W6W!wc zhr8=;xACvw0kVQ6m+uK@w0M_|3*`l1D1SbQ1B%k-HMIa!=~kGkCfuQ8^C^ZQ&7xn%?zUs@ zJv~f?$}gE-(aEgrt|vKx z;}Q@0S-w8jTszP4_+Em>MvCg@+IT%eNk_MIr)gA`;*lhuP%vm}{=>pIah-$r^3{Da zp;l8BZIY#N3v`sN%POMh>Q=e-o^BM2OK_7-ztamrbZ{m49XWXIgg1Gqa+C!XfX?gxVvl@Yc z?lm`jKKariU3($HdVP4LPtp4+4mV=+tw*rjI~_q%R6DfIW|6`<`}My)W_VK!6c^i* zIvi5RI=c%+#{fOc1^%pnKBkmGk{n2 zC<)woa7^dmGd|$2v77jNVg{v9cP;?R<5Hz&w)i1YTrbpNc6%p0{Khx8hi!J94klTx zC9LuDS+2u)()U%ug}~voR<>Cq}#OQfXF2)TCm)4nk4dkJK<{Ji<% zcP30SBMi`eN&Lves%5zi8b`z0j<83Tc~cBqc7F%;N9zZcNAe!JR3!n;@j1h z1lCS;R&Xw6EFbwYNCw_`r4_DiPb}ogRDYy^watxfz7Xy(zQ=RKaRMV#RY}`WgLrrF zVY?S>T2T_0_gmfEc1P>euBpQk$h-TAw(GijhS$+YK=Tg$zQ6?>D}F1vFkHMoukc{a zEy_ED8Uf0r#&yr0HH7|2|B-{vV9-6x6%+AEp3Hd}4fvb`f5|t#1a^r!L``xWv0pYp zK_sWYo?M7Ka~?Ti?_2#VSWzD;+NOTq_0`+=>-+<27aH>r;wtxc2mAJdsVzr(62hGT z)&mW2D1I;#ot)2O9iIWid6J}Na=-qm<@K(sk9ppYVwcO*IkP(P8P9ER7!PsMfNBn& za^K3zdtRPHN^c^l9lmBs5m>rjxgOV7Io|5p!v}X)j;Ax&u7K?;q%XjX_~o%@lPr_8 z*9Uqq$6~D2?gL>l^=mP&+~8z3yT!99Io|+z9QCQwYR2S? z(t}t86UG(B`86l3E&Y`O1p($K!sj_~Szh|(peg0h(+?ymZ?)sk6C*iUD89q@SVAIS z4_&>H|FtF3pZ<_*-;w|rv%!y93`xISUXVWp-T~!8n*#@16?Q}v>{P^~9I69_ z%n*6qXY%Yy!%fWkW5OADjlkEKjP5d$8>`wRrhp=ra6@iEL)prjHQ=o3@+N$WN7maZarII1Zz-rqUrBVRY znukG8!4Q$))$$`IcgoPA;izr~)m2%Wl&%&EHeRmOXUJsiSwge{CQ5;l6K*f{(Y$dK zr+Ms$jZr918R?`Rysv0Z+#6wT~L%t0b;+Q^{rT$Y_J%=|3^Wd zt6$*epNax{<>cRLLyEm2t&MjM8j1U)pYxwc-MDWDwN~$V|G#;ney}e?-YB~f0-n-M zw?G0{JBvufZPvKoY*5O85X8y3)1IFwLkMFr+5G1knQdDje8Y{BGoelP12*9EUN%KY zxk|^L1xHs)rNCp_@p0*`=#9{%r)_7IsX3T&x{b&X;mgnjUOMtgKs#ylC}%kSdtkjl z8!FE;zg-elNMzzYzDjZ0)^Ieq?HW_G)|Sg=4mBA1EloCGZTG(+tr)OPwRZ{J7OY5O z-u^rg$|QACu3Cq*Al+><3gPrW!35XM#YAriTfXw+!m_NkpMN$HY+wKfNr4L9PYUX6 zzlS_jplR*TFaNt8ide7lbsipOGdSE!+zhi$@D8y%FCwjQ$r9L{z>FOk9`c^?Kjmj` zMuYzJ3lU=4n6Q;tr@a$L?%8~af{fraE2*s=hn>Cp;YCQ#>re~C6xoCO7}(mj#Xh*k zba*^&l5yo%qnHQd!W*<-IXZ+8vnMb>c^cM={07F5{v1ulw!aVecf>C42Ir44Vz);s zT-%=b<-{YEZ*nD{U;m4uIi#wyf4G^ggB0@5%#DRIbN7hz&!Bb!hl?A6#(~|dZ%%iN z%o^Sc0oq?wn5_;1HQ*s%km5+`HK!Bq9^dL$ZL7!o2j@&piKs-)bi>dGD9BCC4PSIk zrGJIk0P-Fv?{`4G0`eU>*i`V_XN2xXw%*xTUlVENh%_|iZDkl5p@Y866#=@Xg{cbE zjZtS75AB(^xEogv2B)1x^m!0XZdCqOZ~=~2%7kuI!6E74!u_j2iau*{do^aD^2Vk^O2eW~KSv(BzRD>xw` z&*Gb6ksujl^_Fg<9{Nxn%B8jSv6jcmU+Kw5-Q&psk7EU|G|_)%rogKwNzemwy6QX^ z@ujX`ZkT$alQ%3oWJ2VOJGz{G(ukN|LF&Ga)nKml$M>IY@1F)}2mL&m6~?A)CN|YS zLi^lZj;aN$DQnmlc~AgqcDB7)?<<0=D*JMD zM3%;`BX_AsO%3+;YjwAbOnkT+m^;*q5X>@S2hO@Aa1J zJCCx~6B|ewT}HQECVls)>JqY95!(x8tJTl^D9t}c_G8p6;&167Z{2*+*qbjZdPBKR zwYTwFdQwnL?Q_fZ1S5+O2`Bi&@(s_P_cQY7?>NOU&FL}U5YmlM6yw@TASK}~;pon& z&{?aE)kw+rf)rVR1R!KIA&R@6^&5tt+oJ8h+P)7GWpbZ0xhG1hCCSz8pFjdYT5mJUum4y`e6ST z&@%+@8U+Bx-^#X6vpu~G2`=~;;97zryltTvX_;q&`r%A)oV7(xhxX1-Obw!r%_aBq zXumue@LLi`iFY=9t~-zHYJC&!zW;W6TKK3YgAe-4E5@wu_HwjtlH4Ep5vqLS-2C5$ zSxHdkc#a7g$_vSgCJ_dxxPL&~SeaPflc=j>z18KsBxhHfhSRvim6wzyuJBI@*m2g@ zc2$Hh#1|Nide`x;s zFEY{lfS)AO1(&M2`md$eil6mNBxu2_M(#la)vUt>ub2uO+!3=jb#6Ic2xq$*jBF`n z%L9sP{NK&^17myQl!*yca`I%e*{%{^D5ld#5&5Dbmw2He%xl{Z?Bv@+UmIbjXEHB5 zH5Sh@UPidw19)2ZMmXkn`O@)IsF`Fbj+RLtb$qTJ#B-vXrZ?7??}cA6N56t|TzFj4 z=rAukcL+Zk?vE$J3_QP=HeaZiJ>sPUrar&8Ao}%X-FpDz+o?UsRbtr6!(ES)@vCo94^P>R%u%q(-9wy%Duenrn)jXuW z+2hV;WWLbrH-awRI4^BBwkb{USY=a|U+=L6IJbHc+!%aSb|KB}H$ z?;wmaMfCf`2o^LLsVRHayM++C2aVlLWRbMjawRSh!|`u4I8tjLx>H>?ZR&ba(LJXj z?DRP5gyUNUnznwc)C%qsQ!aTlw6i(@viQ+~|0fLN?FR=&Mz z!m?8%ms9Zm`@?A{S+a>p-JQ}TICnZa{gktp_;s>#3Wv_=7#GC;f$M! z&TRADKS2F7Grq42P=N2(^g3PHSv9Sr5khe~OZap~yE3UUWM-{Fh{H-BGK9MOV3L#y zw*TZQX^enrYRj7iXkEaCLTZF5z%T)MU*{_RxA-*;G{sl{7ry_e1h+X~HM>NyBnnV6 zzcFEEZvv5PId&nY^VG0nqu!l%4Ln9L8OVmkfQi1}=-j_u=t%I1_~|`SZ_zv+SV@2>e1;w+Y$vY75F((`NKQU2vax&tTw!~HE>c2M3z3d>g zk@W;ee$-qtx3IgJ&cQ;-5AmGPIIdtV0YQvcV7G)N!(PWkx#qq=;AiOzb$C@x+Z zu##CR=Q`hVF-LGTr?w9-umq+&6PrkTr)T1CJ!@XV9i+em9sS#E=UO}BNMwuBrCayH zAub{V#`%5ecrycz1$eSV8<2Ikv6CQ5E=h^K%3m6h74APzqFYP{oejD^Y7o_E2b3p| zeA*LbkS?zNs8`f>wX`CuZF=Vcnc?D9l|P;QF8KedIQiHkm!f>Y3}# zl9AL|w=FC#e&CG1Vj1SX@K&6z&wEdwI}i+9}=0 zD)hP8t2qSqGq-zz1>nRbHpsOX+Ou&rc&B>1K5Z`l|60?OVRG!%y@dyXhC`Y)1x&pBnbuTa%|7f^nM;OIHu%(W6&Ci`84e(2e5z z*ThM)rgG_sjP#cQ+Xs8;_5jS%p3?)1Cd0epUI+qH6)RAoaWyIr#O{wWN#wI+_de=e zPHAv`+(8DcYwZezvF?o<#{{xGw05-!dGx*J-i6B-YsG?>W6ke;g4Hg#P+$=@?s0UEI-*Bw6RE<{1I7> zjBlz61z%K{w(Fbs@*+5i`|zyRlh@qP_iu#(*1Wcpz$is&$q|YHc+dRFT7N)#@B@znBGn$2wXOi+ggc5BJ<+2( zlI3ksg*I$2(gaUp4h9pJY${1?hgh6#mU-3e=N{4cTb2V_4R`HbSASd)X&1AJD{hd8 z^}36_R=S?hhh>k{b|Q{V4g^$!<)__{4ZCIAOzE}*nn%8FpA_Bmaub%88)q94qdSj& zU&K}EwoAH(N;V`V{ZfKgP}7P8xX{2STb>)D)y3#SF&&=+6Jz=_o8pqGbBI1lUdL(1 zD2L567hm`YXfrYLV3fz4yv?7yE!3uaicqZ7ufRny<0U&B6qh8bcqsL`r9)-JOxkXy z+l@a1(ptpJ`{M2l$g!g@DX;KZcoPP93JT=vi}|dQ!tn5*k@U)brT5a*!NEAJ2Apj0 z3jNsKvYjiiy-sUG06+A3T)f+N_X|`ZAX$1+M8W1ZaK3Nm6Dd}Xw#CnL+A?Xi*n>}B z+g^J-yeBCQ;(6yjA1~5bLwIzXXp>6syw2d^&DXBrf$G@}~y*QOne;u_UdZD^Cl zXxza$QKpgXzp22W4GZI|8N{0M2?78Z`$wi+S>waN@uSr9`u5+ghvrjfhcjQNuoDp; zk9szfi0j_VBAd2M+55}LBoF!BASF5?QV6q5zf94lQ$2goh8#I@&N4tiMK&5WOgt0H zRiGPL-7G)N zj%2#teK$kweDwBL1+DK?B#>r?tjR02JIr zUq=)|zME?3CA9?-DRGfqM+;h7w&xgGmLjhTAOdy`b%#?iM;>=l7v)^GADOA64 zy}x#1eDIpJ^iQ-mHzp5#R2_{6(~wo;npi>z4tuCy@Z6Ovw1EGFOaCWi{Qog*{?+*F cSLciz6 + \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/media/icon_mic.svg b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/media/icon_mic.svg new file mode 100644 index 000000000..0aeb30d63 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/media/icon_mic.svg @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/media/info.svg b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/media/info.svg new file mode 100644 index 000000000..2210223f4 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/media/info.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/media/layered_image.json b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/media/layered_image.json new file mode 100644 index 000000000..fb4992044 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/media/layered_image.json @@ -0,0 +1,7 @@ +{ + "layered-image": + { + "background" : "$media:background", + "foreground" : "$media:foreground" + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/media/startIcon.png b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/media/startIcon.png new file mode 100644 index 0000000000000000000000000000000000000000..205ad8b5a8a42e8762fbe4899b8e5e31ce822b8b GIT binary patch literal 20093 zcmV)JK)b(*P)AsI{U0tD9;7S&f z3`9H(<`G*WCN>bN493AFOi{!!!L|afI7%o`6&6lXK&2`L1YumJiZTQ+5doQ^Fu|gz zI6Nvw1cME>!8`;4iI*N+z3;u_gZtzG5&vyF~^*1 z?S1yyXYbweAFzGO*PdLxe&gE9j&{c{J=rY}9i1#6cCzdq+ASx~UzXhiC(H6orN{Ar zj;qq$yDTU7NWP@ws1J2_*G}Ykx7%{iE$G@-7-eF^Y3#}`(v#ySiIZdTj}`y+a>=Im9Vq=f1W5yxR*!@kj+Rxz&v=+4_?qb>2v z^P8^zTt$BB=j8B|JpIS7`QY>Jz4z#w<>ZT>lB09T6nS2-t-LNa`Yg!ixr}^gvZsB` z{B;rQ@uVEqwOt7oA8%Sn=e2VBs;^`dNc~|xx$^LKH+*6BuO8<1`K9&UDuw8t_%!FY zoV0NZ!^eH~qhBH?uakr4K4~ZC5VHnAA|L9#J5r^|-)7;Y zUl$mM>pDMqeipwr+7#N+YO&F-3t!twD#tH9_S*S{wQ+C`@f*(uNuw}s=xXMh&DI;Q z;_u$0c(3`5*FEq(O?pz@6#ee_pZMDAFS)(D{hdnlGw+UhHaZ&vMC3y~_HorR=oT!) zD&Jv0*w5!@vBS?MX~$>r(d*!xjZ=9%U3__Gl0?W|%cDAF&TIVSk@)+3cqc!3boGhhYzil=`)k_5%wL2pqQz`Ju@50G)sNfVj zoXGZ|Q(f3+@xx0`O2~K<`L6lJ-SXStp$#*Nk@$Du%RKJ9@n>4_fX zCq4RXG{SB86?4nquk-Hy-E#B;AN86?zpBs|J16`d(I5ZXNB^!~KL7eV0uKN-_1L$Q zfhXMkzP+y=*8|%=cJL*vJ8JS$i*h!V@e z?gp)OZL3q^qPRQ$mTS*l z!1Lo9sgwA)pzOQd7ry0nSAP)8dF^z>J#;@|{wb*sK5UU+HV4!!`0VEJLKou6^E1;q z{-F(t{g8gMTs+F%4CL8B(dE++Be1u} zQa1d_@^?2B{4?(K#G2gBZ2YKxYj^wS1vv8wb2h-K`rtLS+C4j5oS5zZQT6pjk(( zJ4B5)x)C<~DS-Jn#3lX27u>p0yp_M+jn)mGYaUy>+T%Nnb1#0!>tbyAQ%)nklRSgJ z&7=Ic?ks-hoA@5fJ^x~JiY`PYkDmW0C(plGd!Q$Ex;t|N@d~qieC9rdJUa(Jbmg%% zxJoLcUW^RY7oUugb$iXkOVyLI8AJG+ zNchYly!4G7Y^6~5nrXo&e$8p}lUVB0m<1UOEOBY-ht5+)-??6hPx|GZjRV(b``>-$ zM|{PjUt-09)0*964ZWy4qG3A!iZuCL5J4vSq$?ol?wO2=1e&!;9t z{HK#&d2T{`aKZSSV$8nw`5IF+b?d?_&_RB2Nn@S=KEJHRZ&{wfFD-HANt+d!8=g@V${FeVy<@Q=p|RCl}k1iW;RIY+rXYw+ro1J ztScYrS3bq4R+FlcH(!!*-yB2t`NcV#59x0CP?FiqC-VdG1vMIuAg3o=Td=#P|3Z0B%|-@17rLGk-6p<6~!$6~POh1kU3(XXZO`=|>$d z!lw$=5_RyEi#Jr~RP#^%iC^4A^2m;K+VClBHe2;z6Z14*Mk&|$%X0f<_lmdugY8>E zPThfcKaZ0b)2b2Pn1`Dkmvb_pUZ*zC08jjo)ep|hccB`;;R{6kL;Ts-DL%Zk@M}Ec zYe??S-~5VIlRb~$9A!25WQb$>P5#6re$4=RZ7!m^$ICJHQwLq8^3qO zSIW*0ziJfhY2#Np#+5qaD29V6USiSHHu0r%dVQte1>d!Te30L9h<8T(gM1~;2HMmK zAIaG=K2h~u$+A`Ao#yL~^C@rnmi3*Dn>*0%_Q|VFij#Is9D-CUfq|-t52LPSO>Mf;|h8QzG9r>i*kxj)D&%wf12-@hxpQE(boL;`OLW% z&4ra*97R9KXL{m{MVR>LH~jeO-Z?hkb&`yq#K-O6lT$@0DD?-g)^Uzc7T&5n8gw__ z0DpXP`45D@vQE5>CYLA9MXJba02$ioVhjTWVS5bZ6(4zN`ENe`p5>!H^k})NKh(Lb zKhik@lUA-Xx~smjY)TJqEB4J>%kshNC(AGX&hhfC|NQ3id+))>f~iYr%eBS5L6diS z0c(T7VNUk2yzB*+mM{H`dzO#=6GzJf`m=$1G@nblG}%hD(09V$W~@UCQLSS;5BqEV zWae*vfSYo>EH@?Gc;aOFp#GTWmw)f}@_j#ZYkBJ*Le`;RxE%9>G%3oHFxKHSfF_;E zFF&fw_1jO}dg1SWTfI@g(_fZ9_1ee&mj2x4J1a|pX>wLqgaW;Whu>GnNZR9Y^4s;%W zx4i1NzvUU8TZ6Uq$a?oX>%J5^9jAU9em|0;-_C;e(1}uEYG}e zr$t+qTP`-spu!U-M~AgevS79|o^g>`wAc>y@e7Vk`?z91a^qxq>GOBXzxbc8ET8gX z-7Xxv6CigTGJZUUv*`9=vmA1gzg4h49N+Y^ODZ8#@KI9`q-_X zaPu5;fuSS!*@le$mhP;#HK&jK(B1NbUvXvmPhY0_kiYDk{5AHRoIkT@vw@Z8z;F1q z7l7fCCi(MA@@nf@5q}|i{jv8-IsM&M6%o3LI{BfEQREKp4HG$@wUJ1eYx}Q!%BAIh z`K$LWk8838tEq&7|H$p$UeKq__MwZg*U!9Rnw3=(J#1>imzU))z3%$*uKvrZuZ{Wd>ES!5dgNmrfBPTZ zSl;rks&UNFhD?$g9J)KT33%MPXFTyAfBeSP=e+&fch`Iedi2_(FPHhgB&G`tFhZFY^iGZTPO8%A6S;JedWE&6Z7VgKJMLTtbV@Au;oe}a$|fo@8QFpeTE;~ z=(!{4cwATZ_x+vv)3p?oK6COMai}`b-FNw9`G;R}pRW2^Ajgt*_)SjojgA<};ZV-D zH)q&q4iEL*eWU|BFmM=S?>NY;&)5I;`<6?(5sl{jyXGx}^8>dxQX%Vtv5PEo8w6JK zToHH6efQkYp6Q3Mqvhz+s$i(tXF7XpLn?CV%Z6Oqu_p_+nw!5{zT;K*3%heMNzF;f zzun5oTzGVll(CU?9of+U+nP1y(OpU zvv~w9Sr;nLG5?3p<|70ueyyDbUY}Yd!E0=`V+1F2S@%7DUU z!+3G5v_Yp@FhhD(9o{OXys6YM@?dLP0LotS!( zZ~o{ThY!62s*m!Sg&e-XdU0#<$S=0*Pb|w{eYqaXoLkS+K6Rp~Y^EN+{G*Qi6P;tq z8XuKI#YV0>%Nz^2?6yhv9fh2b=evx?JV#`6&=bQOMZM+dz(~P{OOO4g=JV%2_LA3t zIWdLGe~6_L*6U?ZoidN$t=;E~mp$XEY0L*5)a)#9%C_**_ejXj1}SaGL~lF&7ro-L z5_Il{V)fCw*fu?YZqYMj%cgB7z3S~eAahn{_@cQMlFic3)%3UY#Noj!JH4cEvRr#S z^9EDCiHH1&FTSjo9Q4r{^K&2ha-QnFK^=vKuFYqvdxW=7K2uz)M)&XO4}*2S)oU;32*?s`tzhPoNdy zMK~{~T*=4;PVlC()T`0MfB8pTs;kbv+GgKHr(Rq!;3+S|5(B&y+n5*@z^5dLrcGjDVs3` zF=w9B8T=Q$;LA>~9`X4+qVFJ-liI=f8qb5;adlP9$i*t%;M>z~dBL;M7jh(|v1O@a za}jzx7Y{1+b#a=fVe#WfJ$C)~F&^GD!hg8&3xD97hwY{wLOxnA2;wJqo|?br07>n| zdc9}P-SQkmio~mhtX%z&MJycY7!O^|^}~~L*w+vLY!DscBm0>6jPaAr#6u#lPtl}a zn^g8A4RF_SY<9BpclX?P?PZtsH(oFGD^X@u>A2cxb^Xba#{f#>E7Bp? ztFxkR`P@dmpq)Vyx9`@uFnA8e#&tpr-DGb_G^IYIlqLQGW*i-bW1&6e29O6Y4AR#5 zvw3QcRQo|aIrZklmvExE$M4X$oUyA07_9mhM=sXuWE_~5;nT=?xmN7c}VZTZ(}?rL~jVuDCHDd zW0I>4RkJL)P{rpZ{mdS{51lA{3Pf+T`jPlbs|k>vbZN6ZbRkPI+fmPp0DeI6t7Nc~ z$NhZ%nT)>k;6(Zz50&~yf1iG^fs4sKviK#}-Dl{r>Bu~hY2DR;F}T*pmL9|4wUTbw z@xnlPQdFhr&E%R&<~6QfTI+#VgCJrYF+`(acGqTfD_@rASLH)IiT<#`a<+xCqjpL` z>#D>_%Q%UnL=``~nBcrnhfBLfp$0UGM~}`pY-%%xL2Su?1!0>O+=jhV^Q|SHHsi~S zD~0ov1zlYjfNIlt^GFNNb-;qpg1EPAM(ME^ps)?4i@M~QXic5q&!wGA8~zyJ#}kr& z^`4JJ%2R4dCKVL9!V%6$c5)Gv^*q_xt7|K06))bGDUPP7^FtSfX;?h<0|XKb062A zIY|b0!pj0C)Y$7;i^P=d-~9Mh&zQKh^`h&1%>hsw!5hUsnpx4t z<}nU3;cAnu{B7X&Vn5^sgN95?k&<*Nw-dMSz$p_Pc^$xvIFk*X^*T}DEO_*uml7(B z&nEcAJ#m?Xu}#P#5u(vuOElFSM`G;J(?_?d0s0skGYz4+p=0BMwY@=f?C04B`6n16 z7Y+?9wH$J zAxS-==YiY@80*`{n1+s)KEk056AV77g?$%2H0xq(Q))9XS&VWbRL_G=l_J9>UJl0D zL}N3`NDj2QCw^L+J)AKpGPZ04N*&EdoH2o<_uVvg5ExqK?h8cD!pAn(v{$fP*#~QU zh>wrmGmlPAjvv4qPUcCCWLhX|Ka2&~1>W*WY1;yK(tBoXnGCEf#s(&kaR8=O7&`Rb z4)NokexjR!kF~8MOFmU5aQ$lW3aOlWOo#8pn)8ot^lQLVQZO5XoZ}x``u%x;$Cmjs zwt{}jE1RV@QuzczTVvNF(%{QMY#aX3$pievr_W(l1ZA{3C6z9Llh!WOKW`#3*AYhq z-tucRhL5MYjUq^yq;P4yz(j=;Uhu<*6tg}0;12PFp$~4~hxPm_+Zg8Ct>f7*BneZNsSb8?%&Jh@KlZTTrOg zc*d4a&)A=--&QSt^&=aCKtMfi2RM(tjY0_3lN)$zC%(pMOo(G{xaW#VQD)ml*8}*( zn%f398D{+~2NGYgRbLr0gOY-ta%{uQ8}bVGoMs=E!xb*`2zR1d+}H1qgGY~B`-@YJ z>*a;j$od&444i_t&M>U#WibY2>CmtI+6%Qc>JFq&fKMxFac!J|LFhSyp@oAfvh|$Q!ky#K zhS(4BtuuI=bE{5uez>A2b4!3M+hm`g$1$&w|CB6iS~rUj(~}eO8bJK3dJ?_67ebx{ zSHS|R%y8%`=YQMnAR>?_}JgGOix59Mum~lwBBOj7l{Dr%(^B9~CeuB#Ukb0`^qvuU*Y(62BICR)&Tg!A&&-M+!2eTcS zQp|kcb?_I5@TRuW`$zm0SeN?*o>tHfJx!tLIT3p}glz!EcCx$YvH;wLhF24aiOPLh zoyM4vMhXD7pn%KA%I|SJ3pjFVbc&HshPKa%R-zM#w$p3fhA+q*C$x=DN^`o8SMD%{ zlYy6XyKVf(AvWYbX0=U|B7A&%L$qy^lSpgCbq?mNVK#inCYah3&VIO?=1DXw=#`qC zbt3TAho;;JwjNhLV1kW_T;f+5&f5zw$zb{>8{!V`+%h~%KVy-DqlO+=H=VZ=FkY%TPJGOKbO-eUMZb@k`Qw5*kXQI4 zNn-VY-V}k{dvi=NgDj)aFv2b;9&Lhj62jH0Xgt5%4NV`a$nS9VFeZ8jwL3ZT-35mn zvUwAUQ9a=cgBJ%U^%9B`*>UXEt~NPJ9a#K=jILPgIq5_LF4);`bivL2J}%hVmz_pI z&(zfWn4ASNsVrtA?CTky6@SLgnCP>dnQ&s$k2bCduV@v=0M<$2v&?X_w&f?0 zdVL4q!ob4O|06wo;ixOrj>l#y;~Gg=-=WAx*pV-hTSqte=+)3!U&FCJJ(R7IGj_tH zSk_m_@)csRD}7KQl3@|As*N?`C_c!U@vo=O(oUUM9HYTXr$fev>%5uanu%NzjR zCb4pse%58Ff_FbT99ZTs=22SCWBp8Il>D>{j4u>gKeWxhWg0&$HJ{gkdPXCf61P@& ztiI#OvjYd~D)hvhL4pdPanYqKH?T(AS0xsJjcpoa4(T1TJw`VIoTCqRpI?P*;>dsN z5f0BOf=znyxkaZ2tJWn8N$N>lK}c;lWS?W5vOBR=JKko}KC|$3Z%PH$J5|jKJ-NqE z_ZknrZ7W~D$^f(y8P~onU3Oty2J4NY*@llDx%i|JpU9&wHDK(xtG@VU#^kYat*h>i zdSLC^jL7(-#cz$a=M=p%&kPDtW4)wR`B-^()-G4{E(m^LY+5LRq%6%7l<6vOPNhVCyvY=4yUI zIx&MxLE28(nmXlm7viLOLSs$b4|GCD7I{^>sJ)bo<7qB^r=YAS^^JFY6;xwEh zZpDM~;ZEeb0~BvkTQTEG0U3VZL5j9H_mXvxdHwoPMGk8H%GZ$DSUoG};o!Bp*+kXX z`qy7&0LlzDGC5UnIv&!hC5g%LKEG*AaEI$`J|`zF9*~_UC6v2ef%Yt=w?iGS=`x{m`*tc1v}Pz zf~slY{K=p-7He#u7L@_cNMwKhd*f^(-Vaneam*r{gTf>LelwEqaEL>^IXTI3UTi}^ zZkltHCYX)!fRgkGlZFWF0F?CZ*bebcbNh5(fov2_4=P{4lkUMPb=`l~2uhFxu>7&DseW}mFpI(L7m<98w3m<&s^gYwzKLS`@ ziH2UU5yjHI=Sa0E5;z6n)mm>R$Iaaa0HpF2H=cyKrST)6aY5j>Y2EFa4KyaOJpi`Y z0cR0NFVNX;eH&s&2RLs_Wk`!X1Ktl5EXMuVY^M5^Na4ay{PgzMr(hU*GqwVm<`|tx zHqpMHc}$IYj}CnPhO8RSa9ryZ-xY7p0CWe2u`wOua|f#J0CPySsjO015zUoj^|=$R z&P!8a>m2?Q`plg2TfXWox!mch;lqB)b!%4}(i&%-8hjt^C)?8v8krgXwGp&JSbXUmUuKNKj;seLQ@+i{*gD4%I@RALNg?5Nv zHQN3d?-dcg{ZuEQo!};N-E}JHlr|#Z=D+=Y^?ah~?(8cL)5{VsbD?G)a@Zyct*NHxP>~FNNVt39Nz-u{udkt;$vC~g<^Q~(o z@!$ErW946qkAsrqYR=YH5b{$F!kam>41*1>C($G?Qu;QuA8=!KcHIVdWNDr-8-7uK zNuNiULdrZEx{d!~v71dXW?a|C=vhDe#uyuYWb4hW)6k0ypF8ER{BAwTAx;YE-wb!) zU;16Was^(;$OUp5dXvkJY0hDAS|8fn=gyP6&xSuan8cZ0vW)z(=x@DiJPDG%HphC= z- zpYdSh-(EFF=R=BYI@>x#_%jYWdLEjhM|USaBzVpNLG3+y_(R$BD_RmMas$MWs~oG^0ClV~+&9ED$w?cD|Yz+=nu2k$xd2U}uu6PP0V zCo+iBf#`{lqWxs#{-;()(J&9)cV& z*MIxg+j{>(@hd`~jcXbH;1z zth?n%0u(-3tD58KJI#tQPuPp_{T#@NnLsv#(utmIWON>=r)G}FN{F5lNBD@6U;Bn9 z>MqnKn+0+&Jbe!0Sg#XY1|IL>WT_VXUT;oA+Kv6ir{@DlMjpC8`1rDX*N^ifn3Oa- zP>v=r{|3wSjsMrp<+?rvZ1#&IQ%o*?Q%fUy9{OfIvd7w82leqs-`IVe19y5!^8?p+ z%lE(O);9mymq@O`lr{MH-Gap%a!lvK(+9_5!wv_d}s`<0wzR2F;-6sG^f)1 zfAhBE<$Hhn)^a}|--)B-fGBwkg|A}DfUPxB;ADB-k7x(+!4Wu(Z^V|l+qB6&n>1q*9dcD_jHBlT z*vR|+hTp{?KmT(AyX9Nn__#hpI{B~9Yw%ik6(uW2wP}cuI}>`1H0k-6=fBTqX`C$v zyXpzH+GeRX%|8xjW>_S<&=S+Pnr``~H$Jia)W5&2PruNUE@20Cie;tIvIjt59r&b0 zjV=c|+__#ALk??qI+k=+1B_gv^QeSsUl&j? z;p|tZ|KgJ`FMscq_bfcG=0&dhz{tYj7c4!e`8Av9+C(?nNM0J_+A`~hL2+5Y%lGV- zcj`{^cVGXwo}+cX;<;dQvT7u2?0R+qYFq{XM198e*L=}E%d_>lL3~zo=0om&Voy%^ z%h9>f^lD0ytPpr zg~{1jZAiO~^T97J@yeh09w`1xwSh24F`NSEhCjRLSXJn`%mH@4#+$x@;up2ebwIl&_3snm%EJ(YEoj{-clclgY{Q#$UL- z{G^^VuQM1Gu)n(U2vif97a;}2J2D&cm4Ei0<mZtf?9#n|`tkjxXn6KX&EI1=R@*$+Kyw>;|^ zN6TfsKa#H^pu#R*_}$O*#n-X_6q!ggu8IzGT!q@a0d4&GoYsxW{s08 zxcb6`!zl91*VjDiv#}r4pKJ1goci!UFDRc`2%OJ$tT_0@2dCnL<$j-qr9L&M`lL5D z(Jg%h*(2AFmk(S^Onhux>cB?H;>YJE=cKZwR~3}pmJcYob}zo~KupBx=(Nh~M4*nz zFreXsw&7fy?>G)Rb7uLh_>fd0az4fHf;q3Jlg~yVw=Ucr;=5V{Uqw2b-#L3OowL9U z9j+Ix`1q<;8v}WtQ-xXig+I)9(3;nXc|pGNB1^pvR0~0A$kl-?YrweTR}h1GVi

c)ijgxDm}8EsRXFt3h@+Ufr7@DN z^55r2UpdZvo*$)c`MJ_3zXBARbH%T}ifygzYy6g*WBtspGU<*Ccb`wpyW!Ui$gZ}y zo>MwK`K>f-62KfvO2{S zXF|ni6T=gB=C>=mF~5ojWS?I%DBt!ouB^&}v*S8G>5&(6>bM<0W9)PIeSXbv;v2lq zgZx&0)nJZqzUPEz=3RZouldy~VSciFe9|fxrs_KoD#u$hYz3BTu8Twxs@yt>*lp{< zm_XbpVEfL5#v}%x;+@AY<0*cV$ZF-248A&7CXCUG-9e@z7Va=V8J*&{q4I$n{~M-~K{qUmg-Y{N~tC__Y!6wZ`uS zAN=8SKnb`wARia}P{>}4q*mFJ2rt$xz9z}40>2@prKgMpJ4y?1MK zsu;8LLY(s8tNKp-L`??i35r}^567PuI=u8S&*EdFoy9Nf;48%{S#m8d=h|q*N!*Hw zE&QzCc2jn4u4(uar*pTPKCQ7DC)&Cs49?>3$7+X~)XJA`!=HT>p7`~r%@S~FvIWT% zL)t28t$h|BY!xpHnSQNXihG*>p${(0U;hi2mrwZcOUrZh0ee^UiT1oYO{3$5Hop*u zLXEN0l1qM=vD`rN)XOLJdon_5oHz3`AzpsrE1f=|*Mk1={U^)6{EcJ3kodUYZmX=p z&l4~2a)h&L*mG4|<3d+3_?Prr)`vgu$Y1U7EWIl2?@iUEd5K>;n9zxxlFNU^0vTLl zH@o9AcfQkuuVr{d?>6N1tv`70$?|*eKGqA1!uC8^rS(s+P1LOQ9lYFac+7nk_^^=}_9|LQHrRm;gm z#jgtmwd-2xd;fSm;rGSZd-@wbDeXS|)%sP&lv@b1qs`Sf43!0V?3qvsHeeF4^Q(*h z^}o7zxuRcU@`@_U0N4FIMxo}rPTLvJc{K#}XhYWmowJJ2$Yjbl`u)zkPnNIv?#GvR zeQ>x@oZ)FOm|m&l>_ivC(ek;URCk@4f5BINBIPcJedSknv#$7sL09O4r%@qb_M zz2et2d?)PSD|vhJv?jf^coe^7;*5D_(i{GoNjc@GFgNZjMJ5=HK91L-#6s_k5ZsDS zGS%RQ&sF+5eNE*3{W~3);ByDsjH9O)4$S@$?yR>?gy?){V`EPI$n>{$7kZJt&E|jq z@9tl&>KhB0wjiX?fvux_ph<@^P`xU#l~@YcVmvoP|52 zFCDST=db-|m-UT`(xE24+%n&4gZ%FnLi&Yo)!)!<`8*?XqEn@~PlG4oI{hPQc|SBA-3UqQo@Ok7n} zIAZ21l@78Rn`X^sw|ukiJP&AnypS?sjm)BYgRrvd_2vm*-zj>cKd@`Ab&91Yp=>6{)F%4)7auKu@lUJhnvWozKNZb^uG+`E@Y3=U zeK~|@uUf1nf;jWRpXQgYuqA_|MTZQJmcB;TNR^GlS{T8}iC6rO{IH|tWqO{uY5h}C zK^05FmfvX7IMk$1hE*ehH{+tKyHIa1DdB;;rJvHi z@XysN8q8vy7k-&z&tLr~zqICPT-#vO+|kk)bI{UP%}!$rHS^6TDD1uXt~a|@W*~+c z8vo^wJW;Rw34f4ZJkG`2_D~Yj%WRNd2O^Mwn=s<$0*s{9@EYCPT5v)bA~e(n|~6M0EUxGtnrcN&$s(s zzN8S(XWAcol9+ za@NCPqQw`HsBTqo#8>DWj&U^~+CTP~&69^IHqX$ty#E|%_>m7|XO7~asM|V+|Xy_l(fh&fm#RNST>VcoN?=6S_DPi%0~BG=sQt4-78)-@|b)lahBHa~PL<9jHj zNE~dl9PG02qUPM@QPu+cEDu-Af8%z}zB%Ihfge*{9Wd$&G+)E(=&9+o!^CjO`cwNdjVRH+WU`h_MXAOitJp5x3ifW{$igPf9iBj$(b=HI#x==`-hy-E&gI#->XR(BW&pMdcoR19-nNcPkY4s2bR7uK27u z;T-wi{Jv$d3tg^Khr|3zu!D-f$3GV1rd-BjB{h8+psmB&uHFO}3e<>-KnIym}P_oSC zslstp61Dm&1NiV|^pEbaNt}ZX!rh1GA<@OoA~K`yhAgd{@foOROsg!`F}gM(u1!jB zP-&PeM7Vk8W1#d^)-p1e`o(13g|c~w?dj`;4_bZu^_E|g3d=E{cLES;rdxmDH283uG=7WUKG<2~ea{IxU4q0( zBCeM((XD0e;O571>R|^u&Ev*jpsQGwzvm-2(K$^ICifY)?_e`E(umG-isbY(H;sFS z_TV{-u;uIR9OWMt?$V=eCxZbQ9k$3lC>2^A@xz~@XvD&(_uWN31AO=Zpf(=jB!lHh zOT3|j8)NsuFr00(J`~5*Aa@-yCcZDeY#2MK^7+byjE?yuYo4B|14zoWZPTeh8BIOF zi#LZ9-0pPpQq1&2arSg`YF@vQoGhb26RLwnlb*1L_^M-Vlx>giHItHpV-y+pt6ZEK z556G7lZ4?GS?qbNp_S;OAM&IlDs9+mIL@;^vinA)D6z3H9OHAVWxzHP_n^luSJ#<< zbsIty2lS^g(Tp%sL>_Jx%DMrbLPR&IRuN*2au@Mv3b3wQaDyVnmOp4Ma3Q*l1@}l- z7!@6xqcC>X;&3#^WC@2>d~Pt-WCFI;DSS*he8-yHfN>hl!&k7gZRoJWX*}IU_<3Dv zFh%O=_d;$wPTu#$88_QzeaYlJH`gOD^~u}%0AtVi0{v!P<5awgzdH2uJ`V|wUL*2lawezA2~fq&{P;mfB?8T6HUC*4h6A&Uoa8O-j$RT~z$aZBVg6 zzF?cyl6N zdHw?sJ7Tp$XXHMr#>SS7hWS(q4Vv|F6FxR`qoAKa__u1W&%AQI4T^VKan^IyU>zfs zE|$R$NQPNwnbWKcmi{dLjG5%b9r@2i8f!K??SvY4H+*lPY@EblJRiC1P#E;CqroIW z@amJ2xy(A56v{9|GuaTpMMj+DK>H#%Xah4-!k=}#^ zneQH-ALI49-brtya+(0Rs?MoH;W4xa=7q~HKFb7Z1nBuy5&@vrkTKXDY=saRII;oP z3R%&P2^nF-NYearIVR*J3O2Ys934KH3%!qF8Ezacu`vg0S*Oab^yt!p+xLq-xy5gM z#Kw5jI=`XA!CkZ&zAqE&VEj1=NFmPhl*4MSO=PEas`~e2-T71-1sApc|fu*Q}= zsYFnC_DZcy+zSDb@&j)&>t^-n;oK7;%>Y=GI zf;q6^#lf=W>#ky4S#ll)lVVQT_DO*_|C(c%5cIB9nT$1w zdZdwu#x~{=-+@S!Al?*`YqRX_$W)w|mL<42l`iKk-%cwYqIN?eH8`i)kL=}d1?JZx ztLCs2KGwvGug#(X==ud4yo;s5T!B+uNNV9YMyc!;d~C+efEeaJa{IVw7aDzJFOkR6 zSlJt<<>?A3vyx@)YW!;#RD~3cJ<+yt$FWi*K*_8K6|i@y5t3Ja zJ+H|ads>I+vjj95MRGK=^x>=qv2joEMXBp_IFN4`AdHaye#ZCSN+T3ki zEEWhGJ-%>&Q^eAnKgqhuJba{|Jl+AxddOr{Cxi+(@50!IbHi4?hjyY5LQ=XVPTEpb zyqVjwx1@vOf~d3GC@cCi=V6PSGqd|Ua>`SZ|JP5mkUUL?=|EPi{@-nlH?JLkAw z*sMbLgtgvL+o_1?*wJfZjcXpC5>GR~M4yu?y`l7N54Pg1hB01ME2+8Z!14qfU-Yz@ zpP&@C_lf&Q^@(4j;1EbkPV$`KhCay2t@XoalE&DO(HG;)bGsV$(1$|8a365@r{WKw zNW$FkEp^Sm<|7b9uV3Ad{N#D~L@0goVuYqx6L^T_<{Zg#=0otZT7J0Sg93< zJ_mX2IquB#Bm6s#^rsweb>du#$y5q2icb}=oNpi;{UA7T{^iK)*yGw5d6=pq_?*D>mRC&iQRDaItw;A9 zUwyN}YMcO55)^&3H9%p>YklyFuHBgRqrZ5o{^}Fg-RyE2Q&BkPr4P7!;2dsBBY5kZ z6MOo=-HSke#!JD&S`O^!e_!8v^T8YV)+p1?{L!gB{K1puy1vT%sWe=-JBLXqC(&~o zh8QdS8g_rYT88wPo<6+$(H>5CKO8#&q^#c>*j4hprAvR9e{%Kyt8YGf`?u>?8Tz14 zS1k!Et{sV(!ehcu#U^0M9yMmukRS`=W<1D5*Xuj%0?f#3B#i1AuV%Dk0a#p(np`Z z@Ny<>{{ZDV5+@v)mOs>&&;9Vv>-)pHaOkS3YygE%;ePHnZ!h`bKx(H9HZuLnZ`piM z2ii=ClLN3rsu>=c{+jNjKd(=0rLpid^!u4*y(mWJPG6kjm0Yv8i=0jt@0q$c?3SO6 zo`T_+i0(Myt98b;JQvD(PJ8@c_^spR4R6xbATVp;gA^fWJoolt6Viy=aHkR(bL6>a z0*u#QIOR-CHs#1eI_@gp{LgMJH~1i?ZcMM{ufkCb2He+@V%l*Br$@ccN`(OGk)9u)8Cl^IS$70>cnNtJOD;^adIv1mfzOH@{j*A zpUGT+)Iu&-&YD8$81J|E-`Afpo?Sod(=~-f1KG?W4N<>A4H|trX(W)6k{Oa&+m(#9NV~FpO<-jgq5FpLo=R80h%`t-tc094&kfl2?<-(g>J|r?=r^r}OA> zmp&f(`pX~wSI3@L@|*kMoPV!t)up3lQ3afNHGkNJ?ukAA%&S+P!*d|=aQo0Nz5YfK zKR4s_UId|>uzYyqbjJt5=GTt(Ez-yS$U9G{Cqm(9+ajN> zgT~ide(a0*RMefm>R_qQXttNTKUJiWa#G(o>gibbxL(-&eO>l^>-4Yw{;}#f=Ndog zTpjgwLr5GKkp=Bm^VjU9%39U~*@|iCk3RCfSN<|`f4G7d?}tSDTy`AIwQL?;#$97+ ztSvnwvYK=4p}Io0?fv>@g@5oyeJpBc$rtZF^xS26hCWZ4#Yok->p2VeHu^YSPUGG2k^A|XtmgmW>+a9E=9)4OCk5TSW^(Rd;pI_JfySLre zQLOv*sbCN46V?6wuS}=FN|eBT_p(bFq*`MXpIA`Vg(EMp(umI{;a4t?=!xmyYV?&H2P7PMKv=d+vjRBWh(As6Lj0Qcn$#3?!%y6`&&<3aj!!;n$@xk0 z*`QFf2~yb7*ZgYBR84)J;s=KZ&x_vE!tWtII60`G5(@|IFyHPr=5zVG<@(X_<1hTc z_kGCwAo)o&!Uw+XL*A!{f;S*LxN;y5=0e-ZrK)pdNED2liw(!iVbw-%n7!XMpG8kA zGUJMmr0RBj5-MyJddQOpL{O*s7%s{`6u+WXrgQwlI?smCIg$&Q{AYgqCt0wKb7$_% zm%{TugWsEv_{Fa|uJO;}cZ_9uLpG0)>jq*Vhu`WPlbLjiH(IU~Fm-o{X+n|rIebs+ zBK*FBMohVN%r4@=_@qH>4)KXqe5CL#cK)Tu;+Dei@z-rsKEYOe;uO{W-~*^lGv{e} zg4af91r84J?WZul<4pXy&Q9bMAD7uEiayKu@j6WtFdw~+#;%<5b$dDfR;X#?4us;} z-~EhV6zs>~=Rof`?o~=VM~9%M_?8J+n!&AcCV)?AP=;fE71{~UeEA>#S{QucDki=r zzHybu$j{hvT>Nr&n2+r=zY;+&dlw*cHh$KbFJ$UN=-6jIG7AR2vDH_c$iN1FmhpRt z?{%2s!?BZglURd~-k|DP8~&9Flv)o?mLI$Jz3h>-Z8i{UeJRS<(K9vL#!-~$F*1Sp z9>4-|wb7EC2gB>kF9$2`EI#_O(HBeOdGZy+=Ze2BPH_+Mi?qgP47=j(>kB=mJ%oMS z9r<0iE@an9F`Z)KGra&4x%#2EIrCiSSMf=2pI?~4w>$UPbpC{gT;8zlrl=Bb2 zc!MuoiVfHWSDf^|NDlF(^ZW;&*`LSHX6X1EeyW$cIeN{P*pA<}=H;OUB#~>P2l%!Y z!u69#KlsSz*U2UJ{M*;+{q-Mwz4pdlJGFtZ-+TGiS1Ql<#B&y|xO2F8BP#-G95X!= zS3AtF&0v5*jT?Lk8~!j1%0_T}otooBko6is#Sgz&6@Aj7$ONp`$^7Ks*zOGN$=Vl+ z!3WfQyRB%BY(65Ff(S*v1=yWtyJ{I0gB$4W-~OP!g>&~BlI$ss{JeWJ0Y~lvE4La}LgwmJ{B^=-^LrxrR*K+!NY34Y z%M z<9FfUS32e(gAJbEtbl5ub8iasSIo+HYW6cI2(;PPCVrX9hj6>)HIID%gYPzH@6^%v zv^{*@-@5)2n!;y#NN$bBu|)+fn^0}89(_q=8AGE|lG!A3qm}-*G$sPd@g2 zSN`*ry_F8$fdaX8yu3>5_^=Mm3a>SxDq|(W496V3gthog+!l-+gI^0x3>K~U0B9_I z@g1v9#%%cbQY(J<)|7{e%NhR$c6@0R)3;{wt|Y5hT-qAn?23((Ie*Is_;P_4Gx3j1 z3^!RMCcZ=O#~*wM_}}BBm6H6+W|(D1K9`SA_)O&v{7zZehxLm7tBQH}eC`H%|3AL+ zwv$WC=ZSiwBbOHn*aasRMW->jDp-wcQfvqt$sDPv&GGOq`KuGkd^o;c>O`@?JJE_` zdU788%6;TNa;;()znFK!uf=i(n|UXb!}$}T5F5S&N6!Fu`(`Au^2Zij=Z|V?HNBZ# z{Jg_J&>P3Qlh3>HhAVHIXs5)?*?J{TB9TPPY-Gp32p`^F3!lv=`TY2MT!#Dn_EX5YDwXjm4@%zo zyA%j0dpPZ8aUi>rp!dHqyG~d+l6Q>+x9T-*oC&4dQmFv;TYcH~Spj>DJ0esIt zzWNO+#A`{>E5i(Xk;Z0`sjgNLsQM^ePYfMu`tZTDpWqGSgiZetwnduxeT7P8ynTsi zel~9SC}kpn5&t6m<~Z?*-@e9Xw_7%@1cxGiwOUv!*ZAgV{^YpI;WyoHSsAi`#H6j9 zt$aSe;%xY&tQ7Q@%CCLw|GfH*c7B0V=63;TLHuy07aBFXpK@e@kz6>#YSGcv3{ghz zzVXF3=^Q@()T&z5KP7&Q>i!XZTNu&$kfkNQnO!8-_aDL+?R~C8sjF4t! z6x@c9tB)3F@nK85F<=By?G&Gi4}X@LiXJ2XmM&tvDMDVeZJcH{s6W+y1bgFn`9~ZXTFjEjziZ(}(o3vn z`%X>ZGshK%2W48h%Jnqix>9=bSGbGC-{Va~Hp{r_k-l2)R5e=9GXJFTue#GuTPtHLO_kpoE;{;<|N8ou=yCIP zN<{A~WY5T@7mLhsKlK)EER*b9LF?v{dT-&+=Hpvd_~PVB{13->Hs|DD_AU++MKR^? zVbs#s_)ceV^X6!`7vaB08NBAP@4xarcZzYI{jMLv_MN@||G4r!x9+?3(b^}k&qm0m zIJo%3!Mf<)XVROminu6NX7e>E)#+h2O$}L)eu$)~=3}XaGUgyZ_V8KMnK#)7zjPHp z_Ts=j%wK(OAJ%4maf|Pa51wLAKZDR6(r+-k<@J}An;-pDHxE9y+0Rj)g#6$aUwirP zX!kYxQ0mVy-QN2yL-92;)+QS*i|kvrv|fAPK+-?Jmin%y1ZS6N0LGw(w2!|y(vgZ*y#F}>^b>-1db)Nj=f;xC|Ft8@YI zMIq1nn~#0+?)d1{!hey9e+8a5izk@{Oplez2GHqrSUlSN&@^wrvVyP!giSlmuO%9r zW`jOGD83?gYTjdlCEZT%G_f_YKb`yp!)N?Qcc8y6-5c~LFW-9YpKRX@b^v?Vs?#fW z*DlT`JnOH$|Jl3C_q|fP=kqnu&(d`7^YSrkS5(VraZMu&zIv_2t3qXyto_-1d=_pk z^vbJk!~$p|XLVszAW2V_Pv+Y=r{jaEb~--#@C&o@YkYyT{(x!uak=@SdyXFer}KN5 zFTlMk$hvZOMZ0@2f4q3@#*LTjFKs?eK|fUioJEMtmjUO-<02&yOE|p|V-%X=6Xv@X(oCxjr1jf2;npdQ$tQM<2QW z=azp~pZ|S`@O0`r&8O4l#eLPLy7n@?{`u15<>(>(HP?sj)ax^gp0C0^Q@=iWK*f2c zD)fL#sXs~F-K&MVM;neWi6M8@tERwteOT%%cv{JMqtu2a&-F?ld~arKwAH@y=LKKw z#h-2EA?L&VSjQ(K-_mq$Dl8u&b4}hKRXUGo8jtD{dqj15STlZy(C<7sI)2CQ_~fnE k9@EG3{4s5ok?kb>|H;3ubeVRY^#A|>07*qoM6N<$f~C=$asU7T literal 0 HcmV?d00001 diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/profile/backup_config.json b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/profile/backup_config.json new file mode 100644 index 000000000..78f40ae7c --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/profile/backup_config.json @@ -0,0 +1,3 @@ +{ + "allowToBackupRestore": true +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/profile/main_pages.json b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/profile/main_pages.json new file mode 100644 index 000000000..1898d94f5 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/base/profile/main_pages.json @@ -0,0 +1,5 @@ +{ + "src": [ + "pages/Index" + ] +} diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/en_US/element/string.json b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/en_US/element/string.json new file mode 100644 index 000000000..55c8939f3 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/en_US/element/string.json @@ -0,0 +1,16 @@ +{ + "string": [ + { + "name": "module_desc", + "value": "On-device speaker diarization with Next-gen Kaldi" + }, + { + "name": "EntryAbility_desc", + "value": "On-device speaker diarization with Next-gen Kaldi" + }, + { + "name": "EntryAbility_label", + "value": "Speaker diarization" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/rawfile/.gitkeep b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/rawfile/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/zh_CN/element/string.json b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/zh_CN/element/string.json new file mode 100644 index 000000000..d9180dfd1 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/main/resources/zh_CN/element/string.json @@ -0,0 +1,16 @@ +{ + "string": [ + { + "name": "module_desc", + "value": "新一代Kaldi: 本地说话人日志" + }, + { + "name": "EntryAbility_desc", + "value": "新一代Kaldi: 本地说话人日志" + }, + { + "name": "EntryAbility_label", + "value": "说话人日志" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/ohosTest/ets/test/Ability.test.ets b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/ohosTest/ets/test/Ability.test.ets new file mode 100644 index 000000000..8aa374977 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/ohosTest/ets/test/Ability.test.ets @@ -0,0 +1,35 @@ +import hilog from '@ohos.hilog'; +import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from '@ohos/hypium'; + +export default function abilityTest() { + describe('ActsAbilityTest', () => { + // Defines a test suite. Two parameters are supported: test suite name and test suite function. + beforeAll(() => { + // Presets an action, which is performed only once before all test cases of the test suite start. + // This API supports only one parameter: preset action function. + }) + beforeEach(() => { + // Presets an action, which is performed before each unit test case starts. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: preset action function. + }) + afterEach(() => { + // Presets a clear action, which is performed after each unit test case ends. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: clear action function. + }) + afterAll(() => { + // Presets a clear action, which is performed after all test cases of the test suite end. + // This API supports only one parameter: clear action function. + }) + it('assertContain', 0, () => { + // Defines a test case. This API supports three parameters: test case name, filter parameter, and test case function. + hilog.info(0x0000, 'testTag', '%{public}s', 'it begin'); + let a = 'abc'; + let b = 'b'; + // Defines a variety of assertion methods, which are used to declare expected boolean conditions. + expect(a).assertContain(b); + expect(a).assertEqual(a); + }) + }) +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/ohosTest/ets/test/List.test.ets b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/ohosTest/ets/test/List.test.ets new file mode 100644 index 000000000..794c7dc4e --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/ohosTest/ets/test/List.test.ets @@ -0,0 +1,5 @@ +import abilityTest from './Ability.test'; + +export default function testsuite() { + abilityTest(); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/ohosTest/module.json5 b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/ohosTest/module.json5 new file mode 100644 index 000000000..55725a929 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/ohosTest/module.json5 @@ -0,0 +1,13 @@ +{ + "module": { + "name": "entry_test", + "type": "feature", + "deviceTypes": [ + "phone", + "tablet", + "2in1" + ], + "deliveryWithInstall": true, + "installationFree": false + } +} diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/test/List.test.ets b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/test/List.test.ets new file mode 100644 index 000000000..bb5b5c373 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/test/List.test.ets @@ -0,0 +1,5 @@ +import localUnitTest from './LocalUnit.test'; + +export default function testsuite() { + localUnitTest(); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/test/LocalUnit.test.ets b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/test/LocalUnit.test.ets new file mode 100644 index 000000000..165fc1615 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/src/test/LocalUnit.test.ets @@ -0,0 +1,33 @@ +import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from '@ohos/hypium'; + +export default function localUnitTest() { + describe('localUnitTest', () => { + // Defines a test suite. Two parameters are supported: test suite name and test suite function. + beforeAll(() => { + // Presets an action, which is performed only once before all test cases of the test suite start. + // This API supports only one parameter: preset action function. + }); + beforeEach(() => { + // Presets an action, which is performed before each unit test case starts. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: preset action function. + }); + afterEach(() => { + // Presets a clear action, which is performed after each unit test case ends. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: clear action function. + }); + afterAll(() => { + // Presets a clear action, which is performed after all test cases of the test suite end. + // This API supports only one parameter: clear action function. + }); + it('assertContain', 0, () => { + // Defines a test case. This API supports three parameters: test case name, filter parameter, and test case function. + let a = 'abc'; + let b = 'b'; + // Defines a variety of assertion methods, which are used to declare expected boolean conditions. + expect(a).assertContain(b); + expect(a).assertEqual(a); + }); + }); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/hvigor/hvigor-config.json5 b/harmony-os/SherpaOnnxSpeakerDiarization/hvigor/hvigor-config.json5 new file mode 100644 index 000000000..06b278367 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/hvigor/hvigor-config.json5 @@ -0,0 +1,22 @@ +{ + "modelVersion": "5.0.0", + "dependencies": { + }, + "execution": { + // "analyze": "normal", /* Define the build analyze mode. Value: [ "normal" | "advanced" | false ]. Default: "normal" */ + // "daemon": true, /* Enable daemon compilation. Value: [ true | false ]. Default: true */ + // "incremental": true, /* Enable incremental compilation. Value: [ true | false ]. Default: true */ + // "parallel": true, /* Enable parallel compilation. Value: [ true | false ]. Default: true */ + // "typeCheck": false, /* Enable typeCheck. Value: [ true | false ]. Default: false */ + }, + "logging": { + // "level": "info" /* Define the log level. Value: [ "debug" | "info" | "warn" | "error" ]. Default: "info" */ + }, + "debugging": { + // "stacktrace": false /* Disable stacktrace compilation. Value: [ true | false ]. Default: false */ + }, + "nodeOptions": { + // "maxOldSpaceSize": 8192 /* Enable nodeOptions maxOldSpaceSize compilation. Unit M. Used for the daemon process. Default: 8192*/ + // "exposeGC": true /* Enable to trigger garbage collection explicitly. Default: true*/ + } +} diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/hvigorfile.ts b/harmony-os/SherpaOnnxSpeakerDiarization/hvigorfile.ts new file mode 100644 index 000000000..f3cb9f1a8 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/hvigorfile.ts @@ -0,0 +1,6 @@ +import { appTasks } from '@ohos/hvigor-ohos-plugin'; + +export default { + system: appTasks, /* Built-in plugin of Hvigor. It cannot be modified. */ + plugins:[] /* Custom plugin to extend the functionality of Hvigor. */ +} diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/oh-package-lock.json5 b/harmony-os/SherpaOnnxSpeakerDiarization/oh-package-lock.json5 new file mode 100644 index 000000000..f538ae290 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/oh-package-lock.json5 @@ -0,0 +1,19 @@ +{ + "meta": { + "stableOrder": true + }, + "lockfileVersion": 3, + "ATTENTION": "THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.", + "specifiers": { + "@ohos/hypium@1.0.19": "@ohos/hypium@1.0.19" + }, + "packages": { + "@ohos/hypium@1.0.19": { + "name": "@ohos/hypium", + "version": "1.0.19", + "integrity": "sha512-cEjDgLFCm3cWZDeRXk7agBUkPqjWxUo6AQeiu0gEkb3J8ESqlduQLSIXeo3cCsm8U/asL7iKjF85ZyOuufAGSQ==", + "resolved": "https://ohpm.openharmony.cn/ohpm/@ohos/hypium/-/hypium-1.0.19.har", + "registryType": "ohpm" + } + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/oh-package.json5 b/harmony-os/SherpaOnnxSpeakerDiarization/oh-package.json5 new file mode 100644 index 000000000..a79d5300e --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerDiarization/oh-package.json5 @@ -0,0 +1,9 @@ +{ + "modelVersion": "5.0.0", + "description": "Please describe the basic information.", + "dependencies": { + }, + "devDependencies": { + "@ohos/hypium": "1.0.19" + } +} diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index 84b33eb0d..4a11cae29 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -2053,11 +2053,6 @@ SherpaOnnxCreateOfflineSpeakerDiarizationOHOS( auto sd_config = GetOfflineSpeakerDiarizationConfig(config); - if (!sd_config.Validate()) { - SHERPA_ONNX_LOGE("Errors in config"); - return nullptr; - } - SherpaOnnxOfflineSpeakerDiarization *sd = new SherpaOnnxOfflineSpeakerDiarization; diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index a781520ff..4d4a2c4fc 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -1512,10 +1512,10 @@ SHERPA_ONNX_API void SherpaOnnxOfflineSpeakerDiarizationDestroySegment( const SherpaOnnxOfflineSpeakerDiarizationSegment *s); typedef int32_t (*SherpaOnnxOfflineSpeakerDiarizationProgressCallback)( - int32_t num_processed_chunk, int32_t num_total_chunks, void *arg); + int32_t num_processed_chunks, int32_t num_total_chunks, void *arg); typedef int32_t (*SherpaOnnxOfflineSpeakerDiarizationProgressCallbackNoArg)( - int32_t num_processed_chunk, int32_t num_total_chunks); + int32_t num_processed_chunks, int32_t num_total_chunks); // The user has to invoke SherpaOnnxOfflineSpeakerDiarizationDestroyResult() // to free the returned pointer to avoid memory leak. From e011e849d407e9ada939cf3f7dfb4669ab6d8098 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 10 Dec 2024 20:29:33 +0800 Subject: [PATCH 100/183] Release v1.10.34 (#1611) --- CHANGELOG.md | 10 ++++++++++ CMakeLists.txt | 2 +- build-ios-shared.sh | 2 +- dart-api-examples/add-punctuations/pubspec.yaml | 2 +- dart-api-examples/audio-tagging/pubspec.yaml | 2 +- dart-api-examples/keyword-spotter/pubspec.yaml | 2 +- dart-api-examples/non-streaming-asr/pubspec.yaml | 2 +- .../speaker-diarization/pubspec.yaml | 2 +- .../speaker-identification/pubspec.yaml | 2 +- dart-api-examples/streaming-asr/pubspec.yaml | 2 +- dart-api-examples/tts/pubspec.yaml | 2 +- .../vad-with-non-streaming-asr/pubspec.yaml | 2 +- dart-api-examples/vad/pubspec.yaml | 2 +- flutter-examples/streaming_asr/pubspec.yaml | 4 ++-- flutter-examples/tts/pubspec.yaml | 4 ++-- flutter/sherpa_onnx/pubspec.yaml | 12 ++++++------ .../sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec | 2 +- .../macos/sherpa_onnx_macos.podspec | 2 +- harmony-os/README.md | 15 +++++++++------ harmony-os/SherpaOnnxHar/sherpa_onnx/README.md | 5 ++++- .../SherpaOnnxHar/sherpa_onnx/oh-package.json5 | 10 +++++----- .../entry/oh-package.json5 | 2 +- .../entry/oh-package.json5 | 2 +- .../entry/oh-package.json5 | 2 +- harmony-os/SherpaOnnxTts/entry/oh-package.json5 | 2 +- harmony-os/SherpaOnnxVadAsr/entry/README.md | 2 +- .../SherpaOnnxVadAsr/entry/oh-package.json5 | 2 +- new-release.sh | 16 ++++++++-------- nodejs-addon-examples/package.json | 2 +- 29 files changed, 67 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a1ce29af..115f55e25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## 1.10.34 + +* Fix building node-addon package (#1598) +* Update doc links for HarmonyOS (#1601) +* Add on-device real-time ASR demo for HarmonyOS (#1606) +* Add speaker identification APIs for HarmonyOS (#1607) +* Add speaker identification demo for HarmonyOS (#1608) +* Add speaker diarization API for HarmonyOS. (#1609) +* Add speaker diarization demo for HarmonyOS (#1610) + ## 1.10.33 * Add non-streaming ASR support for HarmonyOS. (#1564) diff --git a/CMakeLists.txt b/CMakeLists.txt index cdbdfbc09..37ccb4cc2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ project(sherpa-onnx) # Remember to update # ./CHANGELOG.md # ./new-release.sh -set(SHERPA_ONNX_VERSION "1.10.33") +set(SHERPA_ONNX_VERSION "1.10.34") # Disable warning about # diff --git a/build-ios-shared.sh b/build-ios-shared.sh index 1dee7d76d..0b0acb60a 100755 --- a/build-ios-shared.sh +++ b/build-ios-shared.sh @@ -242,7 +242,7 @@ for d in ios-arm64_x86_64-simulator ios-arm64; do CFBundlePackageType FMWK CFBundleShortVersionString - 1.10.33 + 1.10.34 CFBundleSupportedPlatforms iPhoneOS diff --git a/dart-api-examples/add-punctuations/pubspec.yaml b/dart-api-examples/add-punctuations/pubspec.yaml index ebf064dbb..03f376179 100644 --- a/dart-api-examples/add-punctuations/pubspec.yaml +++ b/dart-api-examples/add-punctuations/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.33 + sherpa_onnx: ^1.10.34 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/audio-tagging/pubspec.yaml b/dart-api-examples/audio-tagging/pubspec.yaml index 20288d20a..a869bfdc3 100644 --- a/dart-api-examples/audio-tagging/pubspec.yaml +++ b/dart-api-examples/audio-tagging/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.33 + sherpa_onnx: ^1.10.34 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/keyword-spotter/pubspec.yaml b/dart-api-examples/keyword-spotter/pubspec.yaml index 7951ff1a0..bf79f3bcf 100644 --- a/dart-api-examples/keyword-spotter/pubspec.yaml +++ b/dart-api-examples/keyword-spotter/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.33 + sherpa_onnx: ^1.10.34 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/non-streaming-asr/pubspec.yaml b/dart-api-examples/non-streaming-asr/pubspec.yaml index d747ba126..b34bee6ec 100644 --- a/dart-api-examples/non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.33 + sherpa_onnx: ^1.10.34 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/speaker-diarization/pubspec.yaml b/dart-api-examples/speaker-diarization/pubspec.yaml index 05ed44f91..c8b045a5a 100644 --- a/dart-api-examples/speaker-diarization/pubspec.yaml +++ b/dart-api-examples/speaker-diarization/pubspec.yaml @@ -8,7 +8,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.33 + sherpa_onnx: ^1.10.34 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/speaker-identification/pubspec.yaml b/dart-api-examples/speaker-identification/pubspec.yaml index 64b1383df..4baa30f8d 100644 --- a/dart-api-examples/speaker-identification/pubspec.yaml +++ b/dart-api-examples/speaker-identification/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.33 + sherpa_onnx: ^1.10.34 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/streaming-asr/pubspec.yaml b/dart-api-examples/streaming-asr/pubspec.yaml index 538641a74..80322e86e 100644 --- a/dart-api-examples/streaming-asr/pubspec.yaml +++ b/dart-api-examples/streaming-asr/pubspec.yaml @@ -11,7 +11,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.33 + sherpa_onnx: ^1.10.34 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/tts/pubspec.yaml b/dart-api-examples/tts/pubspec.yaml index ba4c4de0c..d213371ba 100644 --- a/dart-api-examples/tts/pubspec.yaml +++ b/dart-api-examples/tts/pubspec.yaml @@ -8,7 +8,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.33 + sherpa_onnx: ^1.10.34 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml index 014daad04..4f47c9940 100644 --- a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.33 + sherpa_onnx: ^1.10.34 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad/pubspec.yaml b/dart-api-examples/vad/pubspec.yaml index 230e62677..d6c62a495 100644 --- a/dart-api-examples/vad/pubspec.yaml +++ b/dart-api-examples/vad/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.33 + sherpa_onnx: ^1.10.34 path: ^1.9.0 args: ^2.5.0 diff --git a/flutter-examples/streaming_asr/pubspec.yaml b/flutter-examples/streaming_asr/pubspec.yaml index da2cd2d10..810ae43c6 100644 --- a/flutter-examples/streaming_asr/pubspec.yaml +++ b/flutter-examples/streaming_asr/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' -version: 1.10.33 +version: 1.10.34 topics: - speech-recognition @@ -31,7 +31,7 @@ dependencies: record: ^5.1.0 url_launcher: ^6.2.6 - sherpa_onnx: ^1.10.33 + sherpa_onnx: ^1.10.34 # sherpa_onnx: # path: ../../flutter/sherpa_onnx diff --git a/flutter-examples/tts/pubspec.yaml b/flutter-examples/tts/pubspec.yaml index 646db8906..931641bbc 100644 --- a/flutter-examples/tts/pubspec.yaml +++ b/flutter-examples/tts/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' # Remove this line if you wish to publish to pub.dev -version: 1.10.33 +version: 1.10.34 environment: sdk: ">=2.17.0 <4.0.0" @@ -18,7 +18,7 @@ dependencies: cupertino_icons: ^1.0.6 path_provider: ^2.1.3 path: ^1.9.0 - sherpa_onnx: ^1.10.33 + sherpa_onnx: ^1.10.34 # sherpa_onnx: # path: ../../flutter/sherpa_onnx url_launcher: 6.2.6 diff --git a/flutter/sherpa_onnx/pubspec.yaml b/flutter/sherpa_onnx/pubspec.yaml index 8f7057a47..a63b78ab0 100644 --- a/flutter/sherpa_onnx/pubspec.yaml +++ b/flutter/sherpa_onnx/pubspec.yaml @@ -17,7 +17,7 @@ topics: - voice-activity-detection # remember to change the version in ../sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec -version: 1.10.33 +version: 1.10.34 homepage: https://github.com/k2-fsa/sherpa-onnx @@ -30,23 +30,23 @@ dependencies: flutter: sdk: flutter - sherpa_onnx_android: ^1.10.33 + sherpa_onnx_android: ^1.10.34 # sherpa_onnx_android: # path: ../sherpa_onnx_android - sherpa_onnx_macos: ^1.10.33 + sherpa_onnx_macos: ^1.10.34 # sherpa_onnx_macos: # path: ../sherpa_onnx_macos - sherpa_onnx_linux: ^1.10.33 + sherpa_onnx_linux: ^1.10.34 # sherpa_onnx_linux: # path: ../sherpa_onnx_linux - sherpa_onnx_windows: ^1.10.33 + sherpa_onnx_windows: ^1.10.34 # sherpa_onnx_windows: # path: ../sherpa_onnx_windows - sherpa_onnx_ios: ^1.10.33 + sherpa_onnx_ios: ^1.10.34 # sherpa_onnx_ios: # path: ../sherpa_onnx_ios diff --git a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec index 2ff5a781c..2738be7c7 100644 --- a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec +++ b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec @@ -7,7 +7,7 @@ # https://groups.google.com/g/dart-ffi/c/nUATMBy7r0c Pod::Spec.new do |s| s.name = 'sherpa_onnx_ios' - s.version = '1.10.33' + s.version = '1.10.34' s.summary = 'A new Flutter FFI plugin project.' s.description = <<-DESC A new Flutter FFI plugin project. diff --git a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec index a3ec33d16..6898d644b 100644 --- a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec +++ b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec @@ -4,7 +4,7 @@ # Pod::Spec.new do |s| s.name = 'sherpa_onnx_macos' - s.version = '1.10.33' + s.version = '1.10.34' s.summary = 'sherpa-onnx Flutter FFI plugin project.' s.description = <<-DESC sherpa-onnx Flutter FFI plugin project. diff --git a/harmony-os/README.md b/harmony-os/README.md index 5c1fb9ed7..63a530cc0 100644 --- a/harmony-os/README.md +++ b/harmony-os/README.md @@ -6,15 +6,18 @@ Please refer to our [doc](https://k2-fsa.github.io/sherpa/onnx/harmony-os/how-to-build-har.html) if you want to build `sherpa-onnx` from source. -- [./SherpaOnnxVadAsr](./SherpaOnnxVadAsr) It shows how to use - VAD + Non-streaming ASR for speech recognition. - Please see the doc at - -- [./SherpaOnnxStreamingAsr](./SherpaOnnxStreamingAsr) It shows how to use - streaming ASR models for real-time on-device speech recognition. +- [./SherpaOnnxSpeakerDiarization](./SherpaOnnxSpeakerDiarization) It shows how + to run on-device speaker diarization. - [./SherpaOnnxSpeakerIdentification](./SherpaOnnxSpeakerIdentification) It shows how to use speaker embedding models for on-device speaker identification. +- [./SherpaOnnxStreamingAsr](./SherpaOnnxStreamingAsr) It shows how to use + streaming ASR models for real-time on-device speech recognition. + - [./SherpaOnnxTts](./SherpaOnnxTts) It shows how to run on-device text-to-speech. Please see the doc at + +- [./SherpaOnnxVadAsr](./SherpaOnnxVadAsr) It shows how to use + VAD + Non-streaming ASR for speech recognition. + Please see the doc at diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md b/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md index e94741a01..78ef2c4e1 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md @@ -23,7 +23,7 @@ or update your `oh-package.json5` to include the following: ``` "dependencies": { - "sherpa_onnx": "1.10.33", + "sherpa_onnx": "1.10.34", }, ``` @@ -33,8 +33,11 @@ Note that we recommend always using the latest version. | Demo | URL | Description| |------|-----|------------| +|SherpaOnnxStreamingAsr|[Address](https://github.com/k2-fsa/sherpa-onnx/tree/master/harmony-os/SherpaOnnxStreamingAsr)|On-device real-time/streaming speech recognition with Next-gen Kaldi| |SherpaOnnxVadAsr|[Address](https://github.com/k2-fsa/sherpa-onnx/tree/master/harmony-os/SherpaOnnxVadAsr)|It shows how to use VAD with a non-streaming ASR model for on-device speech recognition without accessing the network | |SherpaOnnxTts|[Address](https://github.com/k2-fsa/sherpa-onnx/tree/master/harmony-os/SherpaOnnxTts)|It shows how to use Next-gen Kaldi for on-device text-to-speech (TTS, i.e., speech synthesis)| +|SherpaOnnxSpeakerDiarization|[Address](https://github.com/k2-fsa/sherpa-onnx/tree/master/harmony-os/SherpaOnnxSpeakerDiarization)|On-device speaker diarization with Next-gen Kaldi| +|SherpaOnnxSpeakerIdentification|[Address](https://github.com/k2-fsa/sherpa-onnx/tree/master/harmony-os/SherpaOnnxSpeakerIdentification)|On-device speaker identification with Next-gen Kaldi| # Documentation diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 index 7e123d17e..13a249052 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 @@ -1,7 +1,7 @@ { "name": "sherpa_onnx", - "version": "1.10.33", - "description": "Speech-to-text, text-to-speech, and speaker diarization using Next-gen Kaldi without internet connection", + "version": "1.10.34", + "description": "On-device speech-to-text, text-to-speech, and speaker diarization using Next-gen Kaldi without Internet connection", "main": "Index.ets", "author": "The next-gen Kaldi team", "license": "Apache-2.0", @@ -13,14 +13,14 @@ "keywords": [ "语音识别", "语音合成", + "说话人日志", "新一代Kaldi", + "不联网", + "本地", "tts", "asr", - "locally", - "diarization", "privacy", "open-source", - "speaker", ], "bugs": { "url": "https://github.com/k2-fsa/sherpa-onnx/issues" diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 b/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 index 97448bbf3..1682f0d10 100644 --- a/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.33" + "sherpa_onnx": "1.10.34" } } diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 b/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 index e57a8e40e..6f404b4ca 100644 --- a/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.33", + "sherpa_onnx": "1.10.34", } } diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 index e57a8e40e..6f404b4ca 100644 --- a/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.33", + "sherpa_onnx": "1.10.34", } } diff --git a/harmony-os/SherpaOnnxTts/entry/oh-package.json5 b/harmony-os/SherpaOnnxTts/entry/oh-package.json5 index e57a8e40e..6f404b4ca 100644 --- a/harmony-os/SherpaOnnxTts/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxTts/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.33", + "sherpa_onnx": "1.10.34", } } diff --git a/harmony-os/SherpaOnnxVadAsr/entry/README.md b/harmony-os/SherpaOnnxVadAsr/entry/README.md index 4862dd945..806291354 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/README.md +++ b/harmony-os/SherpaOnnxVadAsr/entry/README.md @@ -1,6 +1,6 @@ # Introduction -Please download ./sherpa_onnx-v1.10.33.har +Please download ./sherpa_onnx-v1.10.34.har from Hint: For users who have no access to huggingface, please use diff --git a/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 b/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 index b871adad1..dae1ac8c2 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 @@ -7,7 +7,7 @@ "license": "", "dependencies": { // please see https://ohpm.openharmony.cn/#/cn/detail/sherpa_onnx - "sherpa_onnx": "1.10.33", + "sherpa_onnx": "1.10.34", } } diff --git a/new-release.sh b/new-release.sh index 052e9ebf3..82e2fb36e 100755 --- a/new-release.sh +++ b/new-release.sh @@ -2,12 +2,12 @@ set -ex -sed -i.bak 's/1\.10\.32/1\.10\.33/g' ./build-ios-shared.sh -find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.32/1\.10\.33/g' {} \; -find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.32/1\.10\.33/g' {} \; -find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.32/1\.10\.33/g' {} \; -find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.32/1\.10\.33/g' {} \; -find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.32/1\.10\.33/g' {} \; +sed -i.bak 's/1\.10\.33/1\.10\.34/g' ./build-ios-shared.sh +find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.33/1\.10\.34/g' {} \; +find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.33/1\.10\.34/g' {} \; +find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.33/1\.10\.34/g' {} \; +find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.33/1\.10\.34/g' {} \; +find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.33/1\.10\.34/g' {} \; -find harmony-os -name "README.md" -type f -exec sed -i.bak 's/1\.10\.32/1\.10\.33/g' {} \; -find harmony-os -name oh-package.json5 -type f -exec sed -i.bak 's/1\.10\.32/1\.10\.33/g' {} \; +find harmony-os -name "README.md" -type f -exec sed -i.bak 's/1\.10\.33/1\.10\.34/g' {} \; +find harmony-os -name oh-package.json5 -type f -exec sed -i.bak 's/1\.10\.33/1\.10\.34/g' {} \; diff --git a/nodejs-addon-examples/package.json b/nodejs-addon-examples/package.json index 12a5d80ca..3fc50bc2a 100644 --- a/nodejs-addon-examples/package.json +++ b/nodejs-addon-examples/package.json @@ -1,5 +1,5 @@ { "dependencies": { - "sherpa-onnx-node": "^1.10.33" + "sherpa-onnx-node": "^1.10.34" } } From 9d4659fd29d3e8f5bb41471abcd477c22e8cc0f3 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 11 Dec 2024 12:01:13 +0800 Subject: [PATCH 101/183] Add missing changes about speaker identfication demo for HarmonyOS (#1612) --- .../entry/oh-package-lock.json5 | 28 +++ .../entry/src/main/ets/pages/Index.ets | 177 +++++++++++++++--- .../workers/SpeakerIdentificationWorker.ets | 90 ++++++--- 3 files changed, 243 insertions(+), 52 deletions(-) create mode 100644 harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package-lock.json5 diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package-lock.json5 b/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package-lock.json5 new file mode 100644 index 000000000..5a9a42508 --- /dev/null +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package-lock.json5 @@ -0,0 +1,28 @@ +{ + "meta": { + "stableOrder": true + }, + "lockfileVersion": 3, + "ATTENTION": "THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.", + "specifiers": { + "libsherpa_onnx.so@../oh_modules/.ohpm/sherpa_onnx@1y+qvabrznvcerrtte4uydjhwfdt7hfnlsk0jsnicmy=/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx": "libsherpa_onnx.so@../oh_modules/.ohpm/sherpa_onnx@1y+qvabrznvcerrtte4uydjhwfdt7hfnlsk0jsnicmy=/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx", + "sherpa_onnx@sherpa_onnx_2.har": "sherpa_onnx@sherpa_onnx_2.har" + }, + "packages": { + "libsherpa_onnx.so@../oh_modules/.ohpm/sherpa_onnx@1y+qvabrznvcerrtte4uydjhwfdt7hfnlsk0jsnicmy=/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx": { + "name": "libsherpa_onnx.so", + "version": "1.0.0", + "resolved": "../oh_modules/.ohpm/sherpa_onnx@1y+qvabrznvcerrtte4uydjhwfdt7hfnlsk0jsnicmy=/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx", + "registryType": "local" + }, + "sherpa_onnx@sherpa_onnx_2.har": { + "name": "sherpa_onnx", + "version": "1.10.33", + "resolved": "sherpa_onnx_2.har", + "registryType": "local", + "dependencies": { + "libsherpa_onnx.so": "file:./src/main/cpp/types/libsherpa_onnx" + } + } + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/pages/Index.ets b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/pages/Index.ets index ec12f51a6..c17b8b0f6 100644 --- a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/pages/Index.ets +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/pages/Index.ets @@ -72,7 +72,7 @@ struct Index { @State currentIndex: number = 0; - @State message: string = 'Hello World'; + private threshold: string = '0.5'; private workerInstance?: worker.ThreadWorker private readonly scriptURL: string = 'entry/ets/workers/SpeakerIdentificationWorker.ets' @@ -83,15 +83,21 @@ struct Index { @State btnSaveAudioEnabled: boolean = false; @State btnAddEnabled: boolean = false; - private sampleRate: number = 16000; - private sampleList: Float32Array[] = [] + private sampleRate: number = 48000; + private sampleListForAdding: Float32Array[] = [] + private sampleListForTesting: Float32Array[] = [] private mic?: audio.AudioCapturer; @State infoHome: string = ''; @State infoAdd: string = ''; - @State micBtnCaption: string = 'Start recording'; - @State micStarted: boolean = false; + @State micBtnCaptionForAdding: string = 'Start recording'; + @State micStartedForAdding: boolean = false; + @State micBtnEnabledForAdding: boolean = true; + + @State micBtnCaptionForTesting: string = 'Start recording'; + @State micStartedForTesting: boolean = false; + @State micBtnEnabledForTesting: boolean = true; async initMic() { const permissions: Permissions[] = ["ohos.permission.MICROPHONE"]; @@ -158,6 +164,23 @@ struct Index { if (msgType == 'manager-all-speaker-names') { this.allSpeakerNames = e.data['allSpeakers'] as string[]; } + + if (msgType == 'manager-add-speaker-done') { + const ok: boolean = e.data['ok'] as boolean; + const status: string = e.data['status'] as string; + this.infoAdd += '\n' + status; + + if (ok) { + this.sampleListForAdding = []; + this.btnSaveAudioEnabled = false; + this.btnAddEnabled = false; + } + } + + if (msgType == 'manager-search-speaker-done') { + const name = e.data['name'] as string; + this.infoHome = name; + } }; this.workerInstance.postMessage({ msgType: 'init-extractor', context: getContext()}); @@ -181,7 +204,97 @@ struct Index { Tabs({ barPosition: BarPosition.End, controller: this.controller }) { TabContent() { Column({ space: 10 }) { - Button('Home') + Text(this.title).fontSize(this.titleFontSize).fontWeight(FontWeight.Bold); + Row() { + Text('Similary threshold').width('60%'); + + TextInput({ text: this.threshold }).onChange((text) => { + this.threshold = text.trim(); + }).width('20%') + } + Row() { + Button(this.micBtnCaptionForTesting) + .enabled(this.micBtnEnabledForTesting) + .onClick(()=>{ + if (this.allSpeakerNames.length == 0) { + this.infoHome = 'There are no speakers registered. Please add them first'; + return; + } + + let threshold = parseFloat(this.threshold); + if (isNaN(threshold)) { + this.infoHome = 'Please enter a valid threshold'; + return; + } + + if (threshold <= 0) { + this.infoHome = 'Please enter a positive threshold'; + return; + } + console.log(`threshold: ${threshold}`); + + if (this.micStartedForTesting) { + this.micStartedForTesting = false; + this.micBtnCaptionForTesting = 'Start'; + this.micBtnEnabledForAdding = true; + this.mic?.stop(); + + const samples = flatten(this.sampleListForTesting); + const duration = samples.length / this.sampleRate; + if (duration < 0.5) { + this.infoHome = `Please speak for a longer time! Current duration: ${duration}`; + return; + } + if (this.workerInstance) { + this.workerInstance.postMessage({ + msgType: 'manager-search-speaker', + samples: samples, + sampleRate: this.sampleRate, + threshold, + }); + } + } else { + this.sampleListForTesting = []; + this.micStartedForTesting = true; + this.micBtnCaptionForTesting = 'Stop'; + this.micBtnEnabledForAdding = false; + this.mic?.start(); + this.infoHome = `Use threshold: ${threshold}`; + this.infoHome += '\nPlease speak and then click Stop'; + } + }) + + Button('Save audio') + .enabled(!this.micStartedForTesting) + .onClick(()=>{ + if (this.sampleListForTesting.length == 0) { + this.infoHome = 'No audio samples recorded'; + return; + } + const samples = flatten(this.sampleListForTesting); + + if (samples.length == 0) { + this.infoHome = 'Empty samples'; + return; + } + + let uri: string = ''; + + const audioOptions = new picker.AudioSaveOptions(); // audioOptions.newFileNames = ['o.wav']; + + const audioViewPicker = new picker.AudioViewPicker(); + + audioViewPicker.save(audioOptions).then((audioSelectResult: Array) => { + uri = audioSelectResult[0]; + savePcmToWav(uri, toInt16Samples(samples), this.sampleRate); + console.log(`Saved to ${uri}`); + this.infoHome+= `\nSaved to ${uri}`; + }); + }) + } + TextArea({text: this.infoHome}) + .height('100%') + .focusable(false) } }.tabBar(this.TabBuilder('Home', 0, $r('app.media.icon_home'), $r('app.media.icon_home'))) @@ -244,22 +357,25 @@ struct Index { }.width('100%') Row({space: 10}) { - Button(this.micBtnCaption) + Button(this.micBtnCaptionForAdding) + .enabled(this.micBtnEnabledForAdding) .onClick(()=> { if (this.mic) { - if (this.micStarted) { - this.micStarted = false; - this.micBtnCaption = 'Start recording'; + if (this.micStartedForAdding) { + this.micStartedForAdding = false; + this.micBtnEnabledForTesting = true; + this.micBtnCaptionForAdding = 'Start recording'; this.mic.stop(); this.infoAdd = ''; - if (this.sampleList.length > 0) { + if (this.sampleListForAdding.length > 0) { this.btnAddEnabled = true; this.btnSaveAudioEnabled = true; } } else { - this.micStarted = true; - this.micBtnCaption = 'Stop recording'; - this.sampleList = []; + this.micStartedForAdding = true; + this.micBtnEnabledForTesting = false; + this.micBtnCaptionForAdding = 'Stop recording'; + this.sampleListForAdding = []; this.mic.start(); this.infoAdd = ''; @@ -267,30 +383,41 @@ struct Index { this.btnSaveAudioEnabled = false; } } - }) Button('Add') .enabled(this.btnAddEnabled) .onClick(()=>{ if (this.inputSpeakerName.trim() == '') { - this.infoAdd += 'Please input a speaker name first'; + this.infoAdd += '\nPlease input a speaker name first'; return; } - const samples = flatten(this.sampleList); - console.log(`number of samples: ${samples.length}, ${samples.length / this.sampleRate}`); + const samples = flatten(this.sampleListForAdding); + const duration = samples.length / this.sampleRate; + if (duration < 0.5) { + this.infoAdd = `Please speak for a longer time. Current duration: ${duration}`; + return; + } + if (this.workerInstance) { + this.workerInstance.postMessage({ + msgType: 'manager-add-speaker', + name: this.inputSpeakerName, + samples: samples, + sampleRate: this.sampleRate, + }) + } }) Button('Save audio') .enabled(this.btnSaveAudioEnabled) .onClick(()=>{ - if (this.sampleList.length == 0) { + if (this.sampleListForAdding.length == 0) { this.btnSaveAudioEnabled = false; return; } - const samples = flatten(this.sampleList); + const samples = flatten(this.sampleListForAdding); if (samples.length == 0) { this.btnSaveAudioEnabled = false; @@ -352,6 +479,12 @@ https://k2-fsa.github.io/sherpa/social-groups.html samplesFloat[i] = view[i] / 32768.0; } - this.sampleList.push(samplesFloat); + if (this.micStartedForAdding) { + this.sampleListForAdding.push(samplesFloat); + } + + if (this.micStartedForTesting) { + this.sampleListForTesting.push(samplesFloat); + } } -} \ No newline at end of file +} diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/workers/SpeakerIdentificationWorker.ets b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/workers/SpeakerIdentificationWorker.ets index 9dd97d108..5b0679742 100644 --- a/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/workers/SpeakerIdentificationWorker.ets +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/ets/workers/SpeakerIdentificationWorker.ets @@ -1,12 +1,12 @@ -import worker, { ThreadWorkerGlobalScope, MessageEvents, ErrorEvent } from '@ohos.worker'; +import worker, { ErrorEvent, MessageEvents, ThreadWorkerGlobalScope } from '@ohos.worker'; import { + OnlineStream, readWaveFromBinary, Samples, SpeakerEmbeddingExtractor, SpeakerEmbeddingExtractorConfig, SpeakerEmbeddingManager } from 'sherpa_onnx'; -import { fileIo } from '@kit.CoreFileKit'; const workerPort: ThreadWorkerGlobalScope = worker.workerPort; @@ -19,7 +19,19 @@ function readWaveFromRawfile(filename: string, context: Context): Samples { } function initExtractor(context: Context): SpeakerEmbeddingExtractor { - const config = new SpeakerEmbeddingExtractorConfig(); + const config: SpeakerEmbeddingExtractorConfig = new SpeakerEmbeddingExtractorConfig(); + + // Please put the model file inside the directory + // harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/rawfile +/* +(py38) fangjuns-MacBook-Pro:rawfile fangjun$ pwd +/Users/fangjun/open-source/sherpa-onnx/harmony-os/SherpaOnnxSpeakerIdentification/entry/src/main/resources/rawfile +(py38) fangjuns-MacBook-Pro:rawfile fangjun$ ls -lh +total 77336 +-rw-r--r-- 1 fangjun staff 38M Dec 9 19:34 3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx + */ + // You can find more models at + // https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models config.model = '3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx'; config.numThreads = 2; config.debug = true; @@ -28,7 +40,7 @@ function initExtractor(context: Context): SpeakerEmbeddingExtractor { } function extractEmbedding(samples: Samples): Float32Array { - const stream = extractor.createStream(); + const stream: OnlineStream = extractor.createStream(); stream.acceptWaveform(samples); return extractor.compute(stream); } @@ -49,30 +61,6 @@ workerPort.onmessage = (e: MessageEvents) => { extractor = initExtractor(context); manager = new SpeakerEmbeddingManager(extractor.dim); - const filename1 = 'sr-data/enroll/fangjun-sr-1.wav'; - const samples1 = readWaveFromRawfile(filename1, context); - console.log(`sample rate: ${samples1.sampleRate}`); - let ok = manager.add({ name: 'fangjun0', v: extractEmbedding(samples1) }); - ok = manager.add({ name: 'fangjun1', v: extractEmbedding(samples1) }); - /* - ok = manager.add({ name: 'fangjun2', v: extractEmbedding(samples1) }); - ok = manager.add({ name: 'fangjun3', v: extractEmbedding(samples1) }); - ok = manager.add({ name: 'fangjun4', v: extractEmbedding(samples1) }); - ok = manager.add({ name: 'fangjun5', v: extractEmbedding(samples1) }); - ok = manager.add({ name: 'fangjun6', v: extractEmbedding(samples1) }); - ok = manager.add({ name: 'fangjun7', v: extractEmbedding(samples1) }); - ok = manager.add({ name: 'fangjun8', v: extractEmbedding(samples1) }); - ok = manager.add({ name: 'fangjun9', v: extractEmbedding(samples1) }); - ok = manager.add({ name: 'fangjun10', v: extractEmbedding(samples1) }); - */ - - if (ok) { - console.log(`Added fangjun`); - let n = manager.getNumSpeakers(); - console.log(`number of speakers: ${n}`); - console.log(`speaker names: ${manager.getAllSpeakerNames().join('\n')}`); - } - workerPort.postMessage({ msgType: 'manager-all-speaker-names', allSpeakers: manager.getAllSpeakerNames(), }); @@ -80,7 +68,7 @@ workerPort.onmessage = (e: MessageEvents) => { if (msgType == 'manager-delete-speaker') { const name = e.data['name'] as string; - const ok = manager.remove(name); + const ok: boolean = manager.remove(name); if (ok) { console.log(`Removed ${name}.`); @@ -92,6 +80,48 @@ workerPort.onmessage = (e: MessageEvents) => { }); } } + + if (msgType == 'manager-add-speaker') { + const name = e.data['name'] as string; + const samples = e.data['samples'] as Float32Array; + const sampleRate = e.data['sampleRate'] as number; + + const v = extractEmbedding({ samples, sampleRate }); + const ok: boolean = manager.add({ name, v }); + if (ok) { + workerPort.postMessage({ + msgType: 'manager-add-speaker-done', + status: `Added ${name}`, + ok, + }); + workerPort.postMessage({ + msgType: 'manager-all-speaker-names', allSpeakers: manager.getAllSpeakerNames(), + } + ); + } else { + workerPort.postMessage({ + msgType: 'manager-add-speaker-done', + status: `Failed to add ${name}. Possibly due to exsiting speaker name. Please recheck`, + ok, + }); + } + } + + if (msgType == 'manager-search-speaker') { + const threshold = e.data['threshold'] as number; + const samples = e.data['samples'] as Float32Array; + const sampleRate = e.data['sampleRate'] as number; + + const v = extractEmbedding({ samples, sampleRate }); + let name: string = manager.search({ threshold, v }); + if (name == '' || name == undefined) { + name = "======"; + } + workerPort.postMessage({ + msgType: 'manager-search-speaker-done', + name + }); + } } /** @@ -110,4 +140,4 @@ workerPort.onmessageerror = (e: MessageEvents) => { * @param e error message */ workerPort.onerror = (e: ErrorEvent) => { -} \ No newline at end of file +} From 4dc4f1a70846216b6c5483c7f369f7c9964afef0 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 12 Dec 2024 16:59:00 +0800 Subject: [PATCH 102/183] Provide sherpa-onnx.aar for Android (#1615) --- .github/workflows/android.yaml | 145 +++++++++++++- android/SherpaOnnxAar/.gitignore | 15 ++ android/SherpaOnnxAar/README.md | 20 ++ android/SherpaOnnxAar/build.gradle.kts | 6 + android/SherpaOnnxAar/gradle.properties | 23 +++ .../SherpaOnnxAar/gradle/libs.versions.toml | 23 +++ .../gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 59203 bytes .../gradle/wrapper/gradle-wrapper.properties | 6 + android/SherpaOnnxAar/gradlew | 185 +++++++++++++++++ android/SherpaOnnxAar/gradlew.bat | 89 +++++++++ android/SherpaOnnxAar/settings.gradle.kts | 23 +++ android/SherpaOnnxAar/sherpa_onnx/.gitignore | 1 + .../sherpa_onnx/build.gradle.kts | 43 ++++ .../sherpa_onnx/consumer-rules.pro | 0 .../sherpa_onnx/proguard-rules.pro | 21 ++ .../sherpa/onnx/ExampleInstrumentedTest.kt | 24 +++ .../sherpa_onnx/src/main/AndroidManifest.xml | 4 + .../com/k2fsa/sherpa/onnx/AudioTagging.kt | 1 + .../com/k2fsa/sherpa/onnx/FeatureConfig.kt | 1 + .../com/k2fsa/sherpa/onnx/KeywordSpotter.kt | 1 + .../k2fsa/sherpa/onnx/OfflinePunctuation.kt | 1 + .../k2fsa/sherpa/onnx/OfflineRecognizer.kt | 1 + .../sherpa/onnx/OfflineSpeakerDiarization.kt | 1 + .../com/k2fsa/sherpa/onnx/OfflineStream.kt | 1 + .../com/k2fsa/sherpa/onnx/OnlineRecognizer.kt | 1 + .../com/k2fsa/sherpa/onnx/OnlineStream.kt | 1 + .../java/com/k2fsa/sherpa/onnx/Speaker.kt | 1 + .../onnx/SpeakerEmbeddingExtractorConfig.kt | 1 + .../onnx/SpokenLanguageIdentification.kt | 1 + .../main/java/com/k2fsa/sherpa/onnx/Tts.kt | 1 + .../main/java/com/k2fsa/sherpa/onnx/Vad.kt | 1 + .../java/com/k2fsa/sherpa/onnx/WaveReader.kt | 1 + .../src/main/jniLibs/arm64-v8a/.gitkeep | 0 .../src/main/jniLibs/armeabi-v7a/.gitkeep | 0 .../sherpa_onnx/src/main/jniLibs/x86/.gitkeep | 0 .../src/main/jniLibs/x86_64/.gitkeep | 0 .../com/k2fsa/sherpa/onnx/ExampleUnitTest.kt | 17 ++ .../main/java/com/k2fsa/sherpa/onnx/Tts.kt | 188 +----------------- .../com/k2fsa/sherpa/onnx/tts/engine/Tts.kt | 2 +- jitpack.yml | 9 + kotlin-api-examples/Tts.kt | 2 +- new-release.sh | 4 + pom.xml | 19 ++ sherpa-onnx/kotlin-api/Tts.kt | 187 +++++++++++++++++ 44 files changed, 881 insertions(+), 190 deletions(-) create mode 100644 android/SherpaOnnxAar/.gitignore create mode 100644 android/SherpaOnnxAar/README.md create mode 100644 android/SherpaOnnxAar/build.gradle.kts create mode 100644 android/SherpaOnnxAar/gradle.properties create mode 100644 android/SherpaOnnxAar/gradle/libs.versions.toml create mode 100644 android/SherpaOnnxAar/gradle/wrapper/gradle-wrapper.jar create mode 100644 android/SherpaOnnxAar/gradle/wrapper/gradle-wrapper.properties create mode 100755 android/SherpaOnnxAar/gradlew create mode 100644 android/SherpaOnnxAar/gradlew.bat create mode 100644 android/SherpaOnnxAar/settings.gradle.kts create mode 100644 android/SherpaOnnxAar/sherpa_onnx/.gitignore create mode 100644 android/SherpaOnnxAar/sherpa_onnx/build.gradle.kts create mode 100644 android/SherpaOnnxAar/sherpa_onnx/consumer-rules.pro create mode 100644 android/SherpaOnnxAar/sherpa_onnx/proguard-rules.pro create mode 100644 android/SherpaOnnxAar/sherpa_onnx/src/androidTest/java/com/k2fsa/sherpa/onnx/ExampleInstrumentedTest.kt create mode 100644 android/SherpaOnnxAar/sherpa_onnx/src/main/AndroidManifest.xml create mode 120000 android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/AudioTagging.kt create mode 120000 android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.kt create mode 120000 android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotter.kt create mode 120000 android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OfflinePunctuation.kt create mode 120000 android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.kt create mode 120000 android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarization.kt create mode 120000 android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OfflineStream.kt create mode 120000 android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.kt create mode 120000 android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.kt create mode 120000 android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/Speaker.kt create mode 120000 android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractorConfig.kt create mode 120000 android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/SpokenLanguageIdentification.kt create mode 120000 android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/Tts.kt create mode 120000 android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/Vad.kt create mode 120000 android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/WaveReader.kt create mode 100644 android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/arm64-v8a/.gitkeep create mode 100644 android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/armeabi-v7a/.gitkeep create mode 100644 android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/x86/.gitkeep create mode 100644 android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/x86_64/.gitkeep create mode 100644 android/SherpaOnnxAar/sherpa_onnx/src/test/java/com/k2fsa/sherpa/onnx/ExampleUnitTest.kt mode change 100644 => 120000 android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/Tts.kt create mode 100644 jitpack.yml create mode 100644 pom.xml create mode 100644 sherpa-onnx/kotlin-api/Tts.kt diff --git a/.github/workflows/android.yaml b/.github/workflows/android.yaml index ebe9cd90f..0eb00d50a 100644 --- a/.github/workflows/android.yaml +++ b/.github/workflows/android.yaml @@ -32,7 +32,7 @@ concurrency: jobs: build-android-libs: - name: Android for ${{ matrix.os }} + name: Android libs runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -44,6 +44,11 @@ jobs: with: fetch-depth: 0 + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ matrix.os }}-android-jni + - name: Display NDK HOME shell: bash run: | @@ -53,6 +58,9 @@ jobs: - name: build android arm64-v8a shell: bash run: | + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + export ANDROID_NDK=$ANDROID_NDK_LATEST_HOME ./build-android-arm64-v8a.sh mkdir -p jniLibs/arm64-v8a/ @@ -62,6 +70,9 @@ jobs: - name: build android armv7-eabi shell: bash run: | + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + export ANDROID_NDK=$ANDROID_NDK_LATEST_HOME ./build-android-armv7-eabi.sh mkdir -p ./jniLibs/armeabi-v7a/ @@ -71,6 +82,9 @@ jobs: - name: build android x86_64 shell: bash run: | + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + export ANDROID_NDK=$ANDROID_NDK_LATEST_HOME ./build-android-x86-64.sh mkdir -p ./jniLibs/x86_64 @@ -80,6 +94,9 @@ jobs: - name: build android x86 shell: bash run: | + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + export ANDROID_NDK=$ANDROID_NDK_LATEST_HOME ./build-android-x86.sh mkdir -p ./jniLibs/x86 @@ -143,3 +160,129 @@ jobs: file_glob: true overwrite: true file: sherpa-onnx-*-android.tar.bz2 + + build-android-aar: + needs: [build-android-libs] + name: Android AAR + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # https://github.com/actions/setup-java + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' # See 'Supported distributions' for available options + java-version: '21' + + - name: Display NDK HOME + shell: bash + run: | + echo "ANDROID_NDK_LATEST_HOME: ${ANDROID_NDK_LATEST_HOME}" + ls -lh ${ANDROID_NDK_LATEST_HOME} + + - name: Retrieve artifact + uses: actions/download-artifact@v4 + with: + name: sherpa-onnx-android-libs + path: /tmp/jniLibs + + - name: Show jni libs + shell: bash + run: | + ls -lh /tmp/jniLibs + + # drwxr-xr-x 2 runner docker 4.0K Dec 12 06:56 arm64-v8a + # drwxr-xr-x 2 runner docker 4.0K Dec 12 06:56 armeabi-v7a + # drwxr-xr-x 2 runner docker 4.0K Dec 12 06:56 x86 + # drwxr-xr-x 2 runner docker 4.0K Dec 12 06:56 x86_64 + # + - name: Copy libs + shell: bash + run: | + for arch in arm64-v8a armeabi-v7a x86 x86_64; do + cp -v /tmp/jniLibs/$arch/* android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/$arch/ + done + + - name: Check libs + shell: bash + run: | + ls -lh android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/* + + - name: Build aar + shell: bash + run: | + cd android/SherpaOnnxAar + + ./gradlew :sherpa_onnx:assembleRelease + + - name: Display aar + shell: bash + run: | + cd android/SherpaOnnxAar + + ls -lh ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar + cp ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar ../../ + + + - name: Rename aar + shell: bash + run: | + SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + echo "SHERPA_ONNX_VERSION=$SHERPA_ONNX_VERSION" >> "$GITHUB_ENV" + + mv sherpa_onnx-release.aar sherpa-onnx-${SHERPA_ONNX_VERSION}.aar + + - uses: actions/upload-artifact@v4 + with: + name: sherpa-onnx-android-aar + path: ./*.aar + + # https://huggingface.co/docs/hub/spaces-github-actions + - name: Publish to huggingface + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v3 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + du -h -d1 . + ls -lh + + rm -rf huggingface + export GIT_CLONE_PROTECTION_ACTIVE=false + GIT_LFS_SKIP_SMUDGE=1 git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs huggingface + + cd huggingface + dst=android/aar + mkdir -p $dst + + cp -v ../*.aar $dst + + git status + git lfs track "*.aar" + + git add . + + git commit -m "upload sherpa-onnx-${SHERPA_ONNX_VERSION}.aar" + + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs main + + - name: Release android aar + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + overwrite: true + file: ./*.aar diff --git a/android/SherpaOnnxAar/.gitignore b/android/SherpaOnnxAar/.gitignore new file mode 100644 index 000000000..aa724b770 --- /dev/null +++ b/android/SherpaOnnxAar/.gitignore @@ -0,0 +1,15 @@ +*.iml +.gradle +/local.properties +/.idea/caches +/.idea/libraries +/.idea/modules.xml +/.idea/workspace.xml +/.idea/navEditor.xml +/.idea/assetWizardSettings.xml +.DS_Store +/build +/captures +.externalNativeBuild +.cxx +local.properties diff --git a/android/SherpaOnnxAar/README.md b/android/SherpaOnnxAar/README.md new file mode 100644 index 000000000..b7b73837e --- /dev/null +++ b/android/SherpaOnnxAar/README.md @@ -0,0 +1,20 @@ +# Usage of this project + +``` +git clone https://github.com/k2-fsa/sherpa-onnx +cd sherpa-onnx + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.34/sherpa-onnx-v1.10.34-android.tar.bz2 +tar xvf sherpa-onnx-v1.10.34-android.tar.bz2 + +cp -v jniLibs/arm64-v8a/* android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/arm64-v8a/ +cp -v jniLibs/armeabi-v7a/* android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/armeabi-v7a/ +cp -v jniLibs/x86/* android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/x86/ +cp -v jniLibs/x86_64/* android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/x86_64/ + +cd android/SherpaOnnxAar + +./gradlew :sherpa_onnx:assembleRelease +ls -lh ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar +cp ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar ../../sherpa-onnx-1.10.34.aar +``` diff --git a/android/SherpaOnnxAar/build.gradle.kts b/android/SherpaOnnxAar/build.gradle.kts new file mode 100644 index 000000000..e3f8a0741 --- /dev/null +++ b/android/SherpaOnnxAar/build.gradle.kts @@ -0,0 +1,6 @@ +// Top-level build file where you can add configuration options common to all sub-projects/modules. +plugins { + alias(libs.plugins.android.application) apply false + alias(libs.plugins.jetbrains.kotlin.android) apply false + alias(libs.plugins.android.library) apply false +} \ No newline at end of file diff --git a/android/SherpaOnnxAar/gradle.properties b/android/SherpaOnnxAar/gradle.properties new file mode 100644 index 000000000..20e2a0152 --- /dev/null +++ b/android/SherpaOnnxAar/gradle.properties @@ -0,0 +1,23 @@ +# Project-wide Gradle settings. +# IDE (e.g. Android Studio) users: +# Gradle settings configured through the IDE *will override* +# any settings specified in this file. +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8 +# When configured, Gradle will run in incubating parallel mode. +# This option should only be used with decoupled projects. For more details, visit +# https://developer.android.com/r/tools/gradle-multi-project-decoupled-projects +# org.gradle.parallel=true +# AndroidX package structure to make it clearer which packages are bundled with the +# Android operating system, and which are packaged with your app's APK +# https://developer.android.com/topic/libraries/support-library/androidx-rn +android.useAndroidX=true +# Kotlin code style for this project: "official" or "obsolete": +kotlin.code.style=official +# Enables namespacing of each library's R class so that its R class includes only the +# resources declared in the library itself and none from the library's dependencies, +# thereby reducing the size of the R class for that library +android.nonTransitiveRClass=true \ No newline at end of file diff --git a/android/SherpaOnnxAar/gradle/libs.versions.toml b/android/SherpaOnnxAar/gradle/libs.versions.toml new file mode 100644 index 000000000..56172d293 --- /dev/null +++ b/android/SherpaOnnxAar/gradle/libs.versions.toml @@ -0,0 +1,23 @@ +[versions] +agp = "8.4.0" +kotlin = "1.7.20" +coreKtx = "1.15.0" +junit = "4.13.2" +junitVersion = "1.2.1" +espressoCore = "3.6.1" +appcompat = "1.7.0" +material = "1.12.0" + +[libraries] +androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "coreKtx" } +junit = { group = "junit", name = "junit", version.ref = "junit" } +androidx-junit = { group = "androidx.test.ext", name = "junit", version.ref = "junitVersion" } +androidx-espresso-core = { group = "androidx.test.espresso", name = "espresso-core", version.ref = "espressoCore" } +androidx-appcompat = { group = "androidx.appcompat", name = "appcompat", version.ref = "appcompat" } +material = { group = "com.google.android.material", name = "material", version.ref = "material" } + +[plugins] +android-application = { id = "com.android.application", version.ref = "agp" } +jetbrains-kotlin-android = { id = "org.jetbrains.kotlin.android", version.ref = "kotlin" } +android-library = { id = "com.android.library", version.ref = "agp" } + diff --git a/android/SherpaOnnxAar/gradle/wrapper/gradle-wrapper.jar b/android/SherpaOnnxAar/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..e708b1c023ec8b20f512888fe07c5bd3ff77bb8f GIT binary patch literal 59203 zcma&O1CT9Y(k9%tZQHhO+qUh#ZQHhO+qmuS+qP|E@9xZO?0h@l{(r>DQ>P;GjjD{w zH}lENr;dU&FbEU?00aa80D$0M0RRB{U*7-#kbjS|qAG&4l5%47zyJ#WrfA#1$1Ctx zf&Z_d{GW=lf^w2#qRJ|CvSJUi(^E3iv~=^Z(zH}F)3Z%V3`@+rNB7gTVU{Bb~90p|f+0(v;nz01EG7yDMX9@S~__vVgv%rS$+?IH+oZ03D5zYrv|^ zC1J)SruYHmCki$jLBlTaE5&dFG9-kq3!^i>^UQL`%gn6)jz54$WDmeYdsBE9;PqZ_ zoGd=P4+|(-u4U1dbAVQrFWoNgNd;0nrghPFbQrJctO>nwDdI`Q^i0XJDUYm|T|RWc zZ3^Qgo_Qk$%Fvjj-G}1NB#ZJqIkh;kX%V{THPqOyiq)d)0+(r9o(qKlSp*hmK#iIY zA^)Vr$-Hz<#SF=0@tL@;dCQsm`V9s1vYNq}K1B)!XSK?=I1)tX+bUV52$YQu*0%fnWEukW>mxkz+%3-S!oguE8u#MGzST8_Dy^#U?fA@S#K$S@9msUiX!gd_ow>08w5)nX{-KxqMOo7d?k2&?Vf z&diGDtZr(0cwPe9z9FAUSD9KC)7(n^lMWuayCfxzy8EZsns%OEblHFSzP=cL6}?J| z0U$H!4S_TVjj<`6dy^2j`V`)mC;cB%* z8{>_%E1^FH!*{>4a7*C1v>~1*@TMcLK{7nEQ!_igZC}ikJ$*<$yHy>7)oy79A~#xE zWavoJOIOC$5b6*q*F_qN1>2#MY)AXVyr$6x4b=$x^*aqF*L?vmj>Mgv+|ITnw_BoW zO?jwHvNy^prH{9$rrik1#fhyU^MpFqF2fYEt(;4`Q&XWOGDH8k6M=%@fics4ajI;st# zCU^r1CK&|jzUhRMv;+W~6N;u<;#DI6cCw-otsc@IsN3MoSD^O`eNflIoR~l4*&-%RBYk@gb^|-JXs&~KuSEmMxB}xSb z@K76cXD=Y|=I&SNC2E+>Zg?R6E%DGCH5J1nU!A|@eX9oS(WPaMm==k2s_ueCqdZw| z&hqHp)47`c{BgwgvY2{xz%OIkY1xDwkw!<0veB#yF4ZKJyabhyyVS`gZepcFIk%e2 zTcrmt2@-8`7i-@5Nz>oQWFuMC_KlroCl(PLSodswHqJ3fn<;gxg9=}~3x_L3P`9Sn zChIf}8vCHvTriz~T2~FamRi?rh?>3bX1j}%bLH+uFX+p&+^aXbOK7clZxdU~6Uxgy z8R=obwO4dL%pmVo*Ktf=lH6hnlz_5k3cG;m8lgaPp~?eD!Yn2kf)tU6PF{kLyn|oI@eQ`F z3IF7~Blqg8-uwUuWZScRKn%c2_}dXB6Dx_&xR*n9M9LXasJhtZdr$vBY!rP{c@=)& z#!?L$2UrkvClwQO>U*fSMs67oSj2mxiJ$t;E|>q%Kh_GzzWWO&3;ufU%2z%ucBU8H z3WIwr$n)cfCXR&>tyB7BcSInK>=ByZA%;cVEJhcg<#6N{aZC4>K41XF>ZgjG`z_u& zGY?;Ad?-sgiOnI`oppF1o1Gurqbi*;#x2>+SSV6|1^G@ooVy@fg?wyf@0Y!UZ4!}nGuLeC^l)6pwkh|oRY`s1Pm$>zZ3u-83T|9 zGaKJIV3_x+u1>cRibsaJpJqhcm%?0-L;2 zitBrdRxNmb0OO2J%Y&Ym(6*`_P3&&5Bw157{o7LFguvxC$4&zTy#U=W*l&(Q2MNO} zfaUwYm{XtILD$3864IA_nn34oVa_g^FRuHL5wdUd)+W-p-iWCKe8m_cMHk+=? zeKX)M?Dt(|{r5t7IenkAXo%&EXIb-i^w+0CX0D=xApC=|Xy(`xy+QG^UyFe z+#J6h_&T5i#sV)hj3D4WN%z;2+jJcZxcI3*CHXGmOF3^)JD5j&wfX)e?-|V0GPuA+ zQFot%aEqGNJJHn$!_}#PaAvQ^{3-Ye7b}rWwrUmX53(|~i0v{}G_sI9uDch_brX&6 zWl5Ndj-AYg(W9CGfQf<6!YmY>Ey)+uYd_JNXH=>|`OH-CDCmcH(0%iD_aLlNHKH z7bcW-^5+QV$jK?R*)wZ>r9t}loM@XN&M-Pw=F#xn(;u3!(3SXXY^@=aoj70;_=QE9 zGghsG3ekq#N||u{4We_25U=y#T*S{4I{++Ku)> zQ!DZW;pVcn>b;&g2;YE#+V`v*Bl&Y-i@X6D*OpNA{G@JAXho&aOk(_j^weW{#3X5Y z%$q_wpb07EYPdmyH(1^09i$ca{O<}7) zRWncXdSPgBE%BM#by!E>tdnc$8RwUJg1*x($6$}ae$e9Knj8gvVZe#bLi!<+&BkFj zg@nOpDneyc+hU9P-;jmOSMN|*H#>^Ez#?;%C3hg_65leSUm;iz)UkW)jX#p)e&S&M z1|a?wDzV5NVnlhRBCd_;F87wp>6c<&nkgvC+!@KGiIqWY4l}=&1w7|r6{oBN8xyzh zG$b#2=RJp_iq6)#t5%yLkKx(0@D=C3w+oiXtSuaQ%I1WIb-eiE$d~!)b@|4XLy!CZ z9p=t=%3ad@Ep+<9003D2KZ5VyP~_n$=;~r&YUg5UZ0KVD&tR1DHy9x)qWtKJp#Kq# zP*8p#W(8JJ_*h_3W}FlvRam?<4Z+-H77^$Lvi+#vmhL9J zJ<1SV45xi;SrO2f=-OB(7#iNA5)x1uNC-yNxUw|!00vcW2PufRm>e~toH;M0Q85MQLWd?3O{i8H+5VkR@l9Dg-ma ze2fZ%>G(u5(k9EHj2L6!;(KZ8%8|*-1V|B#EagbF(rc+5iL_5;Eu)L4Z-V;0HfK4d z*{utLse_rvHZeQ>V5H=f78M3Ntg1BPxFCVD{HbNA6?9*^YIq;B-DJd{Ca2L#)qWP? zvX^NhFmX?CTWw&Ns}lgs;r3i+Bq@y}Ul+U%pzOS0Fcv9~aB(0!>GT0)NO?p=25LjN z2bh>6RhgqD7bQj#k-KOm@JLgMa6>%-ok1WpOe)FS^XOU{c?d5shG(lIn3GiVBxmg`u%-j=)^v&pX1JecJics3&jvPI)mDut52? z3jEA)DM%}BYbxxKrizVYwq?(P&19EXlwD9^-6J+4!}9{ywR9Gk42jjAURAF&EO|~N z)?s>$Da@ikI4|^z0e{r`J8zIs>SpM~Vn^{3fArRu;?+43>lD+^XtUcY1HidJwnR6+ z!;oG2=B6Z_=M%*{z-RaHc(n|1RTKQdNjjV!Pn9lFt^4w|AeN06*j}ZyhqZ^!-=cyGP_ShV1rGxkx8t zB;8`h!S{LD%ot``700d0@Grql(DTt4Awgmi+Yr0@#jbe=2#UkK%rv=OLqF)9D7D1j z!~McAwMYkeaL$~kI~90)5vBhBzWYc3Cj1WI0RS`z000R8-@ET0dA~*r(gSiCJmQMN&4%1D zyVNf0?}sBH8zNbBLn>~(W{d3%@kL_eQ6jEcR{l>C|JK z(R-fA!z|TTRG40|zv}7E@PqCAXP3n`;%|SCQ|ZS%ym$I{`}t3KPL&^l5`3>yah4*6 zifO#{VNz3)?ZL$be;NEaAk9b#{tV?V7 zP|wf5YA*1;s<)9A4~l3BHzG&HH`1xNr#%){4xZ!jq%o=7nN*wMuXlFV{HaiQLJ`5G zBhDi#D(m`Q1pLh@Tq+L;OwuC52RdW7b8}~60WCOK5iYMUad9}7aWBuILb({5=z~YF zt?*Jr5NG+WadM{mDL>GyiByCuR)hd zA=HM?J6l1Xv0Dl+LW@w$OTcEoOda^nFCw*Sy^I@$sSuneMl{4ys)|RY#9&NxW4S)9 zq|%83IpslTLoz~&vTo!Ga@?rj_kw{|k{nv+w&Ku?fyk4Ki4I?);M|5Axm)t+BaE)D zm(`AQ#k^DWrjbuXoJf2{Aj^KT zFb1zMSqxq|vceV+Mf-)$oPflsO$@*A0n0Z!R{&(xh8s}=;t(lIy zv$S8x>m;vQNHuRzoaOo?eiWFe{0;$s`Bc+Osz~}Van${u;g(su`3lJ^TEfo~nERfP z)?aFzpDgnLYiERsKPu|0tq4l2wT)Atr6Qb%m-AUn6HnCue*yWICp7TjW$@sO zm5rm4aTcPQ(rfi7a`xP7cKCFrJD}*&_~xgLyr^-bmsL}y;A5P|al8J3WUoBSjqu%v zxC;mK!g(7r6RRJ852Z~feoC&sD3(6}^5-uLK8o)9{8L_%%rItZK9C){UxB|;G>JbP zsRRtS4-3B*5c+K2kvmgZK8472%l>3cntWUOVHxB|{Ay~aOg5RN;{PJgeVD*H%ac+y!h#wi%o2bF2Ca8IyMyH{>4#{E_8u^@+l-+n=V}Sq?$O z{091@v%Bd*3pk0^2UtiF9Z+(a@wy6 zUdw8J*ze$K#=$48IBi1U%;hmhO>lu!uU;+RS}p&6@rQila7WftH->*A4=5W|Fmtze z)7E}jh@cbmr9iup^i%*(uF%LG&!+Fyl@LFA-}Ca#bxRfDJAiR2dt6644TaYw1Ma79 zt8&DYj31j^5WPNf5P&{)J?WlCe@<3u^78wnd(Ja4^a>{^Tw}W>|Cjt^If|7l^l)^Q zbz|7~CF(k_9~n|h;ysZ+jHzkXf(*O*@5m zLzUmbHp=x!Q|!9NVXyipZ3)^GuIG$k;D)EK!a5=8MFLI_lpf`HPKl=-Ww%z8H_0$j ztJ||IfFG1lE9nmQ0+jPQy zCBdKkjArH@K7jVcMNz);Q(Q^R{d5G?-kk;Uu_IXSyWB)~KGIizZL(^&qF;|1PI7!E zTP`%l)gpX|OFn&)M%txpQ2F!hdA~hX1Cm5)IrdljqzRg!f{mN%G~H1&oqe`5eJCIF zHdD7O;AX-{XEV(a`gBFJ9ews#CVS2y!&>Cm_dm3C8*n3MA*e67(WC?uP@8TXuMroq z{#w$%z@CBIkRM7?}Xib+>hRjy?%G!fiw8! z8(gB+8J~KOU}yO7UGm&1g_MDJ$IXS!`+*b*QW2x)9>K~Y*E&bYMnjl6h!{17_8d!%&9D`a7r&LKZjC<&XOvTRaKJ1 zUY@hl5^R&kZl3lU3njk`3dPzxj$2foOL26r(9zsVF3n_F#v)s5vv3@dgs|lP#eylq62{<-vczqP!RpVBTgI>@O6&sU>W|do17+#OzQ7o5A$ICH z?GqwqnK^n2%LR;$^oZM;)+>$X3s2n}2jZ7CdWIW0lnGK-b#EG01)P@aU`pg}th&J-TrU`tIpb5t((0eu|!u zQz+3ZiOQ^?RxxK4;zs=l8q!-n7X{@jSwK(iqNFiRColuEOg}!7cyZi`iBX4g1pNBj zAPzL?P^Ljhn;1$r8?bc=#n|Ed7wB&oHcw()&*k#SS#h}jO?ZB246EGItsz*;^&tzp zu^YJ0=lwsi`eP_pU8}6JA7MS;9pfD;DsSsLo~ogzMNP70@@;Fm8f0^;>$Z>~}GWRw!W5J3tNX*^2+1f3hz{~rIzJo z6W%J(H!g-eI_J1>0juX$X4Cl6i+3wbc~k146UIX&G22}WE>0ga#WLsn9tY(&29zBvH1$`iWtTe zG2jYl@P!P)eb<5DsR72BdI7-zP&cZNI{7q3e@?N8IKc4DE#UVr->|-ryuJXk^u^>4 z$3wE~=q390;XuOQP~TNoDR?#|NSPJ%sTMInA6*rJ%go|=YjGe!B>z6u$IhgQSwoV* zjy3F2#I>uK{42{&IqP59)Y(1*Z>>#W8rCf4_eVsH)`v!P#^;BgzKDR`ARGEZzkNX+ zJUQu=*-ol=Xqqt5=`=pA@BIn@6a9G8C{c&`i^(i+BxQO9?YZ3iu%$$da&Kb?2kCCo zo7t$UpSFWqmydXf@l3bVJ=%K?SSw)|?srhJ-1ZdFu*5QhL$~-IQS!K1s@XzAtv6*Y zl8@(5BlWYLt1yAWy?rMD&bwze8bC3-GfNH=p zynNFCdxyX?K&G(ZZ)afguQ2|r;XoV^=^(;Cku#qYn4Lus`UeKt6rAlFo_rU`|Rq z&G?~iWMBio<78of-2X(ZYHx~=U0Vz4btyXkctMKdc9UM!vYr~B-(>)(Hc|D zMzkN4!PBg%tZoh+=Gba!0++d193gbMk2&krfDgcbx0jI92cq?FFESVg0D$>F+bil} zY~$)|>1HZsX=5sAZ2WgPB5P=8X#TI+NQ(M~GqyVB53c6IdX=k>Wu@A0Svf5#?uHaF zsYn|koIi3$(%GZ2+G+7Fv^lHTb#5b8sAHSTnL^qWZLM<(1|9|QFw9pnRU{svj}_Al zL)b9>fN{QiA($8peNEJyy`(a{&uh-T4_kdZFIVsKKVM(?05}76EEz?#W za^fiZOAd14IJ4zLX-n7Lq0qlQ^lW8Cvz4UKkV9~P}>sq0?xD3vg+$4vLm~C(+ zM{-3Z#qnZ09bJ>}j?6ry^h+@PfaD7*jZxBEY4)UG&daWb??6)TP+|3#Z&?GL?1i+280CFsE|vIXQbm| zM}Pk!U`U5NsNbyKzkrul-DzwB{X?n3E6?TUHr{M&+R*2%yOiXdW-_2Yd6?38M9Vy^ z*lE%gA{wwoSR~vN0=no}tP2Ul5Gk5M(Xq`$nw#ndFk`tcpd5A=Idue`XZ!FS>Q zG^0w#>P4pPG+*NC9gLP4x2m=cKP}YuS!l^?sHSFftZy{4CoQrb_ z^20(NnG`wAhMI=eq)SsIE~&Gp9Ne0nD4%Xiu|0Fj1UFk?6avDqjdXz{O1nKao*46y zT8~iA%Exu=G#{x=KD;_C&M+Zx4+n`sHT>^>=-1YM;H<72k>$py1?F3#T1*ef9mLZw z5naLQr?n7K;2l+{_uIw*_1nsTn~I|kkCgrn;|G~##hM;9l7Jy$yJfmk+&}W@JeKcF zx@@Woiz8qdi|D%aH3XTx5*wDlbs?dC1_nrFpm^QbG@wM=i2?Zg;$VK!c^Dp8<}BTI zyRhAq@#%2pGV49*Y5_mV4+OICP|%I(dQ7x=6Ob}>EjnB_-_18*xrY?b%-yEDT(wrO z9RY2QT0`_OpGfMObKHV;QLVnrK%mc?$WAdIT`kJQT^n%GuzE7|9@k3ci5fYOh(287 zuIbg!GB3xLg$YN=n)^pHGB0jH+_iIiC=nUcD;G6LuJsjn2VI1cyZx=a?ShCsF==QK z;q~*m&}L<-cb+mDDXzvvrRsybcgQ;Vg21P(uLv5I+eGc7o7tc6`;OA9{soHFOz zT~2?>Ts}gprIX$wRBb4yE>ot<8+*Bv`qbSDv*VtRi|cyWS>)Fjs>fkNOH-+PX&4(~ z&)T8Zam2L6puQl?;5zg9h<}k4#|yH9czHw;1jw-pwBM*O2hUR6yvHATrI%^mvs9q_ z&ccT0>f#eDG<^WG^q@oVqlJrhxH)dcq2cty@l3~|5#UDdExyXUmLQ}f4#;6fI{f^t zDCsgIJ~0`af%YR%Ma5VQq-p21k`vaBu6WE?66+5=XUd%Ay%D$irN>5LhluRWt7 zov-=f>QbMk*G##&DTQyou$s7UqjjW@k6=!I@!k+S{pP8R(2=e@io;N8E`EOB;OGoI zw6Q+{X1_I{OO0HPpBz!X!@`5YQ2)t{+!?M_iH25X(d~-Zx~cXnS9z>u?+If|iNJbx zyFU2d1!ITX64D|lE0Z{dLRqL1Ajj=CCMfC4lD3&mYR_R_VZ>_7_~|<^o*%_&jevU+ zQ4|qzci=0}Jydw|LXLCrOl1_P6Xf@c0$ieK2^7@A9UbF{@V_0p%lqW|L?5k>bVM8|p5v&2g;~r>B8uo<4N+`B zH{J)h;SYiIVx@#jI&p-v3dwL5QNV1oxPr8J%ooezTnLW>i*3Isb49%5i!&ac_dEXv zvXmVUck^QHmyrF8>CGXijC_R-y(Qr{3Zt~EmW)-nC!tiH`wlw5D*W7Pip;T?&j%kX z6DkZX4&}iw>hE(boLyjOoupf6JpvBG8}jIh!!VhnD0>}KSMMo{1#uU6kiFcA04~|7 zVO8eI&x1`g4CZ<2cYUI(n#wz2MtVFHx47yE5eL~8bot~>EHbevSt}LLMQX?odD{Ux zJMnam{d)W4da{l7&y-JrgiU~qY3$~}_F#G7|MxT)e;G{U`In&?`j<5D->}cb{}{T(4DF0BOk-=1195KB-E*o@c?`>y#4=dMtYtSY=&L{!TAjFVcq0y@AH`vH! z$41+u!Ld&}F^COPgL(EE{0X7LY&%D7-(?!kjFF7=qw<;`V{nwWBq<)1QiGJgUc^Vz ztMUlq1bZqKn17|6x6iAHbWc~l1HcmAxr%$Puv!znW)!JiukwIrqQ00|H$Z)OmGG@= zv%A8*4cq}(?qn4rN6o`$Y))(MyXr8R<2S^J+v(wmFmtac!%VOfN?&(8Nr!T@kV`N; z*Q33V3t`^rN&aBiHet)18wy{*wi1=W!B%B-Q6}SCrUl$~Hl{@!95ydml@FK8P=u4s z4e*7gV2s=YxEvskw2Ju!2%{8h01rx-3`NCPc(O zH&J0VH5etNB2KY6k4R@2Wvl^Ck$MoR3=)|SEclT2ccJ!RI9Nuter7u9@;sWf-%um;GfI!=eEIQ2l2p_YWUd{|6EG ze{yO6;lMc>;2tPrsNdi@&1K6(1;|$xe8vLgiouj%QD%gYk`4p{Ktv9|j+!OF-P?@p z;}SV|oIK)iwlBs+`ROXkhd&NK zzo__r!B>tOXpBJMDcv!Mq54P+n4(@dijL^EpO1wdg~q+!DT3lB<>9AANSe!T1XgC=J^)IP0XEZ()_vpu!!3HQyJhwh?r`Ae%Yr~b% zO*NY9t9#qWa@GCPYOF9aron7thfWT`eujS4`t2uG6)~JRTI;f(ZuoRQwjZjp5Pg34 z)rp$)Kr?R+KdJ;IO;pM{$6|2y=k_siqvp%)2||cHTe|b5Ht8&A{wazGNca zX$Ol?H)E_R@SDi~4{d-|8nGFhZPW;Cts1;08TwUvLLv&_2$O6Vt=M)X;g%HUr$&06 zISZb(6)Q3%?;3r~*3~USIg=HcJhFtHhIV(siOwV&QkQe#J%H9&E21!C*d@ln3E@J* zVqRO^<)V^ky-R|%{(9`l-(JXq9J)1r$`uQ8a}$vr9E^nNiI*thK8=&UZ0dsFN_eSl z(q~lnD?EymWLsNa3|1{CRPW60>DSkY9YQ;$4o3W7Ms&@&lv9eH!tk~N&dhqX&>K@} zi1g~GqglxkZ5pEFkllJ)Ta1I^c&Bt6#r(QLQ02yHTaJB~- zCcE=5tmi`UA>@P=1LBfBiqk)HB4t8D?02;9eXj~kVPwv?m{5&!&TFYhu>3=_ zsGmYZ^mo*-j69-42y&Jj0cBLLEulNRZ9vXE)8~mt9C#;tZs;=#M=1*hebkS;7(aGf zcs7zH(I8Eui9UU4L--))yy`&d&$In&VA2?DAEss4LAPCLd>-$i?lpXvn!gu^JJ$(DoUlc6wE98VLZ*z`QGQov5l4Fm_h?V-;mHLYDVOwKz7>e4+%AzeO>P6v}ndPW| zM>m#6Tnp7K?0mbK=>gV}=@k*0Mr_PVAgGMu$j+pWxzq4MAa&jpCDU&-5eH27Iz>m^ zax1?*HhG%pJ((tkR(V(O(L%7v7L%!_X->IjS3H5kuXQT2!ow(;%FDE>16&3r){!ex zhf==oJ!}YU89C9@mfDq!P3S4yx$aGB?rbtVH?sHpg?J5C->!_FHM%Hl3#D4eplxzQ zRA+<@LD%LKSkTk2NyWCg7u=$%F#;SIL44~S_OGR}JqX}X+=bc@swpiClB`Zbz|f!4 z7Ysah7OkR8liXfI`}IIwtEoL}(URrGe;IM8%{>b1SsqXh)~w}P>yiFRaE>}rEnNkT z!HXZUtxUp1NmFm)Dm@-{FI^aRQqpSkz}ZSyKR%Y}YHNzBk)ZIp} zMtS=aMvkgWKm9&oTcU0?S|L~CDqA+sHpOxwnswF-fEG)cXCzUR?ps@tZa$=O)=L+5 zf%m58cq8g_o}3?Bhh+c!w4(7AjxwQ3>WnVi<{{38g7yFboo>q|+7qs<$8CPXUFAN< zG&}BHbbyQ5n|qqSr?U~GY{@GJ{(Jny{bMaOG{|IkUj7tj^9pa9|FB_<+KHLxSxR;@ zHpS$4V)PP+tx}22fWx(Ku9y+}Ap;VZqD0AZW4gCDTPCG=zgJmF{|x;(rvdM|2|9a}cex6xrMkERnkE;}jvU-kmzd%_J50$M`lIPCKf+^*zL=@LW`1SaEc%=m zQ+lT06Gw+wVwvQ9fZ~#qd430v2HndFsBa9WjD0P}K(rZYdAt^5WQIvb%D^Q|pkVE^ zte$&#~zmULFACGfS#g=2OLOnIf2Of-k!(BIHjs77nr!5Q1*I9 z1%?=~#Oss!rV~?-6Gm~BWJiA4mJ5TY&iPm_$)H1_rTltuU1F3I(qTQ^U$S>%$l z)Wx1}R?ij0idp@8w-p!Oz{&*W;v*IA;JFHA9%nUvVDy7Q8woheC#|8QuDZb-L_5@R zOqHwrh|mVL9b=+$nJxM`3eE{O$sCt$UK^2@L$R(r^-_+z?lOo+me-VW=Zw z-Bn>$4ovfWd%SPY`ab-u9{INc*k2h+yH%toDHIyqQ zO68=u`N}RIIs7lsn1D){)~%>ByF<>i@qFb<-axvu(Z+6t7v<^z&gm9McRB~BIaDn$ z#xSGT!rzgad8o>~kyj#h1?7g96tOcCJniQ+*#=b7wPio>|6a1Z?_(TS{)KrPe}(8j z!#&A=k(&Pj^F;r)CI=Z{LVu>uj!_W1q4b`N1}E(i%;BWjbEcnD=mv$FL$l?zS6bW!{$7j1GR5ocn94P2u{ z70tAAcpqtQo<@cXw~@i-@6B23;317|l~S>CB?hR5qJ%J3EFgyBdJd^fHZu7AzHF(BQ!tyAz^L0`X z23S4Fe{2X$W0$zu9gm%rg~A>ijaE#GlYlrF9$ds^QtaszE#4M(OLVP2O-;XdT(XIC zatwzF*)1c+t~c{L=fMG8Z=k5lv>U0;C{caN1NItnuSMp)6G3mbahu>E#sj&oy94KC zpH}8oEw{G@N3pvHhp{^-YaZeH;K+T_1AUv;IKD<=mv^&Ueegrb!yf`4VlRl$M?wsl zZyFol(2|_QM`e_2lYSABpKR{{NlxlDSYQNkS;J66aT#MSiTx~;tUmvs-b*CrR4w=f z8+0;*th6kfZ3|5!Icx3RV11sp=?`0Jy3Fs0N4GZQMN=8HmT6%x9@{Dza)k}UwL6JT zHRDh;%!XwXr6yuuy`4;Xsn0zlR$k%r%9abS1;_v?`HX_hI|+EibVnlyE@3aL5vhQq zlIG?tN^w@0(v9M*&L+{_+RQZw=o|&BRPGB>e5=ys7H`nc8nx)|-g;s7mRc7hg{GJC zAe^vCIJhajmm7C6g! zL&!WAQ~5d_5)00?w_*|*H>3$loHrvFbitw#WvLB!JASO?#5Ig5$Ys10n>e4|3d;tS zELJ0|R4n3Az(Fl3-r^QiV_C;)lQ1_CW{5bKS15U|E9?ZgLec@%kXr84>5jV2a5v=w z?pB1GPdxD$IQL4)G||B_lI+A=08MUFFR4MxfGOu07vfIm+j=z9tp~5i_6jb`tR>qV z$#`=BQ*jpCjm$F0+F)L%xRlnS%#&gro6PiRfu^l!EVan|r3y}AHJQOORGx4~ z&<)3=K-tx518DZyp%|!EqpU!+X3Et7n2AaC5(AtrkW>_57i}$eqs$rupubg0a1+WO zGHZKLN2L0D;ab%{_S1Plm|hx8R?O14*w*f&2&bB050n!R2by zw!@XOQx$SqZ5I<(Qu$V6g>o#A!JVwErWv#(Pjx=KeS0@hxr4?13zj#oWwPS(7Ro|v z>Mp@Kmxo79q|}!5qtX2-O@U&&@6s~!I&)1WQIl?lTnh6UdKT_1R640S4~f=_xoN3- zI+O)$R@RjV$F=>Ti7BlnG1-cFKCC(t|Qjm{SalS~V-tX#+2ekRhwmN zZr`8{QF6y~Z!D|{=1*2D-JUa<(1Z=;!Ei!KiRNH?o{p5o3crFF=_pX9O-YyJchr$~ zRC`+G+8kx~fD2k*ZIiiIGR<8r&M@3H?%JVOfE>)})7ScOd&?OjgAGT@WVNSCZ8N(p zuQG~76GE3%(%h1*vUXg$vH{ua0b`sQ4f0*y=u~lgyb^!#CcPJa2mkSEHGLsnO^kb$ zru5_l#nu=Y{rSMWiYx?nO{8I!gH+?wEj~UM?IrG}E|bRIBUM>UlY<`T1EHpRr36vv zBi&dG8oxS|J$!zoaq{+JpJy+O^W(nt*|#g32bd&K^w-t>!Vu9N!k9eA8r!Xc{utY> zg9aZ(D2E0gL#W0MdjwES-7~Wa8iubPrd?8-$C4BP?*wok&O8+ykOx{P=Izx+G~hM8 z*9?BYz!T8~dzcZr#ux8kS7u7r@A#DogBH8km8Ry4slyie^n|GrTbO|cLhpqgMdsjX zJ_LdmM#I&4LqqsOUIXK8gW;V0B(7^$y#h3h>J0k^WJfAMeYek%Y-Dcb_+0zPJez!GM zAmJ1u;*rK=FNM0Nf}Y!!P9c4)HIkMnq^b;JFd!S3?_Qi2G#LIQ)TF|iHl~WKK6JmK zbv7rPE6VkYr_%_BT}CK8h=?%pk@3cz(UrZ{@h40%XgThP*-Oeo`T0eq9 zA8BnWZKzCy5e&&_GEsU4*;_k}(8l_&al5K-V*BFM=O~;MgRkYsOs%9eOY6s6AtE*<7GQAR2ulC3RAJrG_P1iQK5Z~&B z&f8X<>yJV6)oDGIlS$Y*D^Rj(cszTy5c81a5IwBr`BtnC6_e`ArI8CaTX_%rx7;cn zR-0?J_LFg*?(#n~G8cXut(1nVF0Oka$A$1FGcERU<^ggx;p@CZc?3UB41RY+wLS`LWFNSs~YP zuw1@DNN3lTd|jDL7gjBsd9}wIw}4xT2+8dBQzI00m<@?c2L%>}QLfK5%r!a-iII`p zX@`VEUH)uj^$;7jVUYdADQ2k*!1O3WdfgF?OMtUXNpQ1}QINamBTKDuv19^{$`8A1 zeq%q*O0mi@(%sZU>Xdb0Ru96CFqk9-L3pzLVsMQ`Xpa~N6CR{9Rm2)A|CI21L(%GW zh&)Y$BNHa=FD+=mBw3{qTgw)j0b!Eahs!rZnpu)z!!E$*eXE~##yaXz`KE5(nQM`s zD!$vW9XH)iMxu9R>r$VlLk9oIR%HxpUiW=BK@4U)|1WNQ=mz9a z^!KkO=>GaJ!GBXm{KJj^;kh-MkUlEQ%lza`-G&}C5y1>La1sR6hT=d*NeCnuK%_LV zOXt$}iP6(YJKc9j-Fxq~*ItVUqljQ8?oaysB-EYtFQp9oxZ|5m0^Hq(qV!S+hq#g( z?|i*H2MIr^Kxgz+3vIljQ*Feejy6S4v~jKEPTF~Qhq!(ms5>NGtRgO5vfPPc4Z^AM zTj!`5xEreIN)vaNxa|q6qWdg>+T`Ol0Uz)ckXBXEGvPNEL3R8hB3=C5`@=SYgAju1 z!)UBr{2~=~xa{b8>x2@C7weRAEuatC)3pkRhT#pMPTpSbA|tan%U7NGMvzmF?c!V8 z=pEWxbdXbTAGtWTyI?Fml%lEr-^AE}w#l(<7OIw;ctw}imYax&vR4UYNJZK6P7ZOd zP87XfhnUHxCUHhM@b*NbTi#(-8|wcv%3BGNs#zRCVV(W?1Qj6^PPQa<{yaBwZ`+<`w|;rqUY_C z&AeyKwwf*q#OW-F()lir=T^<^wjK65Lif$puuU5+tk$;e_EJ;Lu+pH>=-8=PDhkBg z8cWt%@$Sc#C6F$Vd+0507;{OOyT7Hs%nKS88q-W!$f~9*WGBpHGgNp}=C*7!RiZ5s zn1L_DbKF@B8kwhDiLKRB@lsXVVLK|ph=w%_`#owlf@s@V(pa`GY$8h%;-#h@TsO|Y8V=n@*!Rog7<7Cid%apR|x zOjhHCyfbIt%+*PCveTEcuiDi%Wx;O;+K=W?OFUV%)%~6;gl?<0%)?snDDqIvkHF{ zyI02)+lI9ov42^hL>ZRrh*HhjF9B$A@=H94iaBESBF=eC_KT$8A@uB^6$~o?3Wm5t1OIaqF^~><2?4e3c&)@wKn9bD? zoeCs;H>b8DL^F&>Xw-xjZEUFFTv>JD^O#1E#)CMBaG4DX9bD(Wtc8Rzq}9soQ8`jf zeSnHOL}<+WVSKp4kkq&?SbETjq6yr@4%SAqOG=9E(3YeLG9dtV+8vmzq+6PFPk{L; z(&d++iu=^F%b+ea$i2UeTC{R*0Isk;vFK!no<;L+(`y`3&H-~VTdKROkdyowo1iqR zbVW(3`+(PQ2>TKY>N!jGmGo7oeoB8O|P_!Ic@ zZ^;3dnuXo;WJ?S+)%P>{Hcg!Jz#2SI(s&dY4QAy_vRlmOh)QHvs_7c&zkJCmJGVvV zX;Mtb>QE+xp`KyciG$Cn*0?AK%-a|=o!+7x&&yzHQOS>8=B*R=niSnta^Pxp1`=md z#;$pS$4WCT?mbiCYU?FcHGZ#)kHVJTTBt^%XE(Q};aaO=Zik0UgLcc0I(tUpt(>|& zcxB_|fxCF7>&~5eJ=Dpn&5Aj{A^cV^^}(7w#p;HG&Q)EaN~~EqrE1qKrMAc&WXIE;>@<&)5;gD2?={Xf@Mvn@OJKw=8Mgn z!JUFMwD+s==JpjhroT&d{$kQAy%+d`a*XxDEVxy3`NHzmITrE`o!;5ClXNPb4t*8P zzAivdr{j_v!=9!^?T3y?gzmqDWX6mkzhIzJ-3S{T5bcCFMr&RPDryMcdwbBuZbsgN zGrp@^i?rcfN7v0NKGzDPGE#4yszxu=I_`MI%Z|10nFjU-UjQXXA?k8Pk|OE<(?ae) zE%vG#eZAlj*E7_3dx#Zz4kMLj>H^;}33UAankJiDy5ZvEhrjr`!9eMD8COp}U*hP+ zF}KIYx@pkccIgyxFm#LNw~G&`;o&5)2`5aogs`1~7cMZQ7zj!%L4E`2yzlQN6REX20&O<9 zKV6fyr)TScJPPzNTC2gL+0x#=u>(({{D7j)c-%tvqls3#Y?Z1m zV5WUE)zdJ{$p>yX;^P!UcXP?UD~YM;IRa#Rs5~l+*$&nO(;Ers`G=0D!twR(0GF@c zHl9E5DQI}Oz74n zfKP>&$q0($T4y$6w(p=ERAFh+>n%iaeRA%!T%<^+pg?M)@ucY<&59$x9M#n+V&>}=nO9wCV{O~lg&v#+jcUj(tQ z`0u1YH)-`U$15a{pBkGyPL0THv1P|4e@pf@3IBZS4dVJPo#H>pWq%Lr0YS-SeWash z8R7=jb28KPMI|_lo#GEO|5B?N_e``H*23{~a!AmUJ+fb4HX-%QI@lSEUxKlGV7z7Q zSKw@-TR>@1RL%w{x}dW#k1NgW+q4yt2Xf1J62Bx*O^WG8OJ|FqI4&@d3_o8Id@*)4 zYrk=>@!wv~mh7YWv*bZhxqSmFh2Xq)o=m;%n$I?GSz49l1$xRpPu_^N(vZ>*>Z<04 z2+rP70oM=NDysd!@fQdM2OcyT?3T^Eb@lIC-UG=Bw{BjQ&P`KCv$AcJ;?`vdZ4){d z&gkoUK{$!$$K`3*O-jyM1~p-7T*qb)Ys>Myt^;#1&a%O@x8A+E>! zY8=eD`ZG)LVagDLBeHg>=atOG?Kr%h4B%E6m@J^C+U|y)XX@f z8oyJDW|9g=<#f<{JRr{y#~euMnv)`7j=%cHWLc}ngjq~7k**6%4u>Px&W%4D94(r* z+akunK}O0DC2A%Xo9jyF;DobX?!1I(7%}@7F>i%&nk*LMO)bMGg2N+1iqtg+r(70q zF5{Msgsm5GS7DT`kBsjMvOrkx&|EU!{{~gL4d2MWrAT=KBQ-^zQCUq{5PD1orxlIL zq;CvlWx#f1NWvh`hg011I%?T_s!e38l*lWVt|~z-PO4~~1g)SrJ|>*tXh=QfXT)%( z+ex+inPvD&O4Ur;JGz>$sUOnWdpSLcm1X%aQDw4{dB!cnj`^muI$CJ2%p&-kULVCE z>$eMR36kN$wCPR+OFDM3-U(VOrp9k3)lI&YVFqd;Kpz~K)@Fa&FRw}L(SoD z9B4a+hQzZT-BnVltst&=kq6Y(f^S4hIGNKYBgMxGJ^;2yrO}P3;r)(-I-CZ)26Y6? z&rzHI_1GCvGkgy-t1E;r^3Le30|%$ebDRu2+gdLG)r=A~Qz`}~&L@aGJ{}vVs_GE* zVUjFnzHiXfKQbpv&bR&}l2bzIjAooB)=-XNcYmrGmBh(&iu@o!^hn0^#}m2yZZUK8 zufVm7Gq0y`Mj;9b>`c?&PZkU0j4>IL=UL&-Lp3j&47B5pAW4JceG{!XCA)kT<%2nqCxj<)uy6XR_uws~>_MEKPOpAQ!H zkn>FKh)<9DwwS*|Y(q?$^N!6(51O0 z^JM~Ax{AI1Oj$fs-S5d4T7Z_i1?{%0SsIuQ&r8#(JA=2iLcTN+?>wOL532%&dMYkT z*T5xepC+V6zxhS@vNbMoi|i)=rpli@R9~P!39tWbSSb904ekv7D#quKbgFEMTb48P zuq(VJ+&L8aWU(_FCD$3^uD!YM%O^K(dvy~Wm2hUuh6bD|#(I39Xt>N1Y{ZqXL`Fg6 zKQ?T2htHN!(Bx;tV2bfTtIj7e)liN-29s1kew>v(D^@)#v;}C4-G=7x#;-dM4yRWm zyY`cS21ulzMK{PoaQ6xChEZ}o_#}X-o}<&0)$1#3we?+QeLt;aVCjeA)hn!}UaKt< zat1fHEx13y-rXNMvpUUmCVzocPmN~-Y4(YJvQ#db)4|%B!rBsgAe+*yor~}FrNH08 z3V!97S}D7d$zbSD{$z;@IYMxM6aHdypIuS*pr_U6;#Y!_?0i|&yU*@16l z*dcMqDQgfNBf}?quiu4e>H)yTVfsp#f+Du0@=Kc41QockXkCkvu>FBd6Q+@FL!(Yx z2`YuX#eMEiLEDhp+9uFqME_E^faV&~9qjBHJkIp~%$x^bN=N)K@kvSVEMdDuzA0sn z88CBG?`RX1@#hQNd`o^V{37)!w|nA)QfiYBE^m=yQKv-fQF+UCMcuEe1d4BH7$?>b zJl-r9@0^Ie=)guO1vOd=i$_4sz>y3x^R7n4ED!5oXL3@5**h(xr%Hv)_gILarO46q+MaDOF%ChaymKoI6JU5Pg;7#2n9-18|S1;AK+ zgsn6;k6-%!QD>D?cFy}8F;r@z8H9xN1jsOBw2vQONVqBVEbkiNUqgw~*!^##ht>w0 zUOykwH=$LwX2j&nLy=@{hr)2O&-wm-NyjW7n~Zs9UlH;P7iP3 zI}S(r0YFVYacnKH(+{*)Tbw)@;6>%=&Th=+Z6NHo_tR|JCI8TJiXv2N7ei7M^Q+RM z?9o`meH$5Yi;@9XaNR#jIK^&{N|DYNNbtdb)XW1Lv2k{E>;?F`#Pq|&_;gm~&~Zc9 zf+6ZE%{x4|{YdtE?a^gKyzr}dA>OxQv+pq|@IXL%WS0CiX!V zm$fCePA%lU{%pTKD7|5NJHeXg=I0jL@$tOF@K*MI$)f?om)D63K*M|r`gb9edD1~Y zc|w7N)Y%do7=0{RC|AziW7#am$)9jciRJ?IWl9PE{G3U+$%FcyKs_0Cgq`=K3@ttV z9g;M!3z~f_?P%y3-ph%vBMeS@p7P&Ea8M@97+%XEj*(1E6vHj==d zjsoviB>j^$_^OI_DEPvFkVo(BGRo%cJeD){6Uckei=~1}>sp299|IRjhXe)%?uP0I zF5+>?0#Ye}T^Y$u_rc4=lPcq4K^D(TZG-w30-YiEM=dcK+4#o*>lJ8&JLi+3UcpZk z!^?95S^C0ja^jwP`|{<+3cBVog$(mRdQmadS+Vh~z zS@|P}=|z3P6uS+&@QsMp0no9Od&27O&14zHXGAOEy zh~OKpymK5C%;LLb467@KgIiVwYbYd6wFxI{0-~MOGfTq$nBTB!{SrWmL9Hs}C&l&l#m?s*{tA?BHS4mVKHAVMqm63H<|c5n0~k)-kbg zXidai&9ZUy0~WFYYKT;oe~rytRk?)r8bptITsWj(@HLI;@=v5|XUnSls7$uaxFRL+ zRVMGuL3w}NbV1`^=Pw*0?>bm8+xfeY(1PikW*PB>>Tq(FR`91N0c2&>lL2sZo5=VD zQY{>7dh_TX98L2)n{2OV=T10~*YzX27i2Q7W86M4$?gZIXZaBq#sA*{PH8){|GUi;oM>e?ua7eF4WFuFYZSG| zze?srg|5Ti8Og{O zeFxuw9!U+zhyk?@w zjsA6(oKD=Ka;A>Ca)oPORxK+kxH#O@zhC!!XS4@=swnuMk>t+JmLmFiE^1aX3f<)D@`%K0FGK^gg1a1j>zi z2KhV>sjU7AX3F$SEqrXSC}fRx64GDoc%!u2Yag68Lw@w9v;xOONf@o)Lc|Uh3<21ctTYu-mFZuHk*+R{GjXHIGq3p)tFtQp%TYqD=j1&y)>@zxoxUJ!G@ zgI0XKmP6MNzw>nRxK$-Gbzs}dyfFzt>#5;f6oR27ql!%+{tr+(`(>%51|k`ML} zY4eE)Lxq|JMas(;JibNQds1bUB&r}ydMQXBY4x(^&fY_&LlQC)3hylc$~8&~|06-D z#T+%66rYbHX%^KuqJED_wuGB+=h`nWA!>1n0)3wZrBG3%`b^Ozv6__dNa@%V14|!D zQ?o$z5u0^8`giv%qE!BzZ!3j;BlDlJDk)h@9{nSQeEk!z9RGW) z${RSF3phEM*ce*>Xdp}585vj$|40=&S{S-GTiE?Op*vY&Lvr9}BO$XWy80IF+6@%n z5*2ueT_g@ofP#u5pxb7n*fv^Xtt7&?SRc{*2Ka-*!BuOpf}neHGCiHy$@Ka1^Dint z;DkmIL$-e)rj4o2WQV%Gy;Xg(_Bh#qeOsTM2f@KEe~4kJ8kNLQ+;(!j^bgJMcNhvklP5Z6I+9Fq@c&D~8Fb-4rmDT!MB5QC{Dsb;BharP*O;SF4& zc$wj-7Oep7#$WZN!1nznc@Vb<_Dn%ga-O#J(l=OGB`dy=Sy&$(5-n3zzu%d7E#^8`T@}V+5B;PP8J14#4cCPw-SQTdGa2gWL0*zKM z#DfSXs_iWOMt)0*+Y>Lkd=LlyoHjublNLefhKBv@JoC>P7N1_#> zv=mLWe96%EY;!ZGSQDbZWb#;tzqAGgx~uk+-$+2_8U`!ypbwXl z^2E-FkM1?lY@yt8=J3%QK+xaZ6ok=-y%=KXCD^0r!5vUneW>95PzCkOPO*t}p$;-> ze5j-BLT_;)cZQzR2CEsm@rU7GZfFtdp*a|g4wDr%8?2QkIGasRfDWT-Dvy*U{?IHT z*}wGnzdlSptl#ZF^sf)KT|BJs&kLG91^A6ls{CzFprZ6-Y!V0Xysh%9p%iMd7HLsS zN+^Un$tDV)T@i!v?3o0Fsx2qI(AX_$dDkBzQ@fRM%n zRXk6hb9Py#JXUs+7)w@eo;g%QQ95Yq!K_d=z{0dGS+pToEI6=Bo8+{k$7&Z zo4>PH(`ce8E-Ps&uv`NQ;U$%t;w~|@E3WVOCi~R4oj5wP?%<*1C%}Jq%a^q~T7u>K zML5AKfQDv6>PuT`{SrKHRAF+^&edg6+5R_#H?Lz3iGoWo#PCEd0DS;)2U({{X#zU^ zw_xv{4x7|t!S)>44J;KfA|DC?;uQ($l+5Vp7oeqf7{GBF9356nx|&B~gs+@N^gSdd zvb*>&W)|u#F{Z_b`f#GVtQ`pYv3#||N{xj1NgB<#=Odt6{eB%#9RLt5v zIi|0u70`#ai}9fJjKv7dE!9ZrOIX!3{$z_K5FBd-Kp-&e4(J$LD-)NMTp^_pB`RT; zftVVlK2g@+1Ahv2$D){@Y#cL#dUj9*&%#6 zd2m9{1NYp>)6=oAvqdCn5#cx{AJ%S8skUgMglu2*IAtd+z1>B&`MuEAS(D(<6X#Lj z?f4CFx$)M&$=7*>9v1ER4b6!SIz-m0e{o0BfkySREchp?WdVPpQCh!q$t>?rL!&Jg zd#heM;&~A}VEm8Dvy&P|J*eAV&w!&Nx6HFV&B8jJFVTmgLaswn!cx$&%JbTsloz!3 zMEz1d`k==`Ueub_JAy_&`!ogbwx27^ZXgFNAbx=g_I~5nO^r)}&myw~+yY*cJl4$I znNJ32M&K=0(2Dj_>@39`3=FX!v3nZHno_@q^!y}%(yw0PqOo=);6Y@&ylVe>nMOZ~ zd>j#QQSBn3oaWd;qy$&5(5H$Ayi)0haAYO6TH>FR?rhqHmNOO+(})NB zLI@B@v0)eq!ug`>G<@htRlp3n!EpU|n+G+AvXFrWSUsLMBfL*ZB`CRsIVHNTR&b?K zxBgsN0BjfB>UVcJ|x%=-zb%OV7lmZc& zxiupadZVF7)6QuhoY;;FK2b*qL0J-Rn-8!X4ZY$-ZSUXV5DFd7`T41c(#lAeLMoeT z4%g655v@7AqT!i@)Edt5JMbN(=Q-6{=L4iG8RA%}w;&pKmtWvI4?G9pVRp|RTw`g0 zD5c12B&A2&P6Ng~8WM2eIW=wxd?r7A*N+&!Be7PX3s|7~z=APxm=A?5 zt>xB4WG|*Td@VX{Rs)PV0|yK`oI3^xn(4c_j&vgxk_Y3o(-`_5o`V zRTghg6%l@(qodXN;dB#+OKJEEvhfcnc#BeO2|E(5df-!fKDZ!%9!^BJ_4)9P+9Dq5 zK1=(v?KmIp34r?z{NEWnLB3Px{XYwy-akun4F7xTRr2^zeYW{gcK9)>aJDdU5;w5@ zak=<+-PLH-|04pelTb%ULpuuuJC7DgyT@D|p{!V!0v3KpDnRjANN12q6SUR3mb9<- z>2r~IApQGhstZ!3*?5V z8#)hJ0TdZg0M-BK#nGFP>$i=qk82DO z7h;Ft!D5E15OgW)&%lej*?^1~2=*Z5$2VX>V{x8SC+{i10BbtUk9@I#Vi&hX)q
Q!LwySI{Bnv%Sm)yh{^sSVJ8&h_D-BJ_YZe5eCaAWU9b$O2c z$T|{vWVRtOL!xC0DTc(Qbe`ItNtt5hr<)VijD0{U;T#bUEp381_y`%ZIav?kuYG{iyYdEBPW=*xNSc;Rlt6~F4M`5G+VtOjc z*0qGzCb@gME5udTjJA-9O<&TWd~}ysBd(eVT1-H82-doyH9RST)|+Pb{o*;$j9Tjs zhU!IlsPsj8=(x3bAKJTopW3^6AKROHR^7wZ185wJGVhA~hEc|LP;k7NEz-@4p5o}F z`AD6naG3(n=NF9HTH81=F+Q|JOz$7wm9I<+#BSmB@o_cLt2GkW9|?7mM;r!JZp89l zbo!Hp8=n!XH1{GwaDU+k)pGp`C|cXkCU5%vcH)+v@0eK>%7gWxmuMu9YLlChA|_D@ zi#5zovN_!a-0?~pUV-Rj*1P)KwdU-LguR>YM&*Nen+ln8Q$?WFCJg%DY%K}2!!1FE zDv-A%Cbwo^p(lzac&_TZ-l#9kq`mhLcY3h9ZTUVCM(Ad&=EriQY5{jJv<5K&g|*Lk zgV%ILnf1%8V2B0E&;Sp4sYbYOvvMebLwYwzkRQ#F8GpTQq#uv=J`uaSJ34OWITeSGo6+-8Xw znCk*n{kdDEi)Hi&u^)~cs@iyCkFWB2SWZU|Uc%^43ZIZQ-vWNExCCtDWjqHs;;tWf$v{}0{p0Rvxkq``)*>+Akq%|Na zA`@~-Vfe|+(AIlqru+7Ceh4nsVmO9p9jc8}HX^W&ViBDXT+uXbT#R#idPn&L>+#b6 zflC-4C5-X;kUnR~L>PSLh*gvL68}RBsu#2l`s_9KjUWRhiqF`j)`y`2`YU(>3bdBj z?>iyjEhe-~$^I5!nn%B6Wh+I`FvLNvauve~eX<+Ipl&04 zT}};W&1a3%W?dJ2=N#0t?e+aK+%t}5q%jSLvp3jZ%?&F}nOOWr>+{GFIa%wO_2`et z=JzoRR~}iKuuR+azPI8;Gf9)z3kyA4EIOSl!sRR$DlW}0>&?GbgPojmjmnln;cTqCt=ADbE zZ8GAnoM+S1(5$i8^O4t`ue;vO4i}z0wz-QEIVe5_u03;}-!G1NyY8;h^}y;tzY}i5 zqQr#Ur3Fy8sSa$Q0ys+f`!`+>9WbvU_I`Sj;$4{S>O3?#inLHCrtLy~!s#WXV=oVP zeE93*Nc`PBi4q@%Ao$x4lw9vLHM!6mn3-b_cebF|n-2vt-zYVF_&sDE--J-P;2WHo z+@n2areE0o$LjvjlV2X7ZU@j+`{*8zq`JR3gKF#EW|#+{nMyo-a>nFFTg&vhyT=b} zDa8+v0(Dgx0yRL@ZXOYIlVSZ0|MFizy0VPW8;AfA5|pe!#j zX}Py^8fl5SyS4g1WSKKtnyP+_PoOwMMwu`(i@Z)diJp~U54*-miOchy7Z35eL>^M z4p<-aIxH4VUZgS783@H%M7P9hX>t{|RU7$n4T(brCG#h9e9p! z+o`i;EGGq3&pF;~5V~eBD}lC)>if$w%Vf}AFxGqO88|ApfHf&Bvu+xdG)@vuF}Yvk z)o;~k-%+0K0g+L`Wala!$=ZV|z$e%>f0%XoLib%)!R^RoS+{!#X?h-6uu zF&&KxORdZU&EwQFITIRLo(7TA3W}y6X{?Y%y2j0It!ekU#<)$qghZtpcS>L3uh`Uj z7GY;6f$9qKynP#oS3$$a{p^{D+0oJQ71`1?OAn_m8)UGZmj3l*ZI)`V-a>MKGGFG< z&^jg#Ok%(hhm>hSrZ5;Qga4u(?^i>GiW_j9%_7M>j(^|Om$#{k+^*ULnEgzW_1gCICtAD^WpC`A z{9&DXkG#01Xo)U$OC(L5Y$DQ|Q4C6CjUKk1UkPj$nXH##J{c8e#K|&{mA*;b$r0E4 zUNo0jthwA(c&N1l=PEe8Rw_8cEl|-eya9z&H3#n`B$t#+aJ03RFMzrV@gowbe8v(c zIFM60^0&lCFO10NU4w@|61xiZ4CVXeaKjd;d?sv52XM*lS8XiVjgWpRB;&U_C0g+`6B5V&w|O6B*_q zsATxL!M}+$He)1eOWECce#eS@2n^xhlB4<_Nn?yCVEQWDs(r`|@2GqLe<#(|&P0U? z$7V5IgpWf09uIf_RazRwC?qEqRaHyL?iiS05UiGesJy%^>-C{{ypTBI&B0-iUYhk> zIk<5xpsuV@g|z(AZD+C-;A!fTG=df1=<%nxy(a(IS+U{ME4ZbDEBtcD_3V=icT6*_ z)>|J?>&6%nvHhZERBtjK+s4xnut*@>GAmA5m*OTp$!^CHTr}vM4n(X1Q*;{e-Rd2BCF-u@1ZGm z!S8hJ6L=Gl4T_SDa7Xx|-{4mxveJg=ctf`BJ*fy!yF6Dz&?w(Q_6B}WQVtNI!BVBC zKfX<>7vd6C96}XAQmF-Jd?1Q4eTfRB3q7hCh0f!(JkdWT5<{iAE#dKy*Jxq&3a1@~ z8C||Dn2mFNyrUV|<-)C^_y7@8c2Fz+2jrae9deBDu;U}tJ{^xAdxCD248(k;dCJ%o z`y3sADe>U%suxwwv~8A1+R$VB=Q?%U?4joI$um;aH+eCrBqpn- z%79D_7rb;R-;-9RTrwi9dPlg8&@tfWhhZ(Vx&1PQ+6(huX`;M9x~LrW~~#3{j0Bh2kDU$}@!fFQej4VGkJv?M4rU^x!RU zEwhu$!CA_iDjFjrJa`aocySDX16?~;+wgav;}Zut6Mg%C4>}8FL?8)Kgwc(Qlj{@#2Pt0?G`$h7P#M+qoXtlV@d}%c&OzO+QYKK`kyXaK{U(O^2DyIXCZlNQjt0^8~8JzNGrIxhj}}M z&~QZlbx%t;MJ(Vux;2tgNKGlAqphLq%pd}JG9uoVHUo?|hN{pLQ6Em%r*+7t^<);X zm~6=qChlNAVXNN*Sow->*4;}T;l;D1I-5T{Bif@4_}=>l`tK;qqDdt5zvisCKhMAH z#r}`)7VW?LZqfdmXQ%zo5bJ00{Xb9^YKrk0Nf|oIW*K@(=`o2Vndz}ZDyk{!u}PVx zzd--+_WC*U{~DH3{?GI64IB+@On&@9X>EUAo&L+G{L^dozaI4C3G#2wr~hseW@K&g zKWs{uHu-9Je!3;4pE>eBltKUXb^*hG8I&413)$J&{D4N%7PcloU6bn%jPxJyQL?g* z9g+YFFEDiE`8rW^laCNzQmi7CTnPfwyg3VDHRAl>h=In6jeaVOP@!-CP60j3+#vpL zEYmh_oP0{-gTe7Or`L6x)6w?77QVi~jD8lWN@3RHcm80iV%M1A!+Y6iHM)05iC64tb$X2lV_%Txk@0l^hZqi^%Z?#- zE;LE0uFx)R08_S-#(wC=dS&}vj6P4>5ZWjhthP=*Hht&TdLtKDR;rXEX4*z0h74FA zMCINqrh3Vq;s%3MC1YL`{WjIAPkVL#3rj^9Pj9Ss7>7duy!9H0vYF%>1jh)EPqvlr6h%R%CxDsk| z!BACz7E%j?bm=pH6Eaw{+suniuY7C9Ut~1cWfOX9KW9=H><&kQlinPV3h9R>3nJvK z4L9(DRM=x;R&d#a@oFY7mB|m8h4692U5eYfcw|QKwqRsshN(q^v$4$)HgPpAJDJ`I zkqjq(8Cd!K!+wCd=d@w%~e$=gdUgD&wj$LQ1r>-E=O@c ze+Z$x{>6(JA-fNVr)X;*)40Eym1TtUZI1Pwwx1hUi+G1Jlk~vCYeXMNYtr)1?qwyg zsX_e*$h?380O00ou?0R@7-Fc59o$UvyVs4cUbujHUA>sH!}L54>`e` zHUx#Q+Hn&Og#YVOuo*niy*GU3rH;%f``nk#NN5-xrZ34NeH$l`4@t);4(+0|Z#I>Y z)~Kzs#exIAaf--65L0UHT_SvV8O2WYeD>Mq^Y6L!Xu8%vnpofG@w!}R7M28?i1*T&zp3X4^OMCY6(Dg<-! zXmcGQrRgHXGYre7GfTJ)rhl|rs%abKT_Nt24_Q``XH{88NVPW+`x4ZdrMuO0iZ0g` z%p}y};~T5gbb9SeL8BSc`SO#ixC$@QhXxZ=B}L`tP}&k?1oSPS=4%{UOHe0<_XWln zwbl5cn(j-qK`)vGHY5B5C|QZd5)W7c@{bNVXqJ!!n$^ufc?N9C-BF2QK1(kv++h!>$QbAjq)_b$$PcJdV+F7hz0Hu@ zqj+}m0qn{t^tD3DfBb~0B36|Q`bs*xs|$i^G4uNUEBl4g;op-;Wl~iThgga?+dL7s zUP(8lMO?g{GcYpDS{NM!UA8Hco?#}eNEioRBHy4`mq!Pd-9@-97|k$hpEX>xoX+dY zDr$wfm^P&}Wu{!%?)U_(%Mn79$(ywvu*kJ9r4u|MyYLI_67U7%6Gd_vb##Nerf@>& z8W11z$$~xEZt$dPG}+*IZky+os5Ju2eRi;1=rUEeIn>t-AzC_IGM-IXWK3^6QNU+2pe=MBn4I*R@A%-iLDCOHTE-O^wo$sL_h{dcPl=^muAQb`_BRm};=cy{qSkui;`WSsj9%c^+bIDQ z0`_?KX0<-=o!t{u(Ln)v>%VGL z0pC=GB7*AQ?N7N{ut*a%MH-tdtNmNC+Yf$|KS)BW(gQJ*z$d{+{j?(e&hgTy^2|AR9vx1Xre2fagGv0YXWqtNkg*v%40v?BJBt|f9wX5 z{QTlCM}b-0{mV?IG>TW_BdviUKhtosrBqdfq&Frdz>cF~yK{P@(w{Vr7z2qKFwLhc zQuogKO@~YwyS9%+d-zD7mJG~@?EFJLSn!a&mhE5$_4xBl&6QHMzL?CdzEnC~C3$X@ zvY!{_GR06ep5;<#cKCSJ%srxX=+pn?ywDwtJ2{TV;0DKBO2t++B(tIO4)Wh`rD13P z4fE$#%zkd=UzOB74gi=-*CuID&Z3zI^-`4U^S?dHxK8fP*;fE|a(KYMgMUo`THIS1f!*6dOI2 zFjC3O=-AL`6=9pp;`CYPTdVX z8(*?V&%QoipuH0>WKlL8A*zTKckD!paN@~hh zmXzm~qZhMGVdQGd=AG8&20HW0RGV8X{$9LldFZYm zE?}`Q3i?xJRz43S?VFMmqRyvWaS#(~Lempg9nTM$EFDP(Gzx#$r)W&lpFKqcAoJh-AxEw$-bjW>`_+gEi z2w`99#UbFZGiQjS8kj~@PGqpsPX`T{YOj`CaEqTFag;$jY z8_{Wzz>HXx&G*Dx<5skhpETxIdhKH?DtY@b9l8$l?UkM#J-Snmts7bd7xayKTFJ(u zyAT&@6cAYcs{PBfpqZa%sxhJ5nSZBPji?Zlf&}#L?t)vC4X5VLp%~fz2Sx<*oN<7` z?ge=k<=X7r<~F7Tvp9#HB{!mA!QWBOf%EiSJ6KIF8QZNjg&x~-%e*tflL(ji_S^sO ztmib1rp09uon}RcsFi#k)oLs@$?vs(i>5k3YN%$T(5Or(TZ5JW9mA6mIMD08=749$ z!d+l*iu{Il7^Yu}H;lgw=En1sJpCKPSqTCHy4(f&NPelr31^*l%KHq^QE>z>Ks_bH zjbD?({~8Din7IvZeJ>8Ey=e;I?thpzD=zE5UHeO|neioJwG;IyLk?xOz(yO&0DTU~ z^#)xcs|s>Flgmp;SmYJ4g(|HMu3v7#;c*Aa8iF#UZo7CvDq4>8#qLJ|YdZ!AsH%^_7N1IQjCro

K7UpUK$>l@ zw`1S}(D?mUXu_C{wupRS-jiX~w=Uqqhf|Vb3Cm9L=T+w91Cu^ z*&Ty%sN?x*h~mJc4g~k{xD4ZmF%FXZNC;oVDwLZ_WvrnzY|{v8hc1nmx4^}Z;yriXsAf+Lp+OFLbR!&Ox?xABwl zu8w&|5pCxmu#$?Cv2_-Vghl2LZ6m7}VLEfR5o2Ou$x02uA-%QB2$c(c1rH3R9hesc zfpn#oqpbKuVsdfV#cv@5pV4^f_!WS+F>SV6N0JQ9E!T90EX((_{bSSFv9ld%I0&}9 zH&Jd4MEX1e0iqDtq~h?DBrxQX1iI0lIs<|kB$Yrh&cpeK0-^K%=FBsCBT46@h#yi!AyDq1V(#V}^;{{V*@T4WJ&U-NTq43w=|K>z8%pr_nC>%C(Wa_l78Ufib$r8Od)IIN=u>417 z`Hl{9A$mI5A(;+-Q&$F&h-@;NR>Z<2U;Y21>>Z;s@0V@SbkMQQj%_;~+qTuQ?c|AV zcWm3XZQHhP&R%QWarS%mJ!9R^&!_)*s(v+VR@I#QrAT}`17Y+l<`b-nvmDNW`De%y zrwTZ9EJrj1AFA>B`1jYDow}~*dfPs}IZMO3=a{Fy#IOILc8F0;JS4x(k-NSpbN@qM z`@aE_e}5{!$v3+qVs7u?sOV(y@1Os*Fgu`fCW9=G@F_#VQ%xf$hj0~wnnP0$hFI+@ zkQj~v#V>xn)u??YutKsX>pxKCl^p!C-o?+9;!Nug^ z{rP!|+KsP5%uF;ZCa5F;O^9TGac=M|=V z_H(PfkV1rz4jl?gJ(ArXMyWT4y(86d3`$iI4^l9`vLdZkzpznSd5Ikfrs8qcSy&>z zTIZgWZGXw0n9ibQxYWE@gI0(3#KA-dAdPcsL_|hg2@~C!VZDM}5;v_Nykfq!*@*Zf zE_wVgx82GMDryKO{U{D>vSzSc%B~|cjDQrt5BN=Ugpsf8H8f1lR4SGo#hCuXPL;QQ z#~b?C4MoepT3X`qdW2dNn& zo8)K}%Lpu>0tQei+{>*VGErz|qjbK#9 zvtd8rcHplw%YyQCKR{kyo6fgg!)6tHUYT(L>B7er5)41iG`j$qe*kSh$fY!PehLcD zWeKZHn<492B34*JUQh=CY1R~jT9Jt=k=jCU2=SL&&y5QI2uAG2?L8qd2U(^AW#{(x zThSy=C#>k+QMo^7caQcpU?Qn}j-`s?1vXuzG#j8(A+RUAY})F@=r&F(8nI&HspAy4 z4>(M>hI9c7?DCW8rw6|23?qQMSq?*Vx?v30U%luBo)B-k2mkL)Ljk5xUha3pK>EEj z@(;tH|M@xkuN?gsz;*bygizwYR!6=(Xgcg^>WlGtRYCozY<rFX2E>kaZo)O<^J7a`MX8Pf`gBd4vrtD|qKn&B)C&wp0O-x*@-|m*0egT=-t@%dD zgP2D+#WPptnc;_ugD6%zN}Z+X4=c61XNLb7L1gWd8;NHrBXwJ7s0ce#lWnnFUMTR& z1_R9Fin4!d17d4jpKcfh?MKRxxQk$@)*hradH2$3)nyXep5Z;B z?yX+-Bd=TqO2!11?MDtG0n(*T^!CIiF@ZQymqq1wPM_X$Iu9-P=^}v7npvvPBu!d$ z7K?@CsA8H38+zjA@{;{kG)#AHME>Ix<711_iQ@WWMObXyVO)a&^qE1GqpP47Q|_AG zP`(AD&r!V^MXQ^e+*n5~Lp9!B+#y3#f8J^5!iC@3Y@P`;FoUH{G*pj*q7MVV)29+j z>BC`a|1@U_v%%o9VH_HsSnM`jZ-&CDvbiqDg)tQEnV>b%Ptm)T|1?TrpIl)Y$LnG_ zzKi5j2Fx^K^PG1=*?GhK;$(UCF-tM~^=Z*+Wp{FSuy7iHt9#4n(sUuHK??@v+6*|10Csdnyg9hAsC5_OrSL;jVkLlf zHXIPukLqbhs~-*oa^gqgvtpgTk_7GypwH><53riYYL*M=Q@F-yEPLqQ&1Sc zZB%w}T~RO|#jFjMWcKMZccxm-SL)s_ig?OC?y_~gLFj{n8D$J_Kw%{r0oB8?@dWzn zB528d-wUBQzrrSSLq?fR!K%59Zv9J4yCQhhDGwhptpA5O5U?Hjqt>8nOD zi{)0CI|&Gu%zunGI*XFZh(ix)q${jT8wnnzbBMPYVJc4HX*9d^mz|21$=R$J$(y7V zo0dxdbX3N#=F$zjstTf*t8vL)2*{XH!+<2IJ1VVFa67|{?LP&P41h$2i2;?N~RA30LV`BsUcj zfO9#Pg1$t}7zpv#&)8`mis3~o+P(DxOMgz-V*(?wWaxi?R=NhtW}<#^Z?(BhSwyar zG|A#Q7wh4OfK<|DAcl9THc-W4*>J4nTevsD%dkj`U~wSUCh15?_N@uMdF^Kw+{agk zJ`im^wDqj`Ev)W3k3stasP`88-M0ZBs7;B6{-tSm3>I@_e-QfT?7|n0D~0RRqDb^G zyHb=is;IwuQ&ITzL4KsP@Z`b$d%B0Wuhioo1CWttW8yhsER1ZUZzA{F*K=wmi-sb#Ju+j z-l@In^IKnb{bQG}Ps>+Vu_W#grNKNGto+yjA)?>0?~X`4I3T@5G1)RqGUZuP^NJCq&^HykuYtMDD8qq+l8RcZNJsvN(10{ zQ1$XcGt}QH-U^WU!-wRR1d--{B$%vY{JLWIV%P4-KQuxxDeJaF#{eu&&r!3Qu{w}0f--8^H|KwE>)ORrcR+2Qf zb})DRcH>k0zWK8@{RX}NYvTF;E~phK{+F;MkIP$)T$93Ba2R2TvKc>`D??#mv9wg$ zd~|-`Qx5LwwsZ2hb*Rt4S9dsF%Cny5<1fscy~)d;0m2r$f=83<->c~!GNyb!U)PA; zq^!`@@)UaG)Ew(9V?5ZBq#c%dCWZrplmuM`o~TyHjAIMh0*#1{B>K4po-dx$Tk-Cq z=WZDkP5x2W&Os`N8KiYHRH#UY*n|nvd(U>yO=MFI-2BEp?x@=N<~CbLJBf6P)}vLS?xJXYJ2^<3KJUdrwKnJnTp{ zjIi|R=L7rn9b*D#Xxr4*R<3T5AuOS+#U8hNlfo&^9JO{VbH!v9^JbK=TCGR-5EWR@ zN8T-_I|&@A}(hKeL4_*eb!1G8p~&_Im8|wc>Cdir+gg90n1dw?QaXcx6Op_W1r=axRw>4;rM*UOpT#Eb9xU1IiWo@h?|5uP zka>-XW0Ikp@dIe;MN8B01a7+5V@h3WN{J=HJ*pe0uwQ3S&MyWFni47X32Q7SyCTNQ z+sR!_9IZa5!>f&V$`q!%H8ci!a|RMx5}5MA_kr+bhtQy{-^)(hCVa@I!^TV4RBi zAFa!Nsi3y37I5EK;0cqu|9MRj<^r&h1lF}u0KpKQD^5Y+LvFEwM zLU@@v4_Na#Axy6tn3P%sD^5P#<7F;sd$f4a7LBMk zGU^RZHBcxSA%kCx*eH&wgA?Qwazm8>9SCSz_!;MqY-QX<1@p$*T8lc?@`ikEqJ>#w zcG``^CoFMAhdEXT9qt47g0IZkaU)4R7wkGs^Ax}usqJ5HfDYAV$!=6?>J6+Ha1I<5 z|6=9soU4>E))tW$<#>F ziZ$6>KJf0bPfbx_)7-}tMINlc=}|H+$uX)mhC6-Hz+XZxsKd^b?RFB6et}O#+>Wmw9Ec9) z{q}XFWp{3@qmyK*Jvzpyqv57LIR;hPXKsrh{G?&dRjF%Zt5&m20Ll?OyfUYC3WRn{cgQ?^V~UAv+5 z&_m#&nIwffgX1*Z2#5^Kl4DbE#NrD&Hi4|7SPqZ}(>_+JMz=s|k77aEL}<=0Zfb)a z%F(*L3zCA<=xO)2U3B|pcTqDbBoFp>QyAEU(jMu8(jLA61-H!ucI804+B!$E^cQQa z)_ERrW3g!B9iLb3nn3dlkvD7KsY?sRvls3QC0qPi>o<)GHx%4Xb$5a3GBTJ(k@`e@ z$RUa^%S15^1oLEmA=sayrP5;9qtf!Z1*?e$ORVPsXpL{jL<6E)0sj&swP3}NPmR%FM?O>SQgN5XfHE< zo(4#Cv11(%Nnw_{_Ro}r6=gKd{k?NebJ~<~Kv0r(r0qe4n3LFx$5%x(BKvrz$m?LG zjLIc;hbj0FMdb9aH9Lpsof#yG$(0sG2%RL;d(n>;#jb!R_+dad+K;Ccw!|RY?uS(a zj~?=&M!4C(5LnlH6k%aYvz@7?xRa^2gml%vn&eKl$R_lJ+e|xsNfXzr#xuh(>`}9g zLHSyiFwK^-p!;p$yt7$F|3*IfO3Mlu9e>Dpx8O`37?fA`cj`C0B-m9uRhJjs^mRp# zWB;Aj6|G^1V6`jg7#7V9UFvnB4((nIwG?k%c7h`?0tS8J3Bn0t#pb#SA}N-|45$-j z$R>%7cc2ebAClXc(&0UtHX<>pd)akR3Kx_cK+n<}FhzmTx!8e9^u2e4%x{>T6pQ`6 zO182bh$-W5A3^wos0SV_TgPmF4WUP-+D25KjbC{y_6W_9I2_vNKwU(^qSdn&>^=*t z&uvp*@c8#2*paD!ZMCi3;K{Na;I4Q35zw$YrW5U@Kk~)&rw;G?d7Q&c9|x<Hg|CNMsxovmfth*|E*GHezPTWa^Hd^F4!B3sF;)? z(NaPyAhocu1jUe(!5Cy|dh|W2=!@fNmuNOzxi^tE_jAtzNJ0JR-avc_H|ve#KO}#S z#a(8secu|^Tx553d4r@3#6^MHbH)vmiBpn0X^29xEv!Vuh1n(Sr5I0V&`jA2;WS|Y zbf0e}X|)wA-Pf5gBZ>r4YX3Mav1kKY(ulAJ0Q*jB)YhviHK)w!TJsi3^dMa$L@^{` z_De`fF4;M87vM3Ph9SzCoCi$#Fsd38u!^0#*sPful^p5oI(xGU?yeYjn;Hq1!wzFk zG&2w}W3`AX4bxoVm03y>ts{KaDf!}b&7$(P4KAMP=vK5?1In^-YYNtx1f#}+2QK@h zeSeAI@E6Z8a?)>sZ`fbq9_snl6LCu6g>o)rO;ijp3|$vig+4t} zylEo7$SEW<_U+qgVcaVhk+4k+C9THI5V10qV*dOV6pPtAI$)QN{!JRBKh-D zk2^{j@bZ}yqW?<#VVuI_27*cI-V~sJiqQv&m07+10XF+#ZnIJdr8t`9s_EE;T2V;B z4UnQUH9EdX%zwh-5&wflY#ve!IWt0UE-My3?L#^Bh%kcgP1q{&26eXLn zTkjJ*w+(|_>Pq0v8{%nX$QZbf)tbJaLY$03;MO=Ic-uqYUmUCuXD>J>o6BCRF=xa% z3R4SK9#t1!K4I_d>tZgE>&+kZ?Q}1qo4&h%U$GfY058s%*=!kac{0Z+4Hwm!)pFLR zJ+5*OpgWUrm0FPI2ib4NPJ+Sk07j(`diti^i#kh&f}i>P4~|d?RFb#!JN)~D@)beox}bw?4VCf^y*`2{4`-@%SFTry2h z>9VBc9#JxEs1+0i2^LR@B1J`B9Ac=#FW=(?2;5;#U$0E0UNag_!jY$&2diQk_n)bT zl5Me_SUvqUjwCqmVcyb`igygB_4YUB*m$h5oeKv3uIF0sk}~es!{D>4r%PC*F~FN3owq5e0|YeUTSG#Vq%&Gk7uwW z0lDo#_wvflqHeRm*}l?}o;EILszBt|EW*zNPmq#?4A+&i0xx^?9obLyY4xx=Y9&^G;xYXYPxG)DOpPg!i_Ccl#3L}6xAAZzNhPK1XaC_~ z!A|mlo?Be*8Nn=a+FhgpOj@G7yYs(Qk(8&|h@_>w8Y^r&5nCqe0V60rRz?b5%J;GYeBqSAjo|K692GxD4` zRZyM2FdI+-jK2}WAZTZ()w_)V{n5tEb@>+JYluDozCb$fA4H)$bzg(Ux{*hXurjO^ zwAxc+UXu=&JV*E59}h3kzQPG4M)X8E*}#_&}w*KEgtX)cU{vm9b$atHa;s>| z+L6&cn8xUL*OSjx4YGjf6{Eq+Q3{!ZyhrL&^6Vz@jGbI%cAM9GkmFlamTbcQGvOlL zmJ?(FI)c86=JEs|*;?h~o)88>12nXlpMR4@yh%qdwFNpct;vMlc=;{FSo*apJ;p}! zAX~t;3tb~VuP|ZW;z$=IHf->F@Ml)&-&Bnb{iQyE#;GZ@C$PzEf6~q}4D>9jic@mTO5x76ulDz@+XAcm35!VSu zT*Gs>;f0b2TNpjU_BjHZ&S6Sqk6V1370+!eppV2H+FY!q*n=GHQ!9Rn6MjY!Jc77A zG7Y!lFp8?TIHN!LXO?gCnsYM-gQxsm=Ek**VmZu7vnuufD7K~GIxfxbsQ@qv2T zPa`tvHB$fFCyZl>3oYg?_wW)C>^_iDOc^B7klnTOoytQH18WkOk)L2BSD0r%xgRSW zQS9elF^?O=_@|58zKLK;(f77l-Zzu}4{fXed2saq!5k#UZAoDBqYQS{sn@j@Vtp|$ zG%gnZ$U|9@u#w1@11Sjl8ze^Co=)7yS(}=;68a3~g;NDe_X^}yJj;~s8xq9ahQ5_r zxAlTMnep*)w1e(TG%tWsjo3RR;yVGPEO4V{Zp?=a_0R#=V^ioQu4YL=BO4r0$$XTX zZfnw#_$V}sDAIDrezGQ+h?q24St0QNug_?{s-pI(^jg`#JRxM1YBV;a@@JQvH8*>> zIJvku74E0NlXkYe_624>znU0J@L<-c=G#F3k4A_)*;ky!C(^uZfj%WB3-*{*B$?9+ zDm$WFp=0(xnt6`vDQV3Jl5f&R(Mp};;q8d3I%Kn>Kx=^;uSVCw0L=gw53%Bp==8Sw zxtx=cs!^-_+i{2OK`Q;913+AXc_&Z5$@z3<)So0CU3;JAv=H?@Zpi~riQ{z-zLtVL z!oF<}@IgJp)Iyz1zVJ42!SPHSkjYNS4%ulVVIXdRuiZ@5Mx8LJS}J#qD^Zi_xQ@>DKDr-_e#>5h3dtje*NcwH_h;i{Sx7}dkdpuW z(yUCjckQsagv*QGMSi9u1`Z|V^}Wjf7B@q%j2DQXyd0nOyqg%m{CK_lAoKlJ7#8M} z%IvR?Vh$6aDWK2W!=i?*<77q&B8O&3?zP(Cs@kapc)&p7En?J;t-TX9abGT#H?TW? ztO5(lPKRuC7fs}zwcUKbRh=7E8wzTsa#Z{a`WR}?UZ%!HohN}d&xJ=JQhpO1PI#>X zHkb>pW04pU%Bj_mf~U}1F1=wxdBZu1790>3Dm44bQ#F=T4V3&HlOLsGH)+AK$cHk6 zia$=$kog?)07HCL*PI6}DRhpM^*%I*kHM<#1Se+AQ!!xyhcy6j7`iDX7Z-2i73_n# zas*?7LkxS-XSqv;YBa zW_n*32D(HTYQ0$feV_Fru1ZxW0g&iwqixPX3=9t4o)o|kOo79V$?$uh?#8Q8e>4e)V6;_(x&ViUVxma+i25qea;d-oK7ouuDsB^ab{ zu1qjQ%`n56VtxBE#0qAzb7lph`Eb-}TYpXB!H-}3Ykqyp`otprp7{VEuW*^IR2n$Fb99*nAtqT&oOFIf z@w*6>YvOGw@Ja?Pp1=whZqydzx@9X4n^2!n83C5{C?G@|E?&$?p*g68)kNvUTJ)I6 z1Q|(#UuP6pj78GUxq11m-GSszc+)X{C2eo-?8ud9sB=3(D47v?`JAa{V(IF zPZQ_0AY*9M97>Jf<o%#O_%Wq}8>YM=q0|tGY+hlXcpE=Z4Od z`NT7Hu2hnvRoqOw@g1f=bv`+nba{GwA$Ak0INlqI1k<9!x_!sL()h?hEWoWrdU3w` zZ%%)VR+Bc@_v!C#koM1p-3v_^L6)_Ktj4HE>aUh%2XZE@JFMOn)J~c`_7VWNb9c-N z2b|SZMR4Z@E7j&q&9(6H3yjEu6HV7{2!1t0lgizD;mZ9$r(r7W5G$ky@w(T_dFnOD z*p#+z$@pKE+>o@%eT(2-p_C}wbQ5s(%Sn_{$HDN@MB+Ev?t@3dPy`%TZ!z}AThZSu zN<1i$siJhXFdjV zP*y|V<`V8t=h#XTRUR~5`c`Z9^-`*BZf?WAehGdg)E2Je)hqFa!k{V(u+(hTf^Yq& zoruUh2(^3pe)2{bvt4&4Y9CY3js)PUHtd4rVG57}uFJL)D(JfSIo^{P=7liFXG zq5yqgof0V8paQcP!gy+;^pp-DA5pj=gbMN0eW=-eY+N8~y+G>t+x}oa!5r>tW$xhI zPQSv=pi;~653Gvf6~*JcQ%t1xOrH2l3Zy@8AoJ+wz@daW@m7?%LXkr!bw9GY@ns3e zSfuWF_gkWnesv?s3I`@}NgE2xwgs&rj?kH-FEy82=O8`+szN ziHch`vvS`zNfap14!&#i9H@wF7}yIPm=UB%(o(}F{wsZ(wA0nJ2aD^@B41>>o-_U6 zUqD~vdo48S8~FTb^+%#zcbQiiYoDKYcj&$#^;Smmb+Ljp(L=1Kt_J!;0s%1|JK}Wi z;={~oL!foo5n8=}rs6MmUW~R&;SIJO3TL4Ky?kh+b2rT9B1Jl4>#Uh-Bec z`Hsp<==#UEW6pGPhNk8H!!DUQR~#F9jEMI6T*OWfN^Ze&X(4nV$wa8QUJ>oTkruH# zm~O<`J7Wxseo@FqaZMl#Y(mrFW9AHM9Kb|XBMqaZ2a)DvJgYipkDD_VUF_PKd~dT7 z#02}bBfPn9a!X!O#83=lbJSK#E}K&yx-HI#T6ua)6o0{|={*HFusCkHzs|Fn&|C3H zBck1cmfcWVUN&i>X$YU^Sn6k2H;r3zuXbJFz)r5~3$d$tUj(l1?o={MM){kjgqXRO zc5R*#{;V7AQh|G|)jLM@wGAK&rm2~@{Pewv#06pHbKn#wL0P6F1!^qw9g&cW3Z=9} zj)POhOlwsh@eF=>z?#sIs*C-Nl(yU!#DaiaxhEs#iJqQ8w%(?+6lU02MYSeDkr!B- zPjMv+on6OLXgGnAtl(ao>|X2Y8*Hb}GRW5}-IzXnoo-d0!m4Vy$GS!XOLy>3_+UGs z2D|YcQx@M#M|}TDOetGi{9lGo9m-=0-^+nKE^*?$^uHkxZh}I{#UTQd;X!L+W@jm( zDg@N4+lUqI92o_rNk{3P>1gxAL=&O;x)ZT=q1mk0kLlE$WeWuY_$0`0jY-Kkt zP*|m3AF}Ubd=`<>(Xg0har*_@x2YH}bn0Wk*OZz3*e5;Zc;2uBdnl8?&XjupbkOeNZsNh6pvsq_ydmJI+*z**{I{0K)-;p1~k8cpJXL$^t!-`E}=*4G^-E8>H!LjTPxSx zcF+cS`ommfKMhNSbas^@YbTpH1*RFrBuATUR zt{oFWSk^$xU&kbFQ;MCX22RAN5F6eq9UfR$ut`Jw--p2YX)A*J69m^!oYfj2y7NYcH6&r+0~_sH^c^nzeN1AU4Ga7=FlR{S|Mm~MpzY0$Z+p2W(a={b-pR9EO1Rs zB%KY|@wLcAA@)KXi!d2_BxrkhDn`DT1=Dec}V!okd{$+wK z4E{n8R*xKyci1(CnNdhf$Dp2(Jpof0-0%-38X=Dd9PQgT+w%Lshx9+loPS~MOm%ZT zt%2B2iL_KU_ita%N>xjB!#71_3=3c}o zgeW~^U_ZTJQ2!PqXulQd=3b=XOQhwATK$y(9$#1jOQ4}4?~l#&nek)H(04f(Sr=s| zWv7Lu1=%WGk4FSw^;;!8&YPM)pQDCY9DhU`hMty1@sq1=Tj7bFsOOBZOFlpR`W>-J$-(kezWJj;`?x-v>ev{*8V z8p|KXJPV$HyQr1A(9LVrM47u-XpcrIyO`yWvx1pVYc&?154aneRpLqgx)EMvRaa#|9?Wwqs2+W8n5~79G z(}iCiLk;?enn}ew`HzhG+tu+Ru@T+K5juvZN)wY;x6HjvqD!&!)$$;1VAh~7fg0K| zEha#aN=Yv|3^~YFH}cc38ovVb%L|g@9W6fo(JtT6$fa?zf@Ct88e}m?i)b*Jgc{fl zExfdvw-BYDmH6>(4QMt#p0;FUIQqkhD}aH?a7)_%JtA~soqj{ppP_82yi9kaxuK>~ ze_)Zt>1?q=ZH*kF{1iq9sr*tVuy=u>Zev}!gEZx@O6-fjyu9X00gpIl-fS_pzjpqJ z1yqBmf9NF!jaF<+YxgH6oXBdK)sH(>VZ)1siyA$P<#KDt;8NT*l_0{xit~5j1P)FN zI8hhYKhQ)i z37^aP13B~u65?sg+_@2Kr^iWHN=U;EDSZ@2W2!5ALhGNWXnFBY%7W?1 z=HI9JzQ-pLKZDYTv<0-lt|6c-RwhxZ)mU2Os{bsX_i^@*fKUj8*aDO5pks=qn3Dv6 zwggpKLuyRCTVPwmw1r}B#AS}?X7b837UlXwp~E2|PJw2SGVueL7){Y&z!jL!XN=0i zU^Eig`S2`{+gU$68aRdWx?BZ{sU_f=8sn~>s~M?GU~`fH5kCc; z8ICp+INM3(3{#k32RZdv6b9MQYdZXNuk7ed8;G?S2nT+NZBG=Tar^KFl2SvhW$bGW#kdWL-I)s_IqVnCDDM9fm8g;P;8 z7t4yZn3^*NQfx7SwmkzP$=fwdC}bafQSEF@pd&P8@H#`swGy_rz;Z?Ty5mkS%>m#% zp_!m9e<()sfKiY(nF<1zBz&&`ZlJf6QLvLhl`_``%RW&{+O>Xhp;lwSsyRqGf=RWd zpftiR`={2(siiPAS|p}@q=NhVc0ELprt%=fMXO3B)4ryC2LT(o=sLM7hJC!}T1@)E zA3^J$3&1*M6Xq>03FX`R&w*NkrZE?FwU+Muut;>qNhj@bX17ZJxnOlPSZ=Zeiz~T_ zOu#yc3t6ONHB;?|r4w+pI)~KGN;HOGC)txxiUN8#mexj+W(cz%9a4sx|IRG=}ia zuEBuba3AHsV2feqw-3MvuL`I+2|`Ud4~7ZkN=JZ;L20|Oxna5vx1qbIh#k2O4$RQF zo`tL()zxaqibg^GbB+BS5#U{@K;WWQj~GcB1zb}zJkPwH|5hZ9iH2308!>_;%msji zJHSL~s)YHBR=Koa1mLEOHos*`gp=s8KA-C zu0aE+W!#iJ*0xqKm3A`fUGy#O+X+5W36myS>Uh2!R*s$aCU^`K&KKLCCDkejX2p=5 z%o7-fl03x`gaSNyr?3_JLv?2RLS3F*8ub>Jd@^Cc17)v8vYEK4aqo?OS@W9mt%ITJ z9=S2%R8M){CugT@k~~0x`}Vl!svYqX=E)c_oU6o}#Hb^%G1l3BudxA{F*tbjG;W_>=xV73pKY53v%>I)@D36I_@&p$h|Aw zonQS`07z_F#@T-%@-Tb|)7;;anoD_WH>9ewFy(ZcEOM$#Y)8>qi7rCnsH9GO-_7zF zu*C87{Df1P4TEOsnzZ@H%&lvV(3V@;Q!%+OYRp`g05PjY^gL$^$-t0Y>H*CDDs?FZly*oZ&dxvsxaUWF!{em4{A>n@vpXg$dwvt@_rgmHF z-MER`ABa8R-t_H*kv>}CzOpz;!>p^^9ztHMsHL|SRnS<-y5Z*r(_}c4=fXF`l^-i}>e7v!qs_jv zqvWhX^F=2sDNWA9c@P0?lUlr6ecrTKM%pNQ^?*Lq?p-0~?_j50xV%^(+H>sMul#Tw zeciF*1=?a7cI(}352%>LO96pD+?9!fNyl^9v3^v&Y4L)mNGK0FN43&Xf8jUlxW1Bw zyiu2;qW-aGNhs=zbuoxnxiwZ3{PFZM#Kw)9H@(hgX23h(`Wm~m4&TvoZoYp{plb^> z_#?vXcxd>r7K+1HKJvhed>gtK`TAbJUazUWQY6T~t2af%#<+Veyr%7-#*A#@&*;@g58{i|E%6yC_InGXCOd{L0;$)z#?n7M`re zh!kO{6=>7I?*}czyF7_frt#)s1CFJ_XE&VrDA?Dp3XbvF{qsEJgb&OLSNz_5g?HpK z9)8rsr4JN!Af3G9!#Qn(6zaUDqLN(g2g8*M)Djap?WMK9NKlkC)E2|-g|#-rp%!Gz zAHd%`iq|81efi93m3yTBw3g0j#;Yb2X{mhRAI?&KDmbGqou(2xiRNb^sV}%%Wu0?< z?($L>(#BO*)^)rSgyNRni$i`R4v;GhlCZ8$@e^ROX(p=2_v6Y!%^As zu022)fHdv_-~Yu_H6WVPLpHQx!W%^6j)cBhS`O3QBW#x(eX54d&I22op(N59b*&$v zFiSRY6rOc^(dgSV1>a7-5C;(5S5MvKcM2Jm-LD9TGqDpP097%52V+0>Xqq!! zq4e3vj53SE6i8J`XcQB|MZPP8j;PAOnpGnllH6#Ku~vS42xP*Nz@~y%db7Xi8s09P z1)e%8ys6&M8D=Dt6&t`iKG_4X=!kgRQoh%Z`dc&mlOUqXk-k`jKv9@(a^2-Upw>?< zt5*^DV~6Zedbec4NVl($2T{&b)zA@b#dUyd>`2JC0=xa_fIm8{5um zr-!ApXZhC8@=vC2WyxO|!@0Km)h8ep*`^he92$@YwP>VcdoS5OC^s38e#7RPsg4j+ zbVGG}WRSET&ZfrcR(x~k8n1rTP%CnfUNKUonD$P?FtNFF#cn!wEIab-;jU=B1dHK@ z(;(yAQJ`O$sMn>h;pf^8{JISW%d+@v6@CnXh9n5TXGC}?FI9i-D0OMaIg&mAg=0Kn zNJ7oz5*ReJukD55fUsMuaP+H4tDN&V9zfqF@ zr=#ecUk9wu{0;!+gl;3Bw=Vn^)z$ahVhhw)io!na&9}LmWurLb0zubxK=UEnU*{5P z+SP}&*(iBKSO4{alBHaY^)5Q=mZ+2OwIooJ7*Q5XJ+2|q`9#f?6myq!&oz?klihLq z4C)$XP!BNS0G_Z1&TM>?Jk{S~{F3n83ioli=IO6f%wkvCl(RFFw~j0tb{GvXTx>*sB0McY0s&SNvj4+^h`9nJ_wM>F!Uc>X}9PifQekn0sKI2SAJP!a4h z5cyGTuCj3ZBM^&{dRelIlT^9zcfaAuL5Y~bl!ppSf`wZbK$z#6U~rdclk``e+!qhe z6Qspo*%<)eu6?C;Bp<^VuW6JI|Ncvyn+LlSl;Mp22Bl7ARQ0Xc24%29(ZrdsIPw&-=yHQ7_Vle|5h>AST0 zUGX2Zk34vp?U~IHT|;$U86T+UUHl_NE4m|}>E~6q``7hccCaT^#y+?wD##Q%HwPd8 zV3x4L4|qqu`B$4(LXqDJngNy-{&@aFBvVsywt@X^}iH7P%>bR?ciC$I^U-4Foa`YKI^qDyGK7k%E%c_P=yzAi`YnxGA%DeNd++j3*h^ z=rn>oBd0|~lZ<6YvmkKY*ZJlJ;Im0tqgWu&E92eqt;+NYdxx`eS(4Hw_Jb5|yVvBg z*tbdY^!AN;luEyN4VRhS@-_DC{({ziH{&Z}iGElSV~qvT>L-8G%+yEL zX#MFOhj{InyKG=mvW-<1B@c-}x$vA(nU?>S>0*eN#!SLzQ)Ex7fvQ)S4D<8|I#N$3 zT5Ei`Z?cxBODHX8(Xp73v`IsAYC@9b;t}z0wxVuQSY1J^GRwDPN@qbM-ZF48T$GZ< z8WU+;Pqo?{ghI-KZ-i*ydXu`Ep0Xw^McH_KE9J0S7G;x8Fe`DVG?j3Pv=0YzJ}yZR z%2=oqHiUjvuk0~Ca>Kol4CFi0_xQT~;_F?=u+!kIDl-9g`#ZNZ9HCy17Ga1v^Jv9# z{T4Kb1-AzUxq*MutfOWWZgD*HnFfyYg0&e9f(5tZ>krPF6{VikNeHoc{linPPt#Si z&*g>(c54V8rT_AX!J&bNm-!umPvOR}vDai#`CX___J#=zeB*{4<&2WpaDncZsOkp* zsg<%@@rbrMkR_ux9?LsQxzoBa1s%$BBn6vk#{&&zUwcfzeCBJUwFYSF$08qDsB;gWQN*g!p8pxjofWbqNSZOEKOaTx@+* zwdt5*Q47@EOZ~EZL9s?1o?A%9TJT=Ob_13yyugvPg*e&ZU(r6^k4=2+D-@n=Hv5vu zSXG|hM(>h9^zn=eQ=$6`JO&70&2|%V5Lsx>)(%#;pcOfu>*nk_3HB_BNaH$`jM<^S zcSftDU1?nL;jy)+sfonQN}(}gUW?d_ikr*3=^{G)=tjBtEPe>TO|0ddVB zTklrSHiW+!#26frPXQQ(YN8DG$PZo?(po(QUCCf_OJC`pw*uey00%gmH!`WJkrKXj2!#6?`T25mTu9OJp2L8z3! z=arrL$ZqxuE{%yV)14Kd>k}j7pxZ6#$Dz8$@WV5p8kTqN<-7W)Q7Gt2{KoOPK_tZ| zf2WG~O5@{qPI+W<4f_;reuFVdO^5`ADC1!JQE|N`s3cq@(0WB!n0uh@*c{=LAd;~} zyGK@hbF-Oo+!nN)@i*O(`@FA#u?o=~e{`4O#5}z&=UkU*50fOrzi11D^&FOqe>wii z?*k+2|EcUs;Gx{!@KBT~>PAwLrIDT7Th=Utu?~?np@t^gFs?zgX=D${RwOY^WGh-+ z+#4$066ISh8eYW#FXWp~S`<*%O^ZuItL1Tyqt8#tZ zY120E;^VG`!lZn&3sPd$RkdHpU#|w+bYV)pJC|SH9g%|5IkxVTQcBA4CL0}$&}ef@ zW^Vtj%M;;_1xxP9x#ex17&4N*{ksO*_4O}xYu(p*JkL#yr}@7b)t5X?%CY<+s5_MJ zuiqt+N_;A(_)%lumoyRFixWa-M7qK_9s6<1X?JDa9fP!+_6u~~M$5L=ipB=7(j#f< zZ34J%=bs549%~_mA(|={uZNs_0?o7;-LBP(ZRnkd{-^|2|=4vUTmtByHL8 zEph`(LSEzQj68a+`d$V<45J7cyv^#|^|%fD#si1Nx!4NW*`l*{->HEWNh6-|g>-=r zXmQ|-i}Ku$ndUeHQ^&ieT!Lf}vf6GaqW9$DJ2NWrqwPY%%4nip$@vK$nRp*_C-v<| zuKz~ZyN&<%!NS26&x?jhy+@awJipMQ-8(X4#Ae5??U<1QMt1l9R=w9fAnEF}NYu$2 z>6}Vkc zIb*A?G*z8^IvibmBKn_u^5&T_1oey0gZS2~obf(#xk=erZGTEdQnt3DMGM+0oPwss zj5zXD;(oWhB_T@~Ig#9@v)AKtXu3>Inmgf@A|-lD-1U>cNyl3h?ADD9)GG4}zUGPk zZzaXe!~Kf?<~@$G?Uql3t8jy9{2!doq4=J}j9ktTxss{p6!9UdjyDERlA*xZ!=Q)KDs5O)phz>Vq3BNGoM(H|=1*Q4$^2fTZw z(%nq1P|5Rt81}SYJpEEzMPl5VJsV5&4e)ZWKDyoZ>1EwpkHx-AQVQc8%JMz;{H~p{=FXV>jIxvm4X*qv52e?Y-f%DJ zxEA165GikEASQ^fH6K#d!Tpu2HP{sFs%E=e$gYd$aj$+xue6N+Wc(rAz~wUsk2`(b z8Kvmyz%bKQxpP}~baG-rwYcYCvkHOi zlkR<=>ZBTU*8RF_d#Bl@zZsRIhx<%~Z@Z=ik z>adw3!DK(8R|q$vy{FTxw%#xliD~6qXmY^7_9kthVPTF~Xy1CfBqbU~?1QmxmU=+k z(ggxvEuA;0e&+ci-zQR{-f7aO{O(Pz_OsEjLh_K>MbvoZ4nxtk5u{g@nPv)cgW_R} z9}EA4K4@z0?7ue}Z(o~R(X&FjejUI2g~08PH1E4w>9o{)S(?1>Z0XMvTb|;&EuyOE zGvWNpYX)Nv<8|a^;1>bh#&znEcl-r!T#pn= z4$?Yudha6F%4b>*8@=BdtXXY4N+`U4Dmx$}>HeVJk-QdTG@t!tVT#0(LeV0gvqyyw z2sEp^9eY0N`u10Tm4n8No&A=)IeEC|gnmEXoNSzu!1<4R<%-9kY_8~5Ej?zRegMn78wuMs#;i&eUA0Zk_RXQ3b&TT} z;SCI=7-FUB@*&;8|n>(_g^HGf3@QODE3LpmX~ELnymQm{Sx9xrKS zK29p~?v@R$0=v6Dr5aW>-!{+h@?Q58|Kz8{{W`%J+lDAdb&M5VHrX_mDY;1-JLnf)ezmPau$)1;=`-FU=-r-83tX=C`S#}GZufju zQ>sXNT0Ny=k@nc%cFnvA_i4SC)?_ORXHq8B4D%el1uPX`c~uG#S1M7C+*MMqLw78E zhY2dI8@+N^qrMI1+;TUda(vGqGSRyU{Fnm`aqrr7bz42c5xsOO-~oZpkzorD1g}Y<6rk&3>PsSGy}W?MtqFky@A(X# zIuNZK0cK?^=;PUAu>j0#HtjbHCV*6?jzA&OoE$*Jlga*}LF`SF?WLhv1O|zqC<>*> zYB;#lsYKx0&kH@BFpW8n*yDcc6?;_zaJs<-jPSkCsSX-!aV=P5kUgF@Nu<{a%#K*F z134Q{9|YX7X(v$62_cY3^G%t~rD>Q0z@)1|zs)vjJ6Jq9;7#Ki`w+eS**En?7;n&7 zu==V3T&eFboN3ZiMx3D8qYc;VjFUk_H-WWCau(VFXSQf~viH0L$gwD$UfFHqNcgN`x}M+YQ6RnN<+@t>JUp#)9YOkqst-Ga?{FsDpEeX0(5v{0J~SEbWiL zXC2}M4?UH@u&|;%0y`eb33ldo4~z-x8zY!oVmV=c+f$m?RfDC35mdQ2E>Pze7KWP- z>!Bh<&57I+O_^s}9Tg^k)h7{xx@0a0IA~GAOt2yy!X%Q$1rt~LbTB6@Du!_0%HV>N zlf)QI1&gvERKwso23mJ!Ou6ZS#zCS5W`gxE5T>C#E|{i<1D35C222I33?Njaz`On7 zi<+VWFP6D{e-{yiN#M|Jgk<44u1TiMI78S5W`Sdb5f+{zu34s{CfWN7a3Cf^@L%!& zN$?|!!9j2c)j$~+R6n#891w-z8(!oBpL2K=+%a$r2|~8-(vQj5_XT`<0Ksf;oP+tz z9CObS!0m)Tgg`K#xBM8B(|Z)Wb&DYL{WTYv`;A=q6~Nnx2+!lTIXtj8J7dZE!P_{z z#f8w6F}^!?^KE#+ZDv+xd5O&3EmomZzsv?>E-~ygGum45fk!SBN&|eo1rKw^?aZJ4 E2O(~oYXATM literal 0 HcmV?d00001 diff --git a/android/SherpaOnnxAar/gradle/wrapper/gradle-wrapper.properties b/android/SherpaOnnxAar/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..a8b356358 --- /dev/null +++ b/android/SherpaOnnxAar/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Thu Dec 12 14:02:30 CST 2024 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/android/SherpaOnnxAar/gradlew b/android/SherpaOnnxAar/gradlew new file mode 100755 index 000000000..4f906e0c8 --- /dev/null +++ b/android/SherpaOnnxAar/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/android/SherpaOnnxAar/gradlew.bat b/android/SherpaOnnxAar/gradlew.bat new file mode 100644 index 000000000..ac1b06f93 --- /dev/null +++ b/android/SherpaOnnxAar/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/android/SherpaOnnxAar/settings.gradle.kts b/android/SherpaOnnxAar/settings.gradle.kts new file mode 100644 index 000000000..53ee52b54 --- /dev/null +++ b/android/SherpaOnnxAar/settings.gradle.kts @@ -0,0 +1,23 @@ +pluginManagement { + repositories { + google { + content { + includeGroupByRegex("com\\.android.*") + includeGroupByRegex("com\\.google.*") + includeGroupByRegex("androidx.*") + } + } + mavenCentral() + gradlePluginPortal() + } +} +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { + google() + mavenCentral() + } +} + +rootProject.name = "SherpaOnnxAar" +include(":sherpa_onnx") diff --git a/android/SherpaOnnxAar/sherpa_onnx/.gitignore b/android/SherpaOnnxAar/sherpa_onnx/.gitignore new file mode 100644 index 000000000..42afabfd2 --- /dev/null +++ b/android/SherpaOnnxAar/sherpa_onnx/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/android/SherpaOnnxAar/sherpa_onnx/build.gradle.kts b/android/SherpaOnnxAar/sherpa_onnx/build.gradle.kts new file mode 100644 index 000000000..4803cb837 --- /dev/null +++ b/android/SherpaOnnxAar/sherpa_onnx/build.gradle.kts @@ -0,0 +1,43 @@ +plugins { + alias(libs.plugins.android.library) + alias(libs.plugins.jetbrains.kotlin.android) +} + +android { + namespace = "com.k2fsa.sherpa.onnx" + compileSdk = 34 + + defaultConfig { + minSdk = 21 + + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + consumerProguardFiles("consumer-rules.pro") + } + + buildTypes { + release { + isMinifyEnabled = false + proguardFiles( + getDefaultProguardFile("proguard-android-optimize.txt"), + "proguard-rules.pro" + ) + } + } + compileOptions { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + } + kotlinOptions { + jvmTarget = "1.8" + } +} + +dependencies { + + implementation(libs.androidx.core.ktx) + implementation(libs.androidx.appcompat) + implementation(libs.material) + testImplementation(libs.junit) + androidTestImplementation(libs.androidx.junit) + androidTestImplementation(libs.androidx.espresso.core) +} \ No newline at end of file diff --git a/android/SherpaOnnxAar/sherpa_onnx/consumer-rules.pro b/android/SherpaOnnxAar/sherpa_onnx/consumer-rules.pro new file mode 100644 index 000000000..e69de29bb diff --git a/android/SherpaOnnxAar/sherpa_onnx/proguard-rules.pro b/android/SherpaOnnxAar/sherpa_onnx/proguard-rules.pro new file mode 100644 index 000000000..481bb4348 --- /dev/null +++ b/android/SherpaOnnxAar/sherpa_onnx/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile \ No newline at end of file diff --git a/android/SherpaOnnxAar/sherpa_onnx/src/androidTest/java/com/k2fsa/sherpa/onnx/ExampleInstrumentedTest.kt b/android/SherpaOnnxAar/sherpa_onnx/src/androidTest/java/com/k2fsa/sherpa/onnx/ExampleInstrumentedTest.kt new file mode 100644 index 000000000..db1fbefc3 --- /dev/null +++ b/android/SherpaOnnxAar/sherpa_onnx/src/androidTest/java/com/k2fsa/sherpa/onnx/ExampleInstrumentedTest.kt @@ -0,0 +1,24 @@ +package com.k2fsa.sherpa.onnx + +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.ext.junit.runners.AndroidJUnit4 + +import org.junit.Test +import org.junit.runner.RunWith + +import org.junit.Assert.* + +/** + * Instrumented test, which will execute on an Android device. + * + * See [testing documentation](http://d.android.com/tools/testing). + */ +@RunWith(AndroidJUnit4::class) +class ExampleInstrumentedTest { + @Test + fun useAppContext() { + // Context of the app under test. + val appContext = InstrumentationRegistry.getInstrumentation().targetContext + assertEquals("com.k2fsa.sherpa.onnx.test", appContext.packageName) + } +} \ No newline at end of file diff --git a/android/SherpaOnnxAar/sherpa_onnx/src/main/AndroidManifest.xml b/android/SherpaOnnxAar/sherpa_onnx/src/main/AndroidManifest.xml new file mode 100644 index 000000000..a5918e68a --- /dev/null +++ b/android/SherpaOnnxAar/sherpa_onnx/src/main/AndroidManifest.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/AudioTagging.kt b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/AudioTagging.kt new file mode 120000 index 000000000..25c36e396 --- /dev/null +++ b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/AudioTagging.kt @@ -0,0 +1 @@ +../../../../../../../../../../sherpa-onnx/kotlin-api/AudioTagging.kt \ No newline at end of file diff --git a/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.kt b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.kt new file mode 120000 index 000000000..952fae878 --- /dev/null +++ b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.kt @@ -0,0 +1 @@ +../../../../../../../../../../sherpa-onnx/kotlin-api/FeatureConfig.kt \ No newline at end of file diff --git a/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotter.kt b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotter.kt new file mode 120000 index 000000000..4392376a1 --- /dev/null +++ b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotter.kt @@ -0,0 +1 @@ +../../../../../../../../../../sherpa-onnx/kotlin-api/KeywordSpotter.kt \ No newline at end of file diff --git a/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OfflinePunctuation.kt b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OfflinePunctuation.kt new file mode 120000 index 000000000..1eed71678 --- /dev/null +++ b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OfflinePunctuation.kt @@ -0,0 +1 @@ +../../../../../../../../../../sherpa-onnx/kotlin-api/OfflinePunctuation.kt \ No newline at end of file diff --git a/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.kt b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.kt new file mode 120000 index 000000000..faa3ab4ac --- /dev/null +++ b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.kt @@ -0,0 +1 @@ +../../../../../../../../../../sherpa-onnx/kotlin-api/OfflineRecognizer.kt \ No newline at end of file diff --git a/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarization.kt b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarization.kt new file mode 120000 index 000000000..d850dd7fd --- /dev/null +++ b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarization.kt @@ -0,0 +1 @@ +../../../../../../../../../../sherpa-onnx/kotlin-api/OfflineSpeakerDiarization.kt \ No newline at end of file diff --git a/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OfflineStream.kt b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OfflineStream.kt new file mode 120000 index 000000000..2a3aff864 --- /dev/null +++ b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OfflineStream.kt @@ -0,0 +1 @@ +../../../../../../../../../../sherpa-onnx/kotlin-api/OfflineStream.kt \ No newline at end of file diff --git a/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.kt b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.kt new file mode 120000 index 000000000..5bb19ee10 --- /dev/null +++ b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.kt @@ -0,0 +1 @@ +../../../../../../../../../../sherpa-onnx/kotlin-api/OnlineRecognizer.kt \ No newline at end of file diff --git a/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.kt b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.kt new file mode 120000 index 000000000..d4518b89b --- /dev/null +++ b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.kt @@ -0,0 +1 @@ +../../../../../../../../../../sherpa-onnx/kotlin-api/OnlineStream.kt \ No newline at end of file diff --git a/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/Speaker.kt b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/Speaker.kt new file mode 120000 index 000000000..66441dea7 --- /dev/null +++ b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/Speaker.kt @@ -0,0 +1 @@ +../../../../../../../../../../sherpa-onnx/kotlin-api/Speaker.kt \ No newline at end of file diff --git a/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractorConfig.kt b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractorConfig.kt new file mode 120000 index 000000000..754102447 --- /dev/null +++ b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractorConfig.kt @@ -0,0 +1 @@ +../../../../../../../../../../sherpa-onnx/kotlin-api/SpeakerEmbeddingExtractorConfig.kt \ No newline at end of file diff --git a/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/SpokenLanguageIdentification.kt b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/SpokenLanguageIdentification.kt new file mode 120000 index 000000000..de79a7d20 --- /dev/null +++ b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/SpokenLanguageIdentification.kt @@ -0,0 +1 @@ +../../../../../../../../../../sherpa-onnx/kotlin-api/SpokenLanguageIdentification.kt \ No newline at end of file diff --git a/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/Tts.kt b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/Tts.kt new file mode 120000 index 000000000..f1392e771 --- /dev/null +++ b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/Tts.kt @@ -0,0 +1 @@ +../../../../../../../../../../sherpa-onnx/kotlin-api/Tts.kt \ No newline at end of file diff --git a/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/Vad.kt b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/Vad.kt new file mode 120000 index 000000000..761b158ce --- /dev/null +++ b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/Vad.kt @@ -0,0 +1 @@ +../../../../../../../../../../sherpa-onnx/kotlin-api/Vad.kt \ No newline at end of file diff --git a/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/WaveReader.kt b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/WaveReader.kt new file mode 120000 index 000000000..05c8fb246 --- /dev/null +++ b/android/SherpaOnnxAar/sherpa_onnx/src/main/java/com/k2fsa/sherpa/onnx/WaveReader.kt @@ -0,0 +1 @@ +../../../../../../../../../../sherpa-onnx/kotlin-api/WaveReader.kt \ No newline at end of file diff --git a/android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/arm64-v8a/.gitkeep b/android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/arm64-v8a/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/armeabi-v7a/.gitkeep b/android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/armeabi-v7a/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/x86/.gitkeep b/android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/x86/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/x86_64/.gitkeep b/android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/x86_64/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/android/SherpaOnnxAar/sherpa_onnx/src/test/java/com/k2fsa/sherpa/onnx/ExampleUnitTest.kt b/android/SherpaOnnxAar/sherpa_onnx/src/test/java/com/k2fsa/sherpa/onnx/ExampleUnitTest.kt new file mode 100644 index 000000000..05dfcd635 --- /dev/null +++ b/android/SherpaOnnxAar/sherpa_onnx/src/test/java/com/k2fsa/sherpa/onnx/ExampleUnitTest.kt @@ -0,0 +1,17 @@ +package com.k2fsa.sherpa.onnx + +import org.junit.Test + +import org.junit.Assert.* + +/** + * Example local unit test, which will execute on the development machine (host). + * + * See [testing documentation](http://d.android.com/tools/testing). + */ +class ExampleUnitTest { + @Test + fun addition_isCorrect() { + assertEquals(4, 2 + 2) + } +} \ No newline at end of file diff --git a/android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/Tts.kt b/android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/Tts.kt deleted file mode 100644 index 4f9c4b6f6..000000000 --- a/android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/Tts.kt +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright (c) 2023 Xiaomi Corporation -package com.k2fsa.sherpa.onnx - -import android.content.res.AssetManager - -data class OfflineTtsVitsModelConfig( - var model: String, - var lexicon: String = "", - var tokens: String, - var dataDir: String = "", - var dictDir: String = "", - var noiseScale: Float = 0.667f, - var noiseScaleW: Float = 0.8f, - var lengthScale: Float = 1.0f, -) - -data class OfflineTtsModelConfig( - var vits: OfflineTtsVitsModelConfig, - var numThreads: Int = 1, - var debug: Boolean = false, - var provider: String = "cpu", -) - -data class OfflineTtsConfig( - var model: OfflineTtsModelConfig, - var ruleFsts: String = "", - var ruleFars: String = "", - var maxNumSentences: Int = 1, -) - -class GeneratedAudio( - val samples: FloatArray, - val sampleRate: Int, -) { - fun save(filename: String) = - saveImpl(filename = filename, samples = samples, sampleRate = sampleRate) - - private external fun saveImpl( - filename: String, - samples: FloatArray, - sampleRate: Int - ): Boolean -} - -class OfflineTts( - assetManager: AssetManager? = null, - var config: OfflineTtsConfig, -) { - private var ptr: Long - - init { - ptr = if (assetManager != null) { - newFromAsset(assetManager, config) - } else { - newFromFile(config) - } - } - - fun sampleRate() = getSampleRate(ptr) - - fun numSpeakers() = getNumSpeakers(ptr) - - fun generate( - text: String, - sid: Int = 0, - speed: Float = 1.0f - ): GeneratedAudio { - val objArray = generateImpl(ptr, text = text, sid = sid, speed = speed) - return GeneratedAudio( - samples = objArray[0] as FloatArray, - sampleRate = objArray[1] as Int - ) - } - - fun generateWithCallback( - text: String, - sid: Int = 0, - speed: Float = 1.0f, - callback: (samples: FloatArray) -> Int - ): GeneratedAudio { - val objArray = generateWithCallbackImpl( - ptr, - text = text, - sid = sid, - speed = speed, - callback = callback - ) - return GeneratedAudio( - samples = objArray[0] as FloatArray, - sampleRate = objArray[1] as Int - ) - } - - fun allocate(assetManager: AssetManager? = null) { - if (ptr == 0L) { - ptr = if (assetManager != null) { - newFromAsset(assetManager, config) - } else { - newFromFile(config) - } - } - } - - fun free() { - if (ptr != 0L) { - delete(ptr) - ptr = 0 - } - } - - protected fun finalize() { - if (ptr != 0L) { - delete(ptr) - ptr = 0 - } - } - - fun release() = finalize() - - private external fun newFromAsset( - assetManager: AssetManager, - config: OfflineTtsConfig, - ): Long - - private external fun newFromFile( - config: OfflineTtsConfig, - ): Long - - private external fun delete(ptr: Long) - private external fun getSampleRate(ptr: Long): Int - private external fun getNumSpeakers(ptr: Long): Int - - // The returned array has two entries: - // - the first entry is an 1-D float array containing audio samples. - // Each sample is normalized to the range [-1, 1] - // - the second entry is the sample rate - private external fun generateImpl( - ptr: Long, - text: String, - sid: Int = 0, - speed: Float = 1.0f - ): Array - - private external fun generateWithCallbackImpl( - ptr: Long, - text: String, - sid: Int = 0, - speed: Float = 1.0f, - callback: (samples: FloatArray) -> Int - ): Array - - companion object { - init { - System.loadLibrary("sherpa-onnx-jni") - } - } -} - -// please refer to -// https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/index.html -// to download models -fun getOfflineTtsConfig( - modelDir: String, - modelName: String, - lexicon: String, - dataDir: String, - dictDir: String, - ruleFsts: String, - ruleFars: String -): OfflineTtsConfig { - return OfflineTtsConfig( - model = OfflineTtsModelConfig( - vits = OfflineTtsVitsModelConfig( - model = "$modelDir/$modelName", - lexicon = "$modelDir/$lexicon", - tokens = "$modelDir/tokens.txt", - dataDir = dataDir, - dictDir = dictDir, - ), - numThreads = 2, - debug = true, - provider = "cpu", - ), - ruleFsts = ruleFsts, - ruleFars = ruleFars, - ) -} diff --git a/android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/Tts.kt b/android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/Tts.kt new file mode 120000 index 000000000..f1392e771 --- /dev/null +++ b/android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/Tts.kt @@ -0,0 +1 @@ +../../../../../../../../../../sherpa-onnx/kotlin-api/Tts.kt \ No newline at end of file diff --git a/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/Tts.kt b/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/Tts.kt index bc6a22c57..b510f97d3 120000 --- a/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/Tts.kt +++ b/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/Tts.kt @@ -1 +1 @@ -../../../../../../../../../../../SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/Tts.kt \ No newline at end of file +../../../../../../../../../../../../sherpa-onnx/kotlin-api/Tts.kt \ No newline at end of file diff --git a/jitpack.yml b/jitpack.yml new file mode 100644 index 000000000..e2936e496 --- /dev/null +++ b/jitpack.yml @@ -0,0 +1,9 @@ +jdk: + - openjdk17 + +before_install: + - wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.34/sherpa-onnx-1.10.34.aar + +install: + - FILE="-Dfile=sherpa-onnx-1.10.34.aar" + - mvn install:install-file $FILE -DgroupId=com.k2fsa.sherpa.onnx -DartifactId=sherpa-onnx -Dversion=1.10.34 -Dpackaging=aar -DgeneratePom=true diff --git a/kotlin-api-examples/Tts.kt b/kotlin-api-examples/Tts.kt index a89bc93cb..02becdb69 120000 --- a/kotlin-api-examples/Tts.kt +++ b/kotlin-api-examples/Tts.kt @@ -1 +1 @@ -../android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/Tts.kt \ No newline at end of file +../sherpa-onnx/kotlin-api/Tts.kt \ No newline at end of file diff --git a/new-release.sh b/new-release.sh index 82e2fb36e..a305d6a66 100755 --- a/new-release.sh +++ b/new-release.sh @@ -3,6 +3,10 @@ set -ex sed -i.bak 's/1\.10\.33/1\.10\.34/g' ./build-ios-shared.sh +sed -i.bak 's/1\.10\.33/1\.10\.34/g' ./pom.xml +sed -i.bak 's/1\.10\.33/1\.10\.34/g' ./jitpack.yml +sed -i.bak 's/1\.10\.33/1\.10\.34/g' ./android/SherpaOnnxAar/README.md + find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.33/1\.10\.34/g' {} \; find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.33/1\.10\.34/g' {} \; find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.33/1\.10\.34/g' {} \; diff --git a/pom.xml b/pom.xml new file mode 100644 index 000000000..ad8756914 --- /dev/null +++ b/pom.xml @@ -0,0 +1,19 @@ + + + 4.0.0 + com.k2fsa.sherpa.onnx + sherpa-onnx-android + 1.10.34 + https://github.com/k2-fsa/sherpa-onnx + pom + First Android Library + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + diff --git a/sherpa-onnx/kotlin-api/Tts.kt b/sherpa-onnx/kotlin-api/Tts.kt new file mode 100644 index 000000000..4f9c4b6f6 --- /dev/null +++ b/sherpa-onnx/kotlin-api/Tts.kt @@ -0,0 +1,187 @@ +// Copyright (c) 2023 Xiaomi Corporation +package com.k2fsa.sherpa.onnx + +import android.content.res.AssetManager + +data class OfflineTtsVitsModelConfig( + var model: String, + var lexicon: String = "", + var tokens: String, + var dataDir: String = "", + var dictDir: String = "", + var noiseScale: Float = 0.667f, + var noiseScaleW: Float = 0.8f, + var lengthScale: Float = 1.0f, +) + +data class OfflineTtsModelConfig( + var vits: OfflineTtsVitsModelConfig, + var numThreads: Int = 1, + var debug: Boolean = false, + var provider: String = "cpu", +) + +data class OfflineTtsConfig( + var model: OfflineTtsModelConfig, + var ruleFsts: String = "", + var ruleFars: String = "", + var maxNumSentences: Int = 1, +) + +class GeneratedAudio( + val samples: FloatArray, + val sampleRate: Int, +) { + fun save(filename: String) = + saveImpl(filename = filename, samples = samples, sampleRate = sampleRate) + + private external fun saveImpl( + filename: String, + samples: FloatArray, + sampleRate: Int + ): Boolean +} + +class OfflineTts( + assetManager: AssetManager? = null, + var config: OfflineTtsConfig, +) { + private var ptr: Long + + init { + ptr = if (assetManager != null) { + newFromAsset(assetManager, config) + } else { + newFromFile(config) + } + } + + fun sampleRate() = getSampleRate(ptr) + + fun numSpeakers() = getNumSpeakers(ptr) + + fun generate( + text: String, + sid: Int = 0, + speed: Float = 1.0f + ): GeneratedAudio { + val objArray = generateImpl(ptr, text = text, sid = sid, speed = speed) + return GeneratedAudio( + samples = objArray[0] as FloatArray, + sampleRate = objArray[1] as Int + ) + } + + fun generateWithCallback( + text: String, + sid: Int = 0, + speed: Float = 1.0f, + callback: (samples: FloatArray) -> Int + ): GeneratedAudio { + val objArray = generateWithCallbackImpl( + ptr, + text = text, + sid = sid, + speed = speed, + callback = callback + ) + return GeneratedAudio( + samples = objArray[0] as FloatArray, + sampleRate = objArray[1] as Int + ) + } + + fun allocate(assetManager: AssetManager? = null) { + if (ptr == 0L) { + ptr = if (assetManager != null) { + newFromAsset(assetManager, config) + } else { + newFromFile(config) + } + } + } + + fun free() { + if (ptr != 0L) { + delete(ptr) + ptr = 0 + } + } + + protected fun finalize() { + if (ptr != 0L) { + delete(ptr) + ptr = 0 + } + } + + fun release() = finalize() + + private external fun newFromAsset( + assetManager: AssetManager, + config: OfflineTtsConfig, + ): Long + + private external fun newFromFile( + config: OfflineTtsConfig, + ): Long + + private external fun delete(ptr: Long) + private external fun getSampleRate(ptr: Long): Int + private external fun getNumSpeakers(ptr: Long): Int + + // The returned array has two entries: + // - the first entry is an 1-D float array containing audio samples. + // Each sample is normalized to the range [-1, 1] + // - the second entry is the sample rate + private external fun generateImpl( + ptr: Long, + text: String, + sid: Int = 0, + speed: Float = 1.0f + ): Array + + private external fun generateWithCallbackImpl( + ptr: Long, + text: String, + sid: Int = 0, + speed: Float = 1.0f, + callback: (samples: FloatArray) -> Int + ): Array + + companion object { + init { + System.loadLibrary("sherpa-onnx-jni") + } + } +} + +// please refer to +// https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/index.html +// to download models +fun getOfflineTtsConfig( + modelDir: String, + modelName: String, + lexicon: String, + dataDir: String, + dictDir: String, + ruleFsts: String, + ruleFars: String +): OfflineTtsConfig { + return OfflineTtsConfig( + model = OfflineTtsModelConfig( + vits = OfflineTtsVitsModelConfig( + model = "$modelDir/$modelName", + lexicon = "$modelDir/$lexicon", + tokens = "$modelDir/tokens.txt", + dataDir = dataDir, + dictDir = dictDir, + ), + numThreads = 2, + debug = true, + provider = "cpu", + ), + ruleFsts = ruleFsts, + ruleFars = ruleFars, + ) +} From be87f866f3320c863abb6072694506352b6f6fea Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 12 Dec 2024 18:26:54 +0800 Subject: [PATCH 103/183] Use aar in Android Java demo. (#1616) --- android/SherpaOnnxJavaDemo/README.md | 44 ++++++++++ android/SherpaOnnxJavaDemo/app/build.gradle | 13 ++- .../app/src/main/assets/.gitkeep | 0 .../k2fsa/sherpa/onnx/OnlineRecognizer.java | 84 ------------------- .../SpeechSherpaRecognitionService.java | 43 ++++++---- android/SherpaOnnxJavaDemo/settings.gradle | 2 +- android/SherpaOnnxJavaDemo/sherpa/.gitignore | 1 - .../SherpaOnnxJavaDemo/sherpa/build.gradle | 43 ---------- .../sherpa/proguard-rules.pro | 21 ----- .../sherpa/src/main/AndroidManifest.xml | 9 -- .../sherpa/src/main/java/com | 1 - .../sherpa/src/main/res/values/strings.xml | 3 - sherpa-onnx/kotlin-api/AudioTagging.kt | 4 +- sherpa-onnx/kotlin-api/KeywordSpotter.kt | 2 +- sherpa-onnx/kotlin-api/OfflinePunctuation.kt | 2 +- sherpa-onnx/kotlin-api/OfflineRecognizer.kt | 4 +- .../kotlin-api/OfflineSpeakerDiarization.kt | 10 +-- sherpa-onnx/kotlin-api/OnlineRecognizer.kt | 4 +- .../SpeakerEmbeddingExtractorConfig.kt | 2 +- .../SpokenLanguageIdentification.kt | 6 +- sherpa-onnx/kotlin-api/Tts.kt | 8 +- sherpa-onnx/kotlin-api/Vad.kt | 6 +- 22 files changed, 101 insertions(+), 211 deletions(-) create mode 100644 android/SherpaOnnxJavaDemo/README.md create mode 100644 android/SherpaOnnxJavaDemo/app/src/main/assets/.gitkeep delete mode 100644 android/SherpaOnnxJavaDemo/app/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.java delete mode 100644 android/SherpaOnnxJavaDemo/sherpa/.gitignore delete mode 100644 android/SherpaOnnxJavaDemo/sherpa/build.gradle delete mode 100644 android/SherpaOnnxJavaDemo/sherpa/proguard-rules.pro delete mode 100644 android/SherpaOnnxJavaDemo/sherpa/src/main/AndroidManifest.xml delete mode 120000 android/SherpaOnnxJavaDemo/sherpa/src/main/java/com delete mode 100644 android/SherpaOnnxJavaDemo/sherpa/src/main/res/values/strings.xml diff --git a/android/SherpaOnnxJavaDemo/README.md b/android/SherpaOnnxJavaDemo/README.md new file mode 100644 index 000000000..cf7d41557 --- /dev/null +++ b/android/SherpaOnnxJavaDemo/README.md @@ -0,0 +1,44 @@ +# Introduction + +Please run the following commands to download model files before you run this Android demo: + +```bash +# Assume we are inside +# /Users/fangjun/open-source/sherpa-onnx/android/SherpaOnnxJavaDemo + +cd app/src/main/assets/ +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 + +tar xvf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 +rm sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 + +mv sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/encoder-epoch-99-avg-1.int8.onnx ./ +mv sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/decoder-epoch-99-avg-1.onnx ./ +mv sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/joiner-epoch-99-avg-1.int8.onnx ./ +mv sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt ./ + +rm -rf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/* + +mv encoder-epoch-99-avg-1.int8.onnx ./ +mv decoder-epoch-99-avg-1.onnx ./ +mv joiner-epoch-99-avg-1.int8.onnx ./ +mv tokens.txt ./ +``` + +You should have the following directory structure: +``` +(py38) fangjuns-MacBook-Pro:assets fangjun$ pwd +/Users/fangjun/open-source/sherpa-onnx/android/SherpaOnnxJavaDemo/app/src/main/assets + +(py38) fangjuns-MacBook-Pro:assets fangjun$ tree . +. +└── sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 + ├── decoder-epoch-99-avg-1.onnx + ├── encoder-epoch-99-avg-1.int8.onnx + ├── joiner-epoch-99-avg-1.int8.onnx + └── tokens.txt + +1 directory, 4 files +``` + +Remember to remove unused files to reduce the file size of the final APK. diff --git a/android/SherpaOnnxJavaDemo/app/build.gradle b/android/SherpaOnnxJavaDemo/app/build.gradle index 500de341f..a2ee27997 100644 --- a/android/SherpaOnnxJavaDemo/app/build.gradle +++ b/android/SherpaOnnxJavaDemo/app/build.gradle @@ -8,7 +8,7 @@ android { defaultConfig { applicationId "com.k2fsa.sherpa.onnx" minSdk 28 - targetSdk 32 + targetSdk 34 versionCode 1 versionName "1.0" @@ -25,17 +25,14 @@ android { sourceCompatibility JavaVersion.VERSION_1_8 targetCompatibility JavaVersion.VERSION_1_8 } - sourceSets.main{ - jniLibs.srcDirs = ['jniLibs'] - } } dependencies { - implementation 'androidx.appcompat:appcompat:1.3.1' implementation 'com.google.android.material:material:1.3.0' implementation 'androidx.constraintlayout:constraintlayout:1.1.3' implementation 'pub.devrel:easypermissions:3.0.0' - implementation project(path: ':sherpa') - -} \ No newline at end of file + implementation 'androidx.core:core-ktx:1.7.0' + // implementation files('/Users/fangjun/open-source/sherpa-onnx/android/SherpaOnnxAar/sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar') + implementation 'com.github.k2-fsa:sherpa-onnx:master-SNAPSHOT' +} diff --git a/android/SherpaOnnxJavaDemo/app/src/main/assets/.gitkeep b/android/SherpaOnnxJavaDemo/app/src/main/assets/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/android/SherpaOnnxJavaDemo/app/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.java b/android/SherpaOnnxJavaDemo/app/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.java deleted file mode 100644 index 3f6faf338..000000000 --- a/android/SherpaOnnxJavaDemo/app/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.java +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2022-2023 by zhaoming -// Copyright 2024 Xiaomi Corporation - -package com.k2fsa.sherpa.onnx; - -import android.content.res.AssetManager; - -public class OnlineRecognizer { - static { - System.loadLibrary("sherpa-onnx-jni"); - } - - private long ptr = 0; - - public OnlineRecognizer(OnlineRecognizerConfig config) { - ptr = newFromFile(config); - } - - public OnlineRecognizer(AssetManager assetManager, OnlineRecognizerConfig config) { - ptr = newFromAsset(assetManager, config); - } - - public void decode(OnlineStream s) { - decode(ptr, s.getPtr()); - } - - public boolean isReady(OnlineStream s) { - return isReady(ptr, s.getPtr()); - } - - public boolean isEndpoint(OnlineStream s) { - return isEndpoint(ptr, s.getPtr()); - } - - public void reset(OnlineStream s) { - reset(ptr, s.getPtr()); - } - - public OnlineStream createStream() { - long p = createStream(ptr, ""); - return new OnlineStream(p); - } - - @Override - protected void finalize() throws Throwable { - release(); - } - - // You'd better call it manually if it is not used anymore - public void release() { - if (this.ptr == 0) { - return; - } - delete(this.ptr); - this.ptr = 0; - } - - public OnlineRecognizerResult getResult(OnlineStream s) { - Object[] arr = getResult(ptr, s.getPtr()); - String text = (String) arr[0]; - String[] tokens = (String[]) arr[1]; - float[] timestamps = (float[]) arr[2]; - return new OnlineRecognizerResult(text, tokens, timestamps); - } - - - private native void delete(long ptr); - - private native long newFromFile(OnlineRecognizerConfig config); - - private native long newFromAsset(AssetManager assetManager, OnlineRecognizerConfig config); - - private native long createStream(long ptr, String hotwords); - - private native void reset(long ptr, long streamPtr); - - private native void decode(long ptr, long streamPtr); - - private native boolean isEndpoint(long ptr, long streamPtr); - - private native boolean isReady(long ptr, long streamPtr); - - private native Object[] getResult(long ptr, long streamPtr); -} \ No newline at end of file diff --git a/android/SherpaOnnxJavaDemo/app/src/main/java/com/k2fsa/sherpa/onnx/service/SpeechSherpaRecognitionService.java b/android/SherpaOnnxJavaDemo/app/src/main/java/com/k2fsa/sherpa/onnx/service/SpeechSherpaRecognitionService.java index 9326ffdeb..02ad4a15d 100644 --- a/android/SherpaOnnxJavaDemo/app/src/main/java/com/k2fsa/sherpa/onnx/service/SpeechSherpaRecognitionService.java +++ b/android/SherpaOnnxJavaDemo/app/src/main/java/com/k2fsa/sherpa/onnx/service/SpeechSherpaRecognitionService.java @@ -1,6 +1,7 @@ package com.k2fsa.sherpa.onnx.service; import android.Manifest; +import android.annotation.SuppressLint; import android.app.Notification; import android.app.NotificationChannel; import android.app.NotificationManager; @@ -67,6 +68,16 @@ public void onCreate() { appViewModel = Application.getInstance().getViewModel(); int numBytes = AudioRecord.getMinBufferSize(sampleRateInHz, channelConfig, audioFormat); + if (ActivityCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) { + // TODO: Consider calling + // ActivityCompat#requestPermissions + // here to request the missing permissions, and then overriding + // public void onRequestPermissionsResult(int requestCode, String[] permissions, + // int[] grantResults) + // to handle the case where the user grants the permission. See the documentation + // for ActivityCompat#requestPermissions for more details. + return; + } audioRecord = new AudioRecord( audioSource, sampleRateInHz, @@ -81,22 +92,21 @@ public void onCreate() { private void initializeSherpa() { Log.d("Current Directory", System.getProperty("user.dir")); - String modelDir = "sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23"; + String modelDir = "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20"; initializeSherpaDir(modelDir, modelDir); - OnlineTransducerModelConfig onlineTransducerModelConfig = OnlineTransducerModelConfig.builder() - .setEncoder(modelDir + "/encoder-epoch-99-avg-1.int8.onnx") - .setDecoder(modelDir + "/decoder-epoch-99-avg-1.onnx") - .setJoiner(modelDir + "/joiner-epoch-99-avg-1.int8.onnx") - .build(); - - OnlineModelConfig onlineModelConfig = OnlineModelConfig.builder() - .setTransducer(onlineTransducerModelConfig) - .setTokens(modelDir + "/tokens.txt") - .setModelType("zipformer") - .build(); - OnlineRecognizerConfig config = OnlineRecognizerConfig.builder() - .setOnlineModelConfig(onlineModelConfig) - .build(); + OnlineTransducerModelConfig onlineTransducerModelConfig = new OnlineTransducerModelConfig(); + onlineTransducerModelConfig.setEncoder(modelDir + "/encoder-epoch-99-avg-1.int8.onnx"); + onlineTransducerModelConfig.setDecoder(modelDir + "/decoder-epoch-99-avg-1.onnx"); + onlineTransducerModelConfig.setJoiner(modelDir + "/joiner-epoch-99-avg-1.int8.onnx"); + + OnlineModelConfig onlineModelConfig = new OnlineModelConfig(); + onlineModelConfig.setTransducer(onlineTransducerModelConfig); + onlineModelConfig.setTokens(modelDir + "/tokens.txt"); + onlineModelConfig.setModelType("zipformer"); + onlineModelConfig.setDebug(true); + + OnlineRecognizerConfig config = new OnlineRecognizerConfig(); + config.setModelConfig(onlineModelConfig); recognizer = new OnlineRecognizer(getAssets(), config); audioRecord.startRecording(); @@ -110,7 +120,7 @@ private void startRecognition() { } private void processSamples() { - OnlineStream stream = recognizer.createStream(); + OnlineStream stream = recognizer.createStream(""); double interval = 0.1; int bufferSize = (int) (interval * sampleRateInHz); short[] buffer = new short[bufferSize]; @@ -182,6 +192,7 @@ public IBinder onBind(Intent intent) { } + @SuppressLint("ForegroundServiceType") private void startForegroundService() { String channelId = createNotificationChannel(); diff --git a/android/SherpaOnnxJavaDemo/settings.gradle b/android/SherpaOnnxJavaDemo/settings.gradle index a1c973b67..e552eb689 100644 --- a/android/SherpaOnnxJavaDemo/settings.gradle +++ b/android/SherpaOnnxJavaDemo/settings.gradle @@ -10,8 +10,8 @@ dependencyResolutionManagement { repositories { google() mavenCentral() + maven { url 'https://jitpack.io' } } } rootProject.name = "SherpaOnnxJavaDemo" include ':app' -include ':sherpa' diff --git a/android/SherpaOnnxJavaDemo/sherpa/.gitignore b/android/SherpaOnnxJavaDemo/sherpa/.gitignore deleted file mode 100644 index 42afabfd2..000000000 --- a/android/SherpaOnnxJavaDemo/sherpa/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/build \ No newline at end of file diff --git a/android/SherpaOnnxJavaDemo/sherpa/build.gradle b/android/SherpaOnnxJavaDemo/sherpa/build.gradle deleted file mode 100644 index a171d3e45..000000000 --- a/android/SherpaOnnxJavaDemo/sherpa/build.gradle +++ /dev/null @@ -1,43 +0,0 @@ -plugins { - id 'com.android.library' -} - -android { - namespace 'com.k2fsa.sherpa' - compileSdk 34 - - defaultConfig { - minSdk 26 - targetSdk 27 - versionCode 1 - versionName "1.0" - missingDimensionStrategy 'base', 'feature1' - testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" - } - - buildTypes { - release { - minifyEnabled false - proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' - } - } - compileOptions { - sourceCompatibility JavaVersion.VERSION_1_8 - targetCompatibility JavaVersion.VERSION_1_8 - } -} - -dependencies { - -// implementation "androidx.appcompat" -// implementation libs.material -// testImplementation libs.junit -// androidTestImplementation libs.androidx.test.ext.junit -// androidTestImplementation libs.espresso.core - implementation 'androidx.appcompat:appcompat:1.6.1' - implementation 'com.google.android.material:material:1.9.0' - testImplementation 'junit:junit:4.13.2' - androidTestImplementation 'androidx.test.ext:junit:1.1.5' - androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1' - -} \ No newline at end of file diff --git a/android/SherpaOnnxJavaDemo/sherpa/proguard-rules.pro b/android/SherpaOnnxJavaDemo/sherpa/proguard-rules.pro deleted file mode 100644 index 481bb4348..000000000 --- a/android/SherpaOnnxJavaDemo/sherpa/proguard-rules.pro +++ /dev/null @@ -1,21 +0,0 @@ -# Add project specific ProGuard rules here. -# You can control the set of applied configuration files using the -# proguardFiles setting in build.gradle. -# -# For more details, see -# http://developer.android.com/guide/developing/tools/proguard.html - -# If your project uses WebView with JS, uncomment the following -# and specify the fully qualified class name to the JavaScript interface -# class: -#-keepclassmembers class fqcn.of.javascript.interface.for.webview { -# public *; -#} - -# Uncomment this to preserve the line number information for -# debugging stack traces. -#-keepattributes SourceFile,LineNumberTable - -# If you keep the line number information, uncomment this to -# hide the original source file name. -#-renamesourcefileattribute SourceFile \ No newline at end of file diff --git a/android/SherpaOnnxJavaDemo/sherpa/src/main/AndroidManifest.xml b/android/SherpaOnnxJavaDemo/sherpa/src/main/AndroidManifest.xml deleted file mode 100644 index 1aab3af56..000000000 --- a/android/SherpaOnnxJavaDemo/sherpa/src/main/AndroidManifest.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/android/SherpaOnnxJavaDemo/sherpa/src/main/java/com b/android/SherpaOnnxJavaDemo/sherpa/src/main/java/com deleted file mode 120000 index 4b54cb296..000000000 --- a/android/SherpaOnnxJavaDemo/sherpa/src/main/java/com +++ /dev/null @@ -1 +0,0 @@ -../../../../../../../sherpa-onnx/sherpa-onnx/java-api/src/com \ No newline at end of file diff --git a/android/SherpaOnnxJavaDemo/sherpa/src/main/res/values/strings.xml b/android/SherpaOnnxJavaDemo/sherpa/src/main/res/values/strings.xml deleted file mode 100644 index 7c5b69c5f..000000000 --- a/android/SherpaOnnxJavaDemo/sherpa/src/main/res/values/strings.xml +++ /dev/null @@ -1,3 +0,0 @@ - - sherpa - \ No newline at end of file diff --git a/sherpa-onnx/kotlin-api/AudioTagging.kt b/sherpa-onnx/kotlin-api/AudioTagging.kt index 9b241ee56..beec31e6a 100644 --- a/sherpa-onnx/kotlin-api/AudioTagging.kt +++ b/sherpa-onnx/kotlin-api/AudioTagging.kt @@ -15,8 +15,8 @@ data class AudioTaggingModelConfig( ) data class AudioTaggingConfig( - var model: AudioTaggingModelConfig, - var labels: String, + var model: AudioTaggingModelConfig = AudioTaggingModelConfig(), + var labels: String = "", var topK: Int = 5, ) diff --git a/sherpa-onnx/kotlin-api/KeywordSpotter.kt b/sherpa-onnx/kotlin-api/KeywordSpotter.kt index ea2143613..3801d32a8 100644 --- a/sherpa-onnx/kotlin-api/KeywordSpotter.kt +++ b/sherpa-onnx/kotlin-api/KeywordSpotter.kt @@ -5,7 +5,7 @@ import android.content.res.AssetManager data class KeywordSpotterConfig( var featConfig: FeatureConfig = FeatureConfig(), - var modelConfig: OnlineModelConfig, + var modelConfig: OnlineModelConfig = OnlineModelConfig(), var maxActivePaths: Int = 4, var keywordsFile: String = "keywords.txt", var keywordsScore: Float = 1.5f, diff --git a/sherpa-onnx/kotlin-api/OfflinePunctuation.kt b/sherpa-onnx/kotlin-api/OfflinePunctuation.kt index 8dfcbd63c..3b6013495 100644 --- a/sherpa-onnx/kotlin-api/OfflinePunctuation.kt +++ b/sherpa-onnx/kotlin-api/OfflinePunctuation.kt @@ -3,7 +3,7 @@ package com.k2fsa.sherpa.onnx import android.content.res.AssetManager data class OfflinePunctuationModelConfig( - var ctTransformer: String, + var ctTransformer: String = "", var numThreads: Int = 1, var debug: Boolean = false, var provider: String = "cpu", diff --git a/sherpa-onnx/kotlin-api/OfflineRecognizer.kt b/sherpa-onnx/kotlin-api/OfflineRecognizer.kt index 647f2447c..b0f35462d 100644 --- a/sherpa-onnx/kotlin-api/OfflineRecognizer.kt +++ b/sherpa-onnx/kotlin-api/OfflineRecognizer.kt @@ -58,14 +58,14 @@ data class OfflineModelConfig( var debug: Boolean = false, var provider: String = "cpu", var modelType: String = "", - var tokens: String, + var tokens: String = "", var modelingUnit: String = "", var bpeVocab: String = "", ) data class OfflineRecognizerConfig( var featConfig: FeatureConfig = FeatureConfig(), - var modelConfig: OfflineModelConfig, + var modelConfig: OfflineModelConfig = OfflineModelConfig(), // var lmConfig: OfflineLMConfig(), // TODO(fangjun): enable it var decodingMethod: String = "greedy_search", var maxActivePaths: Int = 4, diff --git a/sherpa-onnx/kotlin-api/OfflineSpeakerDiarization.kt b/sherpa-onnx/kotlin-api/OfflineSpeakerDiarization.kt index 9190f8ae5..55113e833 100644 --- a/sherpa-onnx/kotlin-api/OfflineSpeakerDiarization.kt +++ b/sherpa-onnx/kotlin-api/OfflineSpeakerDiarization.kt @@ -3,11 +3,11 @@ package com.k2fsa.sherpa.onnx import android.content.res.AssetManager data class OfflineSpeakerSegmentationPyannoteModelConfig( - var model: String, + var model: String = "", ) data class OfflineSpeakerSegmentationModelConfig( - var pyannote: OfflineSpeakerSegmentationPyannoteModelConfig, + var pyannote: OfflineSpeakerSegmentationPyannoteModelConfig = OfflineSpeakerSegmentationPyannoteModelConfig(), var numThreads: Int = 1, var debug: Boolean = false, var provider: String = "cpu", @@ -19,9 +19,9 @@ data class FastClusteringConfig( ) data class OfflineSpeakerDiarizationConfig( - var segmentation: OfflineSpeakerSegmentationModelConfig, - var embedding: SpeakerEmbeddingExtractorConfig, - var clustering: FastClusteringConfig, + var segmentation: OfflineSpeakerSegmentationModelConfig = OfflineSpeakerSegmentationModelConfig(), + var embedding: SpeakerEmbeddingExtractorConfig = SpeakerEmbeddingExtractorConfig(), + var clustering: FastClusteringConfig = FastClusteringConfig(), var minDurationOn: Float = 0.2f, var minDurationOff: Float = 0.5f, ) diff --git a/sherpa-onnx/kotlin-api/OnlineRecognizer.kt b/sherpa-onnx/kotlin-api/OnlineRecognizer.kt index 7ddefdf32..15476f995 100644 --- a/sherpa-onnx/kotlin-api/OnlineRecognizer.kt +++ b/sherpa-onnx/kotlin-api/OnlineRecognizer.kt @@ -38,7 +38,7 @@ data class OnlineModelConfig( var paraformer: OnlineParaformerModelConfig = OnlineParaformerModelConfig(), var zipformer2Ctc: OnlineZipformer2CtcModelConfig = OnlineZipformer2CtcModelConfig(), var neMoCtc: OnlineNeMoCtcModelConfig = OnlineNeMoCtcModelConfig(), - var tokens: String, + var tokens: String = "", var numThreads: Int = 1, var debug: Boolean = false, var provider: String = "cpu", @@ -60,7 +60,7 @@ data class OnlineCtcFstDecoderConfig( data class OnlineRecognizerConfig( var featConfig: FeatureConfig = FeatureConfig(), - var modelConfig: OnlineModelConfig, + var modelConfig: OnlineModelConfig = OnlineModelConfig(), var lmConfig: OnlineLMConfig = OnlineLMConfig(), var ctcFstDecoderConfig: OnlineCtcFstDecoderConfig = OnlineCtcFstDecoderConfig(), var endpointConfig: EndpointConfig = EndpointConfig(), diff --git a/sherpa-onnx/kotlin-api/SpeakerEmbeddingExtractorConfig.kt b/sherpa-onnx/kotlin-api/SpeakerEmbeddingExtractorConfig.kt index 4977ac029..0ae2bb9b1 100644 --- a/sherpa-onnx/kotlin-api/SpeakerEmbeddingExtractorConfig.kt +++ b/sherpa-onnx/kotlin-api/SpeakerEmbeddingExtractorConfig.kt @@ -1,7 +1,7 @@ package com.k2fsa.sherpa.onnx data class SpeakerEmbeddingExtractorConfig( - val model: String, + val model: String = "", var numThreads: Int = 1, var debug: Boolean = false, var provider: String = "cpu", diff --git a/sherpa-onnx/kotlin-api/SpokenLanguageIdentification.kt b/sherpa-onnx/kotlin-api/SpokenLanguageIdentification.kt index 00caca281..07ddafc4d 100644 --- a/sherpa-onnx/kotlin-api/SpokenLanguageIdentification.kt +++ b/sherpa-onnx/kotlin-api/SpokenLanguageIdentification.kt @@ -3,13 +3,13 @@ package com.k2fsa.sherpa.onnx import android.content.res.AssetManager data class SpokenLanguageIdentificationWhisperConfig( - var encoder: String, - var decoder: String, + var encoder: String = "", + var decoder: String = "", var tailPaddings: Int = -1, ) data class SpokenLanguageIdentificationConfig( - var whisper: SpokenLanguageIdentificationWhisperConfig, + var whisper: SpokenLanguageIdentificationWhisperConfig = SpokenLanguageIdentificationWhisperConfig(), var numThreads: Int = 1, var debug: Boolean = false, var provider: String = "cpu", diff --git a/sherpa-onnx/kotlin-api/Tts.kt b/sherpa-onnx/kotlin-api/Tts.kt index 4f9c4b6f6..6152cd914 100644 --- a/sherpa-onnx/kotlin-api/Tts.kt +++ b/sherpa-onnx/kotlin-api/Tts.kt @@ -4,9 +4,9 @@ package com.k2fsa.sherpa.onnx import android.content.res.AssetManager data class OfflineTtsVitsModelConfig( - var model: String, + var model: String = "", var lexicon: String = "", - var tokens: String, + var tokens: String = "", var dataDir: String = "", var dictDir: String = "", var noiseScale: Float = 0.667f, @@ -15,14 +15,14 @@ data class OfflineTtsVitsModelConfig( ) data class OfflineTtsModelConfig( - var vits: OfflineTtsVitsModelConfig, + var vits: OfflineTtsVitsModelConfig = OfflineTtsVitsModelConfig(), var numThreads: Int = 1, var debug: Boolean = false, var provider: String = "cpu", ) data class OfflineTtsConfig( - var model: OfflineTtsModelConfig, + var model: OfflineTtsModelConfig = OfflineTtsModelConfig(), var ruleFsts: String = "", var ruleFars: String = "", var maxNumSentences: Int = 1, diff --git a/sherpa-onnx/kotlin-api/Vad.kt b/sherpa-onnx/kotlin-api/Vad.kt index 08a458505..cdecc3ca2 100644 --- a/sherpa-onnx/kotlin-api/Vad.kt +++ b/sherpa-onnx/kotlin-api/Vad.kt @@ -4,7 +4,7 @@ package com.k2fsa.sherpa.onnx import android.content.res.AssetManager data class SileroVadModelConfig( - var model: String, + var model: String = "", var threshold: Float = 0.5F, var minSilenceDuration: Float = 0.25F, var minSpeechDuration: Float = 0.25F, @@ -13,7 +13,7 @@ data class SileroVadModelConfig( ) data class VadModelConfig( - var sileroVadModelConfig: SileroVadModelConfig, + var sileroVadModelConfig: SileroVadModelConfig = SileroVadModelConfig(), var sampleRate: Int = 16000, var numThreads: Int = 1, var provider: String = "cpu", @@ -112,5 +112,5 @@ fun getVadModelConfig(type: Int): VadModelConfig? { ) } } - return null; + return null } From 0f4b1f41e24fc48426e8eb4f521619edbd700820 Mon Sep 17 00:00:00 2001 From: windy <81013544+deretame@users.noreply.github.com> Date: Thu, 12 Dec 2024 19:39:43 +0800 Subject: [PATCH 104/183] =?UTF-8?q?=F0=9F=94=A7=20build(portaudio-go):=20F?= =?UTF-8?q?ixed=20version=201.0.3=20(#1614)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: windy --- .../real-time-speech-recognition-from-microphone/go.mod | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/go-api-examples/real-time-speech-recognition-from-microphone/go.mod b/go-api-examples/real-time-speech-recognition-from-microphone/go.mod index 5d6a5b784..636d6f797 100644 --- a/go-api-examples/real-time-speech-recognition-from-microphone/go.mod +++ b/go-api-examples/real-time-speech-recognition-from-microphone/go.mod @@ -1,3 +1,7 @@ module real-time-speech-recognition-from-microphone go 1.12 + +require ( + github.com/csukuangfj/portaudio-go v1.0.3 +) From e54c1f45334ec3099a57ccd78c60b91f8b2aaa08 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 12 Dec 2024 20:07:47 +0800 Subject: [PATCH 105/183] Release v1.10.35 (#1617) --- CHANGELOG.md | 6 +++++ CMakeLists.txt | 2 +- android/SherpaOnnxAar/README.md | 6 ++--- build-ios-shared.sh | 2 +- .../add-punctuations/pubspec.yaml | 2 +- dart-api-examples/audio-tagging/pubspec.yaml | 2 +- .../keyword-spotter/pubspec.yaml | 2 +- .../non-streaming-asr/pubspec.yaml | 2 +- .../speaker-diarization/pubspec.yaml | 2 +- .../speaker-identification/pubspec.yaml | 2 +- dart-api-examples/streaming-asr/pubspec.yaml | 2 +- dart-api-examples/tts/pubspec.yaml | 2 +- .../vad-with-non-streaming-asr/pubspec.yaml | 2 +- dart-api-examples/vad/pubspec.yaml | 2 +- flutter-examples/streaming_asr/pubspec.yaml | 4 ++-- flutter-examples/tts/pubspec.yaml | 4 ++-- flutter/sherpa_onnx/pubspec.yaml | 12 +++++----- .../ios/sherpa_onnx_ios.podspec | 2 +- .../macos/sherpa_onnx_macos.podspec | 2 +- harmony-os/SherpaOnnxHar/release.sh | 9 ++++++++ .../sherpa_onnx/BuildProfile.ets | 2 +- .../SherpaOnnxHar/sherpa_onnx/README.md | 2 +- .../sherpa_onnx/oh-package.json5 | 2 +- .../entry/oh-package.json5 | 2 +- .../entry/oh-package.json5 | 2 +- .../entry/oh-package.json5 | 2 +- .../SherpaOnnxTts/entry/oh-package.json5 | 2 +- harmony-os/SherpaOnnxVadAsr/entry/README.md | 2 +- .../SherpaOnnxVadAsr/entry/oh-package.json5 | 2 +- jitpack.yml | 6 ++--- new-release.sh | 22 +++++++++---------- nodejs-addon-examples/package.json | 2 +- pom.xml | 2 +- 33 files changed, 67 insertions(+), 52 deletions(-) create mode 100755 harmony-os/SherpaOnnxHar/release.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 115f55e25..0023d23f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 1.10.35 + +* Add missing changes about speaker identfication demo for HarmonyOS (#1612) +* Provide sherpa-onnx.aar for Android (#1615) +* Use aar in Android Java demo. (#1616) + ## 1.10.34 * Fix building node-addon package (#1598) diff --git a/CMakeLists.txt b/CMakeLists.txt index 37ccb4cc2..f0b8e41ce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ project(sherpa-onnx) # Remember to update # ./CHANGELOG.md # ./new-release.sh -set(SHERPA_ONNX_VERSION "1.10.34") +set(SHERPA_ONNX_VERSION "1.10.35") # Disable warning about # diff --git a/android/SherpaOnnxAar/README.md b/android/SherpaOnnxAar/README.md index b7b73837e..e17ed49ef 100644 --- a/android/SherpaOnnxAar/README.md +++ b/android/SherpaOnnxAar/README.md @@ -4,8 +4,8 @@ git clone https://github.com/k2-fsa/sherpa-onnx cd sherpa-onnx -wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.34/sherpa-onnx-v1.10.34-android.tar.bz2 -tar xvf sherpa-onnx-v1.10.34-android.tar.bz2 +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.35/sherpa-onnx-v1.10.35-android.tar.bz2 +tar xvf sherpa-onnx-v1.10.35-android.tar.bz2 cp -v jniLibs/arm64-v8a/* android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/arm64-v8a/ cp -v jniLibs/armeabi-v7a/* android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/armeabi-v7a/ @@ -16,5 +16,5 @@ cd android/SherpaOnnxAar ./gradlew :sherpa_onnx:assembleRelease ls -lh ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar -cp ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar ../../sherpa-onnx-1.10.34.aar +cp ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar ../../sherpa-onnx-1.10.35.aar ``` diff --git a/build-ios-shared.sh b/build-ios-shared.sh index 0b0acb60a..11e75743a 100755 --- a/build-ios-shared.sh +++ b/build-ios-shared.sh @@ -242,7 +242,7 @@ for d in ios-arm64_x86_64-simulator ios-arm64; do CFBundlePackageType FMWK CFBundleShortVersionString - 1.10.34 + 1.10.35 CFBundleSupportedPlatforms iPhoneOS diff --git a/dart-api-examples/add-punctuations/pubspec.yaml b/dart-api-examples/add-punctuations/pubspec.yaml index 03f376179..cb52a0161 100644 --- a/dart-api-examples/add-punctuations/pubspec.yaml +++ b/dart-api-examples/add-punctuations/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.34 + sherpa_onnx: ^1.10.35 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/audio-tagging/pubspec.yaml b/dart-api-examples/audio-tagging/pubspec.yaml index a869bfdc3..07d4ccf92 100644 --- a/dart-api-examples/audio-tagging/pubspec.yaml +++ b/dart-api-examples/audio-tagging/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.34 + sherpa_onnx: ^1.10.35 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/keyword-spotter/pubspec.yaml b/dart-api-examples/keyword-spotter/pubspec.yaml index bf79f3bcf..4e2253f9e 100644 --- a/dart-api-examples/keyword-spotter/pubspec.yaml +++ b/dart-api-examples/keyword-spotter/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.34 + sherpa_onnx: ^1.10.35 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/non-streaming-asr/pubspec.yaml b/dart-api-examples/non-streaming-asr/pubspec.yaml index b34bee6ec..b8a507bc8 100644 --- a/dart-api-examples/non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.34 + sherpa_onnx: ^1.10.35 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/speaker-diarization/pubspec.yaml b/dart-api-examples/speaker-diarization/pubspec.yaml index c8b045a5a..9a8d4d8c5 100644 --- a/dart-api-examples/speaker-diarization/pubspec.yaml +++ b/dart-api-examples/speaker-diarization/pubspec.yaml @@ -8,7 +8,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.34 + sherpa_onnx: ^1.10.35 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/speaker-identification/pubspec.yaml b/dart-api-examples/speaker-identification/pubspec.yaml index 4baa30f8d..d75596539 100644 --- a/dart-api-examples/speaker-identification/pubspec.yaml +++ b/dart-api-examples/speaker-identification/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.34 + sherpa_onnx: ^1.10.35 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/streaming-asr/pubspec.yaml b/dart-api-examples/streaming-asr/pubspec.yaml index 80322e86e..0275f03c7 100644 --- a/dart-api-examples/streaming-asr/pubspec.yaml +++ b/dart-api-examples/streaming-asr/pubspec.yaml @@ -11,7 +11,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.34 + sherpa_onnx: ^1.10.35 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/tts/pubspec.yaml b/dart-api-examples/tts/pubspec.yaml index d213371ba..62009d641 100644 --- a/dart-api-examples/tts/pubspec.yaml +++ b/dart-api-examples/tts/pubspec.yaml @@ -8,7 +8,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.34 + sherpa_onnx: ^1.10.35 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml index 4f47c9940..c0ecd6fae 100644 --- a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.34 + sherpa_onnx: ^1.10.35 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad/pubspec.yaml b/dart-api-examples/vad/pubspec.yaml index d6c62a495..55df8e3cd 100644 --- a/dart-api-examples/vad/pubspec.yaml +++ b/dart-api-examples/vad/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.34 + sherpa_onnx: ^1.10.35 path: ^1.9.0 args: ^2.5.0 diff --git a/flutter-examples/streaming_asr/pubspec.yaml b/flutter-examples/streaming_asr/pubspec.yaml index 810ae43c6..2d65e4452 100644 --- a/flutter-examples/streaming_asr/pubspec.yaml +++ b/flutter-examples/streaming_asr/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' -version: 1.10.34 +version: 1.10.35 topics: - speech-recognition @@ -31,7 +31,7 @@ dependencies: record: ^5.1.0 url_launcher: ^6.2.6 - sherpa_onnx: ^1.10.34 + sherpa_onnx: ^1.10.35 # sherpa_onnx: # path: ../../flutter/sherpa_onnx diff --git a/flutter-examples/tts/pubspec.yaml b/flutter-examples/tts/pubspec.yaml index 931641bbc..c89ae2a9a 100644 --- a/flutter-examples/tts/pubspec.yaml +++ b/flutter-examples/tts/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' # Remove this line if you wish to publish to pub.dev -version: 1.10.34 +version: 1.10.35 environment: sdk: ">=2.17.0 <4.0.0" @@ -18,7 +18,7 @@ dependencies: cupertino_icons: ^1.0.6 path_provider: ^2.1.3 path: ^1.9.0 - sherpa_onnx: ^1.10.34 + sherpa_onnx: ^1.10.35 # sherpa_onnx: # path: ../../flutter/sherpa_onnx url_launcher: 6.2.6 diff --git a/flutter/sherpa_onnx/pubspec.yaml b/flutter/sherpa_onnx/pubspec.yaml index a63b78ab0..632369503 100644 --- a/flutter/sherpa_onnx/pubspec.yaml +++ b/flutter/sherpa_onnx/pubspec.yaml @@ -17,7 +17,7 @@ topics: - voice-activity-detection # remember to change the version in ../sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec -version: 1.10.34 +version: 1.10.35 homepage: https://github.com/k2-fsa/sherpa-onnx @@ -30,23 +30,23 @@ dependencies: flutter: sdk: flutter - sherpa_onnx_android: ^1.10.34 + sherpa_onnx_android: ^1.10.35 # sherpa_onnx_android: # path: ../sherpa_onnx_android - sherpa_onnx_macos: ^1.10.34 + sherpa_onnx_macos: ^1.10.35 # sherpa_onnx_macos: # path: ../sherpa_onnx_macos - sherpa_onnx_linux: ^1.10.34 + sherpa_onnx_linux: ^1.10.35 # sherpa_onnx_linux: # path: ../sherpa_onnx_linux - sherpa_onnx_windows: ^1.10.34 + sherpa_onnx_windows: ^1.10.35 # sherpa_onnx_windows: # path: ../sherpa_onnx_windows - sherpa_onnx_ios: ^1.10.34 + sherpa_onnx_ios: ^1.10.35 # sherpa_onnx_ios: # path: ../sherpa_onnx_ios diff --git a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec index 2738be7c7..894ee605b 100644 --- a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec +++ b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec @@ -7,7 +7,7 @@ # https://groups.google.com/g/dart-ffi/c/nUATMBy7r0c Pod::Spec.new do |s| s.name = 'sherpa_onnx_ios' - s.version = '1.10.34' + s.version = '1.10.35' s.summary = 'A new Flutter FFI plugin project.' s.description = <<-DESC A new Flutter FFI plugin project. diff --git a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec index 6898d644b..0c21a6369 100644 --- a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec +++ b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec @@ -4,7 +4,7 @@ # Pod::Spec.new do |s| s.name = 'sherpa_onnx_macos' - s.version = '1.10.34' + s.version = '1.10.35' s.summary = 'sherpa-onnx Flutter FFI plugin project.' s.description = <<-DESC sherpa-onnx Flutter FFI plugin project. diff --git a/harmony-os/SherpaOnnxHar/release.sh b/harmony-os/SherpaOnnxHar/release.sh new file mode 100755 index 000000000..cc33364fb --- /dev/null +++ b/harmony-os/SherpaOnnxHar/release.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -ex + +export PATH=/Users/fangjun/software/command-line-tools/bin:$PATH + +hvigorw clean --no-daemon +hvigorw --mode module -p product=default -p module=sherpa_onnx@default assembleHar --analyze=normal --parallel --incremental --no-daemon + +ohpm publish ./sherpa_onnx/build/default/outputs/default/sherpa_onnx.har diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets index 2558f09c8..8eb22c9a7 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets @@ -1,7 +1,7 @@ /** * Use these variables when you tailor your ArkTS code. They must be of the const type. */ -export const HAR_VERSION = '1.10.33'; +export const HAR_VERSION = '1.10.35'; export const BUILD_MODE_NAME = 'debug'; export const DEBUG = true; export const TARGET_NAME = 'default'; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md b/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md index 78ef2c4e1..97372f2ea 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md @@ -23,7 +23,7 @@ or update your `oh-package.json5` to include the following: ``` "dependencies": { - "sherpa_onnx": "1.10.34", + "sherpa_onnx": "1.10.35", }, ``` diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 index 13a249052..071be0246 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 @@ -1,6 +1,6 @@ { "name": "sherpa_onnx", - "version": "1.10.34", + "version": "1.10.35", "description": "On-device speech-to-text, text-to-speech, and speaker diarization using Next-gen Kaldi without Internet connection", "main": "Index.ets", "author": "The next-gen Kaldi team", diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 b/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 index 1682f0d10..ae5ffd748 100644 --- a/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.34" + "sherpa_onnx": "1.10.35" } } diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 b/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 index 6f404b4ca..4ecc1b57f 100644 --- a/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.34", + "sherpa_onnx": "1.10.35", } } diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 index 6f404b4ca..4ecc1b57f 100644 --- a/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.34", + "sherpa_onnx": "1.10.35", } } diff --git a/harmony-os/SherpaOnnxTts/entry/oh-package.json5 b/harmony-os/SherpaOnnxTts/entry/oh-package.json5 index 6f404b4ca..4ecc1b57f 100644 --- a/harmony-os/SherpaOnnxTts/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxTts/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.34", + "sherpa_onnx": "1.10.35", } } diff --git a/harmony-os/SherpaOnnxVadAsr/entry/README.md b/harmony-os/SherpaOnnxVadAsr/entry/README.md index 806291354..39d6f48f5 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/README.md +++ b/harmony-os/SherpaOnnxVadAsr/entry/README.md @@ -1,6 +1,6 @@ # Introduction -Please download ./sherpa_onnx-v1.10.34.har +Please download ./sherpa_onnx-v1.10.35.har from Hint: For users who have no access to huggingface, please use diff --git a/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 b/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 index dae1ac8c2..1ca57069b 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 @@ -7,7 +7,7 @@ "license": "", "dependencies": { // please see https://ohpm.openharmony.cn/#/cn/detail/sherpa_onnx - "sherpa_onnx": "1.10.34", + "sherpa_onnx": "1.10.35", } } diff --git a/jitpack.yml b/jitpack.yml index e2936e496..8040dfd11 100644 --- a/jitpack.yml +++ b/jitpack.yml @@ -2,8 +2,8 @@ jdk: - openjdk17 before_install: - - wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.34/sherpa-onnx-1.10.34.aar + - wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.35/sherpa-onnx-1.10.35.aar install: - - FILE="-Dfile=sherpa-onnx-1.10.34.aar" - - mvn install:install-file $FILE -DgroupId=com.k2fsa.sherpa.onnx -DartifactId=sherpa-onnx -Dversion=1.10.34 -Dpackaging=aar -DgeneratePom=true + - FILE="-Dfile=sherpa-onnx-1.10.35.aar" + - mvn install:install-file $FILE -DgroupId=com.k2fsa.sherpa.onnx -DartifactId=sherpa-onnx -Dversion=1.10.35 -Dpackaging=aar -DgeneratePom=true diff --git a/new-release.sh b/new-release.sh index a305d6a66..255064cd2 100755 --- a/new-release.sh +++ b/new-release.sh @@ -2,16 +2,16 @@ set -ex -sed -i.bak 's/1\.10\.33/1\.10\.34/g' ./build-ios-shared.sh -sed -i.bak 's/1\.10\.33/1\.10\.34/g' ./pom.xml -sed -i.bak 's/1\.10\.33/1\.10\.34/g' ./jitpack.yml -sed -i.bak 's/1\.10\.33/1\.10\.34/g' ./android/SherpaOnnxAar/README.md +sed -i.bak 's/1\.10\.34/1\.10\.35/g' ./build-ios-shared.sh +sed -i.bak 's/1\.10\.34/1\.10\.35/g' ./pom.xml +sed -i.bak 's/1\.10\.34/1\.10\.35/g' ./jitpack.yml +sed -i.bak 's/1\.10\.34/1\.10\.35/g' ./android/SherpaOnnxAar/README.md -find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.33/1\.10\.34/g' {} \; -find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.33/1\.10\.34/g' {} \; -find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.33/1\.10\.34/g' {} \; -find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.33/1\.10\.34/g' {} \; -find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.33/1\.10\.34/g' {} \; +find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.34/1\.10\.35/g' {} \; +find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.34/1\.10\.35/g' {} \; +find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.34/1\.10\.35/g' {} \; +find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.34/1\.10\.35/g' {} \; +find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.34/1\.10\.35/g' {} \; -find harmony-os -name "README.md" -type f -exec sed -i.bak 's/1\.10\.33/1\.10\.34/g' {} \; -find harmony-os -name oh-package.json5 -type f -exec sed -i.bak 's/1\.10\.33/1\.10\.34/g' {} \; +find harmony-os -name "README.md" -type f -exec sed -i.bak 's/1\.10\.34/1\.10\.35/g' {} \; +find harmony-os -name oh-package.json5 -type f -exec sed -i.bak 's/1\.10\.34/1\.10\.35/g' {} \; diff --git a/nodejs-addon-examples/package.json b/nodejs-addon-examples/package.json index 3fc50bc2a..b9186f5e7 100644 --- a/nodejs-addon-examples/package.json +++ b/nodejs-addon-examples/package.json @@ -1,5 +1,5 @@ { "dependencies": { - "sherpa-onnx-node": "^1.10.34" + "sherpa-onnx-node": "^1.10.35" } } diff --git a/pom.xml b/pom.xml index ad8756914..efb7b8ba3 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.k2fsa.sherpa.onnx sherpa-onnx-android - 1.10.34 + 1.10.35 https://github.com/k2-fsa/sherpa-onnx pom First Android Library From efb505f5788de8d5a4c6e8da770942daddda5176 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 12 Dec 2024 20:51:57 +0800 Subject: [PATCH 106/183] Update AAR version in Android Java demo (#1618) --- android/SherpaOnnxJavaDemo/app/build.gradle | 2 +- new-release.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/android/SherpaOnnxJavaDemo/app/build.gradle b/android/SherpaOnnxJavaDemo/app/build.gradle index a2ee27997..53ee06406 100644 --- a/android/SherpaOnnxJavaDemo/app/build.gradle +++ b/android/SherpaOnnxJavaDemo/app/build.gradle @@ -34,5 +34,5 @@ dependencies { implementation 'pub.devrel:easypermissions:3.0.0' implementation 'androidx.core:core-ktx:1.7.0' // implementation files('/Users/fangjun/open-source/sherpa-onnx/android/SherpaOnnxAar/sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar') - implementation 'com.github.k2-fsa:sherpa-onnx:master-SNAPSHOT' + implementation 'com.github.k2-fsa:sherpa-onnx:v1.10.35' } diff --git a/new-release.sh b/new-release.sh index 255064cd2..af0602250 100755 --- a/new-release.sh +++ b/new-release.sh @@ -7,6 +7,8 @@ sed -i.bak 's/1\.10\.34/1\.10\.35/g' ./pom.xml sed -i.bak 's/1\.10\.34/1\.10\.35/g' ./jitpack.yml sed -i.bak 's/1\.10\.34/1\.10\.35/g' ./android/SherpaOnnxAar/README.md +find android -name build.gradle -type f -exec sed -i.bak 's/sherpa-onnx:v1\.10\.34/sherpa-onnx:v1\.10\.35/g' {} \; + find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.34/1\.10\.35/g' {} \; find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.34/1\.10\.35/g' {} \; find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.34/1\.10\.35/g' {} \; From e639c70d78d0620e59c0b7d65daa76a95029ada7 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sat, 14 Dec 2024 09:53:44 +0800 Subject: [PATCH 107/183] Support linking onnxruntime statically for Android (#1619) --- .github/workflows/android-static.yaml | 298 ++++++++++++++++++++++++++ CMakeLists.txt | 7 +- build-android-arm64-v8a.sh | 70 ++++-- build-android-armv7-eabi.sh | 71 ++++-- build-android-x86-64.sh | 71 ++++-- build-android-x86.sh | 6 + sherpa-onnx/jni/CMakeLists.txt | 2 +- 7 files changed, 479 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/android-static.yaml diff --git a/.github/workflows/android-static.yaml b/.github/workflows/android-static.yaml new file mode 100644 index 000000000..84c05d153 --- /dev/null +++ b/.github/workflows/android-static.yaml @@ -0,0 +1,298 @@ +# static means we link onnxruntime statically +# but we still have libsherpa-onnx-jni.so +name: android-static + +on: + push: + branches: + - master + - android-link-onnxruntime-statically + paths: + - '.github/workflows/android-static.yaml' + - 'CMakeLists.txt' + - 'cmake/**' + - 'sherpa-onnx/csrc/*' + - 'sherpa-onnx/jni/*' + - 'build-android*.sh' + tags: + - 'v[0-9]+.[0-9]+.[0-9]+*' + pull_request: + branches: + - master + paths: + - '.github/workflows/android-static.yaml' + - 'CMakeLists.txt' + - 'cmake/**' + - 'sherpa-onnx/csrc/*' + - 'sherpa-onnx/jni/*' + - 'build-android*.sh' + + workflow_dispatch: + +concurrency: + group: android-static-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-android-static-libs: + name: Android static libs + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ matrix.os }}-android-jni-static + + - name: Display NDK HOME + shell: bash + run: | + echo "ANDROID_NDK_LATEST_HOME: ${ANDROID_NDK_LATEST_HOME}" + ls -lh ${ANDROID_NDK_LATEST_HOME} + + - name: build android arm64-v8a + shell: bash + run: | + export BUILD_SHARED_LIBS=OFF + + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + + export ANDROID_NDK=$ANDROID_NDK_LATEST_HOME + ./build-android-arm64-v8a.sh + mkdir -p jniLibs/arm64-v8a/ + cp -v ./build-android-arm64-v8a-static/install/lib/*.so ./jniLibs/arm64-v8a/ + rm -rf ./build-android-arm64-v8a-static/ + + - name: build android armv7-eabi + shell: bash + run: | + export BUILD_SHARED_LIBS=OFF + + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + + export ANDROID_NDK=$ANDROID_NDK_LATEST_HOME + ./build-android-armv7-eabi.sh + mkdir -p ./jniLibs/armeabi-v7a/ + cp -v ./build-android-armv7-eabi-static/install/lib/*.so ./jniLibs/armeabi-v7a/ + rm -rf ./build-android-armv7-eabi-static + + - name: build android x86_64 + shell: bash + run: | + export BUILD_SHARED_LIBS=OFF + + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + + export ANDROID_NDK=$ANDROID_NDK_LATEST_HOME + ./build-android-x86-64.sh + mkdir -p ./jniLibs/x86_64 + cp -v ./build-android-x86-64-static/install/lib/*.so ./jniLibs/x86_64 + rm -rf ./build-android-x86-64-static + + - name: build android x86 + shell: bash + run: | + export BUILD_SHARED_LIBS=OFF + + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + + export ANDROID_NDK=$ANDROID_NDK_LATEST_HOME + ./build-android-x86.sh + mkdir -p ./jniLibs/x86 + cp -v ./build-android-x86/install/lib/*.so ./jniLibs/x86 + rm -rf ./build-android-x86 + + - name: Copy files + shell: bash + run: | + SHERPA_ONNX_VERSION=v$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + echo "SHERPA_ONNX_VERSION=$SHERPA_ONNX_VERSION" >> "$GITHUB_ENV" + + filename=sherpa-onnx-${SHERPA_ONNX_VERSION}-android-static-link-onnxruntime.tar.bz2 + + tar cjvf $filename ./jniLibs + + ls -lh + + - uses: actions/upload-artifact@v4 + with: + name: sherpa-onnx-android-libs-static + path: ./jniLibs + + # https://huggingface.co/docs/hub/spaces-github-actions + - name: Publish to huggingface + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v3 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + du -h -d1 . + ls -lh + + rm -rf huggingface + export GIT_CLONE_PROTECTION_ACTIVE=false + GIT_LFS_SKIP_SMUDGE=1 git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs huggingface + + cd huggingface + + cp -v ../sherpa-onnx-*-android*.tar.bz2 ./ + + git status + git lfs track "*.bz2" + + git add . + + git commit -m "upload sherpa-onnx-${SHERPA_ONNX_VERSION}-android.tar.bz2" + + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs main + + - name: Release android libs + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + overwrite: true + file: sherpa-onnx-*-android*.tar.bz2 + + build-android-aar-static: + needs: [build-android-static-libs] + name: Android AAR + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # https://github.com/actions/setup-java + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' # See 'Supported distributions' for available options + java-version: '21' + + - name: Display NDK HOME + shell: bash + run: | + echo "ANDROID_NDK_LATEST_HOME: ${ANDROID_NDK_LATEST_HOME}" + ls -lh ${ANDROID_NDK_LATEST_HOME} + + - name: Retrieve artifact + uses: actions/download-artifact@v4 + with: + name: sherpa-onnx-android-libs-static + path: /tmp/jniLibs + + - name: Show jni libs + shell: bash + run: | + ls -lh /tmp/jniLibs + + # drwxr-xr-x 2 runner docker 4.0K Dec 12 06:56 arm64-v8a + # drwxr-xr-x 2 runner docker 4.0K Dec 12 06:56 armeabi-v7a + # drwxr-xr-x 2 runner docker 4.0K Dec 12 06:56 x86 + # drwxr-xr-x 2 runner docker 4.0K Dec 12 06:56 x86_64 + # + - name: Copy libs + shell: bash + run: | + for arch in arm64-v8a armeabi-v7a x86 x86_64; do + cp -v /tmp/jniLibs/$arch/* android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/$arch/ + done + + - name: Check libs + shell: bash + run: | + ls -lh android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/* + + - name: Build aar + shell: bash + run: | + cd android/SherpaOnnxAar + + ./gradlew :sherpa_onnx:assembleRelease + + - name: Display aar + shell: bash + run: | + cd android/SherpaOnnxAar + + ls -lh ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar + cp ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar ../../ + + - name: Rename aar + shell: bash + run: | + SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + echo "SHERPA_ONNX_VERSION=$SHERPA_ONNX_VERSION" >> "$GITHUB_ENV" + + mv sherpa_onnx-release.aar sherpa-onnx-static-link-onnxruntime-${SHERPA_ONNX_VERSION}.aar + + - uses: actions/upload-artifact@v4 + with: + name: sherpa-onnx-android-aar-static + path: ./*.aar + + # https://huggingface.co/docs/hub/spaces-github-actions + - name: Publish to huggingface + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v3 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + du -h -d1 . + ls -lh + + rm -rf huggingface + export GIT_CLONE_PROTECTION_ACTIVE=false + GIT_LFS_SKIP_SMUDGE=1 git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs huggingface + + cd huggingface + dst=android/aar + mkdir -p $dst + + cp -v ../*.aar $dst + + git status + git lfs track "*.aar" + + git add . + + git commit -m "upload sherpa-onnx-${SHERPA_ONNX_VERSION}.aar" + + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs main + + - name: Release android aar + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + overwrite: true + file: ./*.aar diff --git a/CMakeLists.txt b/CMakeLists.txt index f0b8e41ce..f45384d4d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -82,11 +82,6 @@ if(SHERPA_ONNX_ENABLE_PYTHON AND NOT BUILD_SHARED_LIBS) set(BUILD_SHARED_LIBS ON CACHE BOOL "" FORCE) endif() -if(SHERPA_ONNX_ENABLE_JNI AND NOT BUILD_SHARED_LIBS) - message(STATUS "Set BUILD_SHARED_LIBS to ON since SHERPA_ONNX_ENABLE_JNI is ON") - set(BUILD_SHARED_LIBS ON CACHE BOOL "" FORCE) -endif() - if(SHERPA_ONNX_ENABLE_GPU) message(WARNING "\ Compiling for NVIDIA GPU is enabled. Please make sure cudatoolkit @@ -157,7 +152,7 @@ message(STATUS "SHERPA_ONNX_USE_PRE_INSTALLED_ONNXRUNTIME_IF_AVAILABLE ${SHERPA_ message(STATUS "SHERPA_ONNX_ENABLE_SANITIZER: ${SHERPA_ONNX_ENABLE_SANITIZER}") message(STATUS "SHERPA_ONNX_BUILD_C_API_EXAMPLES: ${SHERPA_ONNX_BUILD_C_API_EXAMPLES}") -if(BUILD_SHARED_LIBS) +if(BUILD_SHARED_LIBS OR SHERPA_ONNX_ENABLE_JNI) set(CMAKE_CXX_VISIBILITY_PRESET hidden) set(CMAKE_VISIBILITY_INLINES_HIDDEN 1) set(CMAKE_POSITION_INDEPENDENT_CODE ON) diff --git a/build-android-arm64-v8a.sh b/build-android-arm64-v8a.sh index 7967af018..3d53cc23d 100755 --- a/build-android-arm64-v8a.sh +++ b/build-android-arm64-v8a.sh @@ -1,7 +1,26 @@ #!/usr/bin/env bash set -ex -dir=$PWD/build-android-arm64-v8a +# If BUILD_SHARED_LIBS is ON, we use libonnxruntime.so +# If BUILD_SHARED_LIBS is OFF, we use libonnxruntime.a +# +# In any case, we will have libsherpa-onnx-jni.so +# +# If BUILD_SHARED_LIBS is OFF, then libonnxruntime.a is linked into libsherpa-onnx-jni.so +# and you only need to copy libsherpa-onnx-jni.so to your Android projects. +# +# If BUILD_SHARED_LIBS is ON, then you need to copy both libsherpa-onnx-jni.so +# and libonnxruntime.so to your Android projects +# +if [ -z $BUILD_SHARED_LIBS ]; then + BUILD_SHARED_LIBS=ON +fi + +if [ $BUILD_SHARED_LIBS == ON ]; then + dir=$PWD/build-android-arm64-v8a +else + dir=$PWD/build-android-arm64-v8a-static +fi mkdir -p $dir cd $dir @@ -21,6 +40,9 @@ cd $dir if [ -z $ANDROID_NDK ]; then ANDROID_NDK=/star-fj/fangjun/software/android-sdk/ndk/22.1.7171670 + if [ $BUILD_SHARED_LIBS == OFF ]; then + ANDROID_NDK=/star-fj/fangjun/software/android-sdk/ndk/27.0.11718014 + fi # or use # ANDROID_NDK=/star-fj/fangjun/software/android-ndk # @@ -32,6 +54,10 @@ if [ -z $ANDROID_NDK ]; then # Tools -> SDK manager -> Android SDK # and set "Android SDK location" to /Users/fangjun/software/my-android ANDROID_NDK=/Users/fangjun/software/my-android/ndk/22.1.7171670 + + if [ $BUILD_SHARED_LIBS == OFF ]; then + ANDROID_NDK=/Users/fangjun/software/my-android/ndk/27.0.11718014 + fi fi fi @@ -44,17 +70,29 @@ echo "ANDROID_NDK: $ANDROID_NDK" sleep 1 onnxruntime_version=1.17.1 -if [ ! -f $onnxruntime_version/jni/arm64-v8a/libonnxruntime.so ]; then - mkdir -p $onnxruntime_version - pushd $onnxruntime_version - wget -c -q https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${onnxruntime_version}/onnxruntime-android-${onnxruntime_version}.zip - unzip onnxruntime-android-${onnxruntime_version}.zip - rm onnxruntime-android-${onnxruntime_version}.zip - popd -fi +if [ $BUILD_SHARED_LIBS == ON ]; then + if [ ! -f $onnxruntime_version/jni/arm64-v8a/libonnxruntime.so ]; then + mkdir -p $onnxruntime_version + pushd $onnxruntime_version + wget -c -q https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${onnxruntime_version}/onnxruntime-android-${onnxruntime_version}.zip + unzip onnxruntime-android-${onnxruntime_version}.zip + rm onnxruntime-android-${onnxruntime_version}.zip + popd + fi -export SHERPA_ONNXRUNTIME_LIB_DIR=$dir/$onnxruntime_version/jni/arm64-v8a/ -export SHERPA_ONNXRUNTIME_INCLUDE_DIR=$dir/$onnxruntime_version/headers/ + export SHERPA_ONNXRUNTIME_LIB_DIR=$dir/$onnxruntime_version/jni/arm64-v8a/ + export SHERPA_ONNXRUNTIME_INCLUDE_DIR=$dir/$onnxruntime_version/headers/ +else + if [ ! -f ${onnxruntime_version}-static/lib/libonnxruntime.a ]; then + wget -c -q https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${onnxruntime_version}/onnxruntime-android-arm64-v8a-static_lib-${onnxruntime_version}.zip + unzip onnxruntime-android-arm64-v8a-static_lib-${onnxruntime_version}.zip + rm onnxruntime-android-arm64-v8a-static_lib-${onnxruntime_version}.zip + mv onnxruntime-android-arm64-v8a-static_lib-${onnxruntime_version} ${onnxruntime_version}-static + fi + + export SHERPA_ONNXRUNTIME_LIB_DIR=$dir/$onnxruntime_version-static/lib/ + export SHERPA_ONNXRUNTIME_INCLUDE_DIR=$dir/$onnxruntime_version-static/include/ +fi echo "SHERPA_ONNXRUNTIME_LIB_DIR: $SHERPA_ONNXRUNTIME_LIB_DIR" echo "SHERPA_ONNXRUNTIME_INCLUDE_DIR $SHERPA_ONNXRUNTIME_INCLUDE_DIR" @@ -88,24 +126,30 @@ cmake -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake" -DBUILD_ESPEAK_NG_EXE=OFF \ -DBUILD_ESPEAK_NG_TESTS=OFF \ -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_SHARED_LIBS=ON \ + -DBUILD_SHARED_LIBS=$BUILD_SHARED_LIBS \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ -DSHERPA_ONNX_ENABLE_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_CHECK=OFF \ -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -DSHERPA_ONNX_ENABLE_JNI=$SHERPA_ONNX_ENABLE_JNI \ + -DSHERPA_ONNX_LINK_LIBSTDCPP_STATICALLY=OFF \ -DSHERPA_ONNX_ENABLE_C_API=$SHERPA_ONNX_ENABLE_C_API \ -DCMAKE_INSTALL_PREFIX=./install \ -DANDROID_ABI="arm64-v8a" \ -DANDROID_PLATFORM=android-21 .. + # By default, it links to libc++_static.a + # -DANDROID_STL=c++_shared \ + # Please use -DANDROID_PLATFORM=android-27 if you want to use Android NNAPI # make VERBOSE=1 -j4 make -j4 make install/strip -cp -fv $onnxruntime_version/jni/arm64-v8a/libonnxruntime.so install/lib +cp -fv $onnxruntime_version/jni/arm64-v8a/libonnxruntime.so install/lib 2>/dev/null || true +rm -rf install/share rm -rf install/lib/pkgconfig +rm -rf install/lib/lib*.a # To run the generated binaries on Android, please use the following steps. # diff --git a/build-android-armv7-eabi.sh b/build-android-armv7-eabi.sh index 390f5a844..e9a8487fa 100755 --- a/build-android-armv7-eabi.sh +++ b/build-android-armv7-eabi.sh @@ -1,7 +1,26 @@ #!/usr/bin/env bash set -ex -dir=$PWD/build-android-armv7-eabi +# If BUILD_SHARED_LIBS is ON, we use libonnxruntime.so +# If BUILD_SHARED_LIBS is OFF, we use libonnxruntime.a +# +# In any case, we will have libsherpa-onnx-jni.so +# +# If BUILD_SHARED_LIBS is OFF, then libonnxruntime.a is linked into libsherpa-onnx-jni.so +# and you only need to copy libsherpa-onnx-jni.so to your Android projects. +# +# If BUILD_SHARED_LIBS is ON, then you need to copy both libsherpa-onnx-jni.so +# and libonnxruntime.so to your Android projects +# +if [ -z $BUILD_SHARED_LIBS ]; then + BUILD_SHARED_LIBS=ON +fi + +if [ $BUILD_SHARED_LIBS == ON ]; then + dir=$PWD/build-android-armv7-eabi +else + dir=$PWD/build-android-armv7-eabi-static +fi mkdir -p $dir cd $dir @@ -21,6 +40,9 @@ cd $dir if [ -z $ANDROID_NDK ]; then ANDROID_NDK=/star-fj/fangjun/software/android-sdk/ndk/22.1.7171670 + if [ $BUILD_SHARED_LIBS == OFF ]; then + ANDROID_NDK=/star-fj/fangjun/software/android-sdk/ndk/27.0.11718014 + fi # or use # ANDROID_NDK=/star-fj/fangjun/software/android-ndk # @@ -32,6 +54,10 @@ if [ -z $ANDROID_NDK ]; then # Tools -> SDK manager -> Android SDK # and set "Android SDK location" to /Users/fangjun/software/my-android ANDROID_NDK=/Users/fangjun/software/my-android/ndk/22.1.7171670 + + if [ $BUILD_SHARED_LIBS == OFF ]; then + ANDROID_NDK=/Users/fangjun/software/my-android/ndk/27.0.11718014 + fi fi fi @@ -45,17 +71,29 @@ sleep 1 onnxruntime_version=1.17.1 -if [ ! -f $onnxruntime_version/jni/armeabi-v7a/libonnxruntime.so ]; then - mkdir -p $onnxruntime_version - pushd $onnxruntime_version - wget -c -q https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${onnxruntime_version}/onnxruntime-android-${onnxruntime_version}.zip - unzip onnxruntime-android-${onnxruntime_version}.zip - rm onnxruntime-android-${onnxruntime_version}.zip - popd -fi +if [ $BUILD_SHARED_LIBS == ON ]; then + if [ ! -f $onnxruntime_version/jni/armeabi-v7a/libonnxruntime.so ]; then + mkdir -p $onnxruntime_version + pushd $onnxruntime_version + wget -c -q https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${onnxruntime_version}/onnxruntime-android-${onnxruntime_version}.zip + unzip onnxruntime-android-${onnxruntime_version}.zip + rm onnxruntime-android-${onnxruntime_version}.zip + popd + fi + + export SHERPA_ONNXRUNTIME_LIB_DIR=$dir/$onnxruntime_version/jni/armeabi-v7a/ + export SHERPA_ONNXRUNTIME_INCLUDE_DIR=$dir/$onnxruntime_version/headers/ +else + if [ ! -f ${onnxruntime_version}-static/lib/libonnxruntime.a ]; then + wget -c -q https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${onnxruntime_version}/onnxruntime-android-armeabi-v7a-static_lib-${onnxruntime_version}.zip + unzip onnxruntime-android-armeabi-v7a-static_lib-${onnxruntime_version}.zip + rm onnxruntime-android-armeabi-v7a-static_lib-${onnxruntime_version}.zip + mv onnxruntime-android-armeabi-v7a-static_lib-${onnxruntime_version} ${onnxruntime_version}-static + fi -export SHERPA_ONNXRUNTIME_LIB_DIR=$dir/$onnxruntime_version/jni/armeabi-v7a/ -export SHERPA_ONNXRUNTIME_INCLUDE_DIR=$dir/$onnxruntime_version/headers/ + export SHERPA_ONNXRUNTIME_LIB_DIR=$dir/$onnxruntime_version-static/lib/ + export SHERPA_ONNXRUNTIME_INCLUDE_DIR=$dir/$onnxruntime_version-static/include/ +fi echo "SHERPA_ONNXRUNTIME_LIB_DIR: $SHERPA_ONNXRUNTIME_LIB_DIR" echo "SHERPA_ONNXRUNTIME_INCLUDE_DIR $SHERPA_ONNXRUNTIME_INCLUDE_DIR" @@ -89,18 +127,25 @@ cmake -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake" -DBUILD_ESPEAK_NG_EXE=OFF \ -DBUILD_ESPEAK_NG_TESTS=OFF \ -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_SHARED_LIBS=ON \ + -DBUILD_SHARED_LIBS=$BUILD_SHARED_LIBS \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ -DSHERPA_ONNX_ENABLE_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_CHECK=OFF \ -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -DSHERPA_ONNX_ENABLE_JNI=$SHERPA_ONNX_ENABLE_JNI \ + -DSHERPA_ONNX_LINK_LIBSTDCPP_STATICALLY=OFF \ -DSHERPA_ONNX_ENABLE_C_API=$SHERPA_ONNX_ENABLE_C_API \ -DCMAKE_INSTALL_PREFIX=./install \ -DANDROID_ABI="armeabi-v7a" -DANDROID_ARM_NEON=ON \ -DANDROID_PLATFORM=android-21 .. + + # By default, it links to libc++_static.a + # -DANDROID_STL=c++_shared \ + # make VERBOSE=1 -j4 make -j4 make install/strip -cp -fv $onnxruntime_version/jni/armeabi-v7a/libonnxruntime.so install/lib +cp -fv $onnxruntime_version/jni/armeabi-v7a/libonnxruntime.so install/lib 2>/dev/null || true +rm -rf install/share rm -rf install/lib/pkgconfig +rm -rf install/lib/lib*.a diff --git a/build-android-x86-64.sh b/build-android-x86-64.sh index 3743842cc..9422086c9 100755 --- a/build-android-x86-64.sh +++ b/build-android-x86-64.sh @@ -1,7 +1,26 @@ #!/usr/bin/env bash set -ex -dir=$PWD/build-android-x86-64 +# If BUILD_SHARED_LIBS is ON, we use libonnxruntime.so +# If BUILD_SHARED_LIBS is OFF, we use libonnxruntime.a +# +# In any case, we will have libsherpa-onnx-jni.so +# +# If BUILD_SHARED_LIBS is OFF, then libonnxruntime.a is linked into libsherpa-onnx-jni.so +# and you only need to copy libsherpa-onnx-jni.so to your Android projects. +# +# If BUILD_SHARED_LIBS is ON, then you need to copy both libsherpa-onnx-jni.so +# and libonnxruntime.so to your Android projects +# +if [ -z $BUILD_SHARED_LIBS ]; then + BUILD_SHARED_LIBS=ON +fi + +if [ $BUILD_SHARED_LIBS == ON ]; then + dir=$PWD/build-android-x86-64 +else + dir=$PWD/build-android-x86-64-static +fi mkdir -p $dir cd $dir @@ -21,6 +40,9 @@ cd $dir if [ -z $ANDROID_NDK ]; then ANDROID_NDK=/star-fj/fangjun/software/android-sdk/ndk/22.1.7171670 + if [ $BUILD_SHARED_LIBS == OFF ]; then + ANDROID_NDK=/star-fj/fangjun/software/android-sdk/ndk/27.0.11718014 + fi # or use # ANDROID_NDK=/star-fj/fangjun/software/android-ndk # @@ -32,6 +54,10 @@ if [ -z $ANDROID_NDK ]; then # Tools -> SDK manager -> Android SDK # and set "Android SDK location" to /Users/fangjun/software/my-android ANDROID_NDK=/Users/fangjun/software/my-android/ndk/22.1.7171670 + + if [ $BUILD_SHARED_LIBS == OFF ]; then + ANDROID_NDK=/Users/fangjun/software/my-android/ndk/27.0.11718014 + fi fi fi @@ -45,17 +71,29 @@ sleep 1 onnxruntime_version=1.17.1 -if [ ! -f $onnxruntime_version/jni/x86_64/libonnxruntime.so ]; then - mkdir -p $onnxruntime_version - pushd $onnxruntime_version - wget -c -q https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${onnxruntime_version}/onnxruntime-android-${onnxruntime_version}.zip - unzip onnxruntime-android-${onnxruntime_version}.zip - rm onnxruntime-android-${onnxruntime_version}.zip - popd -fi +if [ $BUILD_SHARED_LIBS == ON ]; then + if [ ! -f $onnxruntime_version/jni/x86_64/libonnxruntime.so ]; then + mkdir -p $onnxruntime_version + pushd $onnxruntime_version + wget -c -q https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${onnxruntime_version}/onnxruntime-android-${onnxruntime_version}.zip + unzip onnxruntime-android-${onnxruntime_version}.zip + rm onnxruntime-android-${onnxruntime_version}.zip + popd + fi + + export SHERPA_ONNXRUNTIME_LIB_DIR=$dir/$onnxruntime_version/jni/x86_64/ + export SHERPA_ONNXRUNTIME_INCLUDE_DIR=$dir/$onnxruntime_version/headers/ +else + if [ ! -f ${onnxruntime_version}-static/lib/libonnxruntime.a ]; then + wget -c -q https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${onnxruntime_version}/onnxruntime-android-x86_64-static_lib-${onnxruntime_version}.zip + unzip onnxruntime-android-x86_64-static_lib-${onnxruntime_version}.zip + rm onnxruntime-android-x86_64-static_lib-${onnxruntime_version}.zip + mv onnxruntime-android-x86_64-static_lib-${onnxruntime_version} ${onnxruntime_version}-static + fi -export SHERPA_ONNXRUNTIME_LIB_DIR=$dir/$onnxruntime_version/jni/x86_64/ -export SHERPA_ONNXRUNTIME_INCLUDE_DIR=$dir/$onnxruntime_version/headers/ + export SHERPA_ONNXRUNTIME_LIB_DIR=$dir/$onnxruntime_version-static/lib/ + export SHERPA_ONNXRUNTIME_INCLUDE_DIR=$dir/$onnxruntime_version-static/include/ +fi echo "SHERPA_ONNXRUNTIME_LIB_DIR: $SHERPA_ONNXRUNTIME_LIB_DIR" echo "SHERPA_ONNXRUNTIME_INCLUDE_DIR $SHERPA_ONNXRUNTIME_INCLUDE_DIR" @@ -89,20 +127,27 @@ cmake -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake" -DBUILD_ESPEAK_NG_EXE=OFF \ -DBUILD_ESPEAK_NG_TESTS=OFF \ -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_SHARED_LIBS=ON \ + -DBUILD_SHARED_LIBS=$BUILD_SHARED_LIBS \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ -DSHERPA_ONNX_ENABLE_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_CHECK=OFF \ -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -DSHERPA_ONNX_ENABLE_JNI=$SHERPA_ONNX_ENABLE_JNI \ + -DSHERPA_ONNX_LINK_LIBSTDCPP_STATICALLY=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ -DANDROID_ABI="x86_64" \ -DSHERPA_ONNX_ENABLE_C_API=$SHERPA_ONNX_ENABLE_C_API \ -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ -DANDROID_PLATFORM=android-21 .. + # By default, it links to libc++_static.a + # -DANDROID_STL=c++_shared \ + # make VERBOSE=1 -j4 make -j4 make install/strip -cp -fv $onnxruntime_version/jni/x86_64/libonnxruntime.so install/lib + +cp -fv $onnxruntime_version/jni/x86_64/libonnxruntime.so install/lib 2>/dev/null || true +rm -rf install/share rm -rf install/lib/pkgconfig +rm -rf install/lib/lib*.a diff --git a/build-android-x86.sh b/build-android-x86.sh index f37f84c4e..bf146352f 100755 --- a/build-android-x86.sh +++ b/build-android-x86.sh @@ -1,6 +1,12 @@ #!/usr/bin/env bash set -ex +if [ x$BUILD_SHARED_LIBS == xOFF ]; then + echo "BUILD_SHARED_LIBS=OFF is ignored for Android x86." + echo "Always link with libonnxruntime.so" + sleep 2 +fi + dir=$PWD/build-android-x86 mkdir -p $dir diff --git a/sherpa-onnx/jni/CMakeLists.txt b/sherpa-onnx/jni/CMakeLists.txt index 23544c177..d761bf4e0 100644 --- a/sherpa-onnx/jni/CMakeLists.txt +++ b/sherpa-onnx/jni/CMakeLists.txt @@ -39,7 +39,7 @@ if(SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION) ) endif() -add_library(sherpa-onnx-jni ${sources}) +add_library(sherpa-onnx-jni SHARED ${sources}) target_compile_definitions(sherpa-onnx-jni PRIVATE SHERPA_ONNX_BUILD_SHARED_LIBS=1) target_compile_definitions(sherpa-onnx-jni PRIVATE SHERPA_ONNX_BUILD_MAIN_LIB=1) From ed8d8e41f92195aeb91a69076571e74db045e0b2 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 16 Dec 2024 10:47:07 +0800 Subject: [PATCH 108/183] Update readme to include Open-LLM-VTuber (#1622) --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 96c7885c6..a9bbdc39f 100644 --- a/README.md +++ b/README.md @@ -267,6 +267,13 @@ for 新一代 Kaldi **微信交流群** and **QQ 交流群**. ## Projects using sherpa-onnx +### [Open-LLM-VTuber](https://github.com/t41372/Open-LLM-VTuber) + +Talk to any LLM with hands-free voice interaction, voice interruption, and Live2D taking +face running locally across platforms + +See also + ### [voiceapi](https://github.com/ruzhila/voiceapi)

From 5cc60de5c9525981a283c09d6b35248925aa3ab2 Mon Sep 17 00:00:00 2001 From: sawich Date: Mon, 16 Dec 2024 16:37:59 +0200 Subject: [PATCH 109/183] Rename maxNumStences to maxNumSentences (#1625) --- nodejs-addon-examples/test_tts_non_streaming_vits_coqui_de.js | 2 +- nodejs-addon-examples/test_tts_non_streaming_vits_piper_en.js | 2 +- .../test_tts_non_streaming_vits_zh_aishell3.js | 2 +- nodejs-addon-examples/test_tts_non_streaming_vits_zh_ll.js | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nodejs-addon-examples/test_tts_non_streaming_vits_coqui_de.js b/nodejs-addon-examples/test_tts_non_streaming_vits_coqui_de.js index 73ba4b61c..89a913fd3 100644 --- a/nodejs-addon-examples/test_tts_non_streaming_vits_coqui_de.js +++ b/nodejs-addon-examples/test_tts_non_streaming_vits_coqui_de.js @@ -14,7 +14,7 @@ function createOfflineTts() { numThreads: 1, provider: 'cpu', }, - maxNumStences: 1, + maxNumSentences: 1, }; return new sherpa_onnx.OfflineTts(config); } diff --git a/nodejs-addon-examples/test_tts_non_streaming_vits_piper_en.js b/nodejs-addon-examples/test_tts_non_streaming_vits_piper_en.js index 910f12650..6d18fb076 100644 --- a/nodejs-addon-examples/test_tts_non_streaming_vits_piper_en.js +++ b/nodejs-addon-examples/test_tts_non_streaming_vits_piper_en.js @@ -15,7 +15,7 @@ function createOfflineTts() { numThreads: 1, provider: 'cpu', }, - maxNumStences: 1, + maxNumSentences: 1, }; return new sherpa_onnx.OfflineTts(config); } diff --git a/nodejs-addon-examples/test_tts_non_streaming_vits_zh_aishell3.js b/nodejs-addon-examples/test_tts_non_streaming_vits_zh_aishell3.js index 505247cb3..8409bca50 100644 --- a/nodejs-addon-examples/test_tts_non_streaming_vits_zh_aishell3.js +++ b/nodejs-addon-examples/test_tts_non_streaming_vits_zh_aishell3.js @@ -15,7 +15,7 @@ function createOfflineTts() { numThreads: 1, provider: 'cpu', }, - maxNumStences: 1, + maxNumSentences: 1, ruleFsts: './vits-icefall-zh-aishell3/date.fst,./vits-icefall-zh-aishell3/phone.fst,./vits-icefall-zh-aishell3/number.fst,./vits-icefall-zh-aishell3/new_heteronym.fst', ruleFars: './vits-icefall-zh-aishell3/rule.far', diff --git a/nodejs-addon-examples/test_tts_non_streaming_vits_zh_ll.js b/nodejs-addon-examples/test_tts_non_streaming_vits_zh_ll.js index 5e258e2ad..b739c391e 100644 --- a/nodejs-addon-examples/test_tts_non_streaming_vits_zh_ll.js +++ b/nodejs-addon-examples/test_tts_non_streaming_vits_zh_ll.js @@ -16,7 +16,7 @@ function createOfflineTts() { numThreads: 1, provider: 'cpu', }, - maxNumStences: 1, + maxNumSentences: 1, ruleFsts: './sherpa-onnx-vits-zh-ll/date.fst,./sherpa-onnx-vits-zh-ll/phone.fst,./sherpa-onnx-vits-zh-ll/number.fst', }; From 70ee7794100b247d64cfbd03d2b46a4ea45911bd Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 19 Dec 2024 18:19:53 +0800 Subject: [PATCH 110/183] Support using onnxruntime 1.16.0 with CUDA 11.4 on Jetson Orin NX (Linux arm64 GPU). (#1630) * Support using onnxruntime 1.16.0 with CUDA 11.4 on Jetson Orin NX. The pre-built onnxruntime libs are provided by the community using the following command: ```bash ./build.sh --build_shared_lib --config Release --update \ --build --parallel --use_cuda \ --cuda_home /usr/local/cuda \ --cudnn_home /usr/lib/aarch64-linux-gnu 2>&1 | tee my-log.txt ``` See also https://github.com/microsoft/onnxruntime/discussions/11226 --- Info about the board: ``` Model: NVIDIA Orin NX T801-16GB - Jetpack 5.1.4 [L4T 35.6.0] ``` ``` nvidia@nvidia-desktop:~/Downloads$ head -n 1 /etc/nv_tegra_release # R35 (release), REVISION: 6.0, GCID: 37391689, BOARD: t186ref, EABI: aarch64, DATE: Wed Aug 28 09:12:27 UTC 2024 nvidia@nvidia-desktop:~/Downloads$ uname -r 5.10.216-tegra nvidia@nvidia-desktop:~/Downloads$ lsb_release -i -r Distributor ID: Ubuntu Release: 20.04 nvidia@nvidia-desktop:~/Downloads$ nvcc -V nvcc: NVIDIA (R) Cuda compiler driver Copyright (c) 2005-2022 NVIDIA Corporation Built on Wed_Sep_21_10:43:33_PDT_2022 Cuda compilation tools, release 11.8, V11.8.89 Build cuda_11.8.r11.8/compiler.31833905_0 nvidia@nvidia-desktop:~/Downloads$ dpkg -l libcudnn8 Desired=Unknown/Install/Remove/Purge/Hold | Status=Not/Inst/Conf-files/Unpacked/halF-conf/Half-inst/trig-aWait/Trig-pend |/ Err?=(none)/Reinst-required (Status,Err: uppercase=bad) ||/ Name Version Architecture Description +++-==============-====================-============-================================= ii libcudnn8 8.6.0.166-1+cuda11.4 arm64 cuDNN runtime libraries nvidia@nvidia-desktop:~/Downloads$ dpkg -l tensorrt Desired=Unknown/Install/Remove/Purge/Hold | Status=Not/Inst/Conf-files/Unpacked/halF-conf/Half-inst/trig-aWait/Trig-pend |/ Err?=(none)/Reinst-required (Status,Err: uppercase=bad) ||/ Name Version Architecture Description +++-==============-==================-============-================================= ii tensorrt 8.5.2.2-1+cuda11.4 arm64 Meta package for TensorRT ``` --- .../workflows/aarch64-linux-gnu-shared.yaml | 21 ++++++++---- .../workflows/aarch64-linux-gnu-static.yaml | 2 +- .github/workflows/arm-linux-gnueabihf.yaml | 2 +- CMakeLists.txt | 3 ++ build-aarch64-linux-gnu.sh | 32 ++++++++++++++--- cmake/onnxruntime-linux-aarch64-gpu.cmake | 34 ++++++++++++++----- 6 files changed, 73 insertions(+), 21 deletions(-) diff --git a/.github/workflows/aarch64-linux-gnu-shared.yaml b/.github/workflows/aarch64-linux-gnu-shared.yaml index 1f548e237..c347e56b9 100644 --- a/.github/workflows/aarch64-linux-gnu-shared.yaml +++ b/.github/workflows/aarch64-linux-gnu-shared.yaml @@ -34,12 +34,20 @@ concurrency: jobs: aarch64_linux_gnu_shared: runs-on: ${{ matrix.os }} - name: aarch64 shared GPU ${{ matrix.gpu }} + name: aarch64 shared GPU ${{ matrix.gpu }} ${{ matrix.onnxruntime_version }} strategy: fail-fast: false matrix: - os: [ubuntu-latest] - gpu: [ON, OFF] + include: + - os: ubuntu-latest + gpu: ON + onnxruntime_version: "1.11.0" + - os: ubuntu-latest + gpu: ON + onnxruntime_version: "1.16.0" + - os: ubuntu-latest + gpu: OFF + onnxruntime_version: "" steps: - uses: actions/checkout@v4 @@ -62,7 +70,7 @@ jobs: if: steps.cache-qemu.outputs.cache-hit != 'true' run: | sudo apt-get update - sudo apt-get install autoconf automake autotools-dev ninja-build + sudo apt-get install autoconf automake autotools-dev ninja-build libglib2.0-dev. - name: checkout-qemu if: steps.cache-qemu.outputs.cache-hit != 'true' @@ -159,6 +167,7 @@ jobs: export BUILD_SHARED_LIBS=ON export SHERPA_ONNX_ENABLE_GPU=${{ matrix.gpu }} + export SHERPA_ONNX_LINUX_ARM64_GPU_ONNXRUNTIME_VERSION=${{ matrix.onnxruntime_version }} ./build-aarch64-linux-gnu.sh @@ -199,7 +208,7 @@ jobs: if [[ ${{ matrix.gpu }} == OFF ]]; then dst=${dst}-cpu else - dst=${dst}-gpu + dst=${dst}-gpu-onnxruntime-${{ matrix.onnxruntime_version }} fi mkdir $dst @@ -223,7 +232,7 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: sherpa-onnx-linux-aarch64-shared-gpu-${{ matrix.gpu }} + name: sherpa-onnx-linux-aarch64-shared-gpu-${{ matrix.gpu }}-onnxruntime-${{ matrix.onnxruntime_version }} path: sherpa-onnx-*linux-aarch64-shared*.tar.bz2 # https://huggingface.co/docs/hub/spaces-github-actions diff --git a/.github/workflows/aarch64-linux-gnu-static.yaml b/.github/workflows/aarch64-linux-gnu-static.yaml index 6cbfc0e27..749611e56 100644 --- a/.github/workflows/aarch64-linux-gnu-static.yaml +++ b/.github/workflows/aarch64-linux-gnu-static.yaml @@ -61,7 +61,7 @@ jobs: if: steps.cache-qemu.outputs.cache-hit != 'true' run: | sudo apt-get update - sudo apt-get install autoconf automake autotools-dev ninja-build + sudo apt-get install autoconf automake autotools-dev ninja-build libglib2.0-dev. - name: checkout-qemu if: steps.cache-qemu.outputs.cache-hit != 'true' diff --git a/.github/workflows/arm-linux-gnueabihf.yaml b/.github/workflows/arm-linux-gnueabihf.yaml index 6a2874910..a70f2b2f4 100644 --- a/.github/workflows/arm-linux-gnueabihf.yaml +++ b/.github/workflows/arm-linux-gnueabihf.yaml @@ -62,7 +62,7 @@ jobs: if: steps.cache-qemu.outputs.cache-hit != 'true' run: | sudo apt-get update - sudo apt-get install autoconf automake autotools-dev ninja-build + sudo apt-get install autoconf automake autotools-dev ninja-build libglib2.0-dev. - name: checkout-qemu if: steps.cache-qemu.outputs.cache-hit != 'true' diff --git a/CMakeLists.txt b/CMakeLists.txt index f45384d4d..395872c11 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,6 +46,9 @@ option(SHERPA_ONNX_USE_PRE_INSTALLED_ONNXRUNTIME_IF_AVAILABLE "True to use pre-i option(SHERPA_ONNX_ENABLE_SANITIZER "Whether to enable ubsan and asan" OFF) option(SHERPA_ONNX_BUILD_C_API_EXAMPLES "Whether to enable C API examples" ON) +set(SHERPA_ONNX_LINUX_ARM64_GPU_ONNXRUNTIME_VERSION "1.11.0" CACHE STRING "Used only for Linux ARM64 GPU. If you use Jetson nano b01, then please set it to 1.11.0. If you use Jetson Orin NX, then set it to 1.16.0") + + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") diff --git a/build-aarch64-linux-gnu.sh b/build-aarch64-linux-gnu.sh index 62b359c17..cdc48e372 100755 --- a/build-aarch64-linux-gnu.sh +++ b/build-aarch64-linux-gnu.sh @@ -1,4 +1,25 @@ #!/usr/bin/env bash +# +# Usage of this file +# +# (1) Build CPU version of sherpa-onnx +# ./build-aarch64-linux-gnu.sh +# +# (2) Build GPU version of sherpa-onnx +# +# (a) Make sure your board has NVIDIA GPU(s) +# +# (b) For Jetson Nano B01 (using CUDA 10.2) +# +# export SHERPA_ONNX_ENABLE_GPU=ON +# export SHERPA_ONNX_LINUX_ARM64_GPU_ONNXRUNTIME_VERSION=1.11.0 +# ./build-aarch64-linux-gnu.sh +# +# (c) For Jetson Orin NX (using CUDA 11.4) +# +# export SHERPA_ONNX_ENABLE_GPU=ON +# export SHERPA_ONNX_LINUX_ARM64_GPU_ONNXRUNTIME_VERSION=1.16.0 +# ./build-aarch64-linux-gnu.sh if command -v aarch64-none-linux-gnu-gcc &> /dev/null; then ln -svf $(which aarch64-none-linux-gnu-gcc) ./aarch64-linux-gnu-gcc @@ -47,11 +68,6 @@ fi if [[ x"$SHERPA_ONNX_ENABLE_GPU" == x"" ]]; then # By default, use CPU SHERPA_ONNX_ENABLE_GPU=OFF - - # If you use GPU, then please make sure you have NVIDIA GPUs on your board. - # It uses onnxruntime 1.11.0. - # - # Tested on Jetson Nano B01 fi if [[ x"$SHERPA_ONNX_ENABLE_GPU" == x"ON" ]]; then @@ -59,6 +75,11 @@ if [[ x"$SHERPA_ONNX_ENABLE_GPU" == x"ON" ]]; then BUILD_SHARED_LIBS=ON fi +if [[ x"$SHERPA_ONNX_LINUX_ARM64_GPU_ONNXRUNTIME_VERSION" == x"" ]]; then + # Used only when SHERPA_ONNX_ENABLE_GPU is ON + SHERPA_ONNX_LINUX_ARM64_GPU_ONNXRUNTIME_VERSION="1.11.0" +fi + cmake \ -DBUILD_PIPER_PHONMIZE_EXE=OFF \ -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ @@ -75,6 +96,7 @@ cmake \ -DSHERPA_ONNX_ENABLE_JNI=OFF \ -DSHERPA_ONNX_ENABLE_C_API=ON \ -DSHERPA_ONNX_ENABLE_WEBSOCKET=ON \ + -DSHERPA_ONNX_LINUX_ARM64_GPU_ONNXRUNTIME_VERSION=$SHERPA_ONNX_LINUX_ARM64_GPU_ONNXRUNTIME_VERSION \ -DCMAKE_TOOLCHAIN_FILE=../toolchains/aarch64-linux-gnu.toolchain.cmake \ .. diff --git a/cmake/onnxruntime-linux-aarch64-gpu.cmake b/cmake/onnxruntime-linux-aarch64-gpu.cmake index 64db9c22b..5df32c996 100644 --- a/cmake/onnxruntime-linux-aarch64-gpu.cmake +++ b/cmake/onnxruntime-linux-aarch64-gpu.cmake @@ -18,19 +18,37 @@ if(NOT SHERPA_ONNX_ENABLE_GPU) message(FATAL_ERROR "This file is for NVIDIA GPU only. Given SHERPA_ONNX_ENABLE_GPU: ${SHERPA_ONNX_ENABLE_GPU}") endif() -set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.11.0/onnxruntime-linux-aarch64-gpu-1.11.0.tar.bz2") -set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-linux-aarch64-gpu-1.11.0.tar.bz2") -set(onnxruntime_HASH "SHA256=36eded935551e23aead09d4173bdf0bd1e7b01fdec15d77f97d6e34029aa60d7") +message(WARNING "\ +SHERPA_ONNX_LINUX_ARM64_GPU_ONNXRUNTIME_VERSION: ${SHERPA_ONNX_LINUX_ARM64_GPU_ONNXRUNTIME_VERSION} +If you use Jetson nano b01, then please pass + -DSHERPA_ONNX_LINUX_ARM64_GPU_ONNXRUNTIME_VERSION=1.11.0 +to cmake (You need to make sure CUDA 10.2 is available on your board). + +If you use Jetson Orin NX, then please pass + -DSHERPA_ONNX_LINUX_ARM64_GPU_ONNXRUNTIME_VERSION=1.16.0 +to cmake (You need to make sure CUDA 11.4 is available on your board). +") + +set(v ${SHERPA_ONNX_LINUX_ARM64_GPU_ONNXRUNTIME_VERSION}) + +set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${v}/onnxruntime-linux-aarch64-gpu-${v}.tar.bz2") +set(onnxruntime_URL2 "https://hf-mirror.com/csukuangfj/onnxruntime-libs/resolve/main/onnxruntime-linux-aarch64-gpu-${v}.tar.bz2") + +if(v STREQUAL "1.11.0") + set(onnxruntime_HASH "SHA256=36eded935551e23aead09d4173bdf0bd1e7b01fdec15d77f97d6e34029aa60d7") +else() + set(onnxruntime_HASH "SHA256=4c09d5acf2c2682b4eab1dc2f1ad98fc1fde5f5f1960063e337983ba59379a4b") +endif() # If you don't have access to the Internet, # please download onnxruntime to one of the following locations. # You can add more if you want. set(possible_file_locations - $ENV{HOME}/Downloads/onnxruntime-linux-aarch64-gpu-1.11.0.tar.bz2 - ${CMAKE_SOURCE_DIR}/onnxruntime-linux-aarch64-gpu-1.11.0.tar.bz2 - ${CMAKE_BINARY_DIR}/onnxruntime-linux-aarch64-gpu-1.11.0.tar.bz2 - /tmp/onnxruntime-linux-aarch64-gpu-1.11.0.tar.bz2 - /star-fj/fangjun/download/github/onnxruntime-linux-aarch64-gpu-1.11.0.tar.bz2 + $ENV{HOME}/Downloads/onnxruntime-linux-aarch64-gpu-${v}.tar.bz2 + ${CMAKE_SOURCE_DIR}/onnxruntime-linux-aarch64-gpu-${v}.tar.bz2 + ${CMAKE_BINARY_DIR}/onnxruntime-linux-aarch64-gpu-${v}.tar.bz2 + /tmp/onnxruntime-linux-aarch64-gpu-${v}.tar.bz2 + /star-fj/fangjun/download/github/onnxruntime-linux-aarch64-gpu-${v}.tar.bz2 ) foreach(f IN LISTS possible_file_locations) From 86381e12ba999a84982a766a6e0b3c96ca5b8e56 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 19 Dec 2024 18:52:42 +0800 Subject: [PATCH 111/183] Update readme to include jetson orin nx and nano b01 (#1631) --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index a9bbdc39f..cc31f3054 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,8 @@ on the following platforms and operating systems: - HarmonyOS - NodeJS - WebAssembly + - [NVIDIA Jetson Orin NX][NVIDIA Jetson Orin NX] (Support running on both CPU and GPU) + - [NVIDIA Jetson Nano B01][NVIDIA Jetson Nano B01] (Support running on both CPU and GPU) - [Raspberry Pi][Raspberry Pi] - [RV1126][RV1126] - [LicheePi4A][LicheePi4A] @@ -416,3 +418,5 @@ Video demo in Chinese: [爆了！炫神教你开打字挂！真正影响胜率 [sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 [sherpa-onnx-streaming-zipformer-fr-2023-04-14]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-fr-2023-04-14.tar.bz2 [Moonshine tiny]: https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 +[NVIDIA Jetson Orin NX]: https://developer.download.nvidia.com/assets/embedded/secure/jetson/orin_nx/docs/Jetson_Orin_NX_DS-10712-001_v0.5.pdf?RCPGu9Q6OVAOv7a7vgtwc9-BLScXRIWq6cSLuditMALECJ_dOj27DgnqAPGVnT2VpiNpQan9SyFy-9zRykR58CokzbXwjSA7Gj819e91AXPrWkGZR3oS1VLxiDEpJa_Y0lr7UT-N4GnXtb8NlUkP4GkCkkF_FQivGPrAucCUywL481GH_WpP_p7ziHU1Wg==&t=eyJscyI6ImdzZW8iLCJsc2QiOiJodHRwczovL3d3dy5nb29nbGUuY29tLmhrLyJ9 +[NVIDIA Jetson Nano B01]: https://www.seeedstudio.com/blog/2020/01/16/new-revision-of-jetson-nano-dev-kit-now-supports-new-jetson-nano-module/ From 7192e576a9296ed6cfc63aab32bda37036892811 Mon Sep 17 00:00:00 2001 From: thewh1teagle <61390950+thewh1teagle@users.noreply.github.com> Date: Fri, 20 Dec 2024 09:07:45 +0200 Subject: [PATCH 112/183] feat: add checksum action (#1632) --- .github/workflows/checksum.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/checksum.yaml diff --git a/.github/workflows/checksum.yaml b/.github/workflows/checksum.yaml new file mode 100644 index 000000000..07af3768c --- /dev/null +++ b/.github/workflows/checksum.yaml @@ -0,0 +1,20 @@ +name: Create checksum + +on: + schedule: + - cron: "0 1 * * *" # Runs at 1:00 AM UTC daily + workflow_dispatch: + +jobs: + checksum: + runs-on: macos-latest + strategy: + matrix: + tag: [null, asr-models, tts-models, kws-models, speaker-recongition-models, audio-tagging-models, punctuation-models] + steps: + - name: Run checksum action + uses: thewh1teagle/checksum@v1 + with: + tag: ${{ matrix.tag }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From b76cd9033a521ac3b5b664c5765a9696aa494835 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 20 Dec 2024 19:21:32 +0800 Subject: [PATCH 113/183] Support decoding with byte-level BPE (bbpe) models. (#1633) --- scripts/bbpe/.gitignore | 1 + scripts/bbpe/generate_bbpe_table.py | 67 ++++++++++++++ sherpa-onnx/csrc/CMakeLists.txt | 3 +- sherpa-onnx/csrc/bbpe.cc | 61 ++++++++++++ sherpa-onnx/csrc/bbpe.h | 16 ++++ .../csrc/offline-recognizer-ctc-impl.h | 7 +- .../csrc/offline-recognizer-transducer-impl.h | 6 +- sherpa-onnx/csrc/online-recognizer-ctc-impl.h | 11 ++- .../csrc/online-recognizer-transducer-impl.h | 11 ++- sherpa-onnx/csrc/symbol-table.cc | 92 ++++++++++++++++++- sherpa-onnx/csrc/symbol-table.h | 5 + 11 files changed, 270 insertions(+), 10 deletions(-) create mode 100644 scripts/bbpe/.gitignore create mode 100755 scripts/bbpe/generate_bbpe_table.py create mode 100644 sherpa-onnx/csrc/bbpe.cc create mode 100644 sherpa-onnx/csrc/bbpe.h diff --git a/scripts/bbpe/.gitignore b/scripts/bbpe/.gitignore new file mode 100644 index 000000000..fa966a067 --- /dev/null +++ b/scripts/bbpe/.gitignore @@ -0,0 +1 @@ +bbpe.cc diff --git a/scripts/bbpe/generate_bbpe_table.py b/scripts/bbpe/generate_bbpe_table.py new file mode 100755 index 000000000..0b5204241 --- /dev/null +++ b/scripts/bbpe/generate_bbpe_table.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# Copyright 2024 Xiaomi Corp. (authors: Fangjun Kuang) +# +# See https://github.com/facebookresearch/fairseq/blob/main/fairseq/data/encoders/byte_bpe.py#L28 +# and +# https://github.com/k2-fsa/icefall/blob/master/icefall/byte_utils.py +# +# Caution: The PRINTABLE_LATIN from fairseq is different from PRINTABLE_BASE_CHARS from icefall + +import re + +BPE_UNK = chr(8263) +PRINTABLE_BASE_CHARS = ( + list(range(256, 287 + 1)) + + list(range(32, 126 + 1)) + + list(range(288, 305 + 1)) + + list(range(308, 318 + 1)) + + list(range(321, 328 + 1)) + + list(range(330, 382 + 1)) + + list(range(384, 422 + 1)) +) + + +BYTE_TO_BCHAR = {b: chr(PRINTABLE_BASE_CHARS[b]) for b in range(256)} +BCHAR_TO_BYTE = {bc: b for b, bc in BYTE_TO_BCHAR.items()} +BCHAR_TO_BYTE[BPE_UNK] = 32 # map unk to space + + +def main(): + s = "" + s += "// sherpa-onnx/csrc/bbpe.cc\n" + s += "//\n" + s += "// Copyright (c) 2024 Xiaomi Corporation\n" + s += "\n" + s += "// Auto-generated! DO NOT EDIT\n" + s += "\n" + s += '#include "sherpa-onnx/csrc/bbpe.h"\n' + s += "\n" + s += "#include \n" + s += "#include \n" + s += "#include \n" + s += "\n" + s += "const std::unordered_map &GetByteBpeTable() {\n" + s += " static const std::unordered_map table = {\n" + + s += " " + for i, (k, v) in enumerate(BCHAR_TO_BYTE.items()): + s += "{" + if k in ["\\", '"']: + s += f'"\{k}", {v}' + else: + s += f'"{k}", {v}' + s += "}, " + if i > 0 and i % 7 == 0: + s += "\n" + s += " " + s += "};\n" + s += "\n" + s += " return table\n;" + s += "}\n" + + with open("bbpe.cc", "w", encoding="utf-8") as f: + f.write(s) + + +if __name__ == "__main__": + main() diff --git a/sherpa-onnx/csrc/CMakeLists.txt b/sherpa-onnx/csrc/CMakeLists.txt index b3c1617e3..6bfcd2a98 100644 --- a/sherpa-onnx/csrc/CMakeLists.txt +++ b/sherpa-onnx/csrc/CMakeLists.txt @@ -12,6 +12,7 @@ endif() set(sources base64-decode.cc + bbpe.cc cat.cc circular-buffer.cc context-graph.cc @@ -78,11 +79,11 @@ set(sources online-stream.cc online-transducer-decoder.cc online-transducer-greedy-search-decoder.cc + online-transducer-greedy-search-nemo-decoder.cc online-transducer-model-config.cc online-transducer-model.cc online-transducer-modified-beam-search-decoder.cc online-transducer-nemo-model.cc - online-transducer-greedy-search-nemo-decoder.cc online-wenet-ctc-model-config.cc online-wenet-ctc-model.cc online-zipformer-transducer-model.cc diff --git a/sherpa-onnx/csrc/bbpe.cc b/sherpa-onnx/csrc/bbpe.cc new file mode 100644 index 000000000..1aa67ffa8 --- /dev/null +++ b/sherpa-onnx/csrc/bbpe.cc @@ -0,0 +1,61 @@ +// sherpa-onnx/csrc/bbpe.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +// Auto-generated! DO NOT EDIT + +#include "sherpa-onnx/csrc/bbpe.h" + +#include +#include +#include + +const std::unordered_map &GetByteBpeTable() { + static const std::unordered_map table = { + {"Ā", 0}, {"ā", 1}, {"Ă", 2}, {"ă", 3}, {"Ą", 4}, {"ą", 5}, + {"Ć", 6}, {"ć", 7}, {"Ĉ", 8}, {"ĉ", 9}, {"Ċ", 10}, {"ċ", 11}, + {"Č", 12}, {"č", 13}, {"Ď", 14}, {"ď", 15}, {"Đ", 16}, {"đ", 17}, + {"Ē", 18}, {"ē", 19}, {"Ĕ", 20}, {"ĕ", 21}, {"Ė", 22}, {"ė", 23}, + {"Ę", 24}, {"ę", 25}, {"Ě", 26}, {"ě", 27}, {"Ĝ", 28}, {"ĝ", 29}, + {"Ğ", 30}, {"ğ", 31}, {" ", 32}, {"!", 33}, {"\"", 34}, {"#", 35}, + {"$", 36}, {"%", 37}, {"&", 38}, {"'", 39}, {"(", 40}, {")", 41}, + {"*", 42}, {"+", 43}, {",", 44}, {"-", 45}, {".", 46}, {"/", 47}, + {"0", 48}, {"1", 49}, {"2", 50}, {"3", 51}, {"4", 52}, {"5", 53}, + {"6", 54}, {"7", 55}, {"8", 56}, {"9", 57}, {":", 58}, {";", 59}, + {"<", 60}, {"=", 61}, {">", 62}, {"?", 63}, {"@", 64}, {"A", 65}, + {"B", 66}, {"C", 67}, {"D", 68}, {"E", 69}, {"F", 70}, {"G", 71}, + {"H", 72}, {"I", 73}, {"J", 74}, {"K", 75}, {"L", 76}, {"M", 77}, + {"N", 78}, {"O", 79}, {"P", 80}, {"Q", 81}, {"R", 82}, {"S", 83}, + {"T", 84}, {"U", 85}, {"V", 86}, {"W", 87}, {"X", 88}, {"Y", 89}, + {"Z", 90}, {"[", 91}, {"\\", 92}, {"]", 93}, {"^", 94}, {"_", 95}, + {"`", 96}, {"a", 97}, {"b", 98}, {"c", 99}, {"d", 100}, {"e", 101}, + {"f", 102}, {"g", 103}, {"h", 104}, {"i", 105}, {"j", 106}, {"k", 107}, + {"l", 108}, {"m", 109}, {"n", 110}, {"o", 111}, {"p", 112}, {"q", 113}, + {"r", 114}, {"s", 115}, {"t", 116}, {"u", 117}, {"v", 118}, {"w", 119}, + {"x", 120}, {"y", 121}, {"z", 122}, {"{", 123}, {"|", 124}, {"}", 125}, + {"~", 126}, {"Ġ", 127}, {"ġ", 128}, {"Ģ", 129}, {"ģ", 130}, {"Ĥ", 131}, + {"ĥ", 132}, {"Ħ", 133}, {"ħ", 134}, {"Ĩ", 135}, {"ĩ", 136}, {"Ī", 137}, + {"ī", 138}, {"Ĭ", 139}, {"ĭ", 140}, {"Į", 141}, {"į", 142}, {"İ", 143}, + {"ı", 144}, {"Ĵ", 145}, {"ĵ", 146}, {"Ķ", 147}, {"ķ", 148}, {"ĸ", 149}, + {"Ĺ", 150}, {"ĺ", 151}, {"Ļ", 152}, {"ļ", 153}, {"Ľ", 154}, {"ľ", 155}, + {"Ł", 156}, {"ł", 157}, {"Ń", 158}, {"ń", 159}, {"Ņ", 160}, {"ņ", 161}, + {"Ň", 162}, {"ň", 163}, {"Ŋ", 164}, {"ŋ", 165}, {"Ō", 166}, {"ō", 167}, + {"Ŏ", 168}, {"ŏ", 169}, {"Ő", 170}, {"ő", 171}, {"Œ", 172}, {"œ", 173}, + {"Ŕ", 174}, {"ŕ", 175}, {"Ŗ", 176}, {"ŗ", 177}, {"Ř", 178}, {"ř", 179}, + {"Ś", 180}, {"ś", 181}, {"Ŝ", 182}, {"ŝ", 183}, {"Ş", 184}, {"ş", 185}, + {"Š", 186}, {"š", 187}, {"Ţ", 188}, {"ţ", 189}, {"Ť", 190}, {"ť", 191}, + {"Ŧ", 192}, {"ŧ", 193}, {"Ũ", 194}, {"ũ", 195}, {"Ū", 196}, {"ū", 197}, + {"Ŭ", 198}, {"ŭ", 199}, {"Ů", 200}, {"ů", 201}, {"Ű", 202}, {"ű", 203}, + {"Ų", 204}, {"ų", 205}, {"Ŵ", 206}, {"ŵ", 207}, {"Ŷ", 208}, {"ŷ", 209}, + {"Ÿ", 210}, {"Ź", 211}, {"ź", 212}, {"Ż", 213}, {"ż", 214}, {"Ž", 215}, + {"ž", 216}, {"ƀ", 217}, {"Ɓ", 218}, {"Ƃ", 219}, {"ƃ", 220}, {"Ƅ", 221}, + {"ƅ", 222}, {"Ɔ", 223}, {"Ƈ", 224}, {"ƈ", 225}, {"Ɖ", 226}, {"Ɗ", 227}, + {"Ƌ", 228}, {"ƌ", 229}, {"ƍ", 230}, {"Ǝ", 231}, {"Ə", 232}, {"Ɛ", 233}, + {"Ƒ", 234}, {"ƒ", 235}, {"Ɠ", 236}, {"Ɣ", 237}, {"ƕ", 238}, {"Ɩ", 239}, + {"Ɨ", 240}, {"Ƙ", 241}, {"ƙ", 242}, {"ƚ", 243}, {"ƛ", 244}, {"Ɯ", 245}, + {"Ɲ", 246}, {"ƞ", 247}, {"Ɵ", 248}, {"Ơ", 249}, {"ơ", 250}, {"Ƣ", 251}, + {"ƣ", 252}, {"Ƥ", 253}, {"ƥ", 254}, {"Ʀ", 255}, {"⁇", 32}, + }; + + return table; +} diff --git a/sherpa-onnx/csrc/bbpe.h b/sherpa-onnx/csrc/bbpe.h new file mode 100644 index 000000000..0b6a4ecf3 --- /dev/null +++ b/sherpa-onnx/csrc/bbpe.h @@ -0,0 +1,16 @@ +// sherpa-onnx/csrc/bbpe.h +// +// Copyright (c) 2024 Xiaomi Corporation + +#ifndef SHERPA_ONNX_CSRC_BBPE_H_ +#define SHERPA_ONNX_CSRC_BBPE_H_ +#include +#include +#include + +// It is equivalent to the map BCHAR_TO_BYTE +// from +// https://github.com/k2-fsa/icefall/blob/master/icefall/byte_utils.py#L280 +const std::unordered_map &GetByteBpeTable(); + +#endif // SHERPA_ONNX_CSRC_BBPE_H_ diff --git a/sherpa-onnx/csrc/offline-recognizer-ctc-impl.h b/sherpa-onnx/csrc/offline-recognizer-ctc-impl.h index 2721ecdf3..3dca0dfcc 100644 --- a/sherpa-onnx/csrc/offline-recognizer-ctc-impl.h +++ b/sherpa-onnx/csrc/offline-recognizer-ctc-impl.h @@ -41,7 +41,7 @@ static OfflineRecognitionResult Convert(const OfflineCtcDecoderResult &src, text.append(sym); if (sym.size() == 1 && (sym[0] < 0x20 || sym[0] > 0x7e)) { - // for byte bpe models + // for bpe models with byte_fallback // (but don't rewrite printable characters 0x20..0x7e, // which collide with standard BPE units) std::ostringstream os; @@ -52,6 +52,11 @@ static OfflineRecognitionResult Convert(const OfflineCtcDecoderResult &src, r.tokens.push_back(std::move(sym)); } + + if (sym_table.IsByteBpe()) { + text = sym_table.DecodeByteBpe(text); + } + r.text = std::move(text); float frame_shift_s = frame_shift_ms / 1000. * subsampling_factor; diff --git a/sherpa-onnx/csrc/offline-recognizer-transducer-impl.h b/sherpa-onnx/csrc/offline-recognizer-transducer-impl.h index 64f3798fa..158be5622 100644 --- a/sherpa-onnx/csrc/offline-recognizer-transducer-impl.h +++ b/sherpa-onnx/csrc/offline-recognizer-transducer-impl.h @@ -43,7 +43,7 @@ static OfflineRecognitionResult Convert( text.append(sym); if (sym.size() == 1 && (sym[0] < 0x20 || sym[0] > 0x7e)) { - // for byte bpe models, + // for bpe models with byte_fallback, // (but don't rewrite printable characters 0x20..0x7e, // which collide with standard BPE units) std::ostringstream os; @@ -54,6 +54,10 @@ static OfflineRecognitionResult Convert( r.tokens.push_back(std::move(sym)); } + if (sym_table.IsByteBpe()) { + text = sym_table.DecodeByteBpe(text); + } + r.text = std::move(text); float frame_shift_s = frame_shift_ms / 1000. * subsampling_factor; diff --git a/sherpa-onnx/csrc/online-recognizer-ctc-impl.h b/sherpa-onnx/csrc/online-recognizer-ctc-impl.h index 3560b1ab7..797d90f0c 100644 --- a/sherpa-onnx/csrc/online-recognizer-ctc-impl.h +++ b/sherpa-onnx/csrc/online-recognizer-ctc-impl.h @@ -34,13 +34,14 @@ static OnlineRecognizerResult Convert(const OnlineCtcDecoderResult &src, r.tokens.reserve(src.tokens.size()); r.timestamps.reserve(src.tokens.size()); + std::string text; for (auto i : src.tokens) { auto sym = sym_table[i]; - r.text.append(sym); + text.append(sym); if (sym.size() == 1 && (sym[0] < 0x20 || sym[0] > 0x7e)) { - // for byte bpe models + // for bpe models with byte_fallback // (but don't rewrite printable characters 0x20..0x7e, // which collide with standard BPE units) std::ostringstream os; @@ -52,6 +53,12 @@ static OnlineRecognizerResult Convert(const OnlineCtcDecoderResult &src, r.tokens.push_back(std::move(sym)); } + if (sym_table.IsByteBpe()) { + text = sym_table.DecodeByteBpe(text); + } + + r.text = std::move(text); + float frame_shift_s = frame_shift_ms / 1000. * subsampling_factor; for (auto t : src.timestamps) { float time = frame_shift_s * t; diff --git a/sherpa-onnx/csrc/online-recognizer-transducer-impl.h b/sherpa-onnx/csrc/online-recognizer-transducer-impl.h index 2eac3cf84..e6fd0b505 100644 --- a/sherpa-onnx/csrc/online-recognizer-transducer-impl.h +++ b/sherpa-onnx/csrc/online-recognizer-transducer-impl.h @@ -38,13 +38,14 @@ OnlineRecognizerResult Convert(const OnlineTransducerDecoderResult &src, r.tokens.reserve(src.tokens.size()); r.timestamps.reserve(src.tokens.size()); + std::string text; for (auto i : src.tokens) { auto sym = sym_table[i]; - r.text.append(sym); + text.append(sym); if (sym.size() == 1 && (sym[0] < 0x20 || sym[0] > 0x7e)) { - // for byte bpe models + // for bpe models with byte_fallback // (but don't rewrite printable characters 0x20..0x7e, // which collide with standard BPE units) std::ostringstream os; @@ -56,6 +57,12 @@ OnlineRecognizerResult Convert(const OnlineTransducerDecoderResult &src, r.tokens.push_back(std::move(sym)); } + if (sym_table.IsByteBpe()) { + text = sym_table.DecodeByteBpe(text); + } + + r.text = std::move(text); + float frame_shift_s = frame_shift_ms / 1000. * subsampling_factor; for (auto t : src.timestamps) { float time = frame_shift_s * t; diff --git a/sherpa-onnx/csrc/symbol-table.cc b/sherpa-onnx/csrc/symbol-table.cc index 3455d2007..e36bd5e58 100644 --- a/sherpa-onnx/csrc/symbol-table.cc +++ b/sherpa-onnx/csrc/symbol-table.cc @@ -5,6 +5,7 @@ #include "sherpa-onnx/csrc/symbol-table.h" #include +#include #include #include #include @@ -22,8 +23,10 @@ #endif #include "sherpa-onnx/csrc/base64-decode.h" +#include "sherpa-onnx/csrc/bbpe.h" #include "sherpa-onnx/csrc/lexicon.h" #include "sherpa-onnx/csrc/onnx-utils.h" +#include "sherpa-onnx/csrc/text-utils.h" namespace sherpa_onnx { @@ -47,6 +50,59 @@ inline void Trim(std::string *s, const char *t = ws) { TrimRight(s, t); TrimLeft(s, t); } + +bool IsByteBPE(const char *s, int32_t n) { + const uint8_t *p = reinterpret_cast(s); + if (n >= 3 && p[0] == 0xe2 && p[1] == 0x96 && p[2] == 0x81) { + return IsByteBPE(s + 3, n - 3); + } + + for (int32_t i = 0; i != n; ++i) { + if (p[i] > 0xc6) { + return false; + } + } + + return true; +} + +bool IsByteBPE(const std::unordered_map &sym2id) { + uint8_t max_v = 0; + for (const auto &p : sym2id) { + const auto &s = p.first; + if (!IsByteBPE(s.c_str(), s.size())) { + return false; + } + + uint8_t m = 0; + if (s.size() >= 3) { + const uint8_t *p = reinterpret_cast(s.c_str()); + + if (p[0] == 0xe2 && p[1] == 0x96 && p[2] == 0x81) { + if (s.size() > 3) { + m = *std::max_element( + reinterpret_cast(s.data()) + 3, + reinterpret_cast(s.data()) + s.size()); + } else { + m = 0; + } + } else { + m = *std::max_element( + reinterpret_cast(s.data()), + reinterpret_cast(s.data()) + s.size()); + } + } else { + m = *std::max_element( + reinterpret_cast(s.data()), + reinterpret_cast(s.data()) + s.size()); + } + + max_v = (m > max_v) ? m : max_v; + } + + return static_cast(max_v) == 0xc6; +} + } // namespace std::unordered_map ReadTokens( @@ -111,7 +167,10 @@ SymbolTable::SymbolTable(Manager *mgr, const std::string &filename) { Init(is); } -void SymbolTable::Init(std::istream &is) { sym2id_ = ReadTokens(is, &id2sym_); } +void SymbolTable::Init(std::istream &is) { + sym2id_ = ReadTokens(is, &id2sym_); + is_bbpe_ = IsByteBPE(sym2id_); +} std::string SymbolTable::ToString() const { std::ostringstream os; @@ -124,7 +183,7 @@ std::string SymbolTable::ToString() const { const std::string SymbolTable::operator[](int32_t id) const { std::string sym = id2sym_.at(id); - if (sym.size() >= 3) { + if (sym.size() >= 3 && !is_bbpe_) { // For BPE-based models, we replace ▁ with a space // Unicode 9601, hex 0x2581, utf8 0xe29681 const uint8_t *p = reinterpret_cast(sym.c_str()); @@ -133,7 +192,7 @@ const std::string SymbolTable::operator[](int32_t id) const { } } - // for byte-level BPE + // for BPE with byte_fallback // id 0 is blank, id 1 is sos/eos, id 2 is unk // // Note: For moonshine models, 0 is , 1, is , 2 is @@ -172,6 +231,33 @@ void SymbolTable::ApplyBase64Decode() { } } +std::string SymbolTable::DecodeByteBpe(const std::string &text) const { + if (!is_bbpe_) { + return text; + } + auto v = SplitUtf8(text); + + const auto &bbpe_table = GetByteBpeTable(); + std::string ans; + for (const auto &s : v) { + if (s == "▁") { + if (!ans.empty() && ans.back() != ' ' && std::isprint(ans.back())) { + ans.push_back(' '); + } + } else if (bbpe_table.count(s)) { + ans.push_back(bbpe_table.at(s)); + } else if (std::isprint(s[0])) { + ans.append(s); + } else { + // Should not happen + SHERPA_ONNX_LOGE("Skip OOV: %s from %s", s.c_str(), text.c_str()); + } + } + + // TODO(fangjun): Filter invalid utf-8 sequences + return ans; +} + #if __ANDROID_API__ >= 9 template SymbolTable::SymbolTable(AAssetManager *mgr, const std::string &filename); diff --git a/sherpa-onnx/csrc/symbol-table.h b/sherpa-onnx/csrc/symbol-table.h index 20d8d206b..5950d6008 100644 --- a/sherpa-onnx/csrc/symbol-table.h +++ b/sherpa-onnx/csrc/symbol-table.h @@ -56,12 +56,17 @@ class SymbolTable { int32_t NumSymbols() const { return id2sym_.size(); } + std::string DecodeByteBpe(const std::string &text) const; + + bool IsByteBpe() const { return is_bbpe_; } + private: void Init(std::istream &is); private: std::unordered_map sym2id_; std::unordered_map id2sym_; + bool is_bbpe_ = false; }; std::ostream &operator<<(std::ostream &os, const SymbolTable &symbol_table); From 4681bdfd8df2429075274b548b22eff036927e9e Mon Sep 17 00:00:00 2001 From: thewh1teagle <61390950+thewh1teagle@users.noreply.github.com> Date: Fri, 20 Dec 2024 16:42:31 +0200 Subject: [PATCH 114/183] feat: enable c api for android ci (#1635) --- .github/workflows/android.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/android.yaml b/.github/workflows/android.yaml index 0eb00d50a..17b0f6786 100644 --- a/.github/workflows/android.yaml +++ b/.github/workflows/android.yaml @@ -62,6 +62,7 @@ jobs: export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" export ANDROID_NDK=$ANDROID_NDK_LATEST_HOME + export SHERPA_ONNX_ENABLE_C_API=ON ./build-android-arm64-v8a.sh mkdir -p jniLibs/arm64-v8a/ cp -v ./build-android-arm64-v8a/install/lib/*.so ./jniLibs/arm64-v8a/ @@ -74,6 +75,7 @@ jobs: export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" export ANDROID_NDK=$ANDROID_NDK_LATEST_HOME + export SHERPA_ONNX_ENABLE_C_API=ON ./build-android-armv7-eabi.sh mkdir -p ./jniLibs/armeabi-v7a/ cp -v ./build-android-armv7-eabi/install/lib/*.so ./jniLibs/armeabi-v7a/ @@ -86,6 +88,7 @@ jobs: export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" export ANDROID_NDK=$ANDROID_NDK_LATEST_HOME + export SHERPA_ONNX_ENABLE_C_API=ON ./build-android-x86-64.sh mkdir -p ./jniLibs/x86_64 cp -v ./build-android-x86-64/install/lib/*.so ./jniLibs/x86_64 @@ -98,6 +101,7 @@ jobs: export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" export ANDROID_NDK=$ANDROID_NDK_LATEST_HOME + export SHERPA_ONNX_ENABLE_C_API=ON ./build-android-x86.sh mkdir -p ./jniLibs/x86 cp -v ./build-android-x86/install/lib/*.so ./jniLibs/x86 From a3d63130ba9dce1bdb48bff29ba3c6416c178a9e Mon Sep 17 00:00:00 2001 From: Humorousf <845425706@qq.com> Date: Mon, 23 Dec 2024 15:39:00 +0800 Subject: [PATCH 115/183] Update README.md (#1640) --- android/SherpaOnnxJavaDemo/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/android/SherpaOnnxJavaDemo/README.md b/android/SherpaOnnxJavaDemo/README.md index cf7d41557..8d7b84dbd 100644 --- a/android/SherpaOnnxJavaDemo/README.md +++ b/android/SherpaOnnxJavaDemo/README.md @@ -19,10 +19,10 @@ mv sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt ./ rm -rf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/* -mv encoder-epoch-99-avg-1.int8.onnx ./ -mv decoder-epoch-99-avg-1.onnx ./ -mv joiner-epoch-99-avg-1.int8.onnx ./ -mv tokens.txt ./ +mv encoder-epoch-99-avg-1.int8.onnx sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/ +mv decoder-epoch-99-avg-1.onnx sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/ +mv joiner-epoch-99-avg-1.int8.onnx sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/ +mv tokens.txt sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/ ``` You should have the following directory structure: From 6613828d86effa48751028ddb75b6ae280e896be Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 24 Dec 2024 05:07:32 +0200 Subject: [PATCH 116/183] SherpaOnnxVadAsr: Offload runSecondPass to background thread for improved real-time audio processing (#1638) This change ensures that the main audio processing loop is not blocked by long-running operations in `runSecondPass`, improving responsiveness and reducing the risk of missing parts of input speech. --- .../com/k2fsa/sherpa/onnx/MainActivity.kt | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/android/SherpaOnnxVadAsr/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt b/android/SherpaOnnxVadAsr/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt index fb14d072d..2f6527615 100644 --- a/android/SherpaOnnxVadAsr/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt +++ b/android/SherpaOnnxVadAsr/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt @@ -19,6 +19,11 @@ import com.k2fsa.sherpa.onnx.Vad import com.k2fsa.sherpa.onnx.getFeatureConfig import com.k2fsa.sherpa.onnx.getOfflineModelConfig import com.k2fsa.sherpa.onnx.getVadModelConfig +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext import kotlin.concurrent.thread @@ -166,6 +171,8 @@ class MainActivity : AppCompatActivity() { val bufferSize = 512 // in samples val buffer = ShortArray(bufferSize) + val coroutineScope = CoroutineScope(Dispatchers.IO) + while (isRecording) { val ret = audioRecord?.read(buffer, 0, buffer.size) @@ -175,11 +182,15 @@ class MainActivity : AppCompatActivity() { vad.acceptWaveform(samples) while(!vad.empty()) { var segment = vad.front() - val text = runSecondPass(segment.samples) - - if (text.isNotBlank()) { - lastText = "${lastText}\n${idx}: ${text}" - idx += 1 + coroutineScope.launch { + val text = runSecondPass(segment.samples) + if (text.isNotBlank()) { + withContext(Dispatchers.Main) { + lastText = "${lastText}\n${idx}: ${text}" + idx += 1 + textView.text = lastText.lowercase() + } + } } vad.pop(); @@ -192,6 +203,9 @@ class MainActivity : AppCompatActivity() { } } } + + // Clean up the coroutine scope when done + coroutineScope.cancel() } private fun initOfflineRecognizer() { From d00d1c6298df86d5592439e1c933c716d1fbfca0 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 24 Dec 2024 11:34:35 +0800 Subject: [PATCH 117/183] Fix GitHub actions. (#1642) --- .github/workflows/aarch64-linux-gnu-static.yaml | 2 +- .github/workflows/lazarus.yaml | 16 ++++++++-------- .github/workflows/test-build-wheel.yaml | 4 ++-- .github/workflows/test-pip-install.yaml | 4 ++-- .../offline-moonshine-greedy-search-decoder.cc | 8 +++++++- sherpa-onnx/csrc/symbol-table.cc | 1 + 6 files changed, 21 insertions(+), 14 deletions(-) diff --git a/.github/workflows/aarch64-linux-gnu-static.yaml b/.github/workflows/aarch64-linux-gnu-static.yaml index 749611e56..ae3b9eef2 100644 --- a/.github/workflows/aarch64-linux-gnu-static.yaml +++ b/.github/workflows/aarch64-linux-gnu-static.yaml @@ -61,7 +61,7 @@ jobs: if: steps.cache-qemu.outputs.cache-hit != 'true' run: | sudo apt-get update - sudo apt-get install autoconf automake autotools-dev ninja-build libglib2.0-dev. + sudo apt-get install build-essential zlib1g-dev pkg-config libglib2.0-dev binutils-dev libboost-all-dev autoconf libtool libssl-dev libpixman-1-dev ninja-build - name: checkout-qemu if: steps.cache-qemu.outputs.cache-hit != 'true' diff --git a/.github/workflows/lazarus.yaml b/.github/workflows/lazarus.yaml index f327c99e2..16bfdbba6 100644 --- a/.github/workflows/lazarus.yaml +++ b/.github/workflows/lazarus.yaml @@ -43,7 +43,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-20.04, macos-latest, macos-13, windows-latest] + os: [ubuntu-22.04, macos-latest, macos-13, windows-latest] steps: - uses: actions/checkout@v4 @@ -59,7 +59,7 @@ jobs: - uses: gcarreno/setup-lazarus@v3.3.1 with: lazarus-version: "stable" - with-cache: true + with-cache: false - name: Lazarus info shell: bash @@ -79,14 +79,14 @@ jobs: uname -a - name: Install patchelf for ubuntu - if: matrix.os == 'ubuntu-20.04' + if: matrix.os == 'ubuntu-22.04' shell: bash run: | sudo apt-get update -q sudo apt-get install -q -y patchelf - name: Show Patchelf version (ubuntu) - if: matrix.os == 'ubuntu-20.04' + if: matrix.os == 'ubuntu-22.04' shell: bash run: | patchelf --version @@ -104,7 +104,7 @@ jobs: cd build os=${{ matrix.os }} - if [[ $os == 'windows-latest' || $os == 'ubuntu-20.04' ]]; then + if [[ $os == 'windows-latest' || $os == 'ubuntu-22.04' ]]; then BUILD_SHARED_LIBS=ON else BUILD_SHARED_LIBS=OFF @@ -139,7 +139,7 @@ jobs: lazbuild --verbose --build-mode=Release --widgetset=cocoa ./generate_subtitles.lpi elif [[ $os == macos-latest ]]; then lazbuild --verbose --build-mode=Release --widgetset=cocoa --cpu=aarch64 ./generate_subtitles.lpi - elif [[ $os == 'ubuntu-20.04' ]]; then + elif [[ $os == 'ubuntu-22.04' ]]; then lazbuild --verbose --build-mode=Release-Linux ./generate_subtitles.lpi else lazbuild --verbose --build-mode=Release ./generate_subtitles.lpi @@ -152,7 +152,7 @@ jobs: ls -lh - name: Collect generating subtitles (Ubuntu) - if: matrix.os == 'ubuntu-20.04' + if: matrix.os == 'ubuntu-22.04' shell: bash run: | SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) @@ -223,7 +223,7 @@ jobs: ls -lh /tmp/macos-* - uses: actions/upload-artifact@v4 - if: matrix.os == 'ubuntu-20.04' + if: matrix.os == 'ubuntu-22.04' with: name: linux-x64 path: /tmp/linux-x64 diff --git a/.github/workflows/test-build-wheel.yaml b/.github/workflows/test-build-wheel.yaml index ff6ad0a2d..371d0712c 100644 --- a/.github/workflows/test-build-wheel.yaml +++ b/.github/workflows/test-build-wheel.yaml @@ -139,8 +139,8 @@ jobs: export PATH=/c/hostedtoolcache/windows/Python/3.9.13/x64/bin:$PATH export PATH=/c/hostedtoolcache/windows/Python/3.10.11/x64/bin:$PATH export PATH=/c/hostedtoolcache/windows/Python/3.11.9/x64/bin:$PATH - export PATH=/c/hostedtoolcache/windows/Python/3.12.7/x64/bin:$PATH - export PATH=/c/hostedtoolcache/windows/Python/3.13.0/x64/bin:$PATH + export PATH=/c/hostedtoolcache/windows/Python/3.12.8/x64/bin:$PATH + export PATH=/c/hostedtoolcache/windows/Python/3.13.1/x64/bin:$PATH which sherpa-onnx sherpa-onnx --help diff --git a/.github/workflows/test-pip-install.yaml b/.github/workflows/test-pip-install.yaml index 6d2faa512..139e09a0e 100644 --- a/.github/workflows/test-pip-install.yaml +++ b/.github/workflows/test-pip-install.yaml @@ -110,8 +110,8 @@ jobs: export PATH=/c/hostedtoolcache/windows/Python/3.9.13/x64/bin:$PATH export PATH=/c/hostedtoolcache/windows/Python/3.10.11/x64/bin:$PATH export PATH=/c/hostedtoolcache/windows/Python/3.11.9/x64/bin:$PATH - export PATH=/c/hostedtoolcache/windows/Python/3.12.7/x64/bin:$PATH - export PATH=/c/hostedtoolcache/windows/Python/3.13.0/x64/bin:$PATH + export PATH=/c/hostedtoolcache/windows/Python/3.12.8/x64/bin:$PATH + export PATH=/c/hostedtoolcache/windows/Python/3.13.1/x64/bin:$PATH sherpa-onnx --help sherpa-onnx-keyword-spotter --help diff --git a/sherpa-onnx/csrc/offline-moonshine-greedy-search-decoder.cc b/sherpa-onnx/csrc/offline-moonshine-greedy-search-decoder.cc index 603bdd0cf..3e2d77f69 100644 --- a/sherpa-onnx/csrc/offline-moonshine-greedy-search-decoder.cc +++ b/sherpa-onnx/csrc/offline-moonshine-greedy-search-decoder.cc @@ -73,9 +73,15 @@ OfflineMoonshineGreedySearchDecoder::Decode(Ort::Value encoder_out) { seq_len_tensor = Ort::Value::CreateTensor(memory_info, &seq_len, 1, &seq_len_shape, 1); + // To fix the false alarm of clang-tidy + // error: 'states' used after it was moved + // [bugprone-use-after-move,-warnings-as-errors] + // we use a tmp_states here + std::vector tmp_states{std::move(states)}; + std::tie(logits, states) = model_->ForwardCachedDecoder( std::move(token_tensor), std::move(seq_len_tensor), View(&encoder_out), - std::move(states)); + std::move(tmp_states)); } OfflineMoonshineDecoderResult ans; diff --git a/sherpa-onnx/csrc/symbol-table.cc b/sherpa-onnx/csrc/symbol-table.cc index e36bd5e58..8456cf777 100644 --- a/sherpa-onnx/csrc/symbol-table.cc +++ b/sherpa-onnx/csrc/symbol-table.cc @@ -4,6 +4,7 @@ #include "sherpa-onnx/csrc/symbol-table.h" +#include #include #include #include From 30a17b96fa15821ec52e56282d0eb362dbebb39d Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 24 Dec 2024 11:50:58 +0800 Subject: [PATCH 118/183] Release v1.10.36 (#1643) --- CHANGELOG.md | 16 +++++++++++++ CMakeLists.txt | 2 +- android/SherpaOnnxAar/README.md | 6 ++--- android/SherpaOnnxJavaDemo/app/build.gradle | 2 +- build-ios-shared.sh | 2 +- .../add-punctuations/pubspec.yaml | 2 +- dart-api-examples/audio-tagging/pubspec.yaml | 2 +- .../keyword-spotter/pubspec.yaml | 2 +- .../non-streaming-asr/pubspec.yaml | 2 +- .../speaker-diarization/pubspec.yaml | 2 +- .../speaker-identification/pubspec.yaml | 2 +- dart-api-examples/streaming-asr/pubspec.yaml | 2 +- dart-api-examples/tts/pubspec.yaml | 2 +- .../vad-with-non-streaming-asr/pubspec.yaml | 2 +- dart-api-examples/vad/pubspec.yaml | 2 +- flutter-examples/streaming_asr/pubspec.yaml | 4 ++-- flutter-examples/tts/pubspec.yaml | 4 ++-- flutter/sherpa_onnx/pubspec.yaml | 12 +++++----- .../ios/sherpa_onnx_ios.podspec | 2 +- .../macos/sherpa_onnx_macos.podspec | 2 +- .../SherpaOnnxHar/sherpa_onnx/README.md | 2 +- .../sherpa_onnx/oh-package.json5 | 2 +- .../entry/oh-package.json5 | 2 +- .../entry/oh-package.json5 | 2 +- .../entry/oh-package.json5 | 2 +- .../SherpaOnnxTts/entry/oh-package.json5 | 2 +- harmony-os/SherpaOnnxVadAsr/entry/README.md | 2 +- .../SherpaOnnxVadAsr/entry/oh-package.json5 | 2 +- jitpack.yml | 6 ++--- new-release.sh | 24 +++++++++---------- nodejs-addon-examples/package.json | 2 +- pom.xml | 2 +- 32 files changed, 69 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0023d23f0..450360188 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,19 @@ +## 1.10.36 + +* Update AAR version in Android Java demo (#1618) +* Support linking onnxruntime statically for Android (#1619) +* Update readme to include Open-LLM-VTuber (#1622) +* Rename maxNumStences to maxNumSentences (#1625) +* Support using onnxruntime 1.16.0 with CUDA 11.4 on Jetson Orin NX (Linux arm64 GPU). (#1630) +* Update readme to include jetson orin nx and nano b01 (#1631) +* feat: add checksum action (#1632) +* Support decoding with byte-level BPE (bbpe) models. (#1633) +* feat: enable c api for android ci (#1635) +* Update README.md (#1640) +* SherpaOnnxVadAsr: Offload runSecondPass to background thread for improved real-time audio processing (#1638) +* Fix GitHub actions. (#1642) + + ## 1.10.35 * Add missing changes about speaker identfication demo for HarmonyOS (#1612) diff --git a/CMakeLists.txt b/CMakeLists.txt index 395872c11..7aaba5db8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ project(sherpa-onnx) # Remember to update # ./CHANGELOG.md # ./new-release.sh -set(SHERPA_ONNX_VERSION "1.10.35") +set(SHERPA_ONNX_VERSION "1.10.36") # Disable warning about # diff --git a/android/SherpaOnnxAar/README.md b/android/SherpaOnnxAar/README.md index e17ed49ef..7ccf1f163 100644 --- a/android/SherpaOnnxAar/README.md +++ b/android/SherpaOnnxAar/README.md @@ -4,8 +4,8 @@ git clone https://github.com/k2-fsa/sherpa-onnx cd sherpa-onnx -wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.35/sherpa-onnx-v1.10.35-android.tar.bz2 -tar xvf sherpa-onnx-v1.10.35-android.tar.bz2 +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.36/sherpa-onnx-v1.10.36-android.tar.bz2 +tar xvf sherpa-onnx-v1.10.36-android.tar.bz2 cp -v jniLibs/arm64-v8a/* android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/arm64-v8a/ cp -v jniLibs/armeabi-v7a/* android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/armeabi-v7a/ @@ -16,5 +16,5 @@ cd android/SherpaOnnxAar ./gradlew :sherpa_onnx:assembleRelease ls -lh ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar -cp ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar ../../sherpa-onnx-1.10.35.aar +cp ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar ../../sherpa-onnx-1.10.36.aar ``` diff --git a/android/SherpaOnnxJavaDemo/app/build.gradle b/android/SherpaOnnxJavaDemo/app/build.gradle index 53ee06406..33688b4a1 100644 --- a/android/SherpaOnnxJavaDemo/app/build.gradle +++ b/android/SherpaOnnxJavaDemo/app/build.gradle @@ -34,5 +34,5 @@ dependencies { implementation 'pub.devrel:easypermissions:3.0.0' implementation 'androidx.core:core-ktx:1.7.0' // implementation files('/Users/fangjun/open-source/sherpa-onnx/android/SherpaOnnxAar/sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar') - implementation 'com.github.k2-fsa:sherpa-onnx:v1.10.35' + implementation 'com.github.k2-fsa:sherpa-onnx:v1.10.36' } diff --git a/build-ios-shared.sh b/build-ios-shared.sh index 11e75743a..50609cc2f 100755 --- a/build-ios-shared.sh +++ b/build-ios-shared.sh @@ -242,7 +242,7 @@ for d in ios-arm64_x86_64-simulator ios-arm64; do CFBundlePackageType FMWK CFBundleShortVersionString - 1.10.35 + 1.10.36 CFBundleSupportedPlatforms iPhoneOS diff --git a/dart-api-examples/add-punctuations/pubspec.yaml b/dart-api-examples/add-punctuations/pubspec.yaml index cb52a0161..dcd6b6de3 100644 --- a/dart-api-examples/add-punctuations/pubspec.yaml +++ b/dart-api-examples/add-punctuations/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.35 + sherpa_onnx: ^1.10.36 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/audio-tagging/pubspec.yaml b/dart-api-examples/audio-tagging/pubspec.yaml index 07d4ccf92..23f1c2e8c 100644 --- a/dart-api-examples/audio-tagging/pubspec.yaml +++ b/dart-api-examples/audio-tagging/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.35 + sherpa_onnx: ^1.10.36 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/keyword-spotter/pubspec.yaml b/dart-api-examples/keyword-spotter/pubspec.yaml index 4e2253f9e..640f44ecc 100644 --- a/dart-api-examples/keyword-spotter/pubspec.yaml +++ b/dart-api-examples/keyword-spotter/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.35 + sherpa_onnx: ^1.10.36 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/non-streaming-asr/pubspec.yaml b/dart-api-examples/non-streaming-asr/pubspec.yaml index b8a507bc8..030a5f5ef 100644 --- a/dart-api-examples/non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.35 + sherpa_onnx: ^1.10.36 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/speaker-diarization/pubspec.yaml b/dart-api-examples/speaker-diarization/pubspec.yaml index 9a8d4d8c5..bd41a77ae 100644 --- a/dart-api-examples/speaker-diarization/pubspec.yaml +++ b/dart-api-examples/speaker-diarization/pubspec.yaml @@ -8,7 +8,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.35 + sherpa_onnx: ^1.10.36 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/speaker-identification/pubspec.yaml b/dart-api-examples/speaker-identification/pubspec.yaml index d75596539..19b540114 100644 --- a/dart-api-examples/speaker-identification/pubspec.yaml +++ b/dart-api-examples/speaker-identification/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.35 + sherpa_onnx: ^1.10.36 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/streaming-asr/pubspec.yaml b/dart-api-examples/streaming-asr/pubspec.yaml index 0275f03c7..9e90e8aa9 100644 --- a/dart-api-examples/streaming-asr/pubspec.yaml +++ b/dart-api-examples/streaming-asr/pubspec.yaml @@ -11,7 +11,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.35 + sherpa_onnx: ^1.10.36 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/tts/pubspec.yaml b/dart-api-examples/tts/pubspec.yaml index 62009d641..6fe2973be 100644 --- a/dart-api-examples/tts/pubspec.yaml +++ b/dart-api-examples/tts/pubspec.yaml @@ -8,7 +8,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.35 + sherpa_onnx: ^1.10.36 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml index c0ecd6fae..a915c49bc 100644 --- a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.35 + sherpa_onnx: ^1.10.36 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad/pubspec.yaml b/dart-api-examples/vad/pubspec.yaml index 55df8e3cd..634fdf541 100644 --- a/dart-api-examples/vad/pubspec.yaml +++ b/dart-api-examples/vad/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.35 + sherpa_onnx: ^1.10.36 path: ^1.9.0 args: ^2.5.0 diff --git a/flutter-examples/streaming_asr/pubspec.yaml b/flutter-examples/streaming_asr/pubspec.yaml index 2d65e4452..2118bc734 100644 --- a/flutter-examples/streaming_asr/pubspec.yaml +++ b/flutter-examples/streaming_asr/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' -version: 1.10.35 +version: 1.10.36 topics: - speech-recognition @@ -31,7 +31,7 @@ dependencies: record: ^5.1.0 url_launcher: ^6.2.6 - sherpa_onnx: ^1.10.35 + sherpa_onnx: ^1.10.36 # sherpa_onnx: # path: ../../flutter/sherpa_onnx diff --git a/flutter-examples/tts/pubspec.yaml b/flutter-examples/tts/pubspec.yaml index c89ae2a9a..150098ca8 100644 --- a/flutter-examples/tts/pubspec.yaml +++ b/flutter-examples/tts/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' # Remove this line if you wish to publish to pub.dev -version: 1.10.35 +version: 1.10.36 environment: sdk: ">=2.17.0 <4.0.0" @@ -18,7 +18,7 @@ dependencies: cupertino_icons: ^1.0.6 path_provider: ^2.1.3 path: ^1.9.0 - sherpa_onnx: ^1.10.35 + sherpa_onnx: ^1.10.36 # sherpa_onnx: # path: ../../flutter/sherpa_onnx url_launcher: 6.2.6 diff --git a/flutter/sherpa_onnx/pubspec.yaml b/flutter/sherpa_onnx/pubspec.yaml index 632369503..51cb0d8dd 100644 --- a/flutter/sherpa_onnx/pubspec.yaml +++ b/flutter/sherpa_onnx/pubspec.yaml @@ -17,7 +17,7 @@ topics: - voice-activity-detection # remember to change the version in ../sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec -version: 1.10.35 +version: 1.10.36 homepage: https://github.com/k2-fsa/sherpa-onnx @@ -30,23 +30,23 @@ dependencies: flutter: sdk: flutter - sherpa_onnx_android: ^1.10.35 + sherpa_onnx_android: ^1.10.36 # sherpa_onnx_android: # path: ../sherpa_onnx_android - sherpa_onnx_macos: ^1.10.35 + sherpa_onnx_macos: ^1.10.36 # sherpa_onnx_macos: # path: ../sherpa_onnx_macos - sherpa_onnx_linux: ^1.10.35 + sherpa_onnx_linux: ^1.10.36 # sherpa_onnx_linux: # path: ../sherpa_onnx_linux - sherpa_onnx_windows: ^1.10.35 + sherpa_onnx_windows: ^1.10.36 # sherpa_onnx_windows: # path: ../sherpa_onnx_windows - sherpa_onnx_ios: ^1.10.35 + sherpa_onnx_ios: ^1.10.36 # sherpa_onnx_ios: # path: ../sherpa_onnx_ios diff --git a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec index 894ee605b..33cdc0a3e 100644 --- a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec +++ b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec @@ -7,7 +7,7 @@ # https://groups.google.com/g/dart-ffi/c/nUATMBy7r0c Pod::Spec.new do |s| s.name = 'sherpa_onnx_ios' - s.version = '1.10.35' + s.version = '1.10.36' s.summary = 'A new Flutter FFI plugin project.' s.description = <<-DESC A new Flutter FFI plugin project. diff --git a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec index 0c21a6369..aed63de85 100644 --- a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec +++ b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec @@ -4,7 +4,7 @@ # Pod::Spec.new do |s| s.name = 'sherpa_onnx_macos' - s.version = '1.10.35' + s.version = '1.10.36' s.summary = 'sherpa-onnx Flutter FFI plugin project.' s.description = <<-DESC sherpa-onnx Flutter FFI plugin project. diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md b/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md index 97372f2ea..ca9bdb29f 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md @@ -23,7 +23,7 @@ or update your `oh-package.json5` to include the following: ``` "dependencies": { - "sherpa_onnx": "1.10.35", + "sherpa_onnx": "1.10.36", }, ``` diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 index 071be0246..13f40c024 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 @@ -1,6 +1,6 @@ { "name": "sherpa_onnx", - "version": "1.10.35", + "version": "1.10.36", "description": "On-device speech-to-text, text-to-speech, and speaker diarization using Next-gen Kaldi without Internet connection", "main": "Index.ets", "author": "The next-gen Kaldi team", diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 b/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 index ae5ffd748..cf1c37644 100644 --- a/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.35" + "sherpa_onnx": "1.10.36" } } diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 b/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 index 4ecc1b57f..fbf64d78d 100644 --- a/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.35", + "sherpa_onnx": "1.10.36", } } diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 index 4ecc1b57f..fbf64d78d 100644 --- a/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.35", + "sherpa_onnx": "1.10.36", } } diff --git a/harmony-os/SherpaOnnxTts/entry/oh-package.json5 b/harmony-os/SherpaOnnxTts/entry/oh-package.json5 index 4ecc1b57f..fbf64d78d 100644 --- a/harmony-os/SherpaOnnxTts/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxTts/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.35", + "sherpa_onnx": "1.10.36", } } diff --git a/harmony-os/SherpaOnnxVadAsr/entry/README.md b/harmony-os/SherpaOnnxVadAsr/entry/README.md index 39d6f48f5..f9457676b 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/README.md +++ b/harmony-os/SherpaOnnxVadAsr/entry/README.md @@ -1,6 +1,6 @@ # Introduction -Please download ./sherpa_onnx-v1.10.35.har +Please download ./sherpa_onnx-v1.10.36.har from Hint: For users who have no access to huggingface, please use diff --git a/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 b/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 index 1ca57069b..b5a868d56 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 @@ -7,7 +7,7 @@ "license": "", "dependencies": { // please see https://ohpm.openharmony.cn/#/cn/detail/sherpa_onnx - "sherpa_onnx": "1.10.35", + "sherpa_onnx": "1.10.36", } } diff --git a/jitpack.yml b/jitpack.yml index 8040dfd11..ea04580d2 100644 --- a/jitpack.yml +++ b/jitpack.yml @@ -2,8 +2,8 @@ jdk: - openjdk17 before_install: - - wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.35/sherpa-onnx-1.10.35.aar + - wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.36/sherpa-onnx-1.10.36.aar install: - - FILE="-Dfile=sherpa-onnx-1.10.35.aar" - - mvn install:install-file $FILE -DgroupId=com.k2fsa.sherpa.onnx -DartifactId=sherpa-onnx -Dversion=1.10.35 -Dpackaging=aar -DgeneratePom=true + - FILE="-Dfile=sherpa-onnx-1.10.36.aar" + - mvn install:install-file $FILE -DgroupId=com.k2fsa.sherpa.onnx -DartifactId=sherpa-onnx -Dversion=1.10.36 -Dpackaging=aar -DgeneratePom=true diff --git a/new-release.sh b/new-release.sh index af0602250..86036c38c 100755 --- a/new-release.sh +++ b/new-release.sh @@ -2,18 +2,18 @@ set -ex -sed -i.bak 's/1\.10\.34/1\.10\.35/g' ./build-ios-shared.sh -sed -i.bak 's/1\.10\.34/1\.10\.35/g' ./pom.xml -sed -i.bak 's/1\.10\.34/1\.10\.35/g' ./jitpack.yml -sed -i.bak 's/1\.10\.34/1\.10\.35/g' ./android/SherpaOnnxAar/README.md +sed -i.bak 's/1\.10\.35/1\.10\.36/g' ./build-ios-shared.sh +sed -i.bak 's/1\.10\.35/1\.10\.36/g' ./pom.xml +sed -i.bak 's/1\.10\.35/1\.10\.36/g' ./jitpack.yml +sed -i.bak 's/1\.10\.35/1\.10\.36/g' ./android/SherpaOnnxAar/README.md -find android -name build.gradle -type f -exec sed -i.bak 's/sherpa-onnx:v1\.10\.34/sherpa-onnx:v1\.10\.35/g' {} \; +find android -name build.gradle -type f -exec sed -i.bak 's/sherpa-onnx:v1\.10\.35/sherpa-onnx:v1\.10\.36/g' {} \; -find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.34/1\.10\.35/g' {} \; -find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.34/1\.10\.35/g' {} \; -find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.34/1\.10\.35/g' {} \; -find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.34/1\.10\.35/g' {} \; -find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.34/1\.10\.35/g' {} \; +find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.35/1\.10\.36/g' {} \; +find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.35/1\.10\.36/g' {} \; +find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.35/1\.10\.36/g' {} \; +find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.35/1\.10\.36/g' {} \; +find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.35/1\.10\.36/g' {} \; -find harmony-os -name "README.md" -type f -exec sed -i.bak 's/1\.10\.34/1\.10\.35/g' {} \; -find harmony-os -name oh-package.json5 -type f -exec sed -i.bak 's/1\.10\.34/1\.10\.35/g' {} \; +find harmony-os -name "README.md" -type f -exec sed -i.bak 's/1\.10\.35/1\.10\.36/g' {} \; +find harmony-os -name oh-package.json5 -type f -exec sed -i.bak 's/1\.10\.35/1\.10\.36/g' {} \; diff --git a/nodejs-addon-examples/package.json b/nodejs-addon-examples/package.json index b9186f5e7..064d09f3c 100644 --- a/nodejs-addon-examples/package.json +++ b/nodejs-addon-examples/package.json @@ -1,5 +1,5 @@ { "dependencies": { - "sherpa-onnx-node": "^1.10.35" + "sherpa-onnx-node": "^1.10.36" } } diff --git a/pom.xml b/pom.xml index efb7b8ba3..9ff5f9c1d 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.k2fsa.sherpa.onnx sherpa-onnx-android - 1.10.35 + 1.10.36 https://github.com/k2-fsa/sherpa-onnx pom First Android Library From fe3265aa25a0bacb02076c8f41450ad48aa00bbd Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 24 Dec 2024 15:16:02 +0800 Subject: [PATCH 119/183] Add new tts models for Latvia and Persian+English (#1644) --- scripts/apk/generate-tts-apk-script.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/apk/generate-tts-apk-script.py b/scripts/apk/generate-tts-apk-script.py index 291c4738a..1aa034945 100755 --- a/scripts/apk/generate-tts-apk-script.py +++ b/scripts/apk/generate-tts-apk-script.py @@ -163,6 +163,7 @@ def get_piper_models() -> List[TtsModel]: TtsModel(model_dir="vits-piper-es_MX-claude-high"), TtsModel(model_dir="vits-piper-fa_IR-amir-medium"), TtsModel(model_dir="vits-piper-fa_IR-gyro-medium"), + TtsModel(model_dir="vits-piper-fa_en-rezahedayatfar-ibrahimwalk-medium"), TtsModel(model_dir="vits-piper-fi_FI-harri-low"), TtsModel(model_dir="vits-piper-fi_FI-harri-medium"), # TtsModel(model_dir="vits-piper-fr_FR-mls-medium"), @@ -183,6 +184,7 @@ def get_piper_models() -> List[TtsModel]: TtsModel(model_dir="vits-piper-kk_KZ-iseke-x_low"), TtsModel(model_dir="vits-piper-kk_KZ-issai-high"), TtsModel(model_dir="vits-piper-kk_KZ-raya-x_low"), + TtsModel(model_dir="vits-piper-lv_LV-aivars-medium"), TtsModel(model_dir="vits-piper-lb_LU-marylux-medium"), TtsModel(model_dir="vits-piper-ne_NP-google-medium"), TtsModel(model_dir="vits-piper-ne_NP-google-x_low"), From 08d771337baad95717e1ebb68a468568f2505fe2 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 24 Dec 2024 16:56:49 +0800 Subject: [PATCH 120/183] Add a byte-level BPE Chinese+English non-streaming zipformer model (#1645) --- .github/scripts/test-python.sh | 21 +++++++ .github/workflows/add-new-asr-models.yaml | 61 +++++++++++++++++++ .../main/ets/pages/NonStreamingAsrModels.ets | 12 ++++ scripts/apk/generate-vad-asr-apk-script.py | 20 ++++++ sherpa-onnx/kotlin-api/OfflineRecognizer.kt | 13 ++++ 5 files changed, 127 insertions(+) create mode 100644 .github/workflows/add-new-asr-models.yaml diff --git a/.github/scripts/test-python.sh b/.github/scripts/test-python.sh index 91f6f66bc..f93908d45 100755 --- a/.github/scripts/test-python.sh +++ b/.github/scripts/test-python.sh @@ -8,6 +8,27 @@ log() { echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" } +log "test offline zipformer (byte-level bpe, Chinese+English)" +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-zh-en-2023-11-22.tar.bz2 +tar xvf sherpa-onnx-zipformer-zh-en-2023-11-22.tar.bz2 +rm sherpa-onnx-zipformer-zh-en-2023-11-22.tar.bz2 + +repo=sherpa-onnx-zipformer-zh-en-2023-11-22 + +./python-api-examples/offline-decode-files.py \ + --tokens=$repo/tokens.txt \ + --encoder=$repo/encoder-epoch-34-avg-19.int8.onnx \ + --decoder=$repo/decoder-epoch-34-avg-19.onnx \ + --joiner=$repo/joiner-epoch-34-avg-19.int8.onnx \ + --num-threads=2 \ + --decoding-method=greedy_search \ + --debug=true \ + $repo/test_wavs/0.wav \ + $repo/test_wavs/1.wav \ + $repo/test_wavs/2.wav + +rm -rf sherpa-onnx-zipformer-zh-en-2023-11-22 + log "test offline Moonshine" curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 diff --git a/.github/workflows/add-new-asr-models.yaml b/.github/workflows/add-new-asr-models.yaml new file mode 100644 index 000000000..6bd2230f1 --- /dev/null +++ b/.github/workflows/add-new-asr-models.yaml @@ -0,0 +1,61 @@ +name: add-new-asr-models + +on: + # push: + # branches: + # - new-asr-models + workflow_dispatch: + +concurrency: + group: add-new-asr-models-${{ github.ref }} + cancel-in-progress: true + +jobs: + add-new-asr-models: + runs-on: ${{ matrix.os }} + name: New asr models + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Download icefall-asr-zipformer-multi-zh-en-2023-11-22 + shell: bash + run: | + d=sherpa-onnx-zipformer-zh-en-2023-11-22 + mkdir $d + pushd $d + + wget -q https://huggingface.co/zrjin/icefall-asr-zipformer-multi-zh-en-2023-11-22/resolve/main/data/lang_bbpe_2000/tokens.txt + wget -q https://huggingface.co/zrjin/icefall-asr-zipformer-multi-zh-en-2023-11-22/resolve/main/data/lang_bbpe_2000/bbpe.model + wget -q https://huggingface.co/zrjin/icefall-asr-zipformer-multi-zh-en-2023-11-22/resolve/main/exp/decoder-epoch-34-avg-19.onnx + wget -q https://huggingface.co/zrjin/icefall-asr-zipformer-multi-zh-en-2023-11-22/resolve/main/exp/encoder-epoch-34-avg-19.int8.onnx + wget -q https://huggingface.co/zrjin/icefall-asr-zipformer-multi-zh-en-2023-11-22/resolve/main/exp/encoder-epoch-34-avg-19.onnx + wget -q https://huggingface.co/zrjin/icefall-asr-zipformer-multi-zh-en-2023-11-22/resolve/main/exp/joiner-epoch-34-avg-19.int8.onnx + wget -q https://huggingface.co/zrjin/icefall-asr-zipformer-multi-zh-en-2023-11-22/resolve/main/exp/joiner-epoch-34-avg-19.onnx + + mkdir test_wavs + cd test_wavs + wget -O 0.wav -q https://huggingface.co/zrjin/icefall-asr-zipformer-multi-zh-en-2023-11-22/resolve/main/test_wavs/_1634_210_2577_1_1525157964032_3712259_29.wav + wget -O 1.wav -q https://huggingface.co/zrjin/icefall-asr-zipformer-multi-zh-en-2023-11-22/resolve/main/test_wavs/_1634_210_2577_1_1525157964032_3712259_55.wav + + wget -O 2.wav -q https://huggingface.co/zrjin/icefall-asr-zipformer-multi-zh-en-2023-11-22/resolve/main/test_wavs/_1634_210_2577_1_1525157964032_3712259_75.wav + popd + tar cvjf $d.tar.bz2 $d + ls -lh $d + rm -rf $d + + - name: Release + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + file: ./*.tar.bz2 + overwrite: true + repo_name: k2-fsa/sherpa-onnx + repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} + tag: asr-models diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/NonStreamingAsrModels.ets b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/NonStreamingAsrModels.ets index 4a1af6466..f6a263ead 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/NonStreamingAsrModels.ets +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/NonStreamingAsrModels.ets @@ -229,6 +229,18 @@ export function getOfflineModelConfig(type: number): OfflineModelConfig { break; } + + case 23: { + const modelDir = "sherpa-onnx-zipformer-zh-en-2023-11-22"; + c.transducer.encoder = `${modelDir}/encoder-epoch-34-avg-19.int8.onnx`; + c.transducer.decoder = `${modelDir}/decoder-epoch-34-avg-19.onnx`; + c.transducer.joiner = `${modelDir}/joiner-epoch-34-avg-19.int8.onnx`; + c.tokens = `${modelDir}/tokens.txt`; + c.modelType = "transducer"; + + break; + } + default: { console.log(`Please specify a supported type. Given type ${type}`); } diff --git a/scripts/apk/generate-vad-asr-apk-script.py b/scripts/apk/generate-vad-asr-apk-script.py index fe253841b..27c773a30 100755 --- a/scripts/apk/generate-vad-asr-apk-script.py +++ b/scripts/apk/generate-vad-asr-apk-script.py @@ -420,6 +420,26 @@ def get_models(): ls -lh + popd + """, + ), + Model( + model_name="sherpa-onnx-zipformer-zh-en-2023-11-22", + idx=23, + lang="zh_en", + lang2="Chinese,English", + short_name="zipformer", + cmd=""" + pushd $model_name + + rm -rfv test_wavs + + rm -fv encoder-epoch-34-avg-19.onnx + rm -fv joiner-epoch-34-avg-19.onnx + rm -fv bbpe.model + + ls -lh + popd """, ), diff --git a/sherpa-onnx/kotlin-api/OfflineRecognizer.kt b/sherpa-onnx/kotlin-api/OfflineRecognizer.kt index b0f35462d..3f3293234 100644 --- a/sherpa-onnx/kotlin-api/OfflineRecognizer.kt +++ b/sherpa-onnx/kotlin-api/OfflineRecognizer.kt @@ -451,6 +451,19 @@ fun getOfflineModelConfig(type: Int): OfflineModelConfig? { tokens = "$modelDir/tokens.txt", ) } + + 23 -> { + val modelDir = "sherpa-onnx-zipformer-zh-en-2023-11-22" + return OfflineModelConfig( + transducer = OfflineTransducerModelConfig( + encoder = "$modelDir/encoder-epoch-34-avg-19.int8.onnx", + decoder = "$modelDir/decoder-epoch-34-avg-19.onnx", + joiner = "$modelDir/joiner-epoch-34-avg-19.int8.onnx", + ), + tokens = "$modelDir/tokens.txt", + modelType = "transducer", + ) + } } return null } From b6f0f5fc2eb422d47546f388443c1725b85c728a Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 25 Dec 2024 19:32:13 +0800 Subject: [PATCH 121/183] Support removing invalid utf-8 sequences. (#1648) --- sherpa-onnx/csrc/CMakeLists.txt | 1 + sherpa-onnx/csrc/offline-recognizer-impl.cc | 2 + sherpa-onnx/csrc/online-recognizer-impl.cc | 2 + sherpa-onnx/csrc/text-utils-test.cc | 50 +++++++++ sherpa-onnx/csrc/text-utils.cc | 106 ++++++++++++++++++++ sherpa-onnx/csrc/text-utils.h | 3 + 6 files changed, 164 insertions(+) create mode 100644 sherpa-onnx/csrc/text-utils-test.cc diff --git a/sherpa-onnx/csrc/CMakeLists.txt b/sherpa-onnx/csrc/CMakeLists.txt index 6bfcd2a98..3850c8eb3 100644 --- a/sherpa-onnx/csrc/CMakeLists.txt +++ b/sherpa-onnx/csrc/CMakeLists.txt @@ -545,6 +545,7 @@ if(SHERPA_ONNX_ENABLE_TESTS) pad-sequence-test.cc slice-test.cc stack-test.cc + text-utils-test.cc text2token-test.cc transpose-test.cc unbind-test.cc diff --git a/sherpa-onnx/csrc/offline-recognizer-impl.cc b/sherpa-onnx/csrc/offline-recognizer-impl.cc index b3789849c..1867bf39b 100644 --- a/sherpa-onnx/csrc/offline-recognizer-impl.cc +++ b/sherpa-onnx/csrc/offline-recognizer-impl.cc @@ -488,6 +488,8 @@ OfflineRecognizerImpl::OfflineRecognizerImpl( std::string OfflineRecognizerImpl::ApplyInverseTextNormalization( std::string text) const { + text = RemoveInvalidUtf8Sequences(text); + if (!itn_list_.empty()) { for (const auto &tn : itn_list_) { text = tn->Normalize(text); diff --git a/sherpa-onnx/csrc/online-recognizer-impl.cc b/sherpa-onnx/csrc/online-recognizer-impl.cc index 27168b0f6..652ed2110 100644 --- a/sherpa-onnx/csrc/online-recognizer-impl.cc +++ b/sherpa-onnx/csrc/online-recognizer-impl.cc @@ -194,6 +194,8 @@ OnlineRecognizerImpl::OnlineRecognizerImpl(Manager *mgr, std::string OnlineRecognizerImpl::ApplyInverseTextNormalization( std::string text) const { + text = RemoveInvalidUtf8Sequences(text); + if (!itn_list_.empty()) { for (const auto &tn : itn_list_) { text = tn->Normalize(text); diff --git a/sherpa-onnx/csrc/text-utils-test.cc b/sherpa-onnx/csrc/text-utils-test.cc new file mode 100644 index 000000000..15558f166 --- /dev/null +++ b/sherpa-onnx/csrc/text-utils-test.cc @@ -0,0 +1,50 @@ +// sherpa-onnx/csrc/text-utils-test.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include "sherpa-onnx/csrc/text-utils.h" + +#include "gtest/gtest.h" + +namespace sherpa_onnx { + +TEST(RemoveInvalidUtf8Sequences, Case1) { + std::vector v = { + 0xe4, 0xbb, 0x8a, // 今 + 0xe5, 0xa4, 0xa9, // 天 + 'i', 's', ' ', 'M', 'o', 'd', 'a', 'y', ',', // is Monday, + ' ', 'w', 'i', 'e', ' ', 'h', 'e', 'i', 0xc3, // wie heißen Size + 0x9f, 'e', 'n', ' ', 'S', 'i', 'e', 0xf0, 0x9d, 0x84, 0x81}; + + std::vector v0 = v; + v0[1] = 0xc0; // make the first 3 bytes an invalid utf8 character + std::string s0{v0.begin(), v0.end()}; + EXPECT_EQ(s0.size(), v0.size()); + + auto s = RemoveInvalidUtf8Sequences(s0); // should remove 今 + + v0 = v; + // v0[23] == 0xc3 + // v0[24] == 0x9f + + v0[23] = 0xc1; + + s0 = {v0.begin(), v0.end()}; + s = RemoveInvalidUtf8Sequences(s0); // should remove ß + + EXPECT_EQ(s.size() + 2, v.size()); + + v0 = v; + // v0[31] = 0xf0; + // v0[32] = 0x9d; + // v0[33] = 0x84; + // v0[34] = 0x81; + v0[31] = 0xf5; + + s0 = {v0.begin(), v0.end()}; + s = RemoveInvalidUtf8Sequences(s0); + + EXPECT_EQ(s.size() + 4, v.size()); +} + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/text-utils.cc b/sherpa-onnx/csrc/text-utils.cc index 3f12e1460..7259ed7c4 100644 --- a/sherpa-onnx/csrc/text-utils.cc +++ b/sherpa-onnx/csrc/text-utils.cc @@ -396,4 +396,110 @@ void ToLowerCase(std::string *in_out) { [](unsigned char c) { return std::tolower(c); }); } +static inline bool InRange(uint8_t x, uint8_t low, uint8_t high) { + return low <= x && x <= high; +} + +/* +Please see +https://stackoverflow.com/questions/6555015/check-for-invalid-utf8 + + +Table 3-7. Well-Formed UTF-8 Byte Sequences + +Code Points First Byte Second Byte Third Byte Fourth Byte +U+0000..U+007F 00..7F +U+0080..U+07FF C2..DF 80..BF +U+0800..U+0FFF E0 A0..BF 80..BF +U+1000..U+CFFF E1..EC 80..BF 80..BF +U+D000..U+D7FF ED 80..9F 80..BF +U+E000..U+FFFF EE..EF 80..BF 80..BF +U+10000..U+3FFFF F0 90..BF 80..BF 80..BF +U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF +U+100000..U+10FFFF F4 80..8F 80..BF 80..BF + */ +std::string RemoveInvalidUtf8Sequences(const std::string &text, + bool show_debug_msg /*= false*/) { + int32_t n = static_cast(text.size()); + + std::string ans; + ans.reserve(n); + + int32_t i = 0; + const uint8_t *p = reinterpret_cast(text.data()); + while (i < n) { + if (p[i] <= 0x7f) { + ans.append(text, i, 1); + i += 1; + continue; + } + + if (InRange(p[i], 0xc2, 0xdf) && i + 1 < n && + InRange(p[i + 1], 0x80, 0xbf)) { + ans.append(text, i, 2); + i += 2; + continue; + } + + if (p[i] == 0xe0 && i + 2 < n && InRange(p[i + 1], 0xa0, 0xbf) && + InRange(p[i + 2], 0x80, 0xbf)) { + ans.append(text, i, 3); + i += 3; + continue; + } + + if (InRange(p[i], 0xe1, 0xec) && i + 2 < n && + InRange(p[i + 1], 0x80, 0xbf) && InRange(p[i + 2], 0x80, 0xbf)) { + ans.append(text, i, 3); + i += 3; + continue; + } + + if (p[i] == 0xed && i + 2 < n && InRange(p[i + 1], 0x80, 0x9f) && + InRange(p[i + 2], 0x80, 0xbf)) { + ans.append(text, i, 3); + i += 3; + continue; + } + + if (InRange(p[i], 0xee, 0xef) && i + 2 < n && + InRange(p[i + 1], 0x80, 0xbf) && InRange(p[i + 2], 0x80, 0xbf)) { + ans.append(text, i, 3); + i += 3; + continue; + } + + if (p[i] == 0xf0 && i + 3 < n && InRange(p[i + 1], 0x90, 0xbf) && + InRange(p[i + 2], 0x80, 0xbf) && InRange(p[i + 3], 0x80, 0xbf)) { + ans.append(text, i, 4); + i += 4; + continue; + } + + if (InRange(p[i], 0xf1, 0xf3) && i + 3 < n && + InRange(p[i + 1], 0x80, 0xbf) && InRange(p[i + 2], 0x80, 0xbf) && + InRange(p[i + 3], 0x80, 0xbf)) { + ans.append(text, i, 4); + i += 4; + continue; + } + + if (p[i] == 0xf4 && i + 3 < n && InRange(p[i + 1], 0x80, 0x8f) && + InRange(p[i + 2], 0x80, 0xbf) && InRange(p[i + 3], 0x80, 0xbf)) { + ans.append(text, i, 4); + i += 4; + continue; + } + + if (show_debug_msg) { + SHERPA_ONNX_LOGE("Ignore invalid utf8 sequence at pos: %d, value: %02x", + i, p[i]); + } + + i += 1; + } + + return ans; +} + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/text-utils.h b/sherpa-onnx/csrc/text-utils.h index a0b968d8a..a27137060 100644 --- a/sherpa-onnx/csrc/text-utils.h +++ b/sherpa-onnx/csrc/text-utils.h @@ -124,6 +124,9 @@ std::vector SplitUtf8(const std::string &text); std::string ToLowerCase(const std::string &s); void ToLowerCase(std::string *in_out); +std::string RemoveInvalidUtf8Sequences(const std::string &text, + bool show_debug_msg = false); + } // namespace sherpa_onnx #endif // SHERPA_ONNX_CSRC_TEXT_UTILS_H_ From 268d562135675a7ab913551a488209d608f2be84 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 26 Dec 2024 11:11:03 +0800 Subject: [PATCH 122/183] Add TeleSpeech CTC to non_streaming_server.py (#1649) --- python-api-examples/non_streaming_server.py | 47 +++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/python-api-examples/non_streaming_server.py b/python-api-examples/non_streaming_server.py index 3ffa8b7d5..da05384d0 100755 --- a/python-api-examples/non_streaming_server.py +++ b/python-api-examples/non_streaming_server.py @@ -116,6 +116,16 @@ --sense-voice=./sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/model.int8.onnx \ --tokens=./sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/tokens.txt +(9) Use a Non-streaming telespeech ctc model + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04.tar.bz2 +tar xvf sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04.tar.bz2 +rm sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04.tar.bz2 + +python3 ./python-api-examples/non_streaming_server.py \ + --telespeech-ctc=./sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04/model.int8.onnx \ + --tokens=./sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04/tokens.txt + ---- To use a certificate so that you can use https, please use @@ -250,6 +260,15 @@ def add_nemo_ctc_model_args(parser: argparse.ArgumentParser): ) +def add_telespeech_ctc_model_args(parser: argparse.ArgumentParser): + parser.add_argument( + "--telespeech-ctc", + default="", + type=str, + help="Path to the model.onnx from TeleSpeech CTC", + ) + + def add_wenet_ctc_model_args(parser: argparse.ArgumentParser): parser.add_argument( "--wenet-ctc", @@ -353,6 +372,7 @@ def add_model_args(parser: argparse.ArgumentParser): add_sense_voice_model_args(parser) add_nemo_ctc_model_args(parser) add_wenet_ctc_model_args(parser) + add_telespeech_ctc_model_args(parser) add_tdnn_ctc_model_args(parser) add_whisper_model_args(parser) add_moonshine_model_args(parser) @@ -922,6 +942,7 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: assert len(args.sense_voice) == 0, args.sense_voice assert len(args.nemo_ctc) == 0, args.nemo_ctc assert len(args.wenet_ctc) == 0, args.wenet_ctc + assert len(args.telespeech_ctc) == 0, args.telespeech_ctc assert len(args.whisper_encoder) == 0, args.whisper_encoder assert len(args.whisper_decoder) == 0, args.whisper_decoder assert len(args.tdnn_model) == 0, args.tdnn_model @@ -955,6 +976,7 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: assert len(args.sense_voice) == 0, args.sense_voice assert len(args.nemo_ctc) == 0, args.nemo_ctc assert len(args.wenet_ctc) == 0, args.wenet_ctc + assert len(args.telespeech_ctc) == 0, args.telespeech_ctc assert len(args.whisper_encoder) == 0, args.whisper_encoder assert len(args.whisper_decoder) == 0, args.whisper_decoder assert len(args.tdnn_model) == 0, args.tdnn_model @@ -979,6 +1001,7 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: elif args.sense_voice: assert len(args.nemo_ctc) == 0, args.nemo_ctc assert len(args.wenet_ctc) == 0, args.wenet_ctc + assert len(args.telespeech_ctc) == 0, args.telespeech_ctc assert len(args.whisper_encoder) == 0, args.whisper_encoder assert len(args.whisper_decoder) == 0, args.whisper_decoder assert len(args.tdnn_model) == 0, args.tdnn_model @@ -998,6 +1021,7 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: ) elif args.nemo_ctc: assert len(args.wenet_ctc) == 0, args.wenet_ctc + assert len(args.telespeech_ctc) == 0, args.telespeech_ctc assert len(args.whisper_encoder) == 0, args.whisper_encoder assert len(args.whisper_decoder) == 0, args.whisper_decoder assert len(args.tdnn_model) == 0, args.tdnn_model @@ -1020,6 +1044,7 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: provider=args.provider, ) elif args.wenet_ctc: + assert len(args.telespeech_ctc) == 0, args.telespeech_ctc assert len(args.whisper_encoder) == 0, args.whisper_encoder assert len(args.whisper_decoder) == 0, args.whisper_decoder assert len(args.tdnn_model) == 0, args.tdnn_model @@ -1041,6 +1066,28 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer: decoding_method=args.decoding_method, provider=args.provider, ) + elif args.telespeech_ctc: + assert len(args.whisper_encoder) == 0, args.whisper_encoder + assert len(args.whisper_decoder) == 0, args.whisper_decoder + assert len(args.tdnn_model) == 0, args.tdnn_model + assert len(args.moonshine_preprocessor) == 0, args.moonshine_preprocessor + assert len(args.moonshine_encoder) == 0, args.moonshine_encoder + assert ( + len(args.moonshine_uncached_decoder) == 0 + ), args.moonshine_uncached_decoder + assert len(args.moonshine_cached_decoder) == 0, args.moonshine_cached_decoder + + assert_file_exists(args.telespeech_ctc) + + recognizer = sherpa_onnx.OfflineRecognizer.from_telespeech_ctc( + model=args.telespeech_ctc, + tokens=args.tokens, + num_threads=args.num_threads, + sample_rate=args.sample_rate, + feature_dim=args.feat_dim, + decoding_method=args.decoding_method, + provider=args.provider, + ) elif args.whisper_encoder: assert len(args.tdnn_model) == 0, args.tdnn_model assert_file_exists(args.whisper_encoder) From 38d64a6d8115246355b6810ef27bb578cc95b741 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 27 Dec 2024 18:15:41 +0800 Subject: [PATCH 123/183] Fix building macOS libs (#1656) --- .github/workflows/macos.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml index 9e17491bb..fd26d5b9f 100644 --- a/.github/workflows/macos.yaml +++ b/.github/workflows/macos.yaml @@ -269,11 +269,12 @@ jobs: .github/scripts/test-online-transducer.sh - name: Copy files + if: matrix.build_type == 'Release' shell: bash run: | SHERPA_ONNX_VERSION=v$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) - if [[ ${{ matrix.with_tts }} ]]; then + if [[ ${{ matrix.with_tts }} == ON ]]; then dst=sherpa-onnx-${SHERPA_ONNX_VERSION}-osx-universal2-${{ matrix.lib_type }} else dst=sherpa-onnx-${SHERPA_ONNX_VERSION}-osx-universal2-${{ matrix.lib_type }}-no-tts @@ -290,7 +291,7 @@ jobs: tar cjvf ${dst}.tar.bz2 $dst - name: Release pre-compiled binaries and libs for macOS - if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') + if: matrix.build_type == 'Release' && (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') uses: svenstaro/upload-release-action@v2 with: file_glob: true From 49154c957b74b31116bea0b657342d83ece32cde Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 31 Dec 2024 11:25:32 +0800 Subject: [PATCH 124/183] Add Go API for Keyword spotting (#1662) --- .github/workflows/test-go-package.yaml | 7 + .github/workflows/test-go.yaml | 9 ++ .../keyword-spotting-from-file/go.mod | 4 + .../keyword-spotting-from-file/main.go | 79 ++++++++++ .../keyword-spotting-from-file/run.sh | 13 ++ .../keyword-spotting-from-file/.gitignore | 1 + .../keyword-spotting-from-file/go.mod | 5 + .../keyword-spotting-from-file/main.go | 1 + .../keyword-spotting-from-file/run.sh | 1 + scripts/go/sherpa_onnx.go | 148 ++++++++++++++++++ 10 files changed, 268 insertions(+) create mode 100644 go-api-examples/keyword-spotting-from-file/go.mod create mode 100644 go-api-examples/keyword-spotting-from-file/main.go create mode 100755 go-api-examples/keyword-spotting-from-file/run.sh create mode 100644 scripts/go/_internal/keyword-spotting-from-file/.gitignore create mode 100644 scripts/go/_internal/keyword-spotting-from-file/go.mod create mode 120000 scripts/go/_internal/keyword-spotting-from-file/main.go create mode 120000 scripts/go/_internal/keyword-spotting-from-file/run.sh diff --git a/.github/workflows/test-go-package.yaml b/.github/workflows/test-go-package.yaml index 9074449b4..649735699 100644 --- a/.github/workflows/test-go-package.yaml +++ b/.github/workflows/test-go-package.yaml @@ -68,6 +68,13 @@ jobs: run: | gcc --version + - name: Test Keyword spotting + if: matrix.os != 'windows-latest' + shell: bash + run: | + cd go-api-examples/keyword-spotting-from-file/ + ./run.sh + - name: Test adding punctuation if: matrix.os != 'windows-latest' shell: bash diff --git a/.github/workflows/test-go.yaml b/.github/workflows/test-go.yaml index 9c9951111..eaf561818 100644 --- a/.github/workflows/test-go.yaml +++ b/.github/workflows/test-go.yaml @@ -134,6 +134,15 @@ jobs: name: ${{ matrix.os }}-libs path: to-upload/ + - name: Test Keyword spotting + shell: bash + run: | + cd scripts/go/_internal/keyword-spotting-from-file/ + + ./run.sh + + ls -lh + - name: Test non-streaming decoding files shell: bash run: | diff --git a/go-api-examples/keyword-spotting-from-file/go.mod b/go-api-examples/keyword-spotting-from-file/go.mod new file mode 100644 index 000000000..dbd349a5e --- /dev/null +++ b/go-api-examples/keyword-spotting-from-file/go.mod @@ -0,0 +1,4 @@ +module keyword-spotting-from-file + +go 1.12 + diff --git a/go-api-examples/keyword-spotting-from-file/main.go b/go-api-examples/keyword-spotting-from-file/main.go new file mode 100644 index 000000000..cf6ffa84e --- /dev/null +++ b/go-api-examples/keyword-spotting-from-file/main.go @@ -0,0 +1,79 @@ +package main + +import ( + sherpa "github.com/k2-fsa/sherpa-onnx-go/sherpa_onnx" + "log" +) + +func main() { + log.SetFlags(log.LstdFlags | log.Lmicroseconds) + + config := sherpa.KeywordSpotterConfig{} + + // Please download the models from + // https://github.com/k2-fsa/sherpa-onnx/releases/tag/kws-models + + config.ModelConfig.Transducer.Encoder = "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/encoder-epoch-12-avg-2-chunk-16-left-64.onnx" + config.ModelConfig.Transducer.Decoder = "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/decoder-epoch-12-avg-2-chunk-16-left-64.onnx" + config.ModelConfig.Transducer.Joiner = "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/joiner-epoch-12-avg-2-chunk-16-left-64.onnx" + config.ModelConfig.Tokens = "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/tokens.txt" + config.KeywordsFile = "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/test_wavs/test_keywords.txt" + config.ModelConfig.NumThreads = 1 + config.ModelConfig.Debug = 1 + + spotter := sherpa.NewKeywordSpotter(&config) + defer sherpa.DeleteKeywordSpotter(spotter) + + wave_filename := "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/test_wavs/3.wav" + + wave := sherpa.ReadWave(wave_filename) + if wave == nil { + log.Printf("Failed to read %v\n", wave_filename) + return + } + + log.Println("----------Use pre-defined keywords----------") + + stream := sherpa.NewKeywordStream(spotter) + defer sherpa.DeleteOnlineStream(stream) + + stream.AcceptWaveform(wave.SampleRate, wave.Samples) + + for spotter.IsReady(stream) { + spotter.Decode(stream) + result := spotter.GetResult(stream) + if result.Keyword != "" { + log.Printf("Detected %v\n", result.Keyword) + } + } + + log.Println("----------Use pre-defined keywords + add a new keyword----------") + + stream2 := sherpa.NewKeywordStreamWithKeywords(spotter, "y ǎn y uán @演员") + defer sherpa.DeleteOnlineStream(stream2) + + stream2.AcceptWaveform(wave.SampleRate, wave.Samples) + + for spotter.IsReady(stream2) { + spotter.Decode(stream2) + result := spotter.GetResult(stream2) + if result.Keyword != "" { + log.Printf("Detected %v\n", result.Keyword) + } + } + + log.Println("----------Use pre-defined keywords + add 2 new keywords----------") + + stream3 := sherpa.NewKeywordStreamWithKeywords(spotter, "y ǎn y uán @演员/zh ī m íng @知名") + defer sherpa.DeleteOnlineStream(stream3) + + stream3.AcceptWaveform(wave.SampleRate, wave.Samples) + + for spotter.IsReady(stream3) { + spotter.Decode(stream3) + result := spotter.GetResult(stream3) + if result.Keyword != "" { + log.Printf("Detected %v\n", result.Keyword) + } + } +} diff --git a/go-api-examples/keyword-spotting-from-file/run.sh b/go-api-examples/keyword-spotting-from-file/run.sh new file mode 100755 index 000000000..89411f47a --- /dev/null +++ b/go-api-examples/keyword-spotting-from-file/run.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -ex + +if [ ! -f ./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/tokens.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/kws-models/sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.tar.bz2 + tar xvf sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.tar.bz2 + rm sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.tar.bz2 +fi + +go mod tidy +go build +./keyword-spotting-from-file diff --git a/scripts/go/_internal/keyword-spotting-from-file/.gitignore b/scripts/go/_internal/keyword-spotting-from-file/.gitignore new file mode 100644 index 000000000..2c433c5c7 --- /dev/null +++ b/scripts/go/_internal/keyword-spotting-from-file/.gitignore @@ -0,0 +1 @@ +keyword-spotting-from-file diff --git a/scripts/go/_internal/keyword-spotting-from-file/go.mod b/scripts/go/_internal/keyword-spotting-from-file/go.mod new file mode 100644 index 000000000..9cdbc6321 --- /dev/null +++ b/scripts/go/_internal/keyword-spotting-from-file/go.mod @@ -0,0 +1,5 @@ +module keyword-spotting-from-file + +go 1.12 + +replace github.com/k2-fsa/sherpa-onnx-go/sherpa_onnx => ../ diff --git a/scripts/go/_internal/keyword-spotting-from-file/main.go b/scripts/go/_internal/keyword-spotting-from-file/main.go new file mode 120000 index 000000000..f17d55363 --- /dev/null +++ b/scripts/go/_internal/keyword-spotting-from-file/main.go @@ -0,0 +1 @@ +../../../../go-api-examples/keyword-spotting-from-file/main.go \ No newline at end of file diff --git a/scripts/go/_internal/keyword-spotting-from-file/run.sh b/scripts/go/_internal/keyword-spotting-from-file/run.sh new file mode 120000 index 000000000..a9bb15f88 --- /dev/null +++ b/scripts/go/_internal/keyword-spotting-from-file/run.sh @@ -0,0 +1 @@ +../../../../go-api-examples/keyword-spotting-from-file/run.sh \ No newline at end of file diff --git a/scripts/go/sherpa_onnx.go b/scripts/go/sherpa_onnx.go index cde6513da..17afd32f2 100644 --- a/scripts/go/sherpa_onnx.go +++ b/scripts/go/sherpa_onnx.go @@ -1385,3 +1385,151 @@ func (punc *OfflinePunctuation) AddPunct(text string) string { return text_with_punct } + +// Configuration for the online/streaming recognizer. +type KeywordSpotterConfig struct { + FeatConfig FeatureConfig + ModelConfig OnlineModelConfig + MaxActivePaths int + KeywordsFile string + KeywordsScore float32 + KeywordsThreshold float32 + KeywordsBuf string + KeywordsBufSize int +} + +type KeywordSpotterResult struct { + Keyword string +} + +type KeywordSpotter struct { + impl *C.struct_SherpaOnnxKeywordSpotter +} + +// Free the internal pointer inside the recognizer to avoid memory leak. +func DeleteKeywordSpotter(spotter *KeywordSpotter) { + C.SherpaOnnxDestroyKeywordSpotter(spotter.impl) + spotter.impl = nil +} + +// The user is responsible to invoke [DeleteKeywordSpotter]() to free +// the returned spotter to avoid memory leak +func NewKeywordSpotter(config *KeywordSpotterConfig) *KeywordSpotter { + c := C.struct_SherpaOnnxKeywordSpotterConfig{} + c.feat_config.sample_rate = C.int(config.FeatConfig.SampleRate) + c.feat_config.feature_dim = C.int(config.FeatConfig.FeatureDim) + + c.model_config.transducer.encoder = C.CString(config.ModelConfig.Transducer.Encoder) + defer C.free(unsafe.Pointer(c.model_config.transducer.encoder)) + + c.model_config.transducer.decoder = C.CString(config.ModelConfig.Transducer.Decoder) + defer C.free(unsafe.Pointer(c.model_config.transducer.decoder)) + + c.model_config.transducer.joiner = C.CString(config.ModelConfig.Transducer.Joiner) + defer C.free(unsafe.Pointer(c.model_config.transducer.joiner)) + + c.model_config.paraformer.encoder = C.CString(config.ModelConfig.Paraformer.Encoder) + defer C.free(unsafe.Pointer(c.model_config.paraformer.encoder)) + + c.model_config.paraformer.decoder = C.CString(config.ModelConfig.Paraformer.Decoder) + defer C.free(unsafe.Pointer(c.model_config.paraformer.decoder)) + + c.model_config.zipformer2_ctc.model = C.CString(config.ModelConfig.Zipformer2Ctc.Model) + defer C.free(unsafe.Pointer(c.model_config.zipformer2_ctc.model)) + + c.model_config.tokens = C.CString(config.ModelConfig.Tokens) + defer C.free(unsafe.Pointer(c.model_config.tokens)) + + c.model_config.num_threads = C.int(config.ModelConfig.NumThreads) + + c.model_config.provider = C.CString(config.ModelConfig.Provider) + defer C.free(unsafe.Pointer(c.model_config.provider)) + + c.model_config.debug = C.int(config.ModelConfig.Debug) + + c.model_config.model_type = C.CString(config.ModelConfig.ModelType) + defer C.free(unsafe.Pointer(c.model_config.model_type)) + + c.model_config.modeling_unit = C.CString(config.ModelConfig.ModelingUnit) + defer C.free(unsafe.Pointer(c.model_config.modeling_unit)) + + c.model_config.bpe_vocab = C.CString(config.ModelConfig.BpeVocab) + defer C.free(unsafe.Pointer(c.model_config.bpe_vocab)) + + c.model_config.tokens_buf = C.CString(config.ModelConfig.TokensBuf) + defer C.free(unsafe.Pointer(c.model_config.tokens_buf)) + + c.model_config.tokens_buf_size = C.int(config.ModelConfig.TokensBufSize) + + c.max_active_paths = C.int(config.MaxActivePaths) + + c.keywords_file = C.CString(config.KeywordsFile) + defer C.free(unsafe.Pointer(c.keywords_file)) + + c.keywords_score = C.float(config.KeywordsScore) + + c.keywords_threshold = C.float(config.KeywordsThreshold) + + c.keywords_buf = C.CString(config.KeywordsBuf) + defer C.free(unsafe.Pointer(c.keywords_buf)) + + c.keywords_buf_size = C.int(config.KeywordsBufSize) + + spotter := &KeywordSpotter{} + spotter.impl = C.SherpaOnnxCreateKeywordSpotter(&c) + + return spotter +} + +// The user is responsible to invoke [DeleteOnlineStream]() to free +// the returned stream to avoid memory leak +func NewKeywordStream(spotter *KeywordSpotter) *OnlineStream { + stream := &OnlineStream{} + stream.impl = C.SherpaOnnxCreateKeywordStream(spotter.impl) + return stream +} + +// The user is responsible to invoke [DeleteOnlineStream]() to free +// the returned stream to avoid memory leak +func NewKeywordStreamWithKeywords(spotter *KeywordSpotter, keywords string) *OnlineStream { + stream := &OnlineStream{} + + s := C.CString(keywords) + defer C.free(unsafe.Pointer(s)) + + stream.impl = C.SherpaOnnxCreateKeywordStreamWithKeywords(spotter.impl, s) + return stream +} + +// Check whether the stream has enough feature frames for decoding. +// Return true if this stream is ready for decoding. Return false otherwise. +// +// You will usually use it like below: +// +// for spotter.IsReady(s) { +// spotter.Decode(s) +// } +func (spotter *KeywordSpotter) IsReady(s *OnlineStream) bool { + return C.SherpaOnnxIsKeywordStreamReady(spotter.impl, s.impl) == 1 +} + +// Decode the stream. Before calling this function, you have to ensure +// that spotter.IsReady(s) returns true. Otherwise, you will be SAD. +// +// You usually use it like below: +// +// for spotter.IsReady(s) { +// spotter.Decode(s) +// } +func (spotter *KeywordSpotter) Decode(s *OnlineStream) { + C.SherpaOnnxDecodeKeywordStream(spotter.impl, s.impl) +} + +// Get the current result of stream since the last invoke of Reset() +func (spotter *KeywordSpotter) GetResult(s *OnlineStream) *KeywordSpotterResult { + p := C.SherpaOnnxGetKeywordResult(spotter.impl, s.impl) + defer C.SherpaOnnxDestroyKeywordResult(p) + result := &KeywordSpotterResult{} + result.Keyword = C.GoString(p.keyword) + return result +} From 5c2cc48f50c4dd5c21b910825eaaaa6007f7e082 Mon Sep 17 00:00:00 2001 From: yujinqiu Date: Tue, 31 Dec 2024 11:26:32 +0800 Subject: [PATCH 125/183] Add swift online punctuation (#1661) --- sherpa-onnx/c-api/c-api.cc | 48 +++++++++++++++++++ sherpa-onnx/c-api/c-api.h | 33 +++++++++++++ swift-api-examples/SherpaOnnx.swift | 46 ++++++++++++++++++ .../add-punctuation-online.swift | 35 ++++++++++++++ .../run-add-punctuations-online.sh | 36 ++++++++++++++ 5 files changed, 198 insertions(+) create mode 100644 swift-api-examples/add-punctuation-online.swift create mode 100755 swift-api-examples/run-add-punctuations-online.sh diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index 4a11cae29..4d1bb6625 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -24,6 +24,7 @@ #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/offline-punctuation.h" #include "sherpa-onnx/csrc/offline-recognizer.h" +#include "sherpa-onnx/csrc/online-punctuation.h" #include "sherpa-onnx/csrc/online-recognizer.h" #include "sherpa-onnx/csrc/resample.h" #include "sherpa-onnx/csrc/speaker-embedding-extractor.h" @@ -1717,6 +1718,53 @@ const char *SherpaOfflinePunctuationAddPunct( void SherpaOfflinePunctuationFreeText(const char *text) { delete[] text; } +struct SherpaOnnxOnlinePunctuation { + std::unique_ptr impl; +}; + +const SherpaOnnxOnlinePunctuation *SherpaOnnxCreateOnlinePunctuation( + const SherpaOnnxOnlinePunctuationConfig *config) { + auto p = new SherpaOnnxOnlinePunctuation; + try { + sherpa_onnx::OnlinePunctuationConfig punctuation_config; + punctuation_config.model.cnn_bilstm = SHERPA_ONNX_OR(config->model.cnn_bilstm, ""); + punctuation_config.model.bpe_vocab = SHERPA_ONNX_OR(config->model.bpe_vocab, ""); + punctuation_config.model.num_threads = SHERPA_ONNX_OR(config->model.num_threads, 1); + punctuation_config.model.debug = config->model.debug; + punctuation_config.model.provider = SHERPA_ONNX_OR(config->model.provider, "cpu"); + + p->impl = + std::make_unique(punctuation_config); + } catch (const std::exception &e) { + SHERPA_ONNX_LOGE("Failed to create online punctuation: %s", e.what()); + delete p; + return nullptr; + } + return p; +} + +void SherpaOnnxDestroyOnlinePunctuation(const SherpaOnnxOnlinePunctuation *p) { + delete p; +} + +const char *SherpaOnnxOnlinePunctuationAddPunct( + const SherpaOnnxOnlinePunctuation *punctuation, const char *text) { + if (!punctuation || !text) return nullptr; + + try { + std::string s = punctuation->impl->AddPunctuationWithCase(text); + char *p = new char[s.size() + 1]; + std::copy(s.begin(), s.end(), p); + p[s.size()] = '\0'; + return p; + } catch (const std::exception &e) { + SHERPA_ONNX_LOGE("Failed to add punctuation: %s", e.what()); + return nullptr; + } +} + +void SherpaOnnxOnlinePunctuationFreeText(const char *text) { delete[] text; } + struct SherpaOnnxLinearResampler { std::unique_ptr impl; }; diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index 4d4a2c4fc..990c94cb3 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -1369,6 +1369,39 @@ SHERPA_ONNX_API const char *SherpaOfflinePunctuationAddPunct( SHERPA_ONNX_API void SherpaOfflinePunctuationFreeText(const char *text); +SHERPA_ONNX_API typedef struct SherpaOnnxOnlinePunctuationModelConfig { + const char *cnn_bilstm; + const char *bpe_vocab; + int32_t num_threads; + int32_t debug; + const char *provider; +} SherpaOnnxOnlinePunctuationModelConfig; + +SHERPA_ONNX_API typedef struct SherpaOnnxOnlinePunctuationConfig { + SherpaOnnxOnlinePunctuationModelConfig model; +} SherpaOnnxOnlinePunctuationConfig; + +SHERPA_ONNX_API typedef struct SherpaOnnxOnlinePunctuation SherpaOnnxOnlinePunctuation; + +// Create an online punctuation processor. The user has to invoke +// SherpaOnnxDestroyOnlinePunctuation() to free the returned pointer +// to avoid memory leak +SHERPA_ONNX_API const SherpaOnnxOnlinePunctuation *SherpaOnnxCreateOnlinePunctuation( + const SherpaOnnxOnlinePunctuationConfig *config); + +// Free a pointer returned by SherpaOnnxCreateOnlinePunctuation() +SHERPA_ONNX_API void SherpaOnnxDestroyOnlinePunctuation( + const SherpaOnnxOnlinePunctuation *punctuation); + +// Add punctuations to the input text. The user has to invoke +// SherpaOnnxOnlinePunctuationFreeText() to free the returned pointer +// to avoid memory leak +SHERPA_ONNX_API const char *SherpaOnnxOnlinePunctuationAddPunct( + const SherpaOnnxOnlinePunctuation *punctuation, const char *text); + +// Free a pointer returned by SherpaOnnxOnlinePunctuationAddPunct() +SHERPA_ONNX_API void SherpaOnnxOnlinePunctuationFreeText(const char *text); + // for resampling SHERPA_ONNX_API typedef struct SherpaOnnxLinearResampler SherpaOnnxLinearResampler; diff --git a/swift-api-examples/SherpaOnnx.swift b/swift-api-examples/SherpaOnnx.swift index 661ebba28..b100ef408 100644 --- a/swift-api-examples/SherpaOnnx.swift +++ b/swift-api-examples/SherpaOnnx.swift @@ -1095,6 +1095,52 @@ class SherpaOnnxOfflinePunctuationWrapper { } } +func sherpaOnnxOnlinePunctuationModelConfig( + cnnBiLstm: String, + bpeVocab: String, + numThreads: Int = 1, + debug: Int = 0, + provider: String = "cpu" +) -> SherpaOnnxOnlinePunctuationModelConfig { + return SherpaOnnxOnlinePunctuationModelConfig( + cnn_bilstm: toCPointer(cnnBiLstm), + bpe_vocab: toCPointer(bpeVocab), + num_threads: Int32(numThreads), + debug: Int32(debug), + provider: toCPointer(provider)) +} + +func sherpaOnnxOnlinePunctuationConfig( + model: SherpaOnnxOnlinePunctuationModelConfig +) -> SherpaOnnxOnlinePunctuationConfig { + return SherpaOnnxOnlinePunctuationConfig(model: model) +} + +class SherpaOnnxOnlinePunctuationWrapper { + /// A pointer to the underlying counterpart in C + let ptr: OpaquePointer! + + /// Constructor taking a model config + init( + config: UnsafePointer! + ) { + ptr = SherpaOnnxCreateOnlinePunctuation(config) + } + + deinit { + if let ptr { + SherpaOnnxDestroyOnlinePunctuation(ptr) + } + } + + func addPunct(text: String) -> String { + let cText = SherpaOnnxOnlinePunctuationAddPunct(ptr, toCPointer(text)) + let ans = String(cString: cText!) + SherpaOnnxOnlinePunctuationFreeText(cText) + return ans + } +} + func sherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig(model: String) -> SherpaOnnxOfflineSpeakerSegmentationPyannoteModelConfig { diff --git a/swift-api-examples/add-punctuation-online.swift b/swift-api-examples/add-punctuation-online.swift new file mode 100644 index 000000000..79af921eb --- /dev/null +++ b/swift-api-examples/add-punctuation-online.swift @@ -0,0 +1,35 @@ +func run() { + let model = "./sherpa-onnx-online-punct-en-2024-08-06/model.onnx" + let bpe = "./sherpa-onnx-online-punct-en-2024-08-06/bpe.vocab" + + // Create model config + let modelConfig = sherpaOnnxOnlinePunctuationModelConfig( + cnnBiLstm: model, + bpeVocab: bpe + ) + + // Create punctuation config + var config = sherpaOnnxOnlinePunctuationConfig(model: modelConfig) + + // Create punctuation instance + let punct = SherpaOnnxOnlinePunctuationWrapper(config: &config) + + // Test texts + let textList = [ + "how are you i am fine thank you", + "The African blogosphere is rapidly expanding bringing more voices online in the form of commentaries opinions analyses rants and poetry" + ] + + // Process each text + for i in 0.. Date: Tue, 31 Dec 2024 12:44:14 +0800 Subject: [PATCH 126/183] Add C++ runtime for Matcha-TTS (#1627) --- .github/scripts/test-offline-tts.sh | 34 ++ .github/scripts/test-python.sh | 20 + .github/workflows/linux.yaml | 30 +- .github/workflows/macos.yaml | 18 +- python-api-examples/offline-tts-play.py | 93 ++++- python-api-examples/offline-tts.py | 92 ++++- sherpa-onnx/csrc/CMakeLists.txt | 3 + sherpa-onnx/csrc/hifigan-vocoder.cc | 107 +++++ sherpa-onnx/csrc/hifigan-vocoder.h | 38 ++ sherpa-onnx/csrc/jieba-lexicon.cc | 24 +- sherpa-onnx/csrc/jieba-lexicon.h | 4 +- sherpa-onnx/csrc/offline-tts-impl.cc | 27 +- sherpa-onnx/csrc/offline-tts-impl.h | 4 + sherpa-onnx/csrc/offline-tts-matcha-impl.h | 381 ++++++++++++++++++ .../csrc/offline-tts-matcha-model-config.cc | 143 +++++++ .../csrc/offline-tts-matcha-model-config.h | 56 +++ .../csrc/offline-tts-matcha-model-metadata.h | 28 ++ sherpa-onnx/csrc/offline-tts-matcha-model.cc | 198 +++++++++ sherpa-onnx/csrc/offline-tts-matcha-model.h | 39 ++ sherpa-onnx/csrc/offline-tts-model-config.cc | 8 +- sherpa-onnx/csrc/offline-tts-model-config.h | 4 + sherpa-onnx/csrc/offline-tts-vits-impl.h | 28 +- .../csrc/offline-tts-vits-model-config.cc | 23 +- sherpa-onnx/csrc/offline-tts-vits-model.cc | 4 +- sherpa-onnx/csrc/offline-tts-vits-model.h | 2 +- sherpa-onnx/csrc/session.cc | 5 + sherpa-onnx/csrc/session.h | 3 + sherpa-onnx/csrc/sherpa-onnx-offline-tts.cc | 4 + sherpa-onnx/python/csrc/CMakeLists.txt | 1 + .../csrc/offline-tts-matcha-model-config.cc | 37 ++ .../csrc/offline-tts-matcha-model-config.h | 16 + .../python/csrc/offline-tts-model-config.cc | 8 +- sherpa-onnx/python/sherpa_onnx/__init__.py | 1 + 33 files changed, 1397 insertions(+), 86 deletions(-) create mode 100644 sherpa-onnx/csrc/hifigan-vocoder.cc create mode 100644 sherpa-onnx/csrc/hifigan-vocoder.h create mode 100644 sherpa-onnx/csrc/offline-tts-matcha-impl.h create mode 100644 sherpa-onnx/csrc/offline-tts-matcha-model-config.cc create mode 100644 sherpa-onnx/csrc/offline-tts-matcha-model-config.h create mode 100644 sherpa-onnx/csrc/offline-tts-matcha-model-metadata.h create mode 100644 sherpa-onnx/csrc/offline-tts-matcha-model.cc create mode 100644 sherpa-onnx/csrc/offline-tts-matcha-model.h create mode 100644 sherpa-onnx/python/csrc/offline-tts-matcha-model-config.cc create mode 100644 sherpa-onnx/python/csrc/offline-tts-matcha-model-config.h diff --git a/.github/scripts/test-offline-tts.sh b/.github/scripts/test-offline-tts.sh index d3d35df2c..1aa0340a0 100755 --- a/.github/scripts/test-offline-tts.sh +++ b/.github/scripts/test-offline-tts.sh @@ -18,6 +18,40 @@ which $EXE # test waves are saved in ./tts mkdir ./tts +log "------------------------------------------------------------" +log "matcha-icefall-zh-baker" +log "------------------------------------------------------------" +curl -O -SL https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 +tar xvf matcha-icefall-zh-baker.tar.bz2 +rm matcha-icefall-zh-baker.tar.bz2 + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +$EXE \ + --matcha-acoustic-model=./matcha-icefall-zh-baker/model-steps-3.onnx \ + --matcha-vocoder=./hifigan_v2.onnx \ + --matcha-lexicon=./matcha-icefall-zh-baker/lexicon.txt \ + --matcha-tokens=./matcha-icefall-zh-baker/tokens.txt \ + --matcha-dict-dir=./matcha-icefall-zh-baker/dict \ + --num-threads=2 \ + --debug=1 \ + --output-filename=./tts/matcha-baker-zh-1.wav \ + '小米的使命是，始终坚持做"感动人心、价格厚道"的好产品，让全球每个人都能享受科技带来的美好生活' + +$EXE \ + --matcha-acoustic-model=./matcha-icefall-zh-baker/model-steps-3.onnx \ + --matcha-vocoder=./hifigan_v2.onnx \ + --matcha-lexicon=./matcha-icefall-zh-baker/lexicon.txt \ + --matcha-tokens=./matcha-icefall-zh-baker/tokens.txt \ + --matcha-dict-dir=./matcha-icefall-zh-baker/dict \ + --num-threads=2 \ + --debug=1 \ + --output-filename=./tts/matcha-baker-zh-2.wav \ + "当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转，思念如涟漪荡漾，梦境如画卷展开，我与自然融为一体，沉静在这片宁静的美丽之中，感受着生命的奇迹与温柔。" + +rm hifigan_v2.onnx +rm -rf matcha-icefall-zh-baker + log "------------------------------------------------------------" log "vits-piper-en_US-amy-low" log "------------------------------------------------------------" diff --git a/.github/scripts/test-python.sh b/.github/scripts/test-python.sh index f93908d45..8bfe2c16f 100755 --- a/.github/scripts/test-python.sh +++ b/.github/scripts/test-python.sh @@ -269,6 +269,26 @@ mkdir ./tts log "vits-ljs test" +curl -O -SL https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 +tar xvf matcha-icefall-zh-baker.tar.bz2 +rm matcha-icefall-zh-baker.tar.bz2 + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +python3 ./python-api-examples/offline-tts.py \ + --matcha-acoustic-model=./matcha-icefall-zh-baker/model-steps-3.onnx \ + --matcha-vocoder=./hifigan_v2.onnx \ + --matcha-lexicon=./matcha-icefall-zh-baker/lexicon.txt \ + --matcha-tokens=./matcha-icefall-zh-baker/tokens.txt \ + --tts-rule-fsts=./matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst \ + --matcha-dict-dir=./matcha-icefall-zh-baker/dict \ + --output-filename=./tts/test-matcha.wav \ + "某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。" + +rm -rf matcha-icefall-zh-baker +rm hifigan_v2.onnx + + curl -LS -O https://huggingface.co/csukuangfj/vits-ljs/resolve/main/vits-ljs.onnx curl -LS -O https://huggingface.co/csukuangfj/vits-ljs/resolve/main/lexicon.txt curl -LS -O https://huggingface.co/csukuangfj/vits-ljs/resolve/main/tokens.txt diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml index 98c88e589..ea64662b5 100644 --- a/.github/workflows/linux.yaml +++ b/.github/workflows/linux.yaml @@ -149,6 +149,23 @@ jobs: name: release-${{ matrix.build_type }}-with-shared-lib-${{ matrix.shared_lib }}-with-tts-${{ matrix.with_tts }} path: install/* + - name: Test offline TTS + if: matrix.with_tts == 'ON' + shell: bash + run: | + du -h -d1 . + export PATH=$PWD/build/bin:$PATH + export EXE=sherpa-onnx-offline-tts + + .github/scripts/test-offline-tts.sh + du -h -d1 . + + - uses: actions/upload-artifact@v4 + if: matrix.with_tts == 'ON' + with: + name: tts-generated-test-files-${{ matrix.build_type }}-${{ matrix.shared_lib }}-with-tts-${{ matrix.with_tts }} + path: tts + - name: Test offline Moonshine if: matrix.build_type != 'Debug' shell: bash @@ -309,16 +326,7 @@ jobs: .github/scripts/test-offline-whisper.sh du -h -d1 . - - name: Test offline TTS - if: matrix.with_tts == 'ON' - shell: bash - run: | - du -h -d1 . - export PATH=$PWD/build/bin:$PATH - export EXE=sherpa-onnx-offline-tts - .github/scripts/test-offline-tts.sh - du -h -d1 . - name: Test online paraformer shell: bash @@ -367,8 +375,4 @@ jobs: overwrite: true file: sherpa-onnx-*.tar.bz2 - - uses: actions/upload-artifact@v4 - with: - name: tts-generated-test-files-${{ matrix.build_type }}-${{ matrix.shared_lib }}-with-tts-${{ matrix.with_tts }} - path: tts diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml index fd26d5b9f..e6f627e15 100644 --- a/.github/workflows/macos.yaml +++ b/.github/workflows/macos.yaml @@ -121,6 +121,15 @@ jobs: otool -L build/bin/sherpa-onnx otool -l build/bin/sherpa-onnx + - name: Test offline TTS + if: matrix.with_tts == 'ON' + shell: bash + run: | + export PATH=$PWD/build/bin:$PATH + export EXE=sherpa-onnx-offline-tts + + .github/scripts/test-offline-tts.sh + - name: Test offline Moonshine if: matrix.build_type != 'Debug' shell: bash @@ -226,15 +235,6 @@ jobs: .github/scripts/test-kws.sh - - name: Test offline TTS - if: matrix.with_tts == 'ON' - shell: bash - run: | - export PATH=$PWD/build/bin:$PATH - export EXE=sherpa-onnx-offline-tts - - .github/scripts/test-offline-tts.sh - - name: Test online paraformer shell: bash run: | diff --git a/python-api-examples/offline-tts-play.py b/python-api-examples/offline-tts-play.py index 8457fc45c..e8350ea47 100755 --- a/python-api-examples/offline-tts-play.py +++ b/python-api-examples/offline-tts-play.py @@ -11,7 +11,7 @@ Usage: -Example (1/3) +Example (1/4) wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 tar xf vits-piper-en_US-amy-low.tar.bz2 @@ -23,7 +23,7 @@ --output-filename=./generated.wav \ "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar." -Example (2/3) +Example (2/4) wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-zh-aishell3.tar.bz2 tar xvf vits-zh-aishell3.tar.bz2 @@ -37,7 +37,7 @@ --output-filename=./liubei-21.wav \ "勿以恶小而为之，勿以善小而不为。惟贤惟德，能服于人。122334" -Example (3/3) +Example (3/4) wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-vits-zh-ll.tar.bz2 tar xvf sherpa-onnx-vits-zh-ll.tar.bz2 @@ -53,6 +53,24 @@ --output-filename=./test-2.wav \ "当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转，思念如涟漪荡漾，梦境如画卷展开，我与自然融为一体，沉静在这片宁静的美丽之中，感受着生命的奇迹与温柔。2024年5月11号，拨打110或者18920240511。123456块钱。" +Example (4/4) + +curl -O -SL https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 +tar xvf matcha-icefall-zh-baker.tar.bz2 +rm matcha-icefall-zh-baker.tar.bz2 + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +python3 ./python-api-examples/offline-tts-play.py \ + --matcha-acoustic-model=./matcha-icefall-zh-baker/model-steps-3.onnx \ + --matcha-vocoder=./hifigan_v2.onnx \ + --matcha-lexicon=./matcha-icefall-zh-baker/lexicon.txt \ + --matcha-tokens=./matcha-icefall-zh-baker/tokens.txt \ + --tts-rule-fsts=./matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst \ + --matcha-dict-dir=./matcha-icefall-zh-baker/dict \ + --output-filename=./test-matcha.wav \ + "某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。" + You can find more models at https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models @@ -84,14 +102,11 @@ sys.exit(-1) -def get_args(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - +def add_vits_args(parser): parser.add_argument( "--vits-model", type=str, + default="", help="Path to vits model.onnx", ) @@ -124,6 +139,60 @@ def get_args(): help="Path to the dict directory for models using jieba", ) + +def add_matcha_args(parser): + parser.add_argument( + "--matcha-acoustic-model", + type=str, + default="", + help="Path to model.onnx for matcha", + ) + + parser.add_argument( + "--matcha-vocoder", + type=str, + default="", + help="Path to vocoder for matcha", + ) + + parser.add_argument( + "--matcha-lexicon", + type=str, + default="", + help="Path to lexicon.txt for matcha", + ) + + parser.add_argument( + "--matcha-tokens", + type=str, + default="", + help="Path to tokens.txt for matcha", + ) + + parser.add_argument( + "--matcha-data-dir", + type=str, + default="", + help="""Path to the dict directory of espeak-ng. If it is specified, + --matcha-lexicon and --matcha-tokens are ignored""", + ) + + parser.add_argument( + "--matcha-dict-dir", + type=str, + default="", + help="Path to the dict directory for models using jieba", + ) + + +def get_args(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + add_vits_args(parser) + add_matcha_args(parser) + parser.add_argument( "--tts-rule-fsts", type=str, @@ -313,6 +382,14 @@ def main(): dict_dir=args.vits_dict_dir, tokens=args.vits_tokens, ), + matcha=sherpa_onnx.OfflineTtsMatchaModelConfig( + acoustic_model=args.matcha_acoustic_model, + vocoder=args.matcha_vocoder, + lexicon=args.matcha_lexicon, + tokens=args.matcha_tokens, + data_dir=args.matcha_data_dir, + dict_dir=args.matcha_dict_dir, + ), provider=args.provider, debug=args.debug, num_threads=args.num_threads, diff --git a/python-api-examples/offline-tts.py b/python-api-examples/offline-tts.py index 18ea638e8..aa1cce935 100755 --- a/python-api-examples/offline-tts.py +++ b/python-api-examples/offline-tts.py @@ -12,7 +12,7 @@ Usage: -Example (1/3) +Example (1/4) wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 tar xf vits-piper-en_US-amy-low.tar.bz2 @@ -24,7 +24,7 @@ --output-filename=./generated.wav \ "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar." -Example (2/3) +Example (2/4) wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-icefall-zh-aishell3.tar.bz2 tar xvf vits-icefall-zh-aishell3.tar.bz2 @@ -38,7 +38,7 @@ --output-filename=./liubei-21.wav \ "勿以恶小而为之，勿以善小而不为。惟贤惟德，能服于人。122334" -Example (3/3) +Example (3/4) wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-vits-zh-ll.tar.bz2 tar xvf sherpa-onnx-vits-zh-ll.tar.bz2 @@ -54,6 +54,23 @@ --output-filename=./test-2.wav \ "当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转，思念如涟漪荡漾，梦境如画卷展开，我与自然融为一体，沉静在这片宁静的美丽之中，感受着生命的奇迹与温柔。2024年5月11号，拨打110或者18920240511。123456块钱。" +Example (4/4) + +curl -O -SL https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 +tar xvf matcha-icefall-zh-baker.tar.bz2 +rm matcha-icefall-zh-baker.tar.bz2 + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +python3 ./python-api-examples/offline-tts.py \ + --matcha-acoustic-model=./matcha-icefall-zh-baker/model-steps-3.onnx \ + --matcha-vocoder=./hifigan_v2.onnx \ + --matcha-lexicon=./matcha-icefall-zh-baker/lexicon.txt \ + --matcha-tokens=./matcha-icefall-zh-baker/tokens.txt \ + --tts-rule-fsts=./matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst \ + --matcha-dict-dir=./matcha-icefall-zh-baker/dict \ + --output-filename=./test-matcha.wav \ + "某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。" You can find more models at https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models @@ -71,14 +88,11 @@ import soundfile as sf -def get_args(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - +def add_vits_args(parser): parser.add_argument( "--vits-model", type=str, + default="", help="Path to vits model.onnx", ) @@ -111,6 +125,60 @@ def get_args(): help="Path to the dict directory for models using jieba", ) + +def add_matcha_args(parser): + parser.add_argument( + "--matcha-acoustic-model", + type=str, + default="", + help="Path to model.onnx for matcha", + ) + + parser.add_argument( + "--matcha-vocoder", + type=str, + default="", + help="Path to vocoder for matcha", + ) + + parser.add_argument( + "--matcha-lexicon", + type=str, + default="", + help="Path to lexicon.txt for matcha", + ) + + parser.add_argument( + "--matcha-tokens", + type=str, + default="", + help="Path to tokens.txt for matcha", + ) + + parser.add_argument( + "--matcha-data-dir", + type=str, + default="", + help="""Path to the dict directory of espeak-ng. If it is specified, + --matcha-lexicon and --matcha-tokens are ignored""", + ) + + parser.add_argument( + "--matcha-dict-dir", + type=str, + default="", + help="Path to the dict directory for models using jieba", + ) + + +def get_args(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + add_vits_args(parser) + add_matcha_args(parser) + parser.add_argument( "--tts-rule-fsts", type=str, @@ -196,6 +264,14 @@ def main(): dict_dir=args.vits_dict_dir, tokens=args.vits_tokens, ), + matcha=sherpa_onnx.OfflineTtsMatchaModelConfig( + acoustic_model=args.matcha_acoustic_model, + vocoder=args.matcha_vocoder, + lexicon=args.matcha_lexicon, + tokens=args.matcha_tokens, + data_dir=args.matcha_data_dir, + dict_dir=args.matcha_dict_dir, + ), provider=args.provider, debug=args.debug, num_threads=args.num_threads, diff --git a/sherpa-onnx/csrc/CMakeLists.txt b/sherpa-onnx/csrc/CMakeLists.txt index 3850c8eb3..f146b09e2 100644 --- a/sherpa-onnx/csrc/CMakeLists.txt +++ b/sherpa-onnx/csrc/CMakeLists.txt @@ -151,12 +151,15 @@ list(APPEND sources if(SHERPA_ONNX_ENABLE_TTS) list(APPEND sources + hifigan-vocoder.cc jieba-lexicon.cc lexicon.cc melo-tts-lexicon.cc offline-tts-character-frontend.cc offline-tts-frontend.cc offline-tts-impl.cc + offline-tts-matcha-model-config.cc + offline-tts-matcha-model.cc offline-tts-model-config.cc offline-tts-vits-model-config.cc offline-tts-vits-model.cc diff --git a/sherpa-onnx/csrc/hifigan-vocoder.cc b/sherpa-onnx/csrc/hifigan-vocoder.cc new file mode 100644 index 000000000..b2ff20788 --- /dev/null +++ b/sherpa-onnx/csrc/hifigan-vocoder.cc @@ -0,0 +1,107 @@ +// sherpa-onnx/csrc/hifigan-vocoder.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include "sherpa-onnx/csrc/hifigan-vocoder.h" + +#include +#include +#include + +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + +#include "sherpa-onnx/csrc/macros.h" +#include "sherpa-onnx/csrc/onnx-utils.h" +#include "sherpa-onnx/csrc/session.h" + +namespace sherpa_onnx { + +class HifiganVocoder::Impl { + public: + explicit Impl(int32_t num_threads, const std::string &provider, + const std::string &model) + : env_(ORT_LOGGING_LEVEL_ERROR), + sess_opts_(GetSessionOptions(num_threads, provider)), + allocator_{} { + auto buf = ReadFile(model); + Init(buf.data(), buf.size()); + } + + template + explicit Impl(Manager *mgr, int32_t num_threads, const std::string &provider, + const std::string &model) + : env_(ORT_LOGGING_LEVEL_ERROR), + sess_opts_(GetSessionOptions(num_threads, provider)), + allocator_{} { + auto buf = ReadFile(mgr, model); + Init(buf.data(), buf.size()); + } + + Ort::Value Run(Ort::Value mel) const { + auto out = sess_->Run({}, input_names_ptr_.data(), &mel, 1, + output_names_ptr_.data(), output_names_ptr_.size()); + + return std::move(out[0]); + } + + private: + void Init(void *model_data, size_t model_data_length) { + sess_ = std::make_unique(env_, model_data, model_data_length, + sess_opts_); + + GetInputNames(sess_.get(), &input_names_, &input_names_ptr_); + + GetOutputNames(sess_.get(), &output_names_, &output_names_ptr_); + } + + private: + Ort::Env env_; + Ort::SessionOptions sess_opts_; + Ort::AllocatorWithDefaultOptions allocator_; + + std::unique_ptr sess_; + + std::vector input_names_; + std::vector input_names_ptr_; + + std::vector output_names_; + std::vector output_names_ptr_; +}; + +HifiganVocoder::HifiganVocoder(int32_t num_threads, const std::string &provider, + const std::string &model) + : impl_(std::make_unique(num_threads, provider, model)) {} + +template +HifiganVocoder::HifiganVocoder(Manager *mgr, int32_t num_threads, + const std::string &provider, + const std::string &model) + : impl_(std::make_unique(mgr, num_threads, provider, model)) {} + +HifiganVocoder::~HifiganVocoder() = default; + +Ort::Value HifiganVocoder::Run(Ort::Value mel) const { + return impl_->Run(std::move(mel)); +} + +#if __ANDROID_API__ >= 9 +template HifiganVocoder::HifiganVocoder(AAssetManager *mgr, int32_t num_threads, + const std::string &provider, + const std::string &model); +#endif + +#if __OHOS__ +template HifiganVocoder::HifiganVocoder(NativeResourceManager *mgr, + int32_t num_threads, + const std::string &provider, + const std::string &model); +#endif + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/hifigan-vocoder.h b/sherpa-onnx/csrc/hifigan-vocoder.h new file mode 100644 index 000000000..3d10a2428 --- /dev/null +++ b/sherpa-onnx/csrc/hifigan-vocoder.h @@ -0,0 +1,38 @@ +// sherpa-onnx/csrc/hifigan-vocoder.h +// +// Copyright (c) 2024 Xiaomi Corporation + +#ifndef SHERPA_ONNX_CSRC_HIFIGAN_VOCODER_H_ +#define SHERPA_ONNX_CSRC_HIFIGAN_VOCODER_H_ + +#include +#include + +#include "onnxruntime_cxx_api.h" // NOLINT + +namespace sherpa_onnx { + +class HifiganVocoder { + public: + ~HifiganVocoder(); + + HifiganVocoder(int32_t num_threads, const std::string &provider, + const std::string &model); + + template + HifiganVocoder(Manager *mgr, int32_t num_threads, const std::string &provider, + const std::string &model); + + /** @param mel A float32 tensor of shape (batch_size, feat_dim, num_frames). + * @return Return a float32 tensor of shape (batch_size, num_samples). + */ + Ort::Value Run(Ort::Value mel) const; + + private: + class Impl; + std::unique_ptr impl_; +}; + +} // namespace sherpa_onnx + +#endif // SHERPA_ONNX_CSRC_HIFIGAN_VOCODER_H_ diff --git a/sherpa-onnx/csrc/jieba-lexicon.cc b/sherpa-onnx/csrc/jieba-lexicon.cc index a53f057f0..9ea11f46f 100644 --- a/sherpa-onnx/csrc/jieba-lexicon.cc +++ b/sherpa-onnx/csrc/jieba-lexicon.cc @@ -19,9 +19,8 @@ namespace sherpa_onnx { class JiebaLexicon::Impl { public: Impl(const std::string &lexicon, const std::string &tokens, - const std::string &dict_dir, - const OfflineTtsVitsModelMetaData &meta_data, bool debug) - : meta_data_(meta_data), debug_(debug) { + const std::string &dict_dir, bool debug) + : debug_(debug) { std::string dict = dict_dir + "/jieba.dict.utf8"; std::string hmm = dict_dir + "/hmm_model.utf8"; std::string user_dict = dict_dir + "/user.dict.utf8"; @@ -84,7 +83,6 @@ class JiebaLexicon::Impl { std::vector ans; std::vector this_sentence; - int32_t blank = token2id_.at(" "); for (const auto &w : words) { auto ids = ConvertWordToIds(w); if (ids.empty()) { @@ -93,7 +91,6 @@ class JiebaLexicon::Impl { } this_sentence.insert(this_sentence.end(), ids.begin(), ids.end()); - this_sentence.push_back(blank); if (w == "。" || w == "！" || w == "？" || w == "，") { ans.emplace_back(std::move(this_sentence)); @@ -135,7 +132,9 @@ class JiebaLexicon::Impl { token2id_ = ReadTokens(is); std::vector> puncts = { - {",", "，"}, {".", "。"}, {"!", "！"}, {"?", "？"}}; + {",", "，"}, {".", "。"}, {"!", "！"}, {"?", "？"}, {":", "："}, + {"\"", "“"}, {"\"", "”"}, {"'", "‘"}, {"'", "’"}, {";", "；"}, + }; for (const auto &p : puncts) { if (token2id_.count(p.first) && !token2id_.count(p.second)) { @@ -150,6 +149,10 @@ class JiebaLexicon::Impl { if (!token2id_.count("、") && token2id_.count("，")) { token2id_["、"] = token2id_["，"]; } + + if (!token2id_.count(";") && token2id_.count(",")) { + token2id_[";"] = token2id_[","]; + } } void InitLexicon(std::istream &is) { @@ -195,8 +198,6 @@ class JiebaLexicon::Impl { // tokens.txt is saved in token2id_ std::unordered_map token2id_; - OfflineTtsVitsModelMetaData meta_data_; - std::unique_ptr jieba_; bool debug_ = false; }; @@ -205,11 +206,8 @@ JiebaLexicon::~JiebaLexicon() = default; JiebaLexicon::JiebaLexicon(const std::string &lexicon, const std::string &tokens, - const std::string &dict_dir, - const OfflineTtsVitsModelMetaData &meta_data, - bool debug) - : impl_(std::make_unique(lexicon, tokens, dict_dir, meta_data, - debug)) {} + const std::string &dict_dir, bool debug) + : impl_(std::make_unique(lexicon, tokens, dict_dir, debug)) {} std::vector JiebaLexicon::ConvertTextToTokenIds( const std::string &text, const std::string & /*unused_voice = ""*/) const { diff --git a/sherpa-onnx/csrc/jieba-lexicon.h b/sherpa-onnx/csrc/jieba-lexicon.h index d02e0ee5d..9de104357 100644 --- a/sherpa-onnx/csrc/jieba-lexicon.h +++ b/sherpa-onnx/csrc/jieba-lexicon.h @@ -11,7 +11,6 @@ #include #include "sherpa-onnx/csrc/offline-tts-frontend.h" -#include "sherpa-onnx/csrc/offline-tts-vits-model-metadata.h" namespace sherpa_onnx { @@ -19,8 +18,7 @@ class JiebaLexicon : public OfflineTtsFrontend { public: ~JiebaLexicon() override; JiebaLexicon(const std::string &lexicon, const std::string &tokens, - const std::string &dict_dir, - const OfflineTtsVitsModelMetaData &meta_data, bool debug); + const std::string &dict_dir, bool debug); std::vector ConvertTextToTokenIds( const std::string &text, diff --git a/sherpa-onnx/csrc/offline-tts-impl.cc b/sherpa-onnx/csrc/offline-tts-impl.cc index 62b6eebba..92ccb7fdb 100644 --- a/sherpa-onnx/csrc/offline-tts-impl.cc +++ b/sherpa-onnx/csrc/offline-tts-impl.cc @@ -5,6 +5,7 @@ #include "sherpa-onnx/csrc/offline-tts-impl.h" #include +#include #if __ANDROID_API__ >= 9 #include "android/asset_manager.h" @@ -15,21 +16,39 @@ #include "rawfile/raw_file_manager.h" #endif +#include "sherpa-onnx/csrc/offline-tts-matcha-impl.h" #include "sherpa-onnx/csrc/offline-tts-vits-impl.h" namespace sherpa_onnx { +std::vector OfflineTtsImpl::AddBlank(const std::vector &x, + int32_t blank_id /*= 0*/) const { + // we assume the blank ID is 0 + std::vector buffer(x.size() * 2 + 1, blank_id); + int32_t i = 1; + for (auto k : x) { + buffer[i] = k; + i += 2; + } + return buffer; +} + std::unique_ptr OfflineTtsImpl::Create( const OfflineTtsConfig &config) { - // TODO(fangjun): Support other types - return std::make_unique(config); + if (!config.model.vits.model.empty()) { + return std::make_unique(config); + } + return std::make_unique(config); } template std::unique_ptr OfflineTtsImpl::Create( Manager *mgr, const OfflineTtsConfig &config) { - // TODO(fangjun): Support other types - return std::make_unique(mgr, config); + if (!config.model.vits.model.empty()) { + return std::make_unique(mgr, config); + } + + return std::make_unique(mgr, config); } #if __ANDROID_API__ >= 9 diff --git a/sherpa-onnx/csrc/offline-tts-impl.h b/sherpa-onnx/csrc/offline-tts-impl.h index db8b7162d..061acc747 100644 --- a/sherpa-onnx/csrc/offline-tts-impl.h +++ b/sherpa-onnx/csrc/offline-tts-impl.h @@ -7,6 +7,7 @@ #include #include +#include #include "sherpa-onnx/csrc/offline-tts.h" @@ -32,6 +33,9 @@ class OfflineTtsImpl { // Number of supported speakers. // If it supports only a single speaker, then it return 0 or 1. virtual int32_t NumSpeakers() const = 0; + + std::vector AddBlank(const std::vector &x, + int32_t blank_id = 0) const; }; } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-tts-matcha-impl.h b/sherpa-onnx/csrc/offline-tts-matcha-impl.h new file mode 100644 index 000000000..62c29bb83 --- /dev/null +++ b/sherpa-onnx/csrc/offline-tts-matcha-impl.h @@ -0,0 +1,381 @@ +// sherpa-onnx/csrc/offline-tts-matcha-impl.h +// +// Copyright (c) 2024 Xiaomi Corporation +#ifndef SHERPA_ONNX_CSRC_OFFLINE_TTS_MATCHA_IMPL_H_ +#define SHERPA_ONNX_CSRC_OFFLINE_TTS_MATCHA_IMPL_H_ + +#include +#include +#include +#include +#include + +#include "fst/extensions/far/far.h" +#include "kaldifst/csrc/kaldi-fst-io.h" +#include "kaldifst/csrc/text-normalizer.h" +#include "sherpa-onnx/csrc/hifigan-vocoder.h" +#include "sherpa-onnx/csrc/jieba-lexicon.h" +#include "sherpa-onnx/csrc/lexicon.h" +#include "sherpa-onnx/csrc/macros.h" +#include "sherpa-onnx/csrc/melo-tts-lexicon.h" +#include "sherpa-onnx/csrc/offline-tts-character-frontend.h" +#include "sherpa-onnx/csrc/offline-tts-frontend.h" +#include "sherpa-onnx/csrc/offline-tts-impl.h" +#include "sherpa-onnx/csrc/offline-tts-matcha-model.h" +#include "sherpa-onnx/csrc/onnx-utils.h" +#include "sherpa-onnx/csrc/piper-phonemize-lexicon.h" +#include "sherpa-onnx/csrc/text-utils.h" + +namespace sherpa_onnx { + +class OfflineTtsMatchaImpl : public OfflineTtsImpl { + public: + explicit OfflineTtsMatchaImpl(const OfflineTtsConfig &config) + : config_(config), + model_(std::make_unique(config.model)), + vocoder_(std::make_unique( + config.model.num_threads, config.model.provider, + config.model.matcha.vocoder)) { + InitFrontend(); + + if (!config.rule_fsts.empty()) { + std::vector files; + SplitStringToVector(config.rule_fsts, ",", false, &files); + tn_list_.reserve(files.size()); + for (const auto &f : files) { + if (config.model.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("rule fst: %{public}s", f.c_str()); +#else + SHERPA_ONNX_LOGE("rule fst: %s", f.c_str()); +#endif + } + tn_list_.push_back(std::make_unique(f)); + } + } + + if (!config.rule_fars.empty()) { + if (config.model.debug) { + SHERPA_ONNX_LOGE("Loading FST archives"); + } + std::vector files; + SplitStringToVector(config.rule_fars, ",", false, &files); + + tn_list_.reserve(files.size() + tn_list_.size()); + + for (const auto &f : files) { + if (config.model.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("rule far: %{public}s", f.c_str()); +#else + SHERPA_ONNX_LOGE("rule far: %s", f.c_str()); +#endif + } + std::unique_ptr> reader( + fst::FarReader::Open(f)); + for (; !reader->Done(); reader->Next()) { + std::unique_ptr r( + fst::CastOrConvertToConstFst(reader->GetFst()->Copy())); + + tn_list_.push_back( + std::make_unique(std::move(r))); + } + } + + if (config.model.debug) { + SHERPA_ONNX_LOGE("FST archives loaded!"); + } + } + } + + template + OfflineTtsMatchaImpl(Manager *mgr, const OfflineTtsConfig &config) + : config_(config), + model_(std::make_unique(mgr, config.model)), + vocoder_(std::make_unique( + mgr, config.model.num_threads, config.model.provider, + config.model.matcha.vocoder)) { + InitFrontend(mgr); + + if (!config.rule_fsts.empty()) { + std::vector files; + SplitStringToVector(config.rule_fsts, ",", false, &files); + tn_list_.reserve(files.size()); + for (const auto &f : files) { + if (config.model.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("rule fst: %{public}s", f.c_str()); +#else + SHERPA_ONNX_LOGE("rule fst: %s", f.c_str()); +#endif + } + auto buf = ReadFile(mgr, f); + std::istrstream is(buf.data(), buf.size()); + tn_list_.push_back(std::make_unique(is)); + } + } + + if (!config.rule_fars.empty()) { + std::vector files; + SplitStringToVector(config.rule_fars, ",", false, &files); + tn_list_.reserve(files.size() + tn_list_.size()); + + for (const auto &f : files) { + if (config.model.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("rule far: %{public}s", f.c_str()); +#else + SHERPA_ONNX_LOGE("rule far: %s", f.c_str()); +#endif + } + + auto buf = ReadFile(mgr, f); + + std::unique_ptr s( + new std::istrstream(buf.data(), buf.size())); + + std::unique_ptr> reader( + fst::FarReader::Open(std::move(s))); + + for (; !reader->Done(); reader->Next()) { + std::unique_ptr r( + fst::CastOrConvertToConstFst(reader->GetFst()->Copy())); + + tn_list_.push_back( + std::make_unique(std::move(r))); + } // for (; !reader->Done(); reader->Next()) + } // for (const auto &f : files) + } // if (!config.rule_fars.empty()) + } + + int32_t SampleRate() const override { + return model_->GetMetaData().sample_rate; + } + + int32_t NumSpeakers() const override { + return model_->GetMetaData().num_speakers; + } + + GeneratedAudio Generate( + const std::string &_text, int64_t sid = 0, float speed = 1.0, + GeneratedAudioCallback callback = nullptr) const override { + const auto &meta_data = model_->GetMetaData(); + int32_t num_speakers = meta_data.num_speakers; + + if (num_speakers == 0 && sid != 0) { +#if __OHOS__ + SHERPA_ONNX_LOGE( + "This is a single-speaker model and supports only sid 0. Given sid: " + "%{public}d. sid is ignored", + static_cast(sid)); +#else + SHERPA_ONNX_LOGE( + "This is a single-speaker model and supports only sid 0. Given sid: " + "%d. sid is ignored", + static_cast(sid)); +#endif + } + + if (num_speakers != 0 && (sid >= num_speakers || sid < 0)) { +#if __OHOS__ + SHERPA_ONNX_LOGE( + "This model contains only %{public}d speakers. sid should be in the " + "range [%{public}d, %{public}d]. Given: %{public}d. Use sid=0", + num_speakers, 0, num_speakers - 1, static_cast(sid)); +#else + SHERPA_ONNX_LOGE( + "This model contains only %d speakers. sid should be in the range " + "[%d, %d]. Given: %d. Use sid=0", + num_speakers, 0, num_speakers - 1, static_cast(sid)); +#endif + sid = 0; + } + + std::string text = _text; + if (config_.model.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("Raw text: %{public}s", text.c_str()); +#else + SHERPA_ONNX_LOGE("Raw text: %s", text.c_str()); +#endif + } + + if (!tn_list_.empty()) { + for (const auto &tn : tn_list_) { + text = tn->Normalize(text); + if (config_.model.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("After normalizing: %{public}s", text.c_str()); +#else + SHERPA_ONNX_LOGE("After normalizing: %s", text.c_str()); +#endif + } + } + } + + std::vector token_ids = + frontend_->ConvertTextToTokenIds(text, "en-US"); + + if (token_ids.empty() || + (token_ids.size() == 1 && token_ids[0].tokens.empty())) { +#if __OHOS__ + SHERPA_ONNX_LOGE("Failed to convert '%{public}s' to token IDs", + text.c_str()); +#else + SHERPA_ONNX_LOGE("Failed to convert '%s' to token IDs", text.c_str()); +#endif + return {}; + } + + std::vector> x; + + x.reserve(token_ids.size()); + + for (auto &i : token_ids) { + x.push_back(std::move(i.tokens)); + } + + for (auto &k : x) { + k = AddBlank(k, meta_data.pad_id); + } + + int32_t x_size = static_cast(x.size()); + + if (config_.max_num_sentences <= 0 || x_size <= config_.max_num_sentences) { + auto ans = Process(x, sid, speed); + if (callback) { + callback(ans.samples.data(), ans.samples.size(), 1.0); + } + return ans; + } + + // the input text is too long, we process sentences within it in batches + // to avoid OOM. Batch size is config_.max_num_sentences + std::vector> batch_x; + + int32_t batch_size = config_.max_num_sentences; + batch_x.reserve(config_.max_num_sentences); + int32_t num_batches = x_size / batch_size; + + if (config_.model.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE( + "Text is too long. Split it into %{public}d batches. batch size: " + "%{public}d. Number of sentences: %{public}d", + num_batches, batch_size, x_size); +#else + SHERPA_ONNX_LOGE( + "Text is too long. Split it into %d batches. batch size: %d. Number " + "of sentences: %d", + num_batches, batch_size, x_size); +#endif + } + + GeneratedAudio ans; + + int32_t should_continue = 1; + + int32_t k = 0; + + for (int32_t b = 0; b != num_batches && should_continue; ++b) { + batch_x.clear(); + for (int32_t i = 0; i != batch_size; ++i, ++k) { + batch_x.push_back(std::move(x[k])); + } + + auto audio = Process(batch_x, sid, speed); + ans.sample_rate = audio.sample_rate; + ans.samples.insert(ans.samples.end(), audio.samples.begin(), + audio.samples.end()); + if (callback) { + should_continue = callback(audio.samples.data(), audio.samples.size(), + (b + 1) * 1.0 / num_batches); + // Caution(fangjun): audio is freed when the callback returns, so users + // should copy the data if they want to access the data after + // the callback returns to avoid segmentation fault. + } + } + + batch_x.clear(); + while (k < static_cast(x.size()) && should_continue) { + batch_x.push_back(std::move(x[k])); + + ++k; + } + + if (!batch_x.empty()) { + auto audio = Process(batch_x, sid, speed); + ans.sample_rate = audio.sample_rate; + ans.samples.insert(ans.samples.end(), audio.samples.begin(), + audio.samples.end()); + if (callback) { + callback(audio.samples.data(), audio.samples.size(), 1.0); + // Caution(fangjun): audio is freed when the callback returns, so users + // should copy the data if they want to access the data after + // the callback returns to avoid segmentation fault. + } + } + + return ans; + } + + private: + template + void InitFrontend(Manager *mgr) {} + + void InitFrontend() { + frontend_ = std::make_unique( + config_.model.matcha.lexicon, config_.model.matcha.tokens, + config_.model.matcha.dict_dir, config_.model.debug); + } + + GeneratedAudio Process(const std::vector> &tokens, + int32_t sid, float speed) const { + int32_t num_tokens = 0; + for (const auto &k : tokens) { + num_tokens += k.size(); + } + + std::vector x; + x.reserve(num_tokens); + for (const auto &k : tokens) { + x.insert(x.end(), k.begin(), k.end()); + } + + auto memory_info = + Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault); + + std::array x_shape = {1, static_cast(x.size())}; + Ort::Value x_tensor = Ort::Value::CreateTensor( + memory_info, x.data(), x.size(), x_shape.data(), x_shape.size()); + + Ort::Value mel = model_->Run(std::move(x_tensor), sid, speed); + Ort::Value audio = vocoder_->Run(std::move(mel)); + + std::vector audio_shape = + audio.GetTensorTypeAndShapeInfo().GetShape(); + + int64_t total = 1; + // The output shape may be (1, 1, total) or (1, total) or (total,) + for (auto i : audio_shape) { + total *= i; + } + + const float *p = audio.GetTensorData(); + + GeneratedAudio ans; + ans.sample_rate = model_->GetMetaData().sample_rate; + ans.samples = std::vector(p, p + total); + return ans; + } + + private: + OfflineTtsConfig config_; + std::unique_ptr model_; + std::unique_ptr vocoder_; + std::vector> tn_list_; + std::unique_ptr frontend_; +}; + +} // namespace sherpa_onnx +#endif // SHERPA_ONNX_CSRC_OFFLINE_TTS_MATCHA_IMPL_H_ diff --git a/sherpa-onnx/csrc/offline-tts-matcha-model-config.cc b/sherpa-onnx/csrc/offline-tts-matcha-model-config.cc new file mode 100644 index 000000000..5c736b54d --- /dev/null +++ b/sherpa-onnx/csrc/offline-tts-matcha-model-config.cc @@ -0,0 +1,143 @@ +// sherpa-onnx/csrc/offline-tts-matcha-model-config.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include "sherpa-onnx/csrc/offline-tts-matcha-model-config.h" + +#include + +#include "sherpa-onnx/csrc/file-utils.h" +#include "sherpa-onnx/csrc/macros.h" + +namespace sherpa_onnx { + +void OfflineTtsMatchaModelConfig::Register(ParseOptions *po) { + po->Register("matcha-acoustic-model", &acoustic_model, + "Path to matcha acoustic model"); + po->Register("matcha-vocoder", &vocoder, "Path to matcha vocoder"); + po->Register("matcha-lexicon", &lexicon, + "Path to lexicon.txt for Matcha models"); + po->Register("matcha-tokens", &tokens, + "Path to tokens.txt for Matcha models"); + po->Register("matcha-data-dir", &data_dir, + "Path to the directory containing dict for espeak-ng. If it is " + "given, --matcha-lexicon is ignored."); + po->Register("matcha-dict-dir", &dict_dir, + "Path to the directory containing dict for jieba. Used only for " + "Chinese TTS models using jieba"); + po->Register("matcha-noise-scale", &noise_scale, + "noise_scale for Matcha models"); + po->Register("matcha-length-scale", &length_scale, + "Speech speed. Larger->Slower; Smaller->faster."); +} + +bool OfflineTtsMatchaModelConfig::Validate() const { + if (acoustic_model.empty()) { + SHERPA_ONNX_LOGE("Please provide --matcha-acoustic-model"); + return false; + } + + if (!FileExists(acoustic_model)) { + SHERPA_ONNX_LOGE("--matcha-acoustic-model: '%s' does not exist", + acoustic_model.c_str()); + return false; + } + + if (vocoder.empty()) { + SHERPA_ONNX_LOGE("Please provide --matcha-vocoder"); + return false; + } + + if (!FileExists(vocoder)) { + SHERPA_ONNX_LOGE("--matcha-vocoder: '%s' does not exist", vocoder.c_str()); + return false; + } + + if (tokens.empty()) { + SHERPA_ONNX_LOGE("Please provide --matcha-tokens"); + return false; + } + + if (!FileExists(tokens)) { + SHERPA_ONNX_LOGE("--matcha-tokens: '%s' does not exist", tokens.c_str()); + return false; + } + + if (!data_dir.empty()) { + if (!FileExists(data_dir + "/phontab")) { + SHERPA_ONNX_LOGE( + "'%s/phontab' does not exist. Please check --matcha-data-dir", + data_dir.c_str()); + return false; + } + + if (!FileExists(data_dir + "/phonindex")) { + SHERPA_ONNX_LOGE( + "'%s/phonindex' does not exist. Please check --matcha-data-dir", + data_dir.c_str()); + return false; + } + + if (!FileExists(data_dir + "/phondata")) { + SHERPA_ONNX_LOGE( + "'%s/phondata' does not exist. Please check --matcha-data-dir", + data_dir.c_str()); + return false; + } + + if (!FileExists(data_dir + "/intonations")) { + SHERPA_ONNX_LOGE( + "'%s/intonations' does not exist. Please check --matcha-data-dir", + data_dir.c_str()); + return false; + } + } + + if (!dict_dir.empty()) { + std::vector required_files = { + "jieba.dict.utf8", "hmm_model.utf8", "user.dict.utf8", + "idf.utf8", "stop_words.utf8", + }; + + for (const auto &f : required_files) { + if (!FileExists(dict_dir + "/" + f)) { + SHERPA_ONNX_LOGE( + "'%s/%s' does not exist. Please check --matcha-dict-dir", + dict_dir.c_str(), f.c_str()); + return false; + } + } + + // we require that --matcha-lexicon is not empty + if (lexicon.empty()) { + SHERPA_ONNX_LOGE("Please provide --matcha-lexicon"); + return false; + } + + if (!FileExists(lexicon)) { + SHERPA_ONNX_LOGE("--matcha-lexicon: '%s' does not exist", + lexicon.c_str()); + return false; + } + } + + return true; +} + +std::string OfflineTtsMatchaModelConfig::ToString() const { + std::ostringstream os; + + os << "OfflineTtsMatchaModelConfig("; + os << "acoustic_model=\"" << acoustic_model << "\", "; + os << "vocoder=\"" << vocoder << "\", "; + os << "lexicon=\"" << lexicon << "\", "; + os << "tokens=\"" << tokens << "\", "; + os << "data_dir=\"" << data_dir << "\", "; + os << "dict_dir=\"" << dict_dir << "\", "; + os << "noise_scale=" << noise_scale << ", "; + os << "length_scale=" << length_scale << ")"; + + return os.str(); +} + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-tts-matcha-model-config.h b/sherpa-onnx/csrc/offline-tts-matcha-model-config.h new file mode 100644 index 000000000..f367a7e05 --- /dev/null +++ b/sherpa-onnx/csrc/offline-tts-matcha-model-config.h @@ -0,0 +1,56 @@ +// sherpa-onnx/csrc/offline-tts-matcha-model-config.h +// +// Copyright (c) 2024 Xiaomi Corporation + +#ifndef SHERPA_ONNX_CSRC_OFFLINE_TTS_MATCHA_MODEL_CONFIG_H_ +#define SHERPA_ONNX_CSRC_OFFLINE_TTS_MATCHA_MODEL_CONFIG_H_ + +#include + +#include "sherpa-onnx/csrc/parse-options.h" + +namespace sherpa_onnx { + +struct OfflineTtsMatchaModelConfig { + std::string acoustic_model; + std::string vocoder; + std::string lexicon; + std::string tokens; + + // If data_dir is given, lexicon is ignored + // data_dir is for piper-phonemizer, which uses espeak-ng + std::string data_dir; + + // Used for Chinese TTS models using jieba + std::string dict_dir; + + float noise_scale = 1; + float length_scale = 1; + + OfflineTtsMatchaModelConfig() = default; + + OfflineTtsMatchaModelConfig(const std::string &acoustic_model, + const std::string &vocoder, + const std::string &lexicon, + const std::string &tokens, + const std::string &data_dir, + const std::string &dict_dir, + float noise_scale = 1.0, float length_scale = 1) + : acoustic_model(acoustic_model), + vocoder(vocoder), + lexicon(lexicon), + tokens(tokens), + data_dir(data_dir), + dict_dir(dict_dir), + noise_scale(noise_scale), + length_scale(length_scale) {} + + void Register(ParseOptions *po); + bool Validate() const; + + std::string ToString() const; +}; + +} // namespace sherpa_onnx + +#endif // SHERPA_ONNX_CSRC_OFFLINE_TTS_MATCHA_MODEL_CONFIG_H_ diff --git a/sherpa-onnx/csrc/offline-tts-matcha-model-metadata.h b/sherpa-onnx/csrc/offline-tts-matcha-model-metadata.h new file mode 100644 index 000000000..3147985dd --- /dev/null +++ b/sherpa-onnx/csrc/offline-tts-matcha-model-metadata.h @@ -0,0 +1,28 @@ +// sherpa-onnx/csrc/offline-tts-matcha-model-metadata.h +// +// Copyright (c) 2023 Xiaomi Corporation + +#ifndef SHERPA_ONNX_CSRC_OFFLINE_TTS_MATCHA_MODEL_METADATA_H_ +#define SHERPA_ONNX_CSRC_OFFLINE_TTS_MATCHA_MODEL_METADATA_H_ + +#include +#include + +namespace sherpa_onnx { + +// If you are not sure what each field means, please +// have a look of the Python file in the model directory that +// you have downloaded. +struct OfflineTtsMatchaModelMetaData { + int32_t sample_rate = 0; + int32_t num_speakers = 0; + int32_t version = 1; + int32_t jieba = 0; + int32_t espeak = 0; + int32_t use_eos_bos = 0; + int32_t pad_id = 0; +}; + +} // namespace sherpa_onnx + +#endif // SHERPA_ONNX_CSRC_OFFLINE_TTS_MATCHA_MODEL_METADATA_H_ diff --git a/sherpa-onnx/csrc/offline-tts-matcha-model.cc b/sherpa-onnx/csrc/offline-tts-matcha-model.cc new file mode 100644 index 000000000..066dbd21a --- /dev/null +++ b/sherpa-onnx/csrc/offline-tts-matcha-model.cc @@ -0,0 +1,198 @@ +// sherpa-onnx/csrc/offline-tts-matcha-model.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include "sherpa-onnx/csrc/offline-tts-matcha-model.h" + +#include +#include +#include +#include + +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + +#include "sherpa-onnx/csrc/macros.h" +#include "sherpa-onnx/csrc/onnx-utils.h" +#include "sherpa-onnx/csrc/session.h" + +namespace sherpa_onnx { + +class OfflineTtsMatchaModel::Impl { + public: + explicit Impl(const OfflineTtsModelConfig &config) + : config_(config), + env_(ORT_LOGGING_LEVEL_ERROR), + sess_opts_(GetSessionOptions(config)), + allocator_{} { + auto buf = ReadFile(config.matcha.acoustic_model); + Init(buf.data(), buf.size()); + } + + template + Impl(Manager *mgr, const OfflineTtsModelConfig &config) + : config_(config), + env_(ORT_LOGGING_LEVEL_ERROR), + sess_opts_(GetSessionOptions(config)), + allocator_{} { + auto buf = ReadFile(mgr, config.matcha.acoustic_model); + Init(buf.data(), buf.size()); + } + + const OfflineTtsMatchaModelMetaData &GetMetaData() const { + return meta_data_; + } + + Ort::Value Run(Ort::Value x, int64_t sid, float speed) { + auto memory_info = + Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault); + + std::vector x_shape = x.GetTensorTypeAndShapeInfo().GetShape(); + if (x_shape[0] != 1) { + SHERPA_ONNX_LOGE("Support only batch_size == 1. Given: %d", + static_cast(x_shape[0])); + exit(-1); + } + + int64_t len = x_shape[1]; + int64_t len_shape = 1; + + Ort::Value x_length = + Ort::Value::CreateTensor(memory_info, &len, 1, &len_shape, 1); + + int64_t scale_shape = 1; + float noise_scale = config_.matcha.noise_scale; + float length_scale = config_.matcha.length_scale; + + if (speed != 1 && speed > 0) { + length_scale = 1. / speed; + } + + Ort::Value noise_scale_tensor = + Ort::Value::CreateTensor(memory_info, &noise_scale, 1, &scale_shape, 1); + + Ort::Value length_scale_tensor = Ort::Value::CreateTensor( + memory_info, &length_scale, 1, &scale_shape, 1); + + Ort::Value sid_tensor = + Ort::Value::CreateTensor(memory_info, &sid, 1, &scale_shape, 1); + + std::vector inputs; + inputs.reserve(5); + inputs.push_back(std::move(x)); + inputs.push_back(std::move(x_length)); + inputs.push_back(std::move(noise_scale_tensor)); + inputs.push_back(std::move(length_scale_tensor)); + + if (input_names_.size() == 5 && input_names_.back() == "sid") { + inputs.push_back(std::move(sid_tensor)); + } + + auto out = + sess_->Run({}, input_names_ptr_.data(), inputs.data(), inputs.size(), + output_names_ptr_.data(), output_names_ptr_.size()); + + return std::move(out[0]); + } + + private: + void Init(void *model_data, size_t model_data_length) { + sess_ = std::make_unique(env_, model_data, model_data_length, + sess_opts_); + + GetInputNames(sess_.get(), &input_names_, &input_names_ptr_); + + GetOutputNames(sess_.get(), &output_names_, &output_names_ptr_); + + // get meta data + Ort::ModelMetadata meta_data = sess_->GetModelMetadata(); + if (config_.debug) { + std::ostringstream os; + os << "---matcha model---\n"; + PrintModelMetadata(os, meta_data); + + os << "----------input names----------\n"; + int32_t i = 0; + for (const auto &s : input_names_) { + os << i << " " << s << "\n"; + ++i; + } + os << "----------output names----------\n"; + i = 0; + for (const auto &s : output_names_) { + os << i << " " << s << "\n"; + ++i; + } + +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s\n", os.str().c_str()); +#else + SHERPA_ONNX_LOGE("%s\n", os.str().c_str()); +#endif + } + + Ort::AllocatorWithDefaultOptions allocator; // used in the macro below + SHERPA_ONNX_READ_META_DATA(meta_data_.sample_rate, "sample_rate"); + SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(meta_data_.version, "version", 1); + SHERPA_ONNX_READ_META_DATA(meta_data_.num_speakers, "n_speakers"); + SHERPA_ONNX_READ_META_DATA(meta_data_.jieba, "jieba"); + SHERPA_ONNX_READ_META_DATA(meta_data_.espeak, "has_espeak"); + SHERPA_ONNX_READ_META_DATA(meta_data_.use_eos_bos, "use_eos_bos"); + SHERPA_ONNX_READ_META_DATA(meta_data_.pad_id, "pad_id"); + } + + private: + OfflineTtsModelConfig config_; + Ort::Env env_; + Ort::SessionOptions sess_opts_; + Ort::AllocatorWithDefaultOptions allocator_; + + std::unique_ptr sess_; + + std::vector input_names_; + std::vector input_names_ptr_; + + std::vector output_names_; + std::vector output_names_ptr_; + + OfflineTtsMatchaModelMetaData meta_data_; +}; + +OfflineTtsMatchaModel::OfflineTtsMatchaModel( + const OfflineTtsModelConfig &config) + : impl_(std::make_unique(config)) {} + +template +OfflineTtsMatchaModel::OfflineTtsMatchaModel( + Manager *mgr, const OfflineTtsModelConfig &config) + : impl_(std::make_unique(mgr, config)) {} + +OfflineTtsMatchaModel::~OfflineTtsMatchaModel() = default; + +const OfflineTtsMatchaModelMetaData &OfflineTtsMatchaModel::GetMetaData() + const { + return impl_->GetMetaData(); +} + +Ort::Value OfflineTtsMatchaModel::Run(Ort::Value x, int64_t sid /*= 0*/, + float speed /*= 1.0*/) const { + return impl_->Run(std::move(x), sid, speed); +} + +#if __ANDROID_API__ >= 9 +template OfflineTtsMatchaModel::OfflineTtsMatchaModel( + AAssetManager *mgr, const OfflineTtsModelConfig &config); +#endif + +#if __OHOS__ +template OfflineTtsMatchaModel::OfflineTtsMatchaModel( + NativeResourceManager *mgr, const OfflineTtsModelConfig &config); +#endif + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-tts-matcha-model.h b/sherpa-onnx/csrc/offline-tts-matcha-model.h new file mode 100644 index 000000000..5b02ec9b3 --- /dev/null +++ b/sherpa-onnx/csrc/offline-tts-matcha-model.h @@ -0,0 +1,39 @@ +// sherpa-onnx/csrc/offline-tts-matcha-model.h +// +// Copyright (c) 2024 Xiaomi Corporation + +#ifndef SHERPA_ONNX_CSRC_OFFLINE_TTS_MATCHA_MODEL_H_ +#define SHERPA_ONNX_CSRC_OFFLINE_TTS_MATCHA_MODEL_H_ + +#include +#include + +#include "onnxruntime_cxx_api.h" // NOLINT +#include "sherpa-onnx/csrc/offline-tts-matcha-model-metadata.h" +#include "sherpa-onnx/csrc/offline-tts-model-config.h" + +namespace sherpa_onnx { + +class OfflineTtsMatchaModel { + public: + ~OfflineTtsMatchaModel(); + + explicit OfflineTtsMatchaModel(const OfflineTtsModelConfig &config); + + template + OfflineTtsMatchaModel(Manager *mgr, const OfflineTtsModelConfig &config); + + // Return a float32 tensor containing the mel + // of shape (batch_size, mel_dim, num_frames) + Ort::Value Run(Ort::Value x, int64_t sid = 0, float speed = 1.0) const; + + const OfflineTtsMatchaModelMetaData &GetMetaData() const; + + private: + class Impl; + std::unique_ptr impl_; +}; + +} // namespace sherpa_onnx + +#endif // SHERPA_ONNX_CSRC_OFFLINE_TTS_MATCHA_MODEL_H_ diff --git a/sherpa-onnx/csrc/offline-tts-model-config.cc b/sherpa-onnx/csrc/offline-tts-model-config.cc index f38c681a0..4af179a4b 100644 --- a/sherpa-onnx/csrc/offline-tts-model-config.cc +++ b/sherpa-onnx/csrc/offline-tts-model-config.cc @@ -10,6 +10,7 @@ namespace sherpa_onnx { void OfflineTtsModelConfig::Register(ParseOptions *po) { vits.Register(po); + matcha.Register(po); po->Register("num-threads", &num_threads, "Number of threads to run the neural network"); @@ -27,7 +28,11 @@ bool OfflineTtsModelConfig::Validate() const { return false; } - return vits.Validate(); + if (!vits.model.empty()) { + return vits.Validate(); + } + + return matcha.Validate(); } std::string OfflineTtsModelConfig::ToString() const { @@ -35,6 +40,7 @@ std::string OfflineTtsModelConfig::ToString() const { os << "OfflineTtsModelConfig("; os << "vits=" << vits.ToString() << ", "; + os << "matcha=" << matcha.ToString() << ", "; os << "num_threads=" << num_threads << ", "; os << "debug=" << (debug ? "True" : "False") << ", "; os << "provider=\"" << provider << "\")"; diff --git a/sherpa-onnx/csrc/offline-tts-model-config.h b/sherpa-onnx/csrc/offline-tts-model-config.h index bee50ba12..232686960 100644 --- a/sherpa-onnx/csrc/offline-tts-model-config.h +++ b/sherpa-onnx/csrc/offline-tts-model-config.h @@ -7,6 +7,7 @@ #include +#include "sherpa-onnx/csrc/offline-tts-matcha-model-config.h" #include "sherpa-onnx/csrc/offline-tts-vits-model-config.h" #include "sherpa-onnx/csrc/parse-options.h" @@ -14,6 +15,7 @@ namespace sherpa_onnx { struct OfflineTtsModelConfig { OfflineTtsVitsModelConfig vits; + OfflineTtsMatchaModelConfig matcha; int32_t num_threads = 1; bool debug = false; @@ -22,9 +24,11 @@ struct OfflineTtsModelConfig { OfflineTtsModelConfig() = default; OfflineTtsModelConfig(const OfflineTtsVitsModelConfig &vits, + const OfflineTtsMatchaModelConfig &matcha, int32_t num_threads, bool debug, const std::string &provider) : vits(vits), + matcha(matcha), num_threads(num_threads), debug(debug), provider(provider) {} diff --git a/sherpa-onnx/csrc/offline-tts-vits-impl.h b/sherpa-onnx/csrc/offline-tts-vits-impl.h index 560576357..1cc8d5f95 100644 --- a/sherpa-onnx/csrc/offline-tts-vits-impl.h +++ b/sherpa-onnx/csrc/offline-tts-vits-impl.h @@ -156,17 +156,31 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { int32_t num_speakers = meta_data.num_speakers; if (num_speakers == 0 && sid != 0) { +#if __OHOS__ + SHERPA_ONNX_LOGE( + "This is a single-speaker model and supports only sid 0. Given sid: " + "%{public}d. sid is ignored", + static_cast(sid)); +#else SHERPA_ONNX_LOGE( "This is a single-speaker model and supports only sid 0. Given sid: " "%d. sid is ignored", static_cast(sid)); +#endif } if (num_speakers != 0 && (sid >= num_speakers || sid < 0)) { +#if __OHOS__ + SHERPA_ONNX_LOGE( + "This model contains only %{public}d speakers. sid should be in the " + "range [%{public}d, %{public}d]. Given: %{public}d. Use sid=0", + num_speakers, 0, num_speakers - 1, static_cast(sid)); +#else SHERPA_ONNX_LOGE( "This model contains only %d speakers. sid should be in the range " "[%d, %d]. Given: %d. Use sid=0", num_speakers, 0, num_speakers - 1, static_cast(sid)); +#endif sid = 0; } @@ -389,8 +403,7 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { } else if (meta_data.jieba && !config_.model.vits.dict_dir.empty()) { frontend_ = std::make_unique( config_.model.vits.lexicon, config_.model.vits.tokens, - config_.model.vits.dict_dir, model_->GetMetaData(), - config_.model.debug); + config_.model.vits.dict_dir, config_.model.debug); } else if ((meta_data.is_piper || meta_data.is_coqui || meta_data.is_icefall) && !config_.model.vits.data_dir.empty()) { @@ -410,17 +423,6 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { } } - std::vector AddBlank(const std::vector &x) const { - // we assume the blank ID is 0 - std::vector buffer(x.size() * 2 + 1); - int32_t i = 1; - for (auto k : x) { - buffer[i] = k; - i += 2; - } - return buffer; - } - GeneratedAudio Process(const std::vector> &tokens, const std::vector> &tones, int32_t sid, float speed) const { diff --git a/sherpa-onnx/csrc/offline-tts-vits-model-config.cc b/sherpa-onnx/csrc/offline-tts-vits-model-config.cc index 9eb5b64a3..17c63460b 100644 --- a/sherpa-onnx/csrc/offline-tts-vits-model-config.cc +++ b/sherpa-onnx/csrc/offline-tts-vits-model-config.cc @@ -51,25 +51,30 @@ bool OfflineTtsVitsModelConfig::Validate() const { if (!data_dir.empty()) { if (!FileExists(data_dir + "/phontab")) { - SHERPA_ONNX_LOGE("'%s/phontab' does not exist. Skipping test", - data_dir.c_str()); + SHERPA_ONNX_LOGE( + "'%s/phontab' does not exist. Please check --vits-data-dir", + data_dir.c_str()); return false; } if (!FileExists(data_dir + "/phonindex")) { - SHERPA_ONNX_LOGE("'%s/phonindex' does not exist. Skipping test", - data_dir.c_str()); + SHERPA_ONNX_LOGE( + "'%s/phonindex' does not exist. Please check --vits-data-dir", + data_dir.c_str()); return false; } if (!FileExists(data_dir + "/phondata")) { - SHERPA_ONNX_LOGE("'%s/phondata' does not exist. Skipping test", - data_dir.c_str()); + SHERPA_ONNX_LOGE( + "'%s/phondata' does not exist. Please check --vits-data-dir", + data_dir.c_str()); return false; } if (!FileExists(data_dir + "/intonations")) { - SHERPA_ONNX_LOGE("'%s/intonations' does not exist.", data_dir.c_str()); + SHERPA_ONNX_LOGE( + "'%s/intonations' does not exist. Please check --vits-data-dir", + data_dir.c_str()); return false; } } @@ -82,8 +87,8 @@ bool OfflineTtsVitsModelConfig::Validate() const { for (const auto &f : required_files) { if (!FileExists(dict_dir + "/" + f)) { - SHERPA_ONNX_LOGE("'%s/%s' does not exist.", dict_dir.c_str(), - f.c_str()); + SHERPA_ONNX_LOGE("'%s/%s' does not exist. Please check vits-dict-dir", + dict_dir.c_str(), f.c_str()); return false; } } diff --git a/sherpa-onnx/csrc/offline-tts-vits-model.cc b/sherpa-onnx/csrc/offline-tts-vits-model.cc index eb605a7bd..3587a109d 100644 --- a/sherpa-onnx/csrc/offline-tts-vits-model.cc +++ b/sherpa-onnx/csrc/offline-tts-vits-model.cc @@ -174,7 +174,7 @@ class OfflineTtsVitsModel::Impl { SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(meta_data_.bos_id, "bos_id", 0); SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(meta_data_.eos_id, "eos_id", 0); SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(meta_data_.use_eos_bos, - "use_eos_bos", 0); + "use_eos_bos", 1); SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(meta_data_.pad_id, "pad_id", 0); std::string comment; @@ -362,7 +362,7 @@ Ort::Value OfflineTtsVitsModel::Run(Ort::Value x, int64_t sid /*=0*/, Ort::Value OfflineTtsVitsModel::Run(Ort::Value x, Ort::Value tones, int64_t sid /*= 0*/, - float speed /*= 1.0*/) { + float speed /*= 1.0*/) const { return impl_->Run(std::move(x), std::move(tones), sid, speed); } diff --git a/sherpa-onnx/csrc/offline-tts-vits-model.h b/sherpa-onnx/csrc/offline-tts-vits-model.h index a880934ef..30e4205dc 100644 --- a/sherpa-onnx/csrc/offline-tts-vits-model.h +++ b/sherpa-onnx/csrc/offline-tts-vits-model.h @@ -37,7 +37,7 @@ class OfflineTtsVitsModel { // This is for MeloTTS Ort::Value Run(Ort::Value x, Ort::Value tones, int64_t sid = 0, - float speed = 1.0); + float speed = 1.0) const; const OfflineTtsVitsModelMetaData &GetMetaData() const; diff --git a/sherpa-onnx/csrc/session.cc b/sherpa-onnx/csrc/session.cc index cd9eb516f..a33594f0b 100644 --- a/sherpa-onnx/csrc/session.cc +++ b/sherpa-onnx/csrc/session.cc @@ -273,4 +273,9 @@ Ort::SessionOptions GetSessionOptions(const OnlineLMConfig &config) { return GetSessionOptionsImpl(config.lm_num_threads, config.lm_provider); } +Ort::SessionOptions GetSessionOptions(int32_t num_threads, + const std::string &provider_str) { + return GetSessionOptionsImpl(num_threads, provider_str); +} + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/session.h b/sherpa-onnx/csrc/session.h index e19db6c20..131023e88 100644 --- a/sherpa-onnx/csrc/session.h +++ b/sherpa-onnx/csrc/session.h @@ -26,6 +26,9 @@ Ort::SessionOptions GetSessionOptions(const OnlineModelConfig &config); Ort::SessionOptions GetSessionOptions(const OnlineModelConfig &config, const std::string &model_type); +Ort::SessionOptions GetSessionOptions(int32_t num_threads, + const std::string &provider_str); + template Ort::SessionOptions GetSessionOptions(const T &config) { return GetSessionOptionsImpl(config.num_threads, config.provider); diff --git a/sherpa-onnx/csrc/sherpa-onnx-offline-tts.cc b/sherpa-onnx/csrc/sherpa-onnx-offline-tts.cc index 1ab8b68de..92feb8eac 100644 --- a/sherpa-onnx/csrc/sherpa-onnx-offline-tts.cc +++ b/sherpa-onnx/csrc/sherpa-onnx-offline-tts.cc @@ -72,6 +72,10 @@ or details. exit(EXIT_FAILURE); } + if (config.model.debug) { + fprintf(stderr, "%s\n", config.model.ToString().c_str()); + } + if (!config.Validate()) { fprintf(stderr, "Errors in config!\n"); exit(EXIT_FAILURE); diff --git a/sherpa-onnx/python/csrc/CMakeLists.txt b/sherpa-onnx/python/csrc/CMakeLists.txt index 21f77f29d..38d32de50 100644 --- a/sherpa-onnx/python/csrc/CMakeLists.txt +++ b/sherpa-onnx/python/csrc/CMakeLists.txt @@ -54,6 +54,7 @@ endif() if(SHERPA_ONNX_ENABLE_TTS) list(APPEND srcs + offline-tts-matcha-model-config.cc offline-tts-model-config.cc offline-tts-vits-model-config.cc offline-tts.cc diff --git a/sherpa-onnx/python/csrc/offline-tts-matcha-model-config.cc b/sherpa-onnx/python/csrc/offline-tts-matcha-model-config.cc new file mode 100644 index 000000000..2c932174e --- /dev/null +++ b/sherpa-onnx/python/csrc/offline-tts-matcha-model-config.cc @@ -0,0 +1,37 @@ +// sherpa-onnx/python/csrc/offline-tts-matcha-model-config.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include "sherpa-onnx/python/csrc/offline-tts-matcha-model-config.h" + +#include + +#include "sherpa-onnx/csrc/offline-tts-matcha-model-config.h" + +namespace sherpa_onnx { + +void PybindOfflineTtsMatchaModelConfig(py::module *m) { + using PyClass = OfflineTtsMatchaModelConfig; + + py::class_(*m, "OfflineTtsMatchaModelConfig") + .def(py::init<>()) + .def(py::init(), + py::arg("acoustic_model"), py::arg("vocoder"), py::arg("lexicon"), + py::arg("tokens"), py::arg("data_dir") = "", + py::arg("dict_dir") = "", py::arg("noise_scale") = 1.0, + py::arg("length_scale") = 1.0) + .def_readwrite("acoustic_model", &PyClass::acoustic_model) + .def_readwrite("vocoder", &PyClass::vocoder) + .def_readwrite("lexicon", &PyClass::lexicon) + .def_readwrite("tokens", &PyClass::tokens) + .def_readwrite("data_dir", &PyClass::data_dir) + .def_readwrite("dict_dir", &PyClass::dict_dir) + .def_readwrite("noise_scale", &PyClass::noise_scale) + .def_readwrite("length_scale", &PyClass::length_scale) + .def("__str__", &PyClass::ToString) + .def("validate", &PyClass::Validate); +} + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/python/csrc/offline-tts-matcha-model-config.h b/sherpa-onnx/python/csrc/offline-tts-matcha-model-config.h new file mode 100644 index 000000000..09b0c5c98 --- /dev/null +++ b/sherpa-onnx/python/csrc/offline-tts-matcha-model-config.h @@ -0,0 +1,16 @@ +// sherpa-onnx/python/csrc/offline-tts-matcha-model-config.h +// +// Copyright (c) 2024 Xiaomi Corporation + +#ifndef SHERPA_ONNX_PYTHON_CSRC_OFFLINE_TTS_MATCHA_MODEL_CONFIG_H_ +#define SHERPA_ONNX_PYTHON_CSRC_OFFLINE_TTS_MATCHA_MODEL_CONFIG_H_ + +#include "sherpa-onnx/python/csrc/sherpa-onnx.h" + +namespace sherpa_onnx { + +void PybindOfflineTtsMatchaModelConfig(py::module *m); + +} + +#endif // SHERPA_ONNX_PYTHON_CSRC_OFFLINE_TTS_MATCHA_MODEL_CONFIG_H_ diff --git a/sherpa-onnx/python/csrc/offline-tts-model-config.cc b/sherpa-onnx/python/csrc/offline-tts-model-config.cc index e5e86d968..fd19e4f85 100644 --- a/sherpa-onnx/python/csrc/offline-tts-model-config.cc +++ b/sherpa-onnx/python/csrc/offline-tts-model-config.cc @@ -7,22 +7,26 @@ #include #include "sherpa-onnx/csrc/offline-tts-model-config.h" +#include "sherpa-onnx/python/csrc/offline-tts-matcha-model-config.h" #include "sherpa-onnx/python/csrc/offline-tts-vits-model-config.h" namespace sherpa_onnx { void PybindOfflineTtsModelConfig(py::module *m) { PybindOfflineTtsVitsModelConfig(m); + PybindOfflineTtsMatchaModelConfig(m); using PyClass = OfflineTtsModelConfig; py::class_(*m, "OfflineTtsModelConfig") .def(py::init<>()) - .def(py::init(), - py::arg("vits"), py::arg("num_threads") = 1, + py::arg("vits"), py::arg("matcha"), py::arg("num_threads") = 1, py::arg("debug") = false, py::arg("provider") = "cpu") .def_readwrite("vits", &PyClass::vits) + .def_readwrite("matcha", &PyClass::matcha) .def_readwrite("num_threads", &PyClass::num_threads) .def_readwrite("debug", &PyClass::debug) .def_readwrite("provider", &PyClass::provider) diff --git a/sherpa-onnx/python/sherpa_onnx/__init__.py b/sherpa-onnx/python/sherpa_onnx/__init__.py index 2d5e456dc..330c8d2df 100644 --- a/sherpa-onnx/python/sherpa_onnx/__init__.py +++ b/sherpa-onnx/python/sherpa_onnx/__init__.py @@ -20,6 +20,7 @@ OfflineStream, OfflineTts, OfflineTtsConfig, + OfflineTtsMatchaModelConfig, OfflineTtsModelConfig, OfflineTtsVitsModelConfig, OfflineZipformerAudioTaggingModelConfig, From b2ad6f63f75cebe616772dda2f76328be62b15eb Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 31 Dec 2024 12:50:38 +0800 Subject: [PATCH 127/183] Release v1.10.37 (#1663) --- CHANGELOG.md | 11 +++++++++ CMakeLists.txt | 2 +- android/SherpaOnnxAar/README.md | 6 ++--- android/SherpaOnnxJavaDemo/app/build.gradle | 2 +- build-ios-shared.sh | 2 +- .../add-punctuations/pubspec.yaml | 2 +- dart-api-examples/audio-tagging/pubspec.yaml | 2 +- .../keyword-spotter/pubspec.yaml | 2 +- .../non-streaming-asr/pubspec.yaml | 2 +- .../speaker-diarization/pubspec.yaml | 2 +- .../speaker-identification/pubspec.yaml | 2 +- dart-api-examples/streaming-asr/pubspec.yaml | 2 +- dart-api-examples/tts/pubspec.yaml | 2 +- .../vad-with-non-streaming-asr/pubspec.yaml | 2 +- dart-api-examples/vad/pubspec.yaml | 2 +- flutter-examples/streaming_asr/pubspec.yaml | 4 ++-- flutter-examples/tts/pubspec.yaml | 4 ++-- flutter/sherpa_onnx/pubspec.yaml | 12 +++++----- .../ios/sherpa_onnx_ios.podspec | 2 +- .../macos/sherpa_onnx_macos.podspec | 2 +- .../SherpaOnnxHar/sherpa_onnx/README.md | 2 +- .../sherpa_onnx/oh-package.json5 | 2 +- .../entry/oh-package.json5 | 2 +- .../entry/oh-package.json5 | 2 +- .../entry/oh-package.json5 | 2 +- .../SherpaOnnxTts/entry/oh-package.json5 | 2 +- harmony-os/SherpaOnnxVadAsr/entry/README.md | 2 +- .../SherpaOnnxVadAsr/entry/oh-package.json5 | 2 +- jitpack.yml | 6 ++--- new-release.sh | 24 +++++++++---------- nodejs-addon-examples/package.json | 2 +- pom.xml | 2 +- 32 files changed, 64 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 450360188..b64604100 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +## 1.10.37 + +* Add new tts models for Latvia and Persian+English (#1644) +* Add a byte-level BPE Chinese+English non-streaming zipformer model (#1645) +* Support removing invalid utf-8 sequences. (#1648) +* Add TeleSpeech CTC to non_streaming_server.py (#1649) +* Fix building macOS libs (#1656) +* Add Go API for Keyword spotting (#1662) +* Add Swift online punctuation (#1661) +* Add C++ runtime for Matcha-TTS (#1627) + ## 1.10.36 * Update AAR version in Android Java demo (#1618) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7aaba5db8..b287dc9d7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ project(sherpa-onnx) # Remember to update # ./CHANGELOG.md # ./new-release.sh -set(SHERPA_ONNX_VERSION "1.10.36") +set(SHERPA_ONNX_VERSION "1.10.37") # Disable warning about # diff --git a/android/SherpaOnnxAar/README.md b/android/SherpaOnnxAar/README.md index 7ccf1f163..432777ab3 100644 --- a/android/SherpaOnnxAar/README.md +++ b/android/SherpaOnnxAar/README.md @@ -4,8 +4,8 @@ git clone https://github.com/k2-fsa/sherpa-onnx cd sherpa-onnx -wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.36/sherpa-onnx-v1.10.36-android.tar.bz2 -tar xvf sherpa-onnx-v1.10.36-android.tar.bz2 +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.37/sherpa-onnx-v1.10.37-android.tar.bz2 +tar xvf sherpa-onnx-v1.10.37-android.tar.bz2 cp -v jniLibs/arm64-v8a/* android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/arm64-v8a/ cp -v jniLibs/armeabi-v7a/* android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/armeabi-v7a/ @@ -16,5 +16,5 @@ cd android/SherpaOnnxAar ./gradlew :sherpa_onnx:assembleRelease ls -lh ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar -cp ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar ../../sherpa-onnx-1.10.36.aar +cp ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar ../../sherpa-onnx-1.10.37.aar ``` diff --git a/android/SherpaOnnxJavaDemo/app/build.gradle b/android/SherpaOnnxJavaDemo/app/build.gradle index 33688b4a1..7ddf31c10 100644 --- a/android/SherpaOnnxJavaDemo/app/build.gradle +++ b/android/SherpaOnnxJavaDemo/app/build.gradle @@ -34,5 +34,5 @@ dependencies { implementation 'pub.devrel:easypermissions:3.0.0' implementation 'androidx.core:core-ktx:1.7.0' // implementation files('/Users/fangjun/open-source/sherpa-onnx/android/SherpaOnnxAar/sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar') - implementation 'com.github.k2-fsa:sherpa-onnx:v1.10.36' + implementation 'com.github.k2-fsa:sherpa-onnx:v1.10.37' } diff --git a/build-ios-shared.sh b/build-ios-shared.sh index 50609cc2f..5fb90c691 100755 --- a/build-ios-shared.sh +++ b/build-ios-shared.sh @@ -242,7 +242,7 @@ for d in ios-arm64_x86_64-simulator ios-arm64; do CFBundlePackageType FMWK CFBundleShortVersionString - 1.10.36 + 1.10.37 CFBundleSupportedPlatforms iPhoneOS diff --git a/dart-api-examples/add-punctuations/pubspec.yaml b/dart-api-examples/add-punctuations/pubspec.yaml index dcd6b6de3..78b65ca03 100644 --- a/dart-api-examples/add-punctuations/pubspec.yaml +++ b/dart-api-examples/add-punctuations/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.36 + sherpa_onnx: ^1.10.37 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/audio-tagging/pubspec.yaml b/dart-api-examples/audio-tagging/pubspec.yaml index 23f1c2e8c..0115353ed 100644 --- a/dart-api-examples/audio-tagging/pubspec.yaml +++ b/dart-api-examples/audio-tagging/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.36 + sherpa_onnx: ^1.10.37 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/keyword-spotter/pubspec.yaml b/dart-api-examples/keyword-spotter/pubspec.yaml index 640f44ecc..13c34aaa7 100644 --- a/dart-api-examples/keyword-spotter/pubspec.yaml +++ b/dart-api-examples/keyword-spotter/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.36 + sherpa_onnx: ^1.10.37 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/non-streaming-asr/pubspec.yaml b/dart-api-examples/non-streaming-asr/pubspec.yaml index 030a5f5ef..75523f007 100644 --- a/dart-api-examples/non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.36 + sherpa_onnx: ^1.10.37 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/speaker-diarization/pubspec.yaml b/dart-api-examples/speaker-diarization/pubspec.yaml index bd41a77ae..9b9544cf0 100644 --- a/dart-api-examples/speaker-diarization/pubspec.yaml +++ b/dart-api-examples/speaker-diarization/pubspec.yaml @@ -8,7 +8,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.36 + sherpa_onnx: ^1.10.37 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/speaker-identification/pubspec.yaml b/dart-api-examples/speaker-identification/pubspec.yaml index 19b540114..368c1dde5 100644 --- a/dart-api-examples/speaker-identification/pubspec.yaml +++ b/dart-api-examples/speaker-identification/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.36 + sherpa_onnx: ^1.10.37 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/streaming-asr/pubspec.yaml b/dart-api-examples/streaming-asr/pubspec.yaml index 9e90e8aa9..451e92163 100644 --- a/dart-api-examples/streaming-asr/pubspec.yaml +++ b/dart-api-examples/streaming-asr/pubspec.yaml @@ -11,7 +11,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.36 + sherpa_onnx: ^1.10.37 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/tts/pubspec.yaml b/dart-api-examples/tts/pubspec.yaml index 6fe2973be..b64d7ef83 100644 --- a/dart-api-examples/tts/pubspec.yaml +++ b/dart-api-examples/tts/pubspec.yaml @@ -8,7 +8,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.36 + sherpa_onnx: ^1.10.37 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml index a915c49bc..aedcc1f5a 100644 --- a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.36 + sherpa_onnx: ^1.10.37 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad/pubspec.yaml b/dart-api-examples/vad/pubspec.yaml index 634fdf541..3e16d5b92 100644 --- a/dart-api-examples/vad/pubspec.yaml +++ b/dart-api-examples/vad/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.36 + sherpa_onnx: ^1.10.37 path: ^1.9.0 args: ^2.5.0 diff --git a/flutter-examples/streaming_asr/pubspec.yaml b/flutter-examples/streaming_asr/pubspec.yaml index 2118bc734..78748432b 100644 --- a/flutter-examples/streaming_asr/pubspec.yaml +++ b/flutter-examples/streaming_asr/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' -version: 1.10.36 +version: 1.10.37 topics: - speech-recognition @@ -31,7 +31,7 @@ dependencies: record: ^5.1.0 url_launcher: ^6.2.6 - sherpa_onnx: ^1.10.36 + sherpa_onnx: ^1.10.37 # sherpa_onnx: # path: ../../flutter/sherpa_onnx diff --git a/flutter-examples/tts/pubspec.yaml b/flutter-examples/tts/pubspec.yaml index 150098ca8..854492f80 100644 --- a/flutter-examples/tts/pubspec.yaml +++ b/flutter-examples/tts/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' # Remove this line if you wish to publish to pub.dev -version: 1.10.36 +version: 1.10.37 environment: sdk: ">=2.17.0 <4.0.0" @@ -18,7 +18,7 @@ dependencies: cupertino_icons: ^1.0.6 path_provider: ^2.1.3 path: ^1.9.0 - sherpa_onnx: ^1.10.36 + sherpa_onnx: ^1.10.37 # sherpa_onnx: # path: ../../flutter/sherpa_onnx url_launcher: 6.2.6 diff --git a/flutter/sherpa_onnx/pubspec.yaml b/flutter/sherpa_onnx/pubspec.yaml index 51cb0d8dd..dd69368c6 100644 --- a/flutter/sherpa_onnx/pubspec.yaml +++ b/flutter/sherpa_onnx/pubspec.yaml @@ -17,7 +17,7 @@ topics: - voice-activity-detection # remember to change the version in ../sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec -version: 1.10.36 +version: 1.10.37 homepage: https://github.com/k2-fsa/sherpa-onnx @@ -30,23 +30,23 @@ dependencies: flutter: sdk: flutter - sherpa_onnx_android: ^1.10.36 + sherpa_onnx_android: ^1.10.37 # sherpa_onnx_android: # path: ../sherpa_onnx_android - sherpa_onnx_macos: ^1.10.36 + sherpa_onnx_macos: ^1.10.37 # sherpa_onnx_macos: # path: ../sherpa_onnx_macos - sherpa_onnx_linux: ^1.10.36 + sherpa_onnx_linux: ^1.10.37 # sherpa_onnx_linux: # path: ../sherpa_onnx_linux - sherpa_onnx_windows: ^1.10.36 + sherpa_onnx_windows: ^1.10.37 # sherpa_onnx_windows: # path: ../sherpa_onnx_windows - sherpa_onnx_ios: ^1.10.36 + sherpa_onnx_ios: ^1.10.37 # sherpa_onnx_ios: # path: ../sherpa_onnx_ios diff --git a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec index 33cdc0a3e..dc0b24ecb 100644 --- a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec +++ b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec @@ -7,7 +7,7 @@ # https://groups.google.com/g/dart-ffi/c/nUATMBy7r0c Pod::Spec.new do |s| s.name = 'sherpa_onnx_ios' - s.version = '1.10.36' + s.version = '1.10.37' s.summary = 'A new Flutter FFI plugin project.' s.description = <<-DESC A new Flutter FFI plugin project. diff --git a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec index aed63de85..849ead253 100644 --- a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec +++ b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec @@ -4,7 +4,7 @@ # Pod::Spec.new do |s| s.name = 'sherpa_onnx_macos' - s.version = '1.10.36' + s.version = '1.10.37' s.summary = 'sherpa-onnx Flutter FFI plugin project.' s.description = <<-DESC sherpa-onnx Flutter FFI plugin project. diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md b/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md index ca9bdb29f..76e44a733 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md @@ -23,7 +23,7 @@ or update your `oh-package.json5` to include the following: ``` "dependencies": { - "sherpa_onnx": "1.10.36", + "sherpa_onnx": "1.10.37", }, ``` diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 index 13f40c024..3c365ced4 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 @@ -1,6 +1,6 @@ { "name": "sherpa_onnx", - "version": "1.10.36", + "version": "1.10.37", "description": "On-device speech-to-text, text-to-speech, and speaker diarization using Next-gen Kaldi without Internet connection", "main": "Index.ets", "author": "The next-gen Kaldi team", diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 b/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 index cf1c37644..68c0d2d8f 100644 --- a/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.36" + "sherpa_onnx": "1.10.37" } } diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 b/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 index fbf64d78d..ebcc97a46 100644 --- a/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.36", + "sherpa_onnx": "1.10.37", } } diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 index fbf64d78d..ebcc97a46 100644 --- a/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.36", + "sherpa_onnx": "1.10.37", } } diff --git a/harmony-os/SherpaOnnxTts/entry/oh-package.json5 b/harmony-os/SherpaOnnxTts/entry/oh-package.json5 index fbf64d78d..ebcc97a46 100644 --- a/harmony-os/SherpaOnnxTts/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxTts/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.36", + "sherpa_onnx": "1.10.37", } } diff --git a/harmony-os/SherpaOnnxVadAsr/entry/README.md b/harmony-os/SherpaOnnxVadAsr/entry/README.md index f9457676b..7b5390da4 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/README.md +++ b/harmony-os/SherpaOnnxVadAsr/entry/README.md @@ -1,6 +1,6 @@ # Introduction -Please download ./sherpa_onnx-v1.10.36.har +Please download ./sherpa_onnx-v1.10.37.har from Hint: For users who have no access to huggingface, please use diff --git a/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 b/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 index b5a868d56..223a04081 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 @@ -7,7 +7,7 @@ "license": "", "dependencies": { // please see https://ohpm.openharmony.cn/#/cn/detail/sherpa_onnx - "sherpa_onnx": "1.10.36", + "sherpa_onnx": "1.10.37", } } diff --git a/jitpack.yml b/jitpack.yml index ea04580d2..45d46be6e 100644 --- a/jitpack.yml +++ b/jitpack.yml @@ -2,8 +2,8 @@ jdk: - openjdk17 before_install: - - wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.36/sherpa-onnx-1.10.36.aar + - wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.37/sherpa-onnx-1.10.37.aar install: - - FILE="-Dfile=sherpa-onnx-1.10.36.aar" - - mvn install:install-file $FILE -DgroupId=com.k2fsa.sherpa.onnx -DartifactId=sherpa-onnx -Dversion=1.10.36 -Dpackaging=aar -DgeneratePom=true + - FILE="-Dfile=sherpa-onnx-1.10.37.aar" + - mvn install:install-file $FILE -DgroupId=com.k2fsa.sherpa.onnx -DartifactId=sherpa-onnx -Dversion=1.10.37 -Dpackaging=aar -DgeneratePom=true diff --git a/new-release.sh b/new-release.sh index 86036c38c..abb13fb8a 100755 --- a/new-release.sh +++ b/new-release.sh @@ -2,18 +2,18 @@ set -ex -sed -i.bak 's/1\.10\.35/1\.10\.36/g' ./build-ios-shared.sh -sed -i.bak 's/1\.10\.35/1\.10\.36/g' ./pom.xml -sed -i.bak 's/1\.10\.35/1\.10\.36/g' ./jitpack.yml -sed -i.bak 's/1\.10\.35/1\.10\.36/g' ./android/SherpaOnnxAar/README.md +sed -i.bak 's/1\.10\.36/1\.10\.37/g' ./build-ios-shared.sh +sed -i.bak 's/1\.10\.36/1\.10\.37/g' ./pom.xml +sed -i.bak 's/1\.10\.36/1\.10\.37/g' ./jitpack.yml +sed -i.bak 's/1\.10\.36/1\.10\.37/g' ./android/SherpaOnnxAar/README.md -find android -name build.gradle -type f -exec sed -i.bak 's/sherpa-onnx:v1\.10\.35/sherpa-onnx:v1\.10\.36/g' {} \; +find android -name build.gradle -type f -exec sed -i.bak 's/sherpa-onnx:v1\.10\.36/sherpa-onnx:v1\.10\.37/g' {} \; -find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.35/1\.10\.36/g' {} \; -find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.35/1\.10\.36/g' {} \; -find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.35/1\.10\.36/g' {} \; -find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.35/1\.10\.36/g' {} \; -find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.35/1\.10\.36/g' {} \; +find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.36/1\.10\.37/g' {} \; +find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.36/1\.10\.37/g' {} \; +find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.36/1\.10\.37/g' {} \; +find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.36/1\.10\.37/g' {} \; +find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.36/1\.10\.37/g' {} \; -find harmony-os -name "README.md" -type f -exec sed -i.bak 's/1\.10\.35/1\.10\.36/g' {} \; -find harmony-os -name oh-package.json5 -type f -exec sed -i.bak 's/1\.10\.35/1\.10\.36/g' {} \; +find harmony-os -name "README.md" -type f -exec sed -i.bak 's/1\.10\.36/1\.10\.37/g' {} \; +find harmony-os -name oh-package.json5 -type f -exec sed -i.bak 's/1\.10\.36/1\.10\.37/g' {} \; diff --git a/nodejs-addon-examples/package.json b/nodejs-addon-examples/package.json index 064d09f3c..592f1a7ab 100644 --- a/nodejs-addon-examples/package.json +++ b/nodejs-addon-examples/package.json @@ -1,5 +1,5 @@ { "dependencies": { - "sherpa-onnx-node": "^1.10.36" + "sherpa-onnx-node": "^1.10.37" } } diff --git a/pom.xml b/pom.xml index 9ff5f9c1d..05f74237c 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.k2fsa.sherpa.onnx sherpa-onnx-android - 1.10.36 + 1.10.37 https://github.com/k2-fsa/sherpa-onnx pom First Android Library From d3538531c4659da7a16724096d4d1a62047d51cd Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 31 Dec 2024 15:14:56 +0800 Subject: [PATCH 128/183] Fix initialize TTS in Python. (#1664) --- sherpa-onnx/python/csrc/offline-tts-model-config.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sherpa-onnx/python/csrc/offline-tts-model-config.cc b/sherpa-onnx/python/csrc/offline-tts-model-config.cc index fd19e4f85..ed6a6e090 100644 --- a/sherpa-onnx/python/csrc/offline-tts-model-config.cc +++ b/sherpa-onnx/python/csrc/offline-tts-model-config.cc @@ -23,8 +23,10 @@ void PybindOfflineTtsModelConfig(py::module *m) { .def(py::init(), - py::arg("vits"), py::arg("matcha"), py::arg("num_threads") = 1, - py::arg("debug") = false, py::arg("provider") = "cpu") + py::arg("vits") = OfflineTtsVitsModelConfig{}, + py::arg("matcha") = OfflineTtsMatchaModelConfig{}, + py::arg("num_threads") = 1, py::arg("debug") = false, + py::arg("provider") = "cpu") .def_readwrite("vits", &PyClass::vits) .def_readwrite("matcha", &PyClass::matcha) .def_readwrite("num_threads", &PyClass::num_threads) From ebe92e523d92319163b73655c17f80e1f6d1e813 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 31 Dec 2024 16:06:27 +0800 Subject: [PATCH 129/183] Remove spaces after punctuations for TTS (#1666) --- sherpa-onnx/csrc/jieba-lexicon.cc | 76 +++++++++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 3 deletions(-) diff --git a/sherpa-onnx/csrc/jieba-lexicon.cc b/sherpa-onnx/csrc/jieba-lexicon.cc index 9ea11f46f..11bd1f20f 100644 --- a/sherpa-onnx/csrc/jieba-lexicon.cc +++ b/sherpa-onnx/csrc/jieba-lexicon.cc @@ -6,6 +6,7 @@ #include #include // NOLINT +#include #include #include "cppjieba/Jieba.hpp" @@ -16,6 +17,14 @@ namespace sherpa_onnx { +static bool IsPunct(const std::string &s) { + static const std::unordered_set puncts = { + ",", ".", "!", "?", ":", "\"", "'", "，", + "。", "！", "？", "“", "”", "‘", "’", + }; + return puncts.count(s); +} + class JiebaLexicon::Impl { public: Impl(const std::string &lexicon, const std::string &tokens, @@ -67,8 +76,13 @@ class JiebaLexicon::Impl { jieba_->Cut(text, words, is_hmm); if (debug_) { - SHERPA_ONNX_LOGE("input text: %s", text.c_str()); - SHERPA_ONNX_LOGE("after replacing punctuations: %s", s.c_str()); +#if __OHOS__ + SHERPA_ONNX_LOGE("input text:\n%{public}s", text.c_str()); + SHERPA_ONNX_LOGE("after replacing punctuations:\n%{public}s", s.c_str()); +#else + SHERPA_ONNX_LOGE("input text:\n%s", text.c_str()); + SHERPA_ONNX_LOGE("after replacing punctuations:\n%s", s.c_str()); +#endif std::ostringstream os; std::string sep = ""; @@ -77,7 +91,52 @@ class JiebaLexicon::Impl { sep = "_"; } - SHERPA_ONNX_LOGE("after jieba processing: %s", os.str().c_str()); +#if __OHOS__ + SHERPA_ONNX_LOGE("after jieba processing:\n%{public}s", os.str().c_str()); +#else + SHERPA_ONNX_LOGE("after jieba processing:\n%s", os.str().c_str()); +#endif + } + + // remove spaces after punctuations + std::vector words2 = std::move(words); + words.reserve(words2.size()); + + for (int32_t i = 0; i < words2.size(); ++i) { + if (i == 0) { + words.push_back(std::move(words2[i])); + } else if (words2[i] == " ") { + if (words.back() == " " || IsPunct(words.back())) { + continue; + } else { + words.push_back(std::move(words2[i])); + } + } else if (IsPunct(words2[i])) { + if (words.back() == " " || IsPunct(words.back())) { + continue; + } else { + words.push_back(std::move(words2[i])); + } + } else { + words.push_back(std::move(words2[i])); + } + } + + if (debug_) { + std::ostringstream os; + std::string sep = ""; + for (const auto &w : words) { + os << sep << w; + sep = "_"; + } + +#if __OHOS__ + SHERPA_ONNX_LOGE("after removing spaces after punctuations:\n%{public}s", + os.str().c_str()); +#else + SHERPA_ONNX_LOGE("after removing spaces after punctuations:\n%s", + os.str().c_str()); +#endif } std::vector ans; @@ -86,7 +145,11 @@ class JiebaLexicon::Impl { for (const auto &w : words) { auto ids = ConvertWordToIds(w); if (ids.empty()) { +#if __OHOS__ + SHERPA_ONNX_LOGE("Ignore OOV '%{public}s'", w.c_str()); +#else SHERPA_ONNX_LOGE("Ignore OOV '%s'", w.c_str()); +#endif continue; } @@ -173,8 +236,15 @@ class JiebaLexicon::Impl { ToLowerCase(&word); if (word2ids_.count(word)) { +#if __OHOS__ + SHERPA_ONNX_LOGE( + "Duplicated word: %{public}s at line %{public}d:%{public}s. Ignore " + "it.", + word.c_str(), line_num, line.c_str()); +#else SHERPA_ONNX_LOGE("Duplicated word: %s at line %d:%s. Ignore it.", word.c_str(), line_num, line.c_str()); +#endif continue; } From 0a43e9c8790639c2b1898ecc63b8e0a6dc3f0d3d Mon Sep 17 00:00:00 2001 From: w-rui Date: Tue, 31 Dec 2024 18:07:52 +0800 Subject: [PATCH 130/183] Add constructor fromPtr() for all flutter class with factory ctor. (#1667) Co-authored-by: wangrui --- flutter/sherpa_onnx/lib/src/audio_tagging.dart | 2 ++ flutter/sherpa_onnx/lib/src/keyword_spotter.dart | 2 ++ flutter/sherpa_onnx/lib/src/offline_recognizer.dart | 2 ++ flutter/sherpa_onnx/lib/src/offline_speaker_diarization.dart | 3 +++ flutter/sherpa_onnx/lib/src/online_recognizer.dart | 2 ++ flutter/sherpa_onnx/lib/src/punctuation.dart | 2 ++ flutter/sherpa_onnx/lib/src/speaker_identification.dart | 4 ++++ flutter/sherpa_onnx/lib/src/tts.dart | 2 ++ flutter/sherpa_onnx/lib/src/vad.dart | 4 ++++ 9 files changed, 23 insertions(+) diff --git a/flutter/sherpa_onnx/lib/src/audio_tagging.dart b/flutter/sherpa_onnx/lib/src/audio_tagging.dart index 6c650b30c..3e3dbed2f 100644 --- a/flutter/sherpa_onnx/lib/src/audio_tagging.dart +++ b/flutter/sherpa_onnx/lib/src/audio_tagging.dart @@ -62,6 +62,8 @@ class AudioEvent { } class AudioTagging { + AudioTagging.fromPtr({required this.ptr, required this.config}); + AudioTagging._({required this.ptr, required this.config}); // The user has to invoke AudioTagging.free() to avoid memory leak. diff --git a/flutter/sherpa_onnx/lib/src/keyword_spotter.dart b/flutter/sherpa_onnx/lib/src/keyword_spotter.dart index c09867995..6e2c669ae 100644 --- a/flutter/sherpa_onnx/lib/src/keyword_spotter.dart +++ b/flutter/sherpa_onnx/lib/src/keyword_spotter.dart @@ -53,6 +53,8 @@ class KeywordResult { } class KeywordSpotter { + KeywordSpotter.fromPtr({required this.ptr, required this.config}); + KeywordSpotter._({required this.ptr, required this.config}); /// The user is responsible to call the OnlineRecognizer.free() diff --git a/flutter/sherpa_onnx/lib/src/offline_recognizer.dart b/flutter/sherpa_onnx/lib/src/offline_recognizer.dart index 2f6f167d4..01bceccce 100644 --- a/flutter/sherpa_onnx/lib/src/offline_recognizer.dart +++ b/flutter/sherpa_onnx/lib/src/offline_recognizer.dart @@ -227,6 +227,8 @@ class OfflineRecognizerResult { } class OfflineRecognizer { + OfflineRecognizer.fromPtr({required this.ptr, required this.config}); + OfflineRecognizer._({required this.ptr, required this.config}); void free() { diff --git a/flutter/sherpa_onnx/lib/src/offline_speaker_diarization.dart b/flutter/sherpa_onnx/lib/src/offline_speaker_diarization.dart index 5981e3c04..fe046a166 100644 --- a/flutter/sherpa_onnx/lib/src/offline_speaker_diarization.dart +++ b/flutter/sherpa_onnx/lib/src/offline_speaker_diarization.dart @@ -94,6 +94,9 @@ class OfflineSpeakerDiarizationConfig { } class OfflineSpeakerDiarization { + OfflineSpeakerDiarization.fromPtr( + {required this.ptr, required this.config, required this.sampleRate}); + OfflineSpeakerDiarization._( {required this.ptr, required this.config, required this.sampleRate}); diff --git a/flutter/sherpa_onnx/lib/src/online_recognizer.dart b/flutter/sherpa_onnx/lib/src/online_recognizer.dart index 18d5a6000..69ed93894 100644 --- a/flutter/sherpa_onnx/lib/src/online_recognizer.dart +++ b/flutter/sherpa_onnx/lib/src/online_recognizer.dart @@ -162,6 +162,8 @@ class OnlineRecognizerResult { } class OnlineRecognizer { + OnlineRecognizer.fromPtr({required this.ptr, required this.config}); + OnlineRecognizer._({required this.ptr, required this.config}); /// The user is responsible to call the OnlineRecognizer.free() diff --git a/flutter/sherpa_onnx/lib/src/punctuation.dart b/flutter/sherpa_onnx/lib/src/punctuation.dart index b4197fa46..dd38a2445 100644 --- a/flutter/sherpa_onnx/lib/src/punctuation.dart +++ b/flutter/sherpa_onnx/lib/src/punctuation.dart @@ -36,6 +36,8 @@ class OfflinePunctuationConfig { } class OfflinePunctuation { + OfflinePunctuation.fromPtr({required this.ptr, required this.config}); + OfflinePunctuation._({required this.ptr, required this.config}); // The user has to invoke OfflinePunctuation.free() to avoid memory leak. diff --git a/flutter/sherpa_onnx/lib/src/speaker_identification.dart b/flutter/sherpa_onnx/lib/src/speaker_identification.dart index 5c2e10744..8b27dbc69 100644 --- a/flutter/sherpa_onnx/lib/src/speaker_identification.dart +++ b/flutter/sherpa_onnx/lib/src/speaker_identification.dart @@ -25,6 +25,8 @@ class SpeakerEmbeddingExtractorConfig { } class SpeakerEmbeddingExtractor { + SpeakerEmbeddingExtractor.fromPtr({required this.ptr, required this.dim}); + SpeakerEmbeddingExtractor._({required this.ptr, required this.dim}); /// The user is responsible to call the SpeakerEmbeddingExtractor.free() @@ -101,6 +103,8 @@ class SpeakerEmbeddingExtractor { } class SpeakerEmbeddingManager { + SpeakerEmbeddingManager.fromPtr({required this.ptr, required this.dim}); + SpeakerEmbeddingManager._({required this.ptr, required this.dim}); // The user has to use SpeakerEmbeddingManager.free() to avoid memory leak diff --git a/flutter/sherpa_onnx/lib/src/tts.dart b/flutter/sherpa_onnx/lib/src/tts.dart index f779188b7..2f550e273 100644 --- a/flutter/sherpa_onnx/lib/src/tts.dart +++ b/flutter/sherpa_onnx/lib/src/tts.dart @@ -82,6 +82,8 @@ class GeneratedAudio { } class OfflineTts { + OfflineTts.fromPtr({required this.ptr, required this.config}); + OfflineTts._({required this.ptr, required this.config}); /// The user is responsible to call the OfflineTts.free() diff --git a/flutter/sherpa_onnx/lib/src/vad.dart b/flutter/sherpa_onnx/lib/src/vad.dart index 10fac5a45..7db0e55e0 100644 --- a/flutter/sherpa_onnx/lib/src/vad.dart +++ b/flutter/sherpa_onnx/lib/src/vad.dart @@ -54,6 +54,8 @@ class SpeechSegment { } class CircularBuffer { + CircularBuffer.fromPtr({required this.ptr}); + CircularBuffer._({required this.ptr}); /// The user has to invoke CircularBuffer.free() on the returned instance @@ -115,6 +117,8 @@ class CircularBuffer { } class VoiceActivityDetector { + VoiceActivityDetector.fromPtr({required this.ptr, required this.config}); + VoiceActivityDetector._({required this.ptr, required this.config}); // The user has to invoke VoiceActivityDetector.free() to avoid memory leak. From 3422b9388dfec1824d0970ef0d753e963ab0b76e Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 31 Dec 2024 19:20:52 +0800 Subject: [PATCH 131/183] Add Kotlin API for Matcha-TTS models. (#1668) --- .github/workflows/jni.yaml | 5 ++++ .gitignore | 1 + kotlin-api-examples/run.sh | 10 +++++++ kotlin-api-examples/test_tts.kt | 29 ++++++++++++++++-- sherpa-onnx/c-api/c-api.cc | 12 +++++--- sherpa-onnx/c-api/c-api.h | 6 ++-- sherpa-onnx/csrc/jieba-lexicon.cc | 2 +- sherpa-onnx/jni/offline-tts.cc | 49 +++++++++++++++++++++++++++++++ sherpa-onnx/kotlin-api/Tts.kt | 12 ++++++++ 9 files changed, 117 insertions(+), 9 deletions(-) diff --git a/.github/workflows/jni.yaml b/.github/workflows/jni.yaml index a0f769393..0dc775f4c 100644 --- a/.github/workflows/jni.yaml +++ b/.github/workflows/jni.yaml @@ -75,3 +75,8 @@ jobs: cd ./kotlin-api-examples ./run.sh + + - uses: actions/upload-artifact@v4 + with: + name: tts-files-${{ matrix.os }} + path: kotlin-api-examples/test-*.wav diff --git a/.gitignore b/.gitignore index cfb6fa57c..eeec52d9c 100644 --- a/.gitignore +++ b/.gitignore @@ -125,3 +125,4 @@ sherpa-onnx-moonshine-tiny-en-int8 sherpa-onnx-moonshine-base-en-int8 harmony-os/SherpaOnnxHar/sherpa_onnx/LICENSE harmony-os/SherpaOnnxHar/sherpa_onnx/CHANGELOG.md +matcha-icefall-zh-baker diff --git a/kotlin-api-examples/run.sh b/kotlin-api-examples/run.sh index 3b3d15938..63ea224d1 100755 --- a/kotlin-api-examples/run.sh +++ b/kotlin-api-examples/run.sh @@ -105,6 +105,16 @@ function testTts() { rm vits-piper-en_US-amy-low.tar.bz2 fi + if [ ! -f ./matcha-icefall-zh-baker/model-steps-3.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 + tar xvf matcha-icefall-zh-baker.tar.bz2 + rm matcha-icefall-zh-baker.tar.bz2 + fi + + if [ ! -f ./hifigan_v2.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + fi + out_filename=test_tts.jar kotlinc-jvm -include-runtime -d $out_filename \ test_tts.kt \ diff --git a/kotlin-api-examples/test_tts.kt b/kotlin-api-examples/test_tts.kt index 8602a4461..3865c33e3 100644 --- a/kotlin-api-examples/test_tts.kt +++ b/kotlin-api-examples/test_tts.kt @@ -1,10 +1,35 @@ package com.k2fsa.sherpa.onnx fun main() { - testTts() + testVits() + testMatcha() } -fun testTts() { +fun testMatcha() { + // see https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models + // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 + var config = OfflineTtsConfig( + model=OfflineTtsModelConfig( + matcha=OfflineTtsMatchaModelConfig( + acousticModel="./matcha-icefall-zh-baker/model-steps-3.onnx", + vocoder="./hifigan_v2.onnx", + tokens="./matcha-icefall-zh-baker/tokens.txt", + lexicon="./matcha-icefall-zh-baker/lexicon.txt", + dictDir="./matcha-icefall-zh-baker/dict", + ), + numThreads=1, + debug=true, + ), + ruleFsts="./matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst", + ) + val tts = OfflineTts(config=config) + val audio = tts.generateWithCallback(text="某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。", callback=::callback) + audio.save(filename="test-zh.wav") + tts.release() + println("Saved to test-zh.wav") +} + +fun testVits() { // see https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 var config = OfflineTtsConfig( diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index 4d1bb6625..703380730 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -1727,11 +1727,15 @@ const SherpaOnnxOnlinePunctuation *SherpaOnnxCreateOnlinePunctuation( auto p = new SherpaOnnxOnlinePunctuation; try { sherpa_onnx::OnlinePunctuationConfig punctuation_config; - punctuation_config.model.cnn_bilstm = SHERPA_ONNX_OR(config->model.cnn_bilstm, ""); - punctuation_config.model.bpe_vocab = SHERPA_ONNX_OR(config->model.bpe_vocab, ""); - punctuation_config.model.num_threads = SHERPA_ONNX_OR(config->model.num_threads, 1); + punctuation_config.model.cnn_bilstm = + SHERPA_ONNX_OR(config->model.cnn_bilstm, ""); + punctuation_config.model.bpe_vocab = + SHERPA_ONNX_OR(config->model.bpe_vocab, ""); + punctuation_config.model.num_threads = + SHERPA_ONNX_OR(config->model.num_threads, 1); punctuation_config.model.debug = config->model.debug; - punctuation_config.model.provider = SHERPA_ONNX_OR(config->model.provider, "cpu"); + punctuation_config.model.provider = + SHERPA_ONNX_OR(config->model.provider, "cpu"); p->impl = std::make_unique(punctuation_config); diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index 990c94cb3..167051cd9 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -1381,12 +1381,14 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOnlinePunctuationConfig { SherpaOnnxOnlinePunctuationModelConfig model; } SherpaOnnxOnlinePunctuationConfig; -SHERPA_ONNX_API typedef struct SherpaOnnxOnlinePunctuation SherpaOnnxOnlinePunctuation; +SHERPA_ONNX_API typedef struct SherpaOnnxOnlinePunctuation + SherpaOnnxOnlinePunctuation; // Create an online punctuation processor. The user has to invoke // SherpaOnnxDestroyOnlinePunctuation() to free the returned pointer // to avoid memory leak -SHERPA_ONNX_API const SherpaOnnxOnlinePunctuation *SherpaOnnxCreateOnlinePunctuation( +SHERPA_ONNX_API const SherpaOnnxOnlinePunctuation * +SherpaOnnxCreateOnlinePunctuation( const SherpaOnnxOnlinePunctuationConfig *config); // Free a pointer returned by SherpaOnnxCreateOnlinePunctuation() diff --git a/sherpa-onnx/csrc/jieba-lexicon.cc b/sherpa-onnx/csrc/jieba-lexicon.cc index 11bd1f20f..57b77666b 100644 --- a/sherpa-onnx/csrc/jieba-lexicon.cc +++ b/sherpa-onnx/csrc/jieba-lexicon.cc @@ -155,7 +155,7 @@ class JiebaLexicon::Impl { this_sentence.insert(this_sentence.end(), ids.begin(), ids.end()); - if (w == "。" || w == "！" || w == "？" || w == "，") { + if (IsPunct(w)) { ans.emplace_back(std::move(this_sentence)); this_sentence = {}; } diff --git a/sherpa-onnx/jni/offline-tts.cc b/sherpa-onnx/jni/offline-tts.cc index 4d67afc27..985d581ef 100644 --- a/sherpa-onnx/jni/offline-tts.cc +++ b/sherpa-onnx/jni/offline-tts.cc @@ -20,6 +20,7 @@ static OfflineTtsConfig GetOfflineTtsConfig(JNIEnv *env, jobject config) { jobject model = env->GetObjectField(config, fid); jclass model_config_cls = env->GetObjectClass(model); + // vits fid = env->GetFieldID(model_config_cls, "vits", "Lcom/k2fsa/sherpa/onnx/OfflineTtsVitsModelConfig;"); jobject vits = env->GetObjectField(model, fid); @@ -64,6 +65,54 @@ static OfflineTtsConfig GetOfflineTtsConfig(JNIEnv *env, jobject config) { fid = env->GetFieldID(vits_cls, "lengthScale", "F"); ans.model.vits.length_scale = env->GetFloatField(vits, fid); + // matcha + fid = env->GetFieldID(model_config_cls, "matcha", + "Lcom/k2fsa/sherpa/onnx/OfflineTtsMatchaModelConfig;"); + jobject matcha = env->GetObjectField(model, fid); + jclass matcha_cls = env->GetObjectClass(matcha); + + fid = env->GetFieldID(matcha_cls, "acousticModel", "Ljava/lang/String;"); + s = (jstring)env->GetObjectField(matcha, fid); + p = env->GetStringUTFChars(s, nullptr); + ans.model.matcha.acoustic_model = p; + env->ReleaseStringUTFChars(s, p); + + fid = env->GetFieldID(matcha_cls, "vocoder", "Ljava/lang/String;"); + s = (jstring)env->GetObjectField(matcha, fid); + p = env->GetStringUTFChars(s, nullptr); + ans.model.matcha.vocoder = p; + env->ReleaseStringUTFChars(s, p); + + fid = env->GetFieldID(matcha_cls, "lexicon", "Ljava/lang/String;"); + s = (jstring)env->GetObjectField(matcha, fid); + p = env->GetStringUTFChars(s, nullptr); + ans.model.matcha.lexicon = p; + env->ReleaseStringUTFChars(s, p); + + fid = env->GetFieldID(matcha_cls, "tokens", "Ljava/lang/String;"); + s = (jstring)env->GetObjectField(matcha, fid); + p = env->GetStringUTFChars(s, nullptr); + ans.model.matcha.tokens = p; + env->ReleaseStringUTFChars(s, p); + + fid = env->GetFieldID(matcha_cls, "dataDir", "Ljava/lang/String;"); + s = (jstring)env->GetObjectField(matcha, fid); + p = env->GetStringUTFChars(s, nullptr); + ans.model.matcha.data_dir = p; + env->ReleaseStringUTFChars(s, p); + + fid = env->GetFieldID(matcha_cls, "dictDir", "Ljava/lang/String;"); + s = (jstring)env->GetObjectField(matcha, fid); + p = env->GetStringUTFChars(s, nullptr); + ans.model.matcha.dict_dir = p; + env->ReleaseStringUTFChars(s, p); + + fid = env->GetFieldID(matcha_cls, "noiseScale", "F"); + ans.model.matcha.noise_scale = env->GetFloatField(matcha, fid); + + fid = env->GetFieldID(matcha_cls, "lengthScale", "F"); + ans.model.matcha.length_scale = env->GetFloatField(matcha, fid); + fid = env->GetFieldID(model_config_cls, "numThreads", "I"); ans.model.num_threads = env->GetIntField(model, fid); diff --git a/sherpa-onnx/kotlin-api/Tts.kt b/sherpa-onnx/kotlin-api/Tts.kt index 6152cd914..231b87d81 100644 --- a/sherpa-onnx/kotlin-api/Tts.kt +++ b/sherpa-onnx/kotlin-api/Tts.kt @@ -14,8 +14,20 @@ data class OfflineTtsVitsModelConfig( var lengthScale: Float = 1.0f, ) +data class OfflineTtsMatchaModelConfig( + var acousticModel: String = "", + var vocoder: String = "", + var lexicon: String = "", + var tokens: String = "", + var dataDir: String = "", + var dictDir: String = "", + var noiseScale: Float = 1.0f, + var lengthScale: Float = 1.0f, +) + data class OfflineTtsModelConfig( var vits: OfflineTtsVitsModelConfig = OfflineTtsVitsModelConfig(), + var matcha: OfflineTtsMatchaModelConfig = OfflineTtsMatchaModelConfig(), var numThreads: Int = 1, var debug: Boolean = false, var provider: String = "cpu", From f457baea42b3efedfb55a8e4cda1d9b1538512ac Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 2 Jan 2025 13:46:43 +0800 Subject: [PATCH 132/183] Support Matcha-TTS models using espeak-ng (#1672) --- .github/scripts/test-offline-tts.sh | 22 +++ .github/scripts/test-python.sh | 25 ++- python-api-examples/offline-tts-play.py | 25 ++- python-api-examples/offline-tts.py | 25 ++- sherpa-onnx/csrc/macros.h | 32 ++-- sherpa-onnx/csrc/offline-tts-matcha-impl.h | 41 ++++- .../csrc/offline-tts-matcha-model-metadata.h | 2 +- sherpa-onnx/csrc/offline-tts-matcha-model.cc | 2 +- sherpa-onnx/csrc/piper-phonemize-lexicon.cc | 147 +++++++++++++++--- sherpa-onnx/csrc/piper-phonemize-lexicon.h | 24 ++- 10 files changed, 288 insertions(+), 57 deletions(-) diff --git a/.github/scripts/test-offline-tts.sh b/.github/scripts/test-offline-tts.sh index 1aa0340a0..70fd2247e 100755 --- a/.github/scripts/test-offline-tts.sh +++ b/.github/scripts/test-offline-tts.sh @@ -18,6 +18,28 @@ which $EXE # test waves are saved in ./tts mkdir ./tts +log "------------------------------------------------------------" +log "matcha-icefall-en_US-ljspeech" +log "------------------------------------------------------------" +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 +tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 +rm matcha-icefall-en_US-ljspeech.tar.bz2 + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +$EXE \ + --matcha-acoustic-model=./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ + --matcha-vocoder=./hifigan_v2.onnx \ + --matcha-tokens=./matcha-icefall-en_US-ljspeech/tokens.txt \ + --matcha-data-dir=./matcha-icefall-en_US-ljspeech/espeak-ng-data \ + --num-threads=2 \ + --output-filename=./tts/matcha-ljspeech-1.wav \ + --debug=1 \ + "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar." + +rm hifigan_v2.onnx +rm -rf matcha-icefall-en_US-ljspeech + log "------------------------------------------------------------" log "matcha-icefall-zh-baker" log "------------------------------------------------------------" diff --git a/.github/scripts/test-python.sh b/.github/scripts/test-python.sh index 8bfe2c16f..350d9c185 100755 --- a/.github/scripts/test-python.sh +++ b/.github/scripts/test-python.sh @@ -267,7 +267,27 @@ log "Offline TTS test" # test waves are saved in ./tts mkdir ./tts -log "vits-ljs test" +log "matcha-ljspeech-en test" + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 +tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 +rm matcha-icefall-en_US-ljspeech.tar.bz2 + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +python3 ./python-api-examples/offline-tts.py \ + --matcha-acoustic-model=./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ + --matcha-vocoder=./hifigan_v2.onnx \ + --matcha-tokens=./matcha-icefall-en_US-ljspeech/tokens.txt \ + --matcha-data-dir=./matcha-icefall-en_US-ljspeech/espeak-ng-data \ + --output-filename=./tts/test-matcha-ljspeech-en.wav \ + --num-threads=2 \ + "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar." + +rm hifigan_v2.onnx +rm -rf matcha-icefall-en_US-ljspeech + +log "matcha-baker-zh test" curl -O -SL https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 tar xvf matcha-icefall-zh-baker.tar.bz2 @@ -282,12 +302,13 @@ python3 ./python-api-examples/offline-tts.py \ --matcha-tokens=./matcha-icefall-zh-baker/tokens.txt \ --tts-rule-fsts=./matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst \ --matcha-dict-dir=./matcha-icefall-zh-baker/dict \ - --output-filename=./tts/test-matcha.wav \ + --output-filename=./tts/test-matcha-baker-zh.wav \ "某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。" rm -rf matcha-icefall-zh-baker rm hifigan_v2.onnx +log "vits-ljs test" curl -LS -O https://huggingface.co/csukuangfj/vits-ljs/resolve/main/vits-ljs.onnx curl -LS -O https://huggingface.co/csukuangfj/vits-ljs/resolve/main/lexicon.txt diff --git a/python-api-examples/offline-tts-play.py b/python-api-examples/offline-tts-play.py index e8350ea47..09d03dae6 100755 --- a/python-api-examples/offline-tts-play.py +++ b/python-api-examples/offline-tts-play.py @@ -11,7 +11,7 @@ Usage: -Example (1/4) +Example (1/5) wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 tar xf vits-piper-en_US-amy-low.tar.bz2 @@ -23,7 +23,7 @@ --output-filename=./generated.wav \ "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar." -Example (2/4) +Example (2/5) wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-zh-aishell3.tar.bz2 tar xvf vits-zh-aishell3.tar.bz2 @@ -37,7 +37,7 @@ --output-filename=./liubei-21.wav \ "勿以恶小而为之，勿以善小而不为。惟贤惟德，能服于人。122334" -Example (3/4) +Example (3/5) wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-vits-zh-ll.tar.bz2 tar xvf sherpa-onnx-vits-zh-ll.tar.bz2 @@ -53,7 +53,7 @@ --output-filename=./test-2.wav \ "当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转，思念如涟漪荡漾，梦境如画卷展开，我与自然融为一体，沉静在这片宁静的美丽之中，感受着生命的奇迹与温柔。2024年5月11号，拨打110或者18920240511。123456块钱。" -Example (4/4) +Example (4/5) curl -O -SL https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 tar xvf matcha-icefall-zh-baker.tar.bz2 @@ -71,6 +71,23 @@ --output-filename=./test-matcha.wav \ "某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。" +Example (5/5) + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 +tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 +rm matcha-icefall-en_US-ljspeech.tar.bz2 + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +python3 ./python-api-examples/offline-tts-play.py \ + --matcha-acoustic-model=./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ + --matcha-vocoder=./hifigan_v2.onnx \ + --matcha-tokens=./matcha-icefall-en_US-ljspeech/tokens.txt \ + --matcha-data-dir=./matcha-icefall-en_US-ljspeech/espeak-ng-data \ + --output-filename=./test-matcha-ljspeech-en.wav \ + --num-threads=2 \ + "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar." + You can find more models at https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models diff --git a/python-api-examples/offline-tts.py b/python-api-examples/offline-tts.py index aa1cce935..72bf77959 100755 --- a/python-api-examples/offline-tts.py +++ b/python-api-examples/offline-tts.py @@ -12,7 +12,7 @@ Usage: -Example (1/4) +Example (1/5) wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 tar xf vits-piper-en_US-amy-low.tar.bz2 @@ -24,7 +24,7 @@ --output-filename=./generated.wav \ "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar." -Example (2/4) +Example (2/5) wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-icefall-zh-aishell3.tar.bz2 tar xvf vits-icefall-zh-aishell3.tar.bz2 @@ -38,7 +38,7 @@ --output-filename=./liubei-21.wav \ "勿以恶小而为之，勿以善小而不为。惟贤惟德，能服于人。122334" -Example (3/4) +Example (3/5) wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-vits-zh-ll.tar.bz2 tar xvf sherpa-onnx-vits-zh-ll.tar.bz2 @@ -54,7 +54,7 @@ --output-filename=./test-2.wav \ "当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转，思念如涟漪荡漾，梦境如画卷展开，我与自然融为一体，沉静在这片宁静的美丽之中，感受着生命的奇迹与温柔。2024年5月11号，拨打110或者18920240511。123456块钱。" -Example (4/4) +Example (4/5) curl -O -SL https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 tar xvf matcha-icefall-zh-baker.tar.bz2 @@ -72,6 +72,23 @@ --output-filename=./test-matcha.wav \ "某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。" +Example (5/5) + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 +tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 +rm matcha-icefall-en_US-ljspeech.tar.bz2 + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +python3 ./python-api-examples/offline-tts.py \ + --matcha-acoustic-model=./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ + --matcha-vocoder=./hifigan_v2.onnx \ + --matcha-tokens=./matcha-icefall-en_US-ljspeech/tokens.txt \ + --matcha-data-dir=./matcha-icefall-en_US-ljspeech/espeak-ng-data \ + --output-filename=./test-matcha-ljspeech-en.wav \ + --num-threads=2 \ + "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar." + You can find more models at https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models diff --git a/sherpa-onnx/csrc/macros.h b/sherpa-onnx/csrc/macros.h index ac11d6a79..2788292df 100644 --- a/sherpa-onnx/csrc/macros.h +++ b/sherpa-onnx/csrc/macros.h @@ -49,19 +49,21 @@ } while (0) #endif +#define SHERPA_ONNX_EXIT(code) exit(code) + // Read an integer #define SHERPA_ONNX_READ_META_DATA(dst, src_key) \ do { \ auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \ if (value.empty()) { \ SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ - exit(-1); \ + SHERPA_ONNX_EXIT(-1); \ } \ \ dst = atoi(value.c_str()); \ if (dst < 0) { \ SHERPA_ONNX_LOGE("Invalid value %d for '%s'", dst, src_key); \ - exit(-1); \ + SHERPA_ONNX_EXIT(-1); \ } \ } while (0) @@ -74,7 +76,7 @@ dst = atoi(value.c_str()); \ if (dst < 0) { \ SHERPA_ONNX_LOGE("Invalid value %d for '%s'", dst, src_key); \ - exit(-1); \ + SHERPA_ONNX_EXIT(-1); \ } \ } \ } while (0) @@ -85,13 +87,13 @@ auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \ if (value.empty()) { \ SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ - exit(-1); \ + SHERPA_ONNX_EXIT(-1); \ } \ \ bool ret = SplitStringToIntegers(value.c_str(), ",", true, &dst); \ if (!ret) { \ SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'", value.c_str(), src_key); \ - exit(-1); \ + SHERPA_ONNX_EXIT(-1); \ } \ } while (0) @@ -101,13 +103,13 @@ auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \ if (value.empty()) { \ SHERPA_ONNX_LOGE("%s does not exist in the metadata", src_key); \ - exit(-1); \ + SHERPA_ONNX_EXIT(-1); \ } \ \ bool ret = SplitStringToFloats(value.c_str(), ",", true, &dst); \ if (!ret) { \ SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'", value.c_str(), src_key); \ - exit(-1); \ + SHERPA_ONNX_EXIT(-1); \ } \ } while (0) @@ -117,14 +119,14 @@ auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \ if (value.empty()) { \ SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ - exit(-1); \ + SHERPA_ONNX_EXIT(-1); \ } \ SplitStringToVector(value.c_str(), ",", false, &dst); \ \ if (dst.empty()) { \ SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'. Empty vector!", \ value.c_str(), src_key); \ - exit(-1); \ + SHERPA_ONNX_EXIT(-1); \ } \ } while (0) @@ -134,14 +136,14 @@ auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \ if (value.empty()) { \ SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ - exit(-1); \ + SHERPA_ONNX_EXIT(-1); \ } \ SplitStringToVector(value.c_str(), sep, false, &dst); \ \ if (dst.empty()) { \ SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'. Empty vector!", \ value.c_str(), src_key); \ - exit(-1); \ + SHERPA_ONNX_EXIT(-1); \ } \ } while (0) @@ -151,13 +153,13 @@ auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \ if (value.empty()) { \ SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \ - exit(-1); \ + SHERPA_ONNX_EXIT(-1); \ } \ \ dst = std::move(value); \ if (dst.empty()) { \ SHERPA_ONNX_LOGE("Invalid value for '%s'\n", src_key); \ - exit(-1); \ + SHERPA_ONNX_EXIT(-1); \ } \ } while (0) @@ -178,11 +180,9 @@ dst = std::move(value); \ if (dst.empty()) { \ SHERPA_ONNX_LOGE("Invalid value for '%s'\n", src_key); \ - exit(-1); \ + SHERPA_ONNX_EXIT(-1); \ } \ } \ } while (0) -#define SHERPA_ONNX_EXIT(code) exit(code) - #endif // SHERPA_ONNX_CSRC_MACROS_H_ diff --git a/sherpa-onnx/csrc/offline-tts-matcha-impl.h b/sherpa-onnx/csrc/offline-tts-matcha-impl.h index 62c29bb83..a4f47fadb 100644 --- a/sherpa-onnx/csrc/offline-tts-matcha-impl.h +++ b/sherpa-onnx/csrc/offline-tts-matcha-impl.h @@ -321,12 +321,45 @@ class OfflineTtsMatchaImpl : public OfflineTtsImpl { private: template - void InitFrontend(Manager *mgr) {} + void InitFrontend(Manager *mgr) { + // for piper phonemizer + // we require that you copy espeak_ng_data + // from assets to disk + // + // for jieba + // we require that you copy tokens.txt, lexicon.txt and dict + // from assets to disk + const auto &meta_data = model_->GetMetaData(); + + if (meta_data.jieba && !meta_data.has_espeak) { + frontend_ = std::make_unique( + config_.model.matcha.lexicon, config_.model.matcha.tokens, + config_.model.matcha.dict_dir, config_.model.debug); + } else if (meta_data.has_espeak && !meta_data.jieba) { + frontend_ = std::make_unique( + mgr, config_.model.matcha.tokens, config_.model.matcha.data_dir, + meta_data); + } else { + SHERPA_ONNX_LOGE("jieba + espeaker-ng is not supported yet"); + SHERPA_ONNX_EXIT(-1); + } + } void InitFrontend() { - frontend_ = std::make_unique( - config_.model.matcha.lexicon, config_.model.matcha.tokens, - config_.model.matcha.dict_dir, config_.model.debug); + const auto &meta_data = model_->GetMetaData(); + + if (meta_data.jieba && !meta_data.has_espeak) { + frontend_ = std::make_unique( + config_.model.matcha.lexicon, config_.model.matcha.tokens, + config_.model.matcha.dict_dir, config_.model.debug); + } else if (meta_data.has_espeak && !meta_data.jieba) { + frontend_ = std::make_unique( + config_.model.matcha.tokens, config_.model.matcha.data_dir, + meta_data); + } else { + SHERPA_ONNX_LOGE("jieba + espeaker-ng is not supported yet"); + SHERPA_ONNX_EXIT(-1); + } } GeneratedAudio Process(const std::vector> &tokens, diff --git a/sherpa-onnx/csrc/offline-tts-matcha-model-metadata.h b/sherpa-onnx/csrc/offline-tts-matcha-model-metadata.h index 3147985dd..c5cee9465 100644 --- a/sherpa-onnx/csrc/offline-tts-matcha-model-metadata.h +++ b/sherpa-onnx/csrc/offline-tts-matcha-model-metadata.h @@ -18,7 +18,7 @@ struct OfflineTtsMatchaModelMetaData { int32_t num_speakers = 0; int32_t version = 1; int32_t jieba = 0; - int32_t espeak = 0; + int32_t has_espeak = 0; int32_t use_eos_bos = 0; int32_t pad_id = 0; }; diff --git a/sherpa-onnx/csrc/offline-tts-matcha-model.cc b/sherpa-onnx/csrc/offline-tts-matcha-model.cc index 066dbd21a..afea546d0 100644 --- a/sherpa-onnx/csrc/offline-tts-matcha-model.cc +++ b/sherpa-onnx/csrc/offline-tts-matcha-model.cc @@ -142,7 +142,7 @@ class OfflineTtsMatchaModel::Impl { SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(meta_data_.version, "version", 1); SHERPA_ONNX_READ_META_DATA(meta_data_.num_speakers, "n_speakers"); SHERPA_ONNX_READ_META_DATA(meta_data_.jieba, "jieba"); - SHERPA_ONNX_READ_META_DATA(meta_data_.espeak, "has_espeak"); + SHERPA_ONNX_READ_META_DATA(meta_data_.has_espeak, "has_espeak"); SHERPA_ONNX_READ_META_DATA(meta_data_.use_eos_bos, "use_eos_bos"); SHERPA_ONNX_READ_META_DATA(meta_data_.pad_id, "pad_id"); } diff --git a/sherpa-onnx/csrc/piper-phonemize-lexicon.cc b/sherpa-onnx/csrc/piper-phonemize-lexicon.cc index 298274654..9bc93ce98 100644 --- a/sherpa-onnx/csrc/piper-phonemize-lexicon.cc +++ b/sherpa-onnx/csrc/piper-phonemize-lexicon.cc @@ -32,6 +32,18 @@ namespace sherpa_onnx { +static void CallPhonemizeEspeak( + const std::string &text, + piper::eSpeakPhonemeConfig &config, // NOLINT + std::vector> *phonemes) { + static std::mutex espeak_mutex; + + std::lock_guard lock(espeak_mutex); + + // keep multi threads from calling into piper::phonemize_eSpeak + piper::phonemize_eSpeak(text, config, *phonemes); +} + static std::unordered_map ReadTokens(std::istream &is) { std::wstring_convert, char32_t> conv; std::unordered_map token2id; @@ -87,7 +99,7 @@ static std::unordered_map ReadTokens(std::istream &is) { // see the function "phonemes_to_ids" from // https://github.com/rhasspy/piper/blob/master/notebooks/piper_inference_(ONNX).ipynb -static std::vector PiperPhonemesToIds( +static std::vector PiperPhonemesToIdsVits( const std::unordered_map &token2id, const std::vector &phonemes) { // see @@ -114,17 +126,46 @@ static std::vector PiperPhonemesToIds( return ans; } +static std::vector PiperPhonemesToIdsMatcha( + const std::unordered_map &token2id, + const std::vector &phonemes, bool use_eos_bos) { + std::vector ans; + ans.reserve(phonemes.size()); + + int32_t bos = token2id.at(U'^'); + int32_t eos = token2id.at(U'$'); + + if (use_eos_bos) { + ans.push_back(bos); + } + + for (auto p : phonemes) { + if (token2id.count(p)) { + ans.push_back(token2id.at(p)); + } else { + SHERPA_ONNX_LOGE("Skip unknown phonemes. Unicode codepoint: \\U+%04x.", + static_cast(p)); + } + } + + if (use_eos_bos) { + ans.push_back(eos); + } + + return ans; +} + static std::vector CoquiPhonemesToIds( const std::unordered_map &token2id, const std::vector &phonemes, - const OfflineTtsVitsModelMetaData &meta_data) { + const OfflineTtsVitsModelMetaData &vits_meta_data) { // see // https://github.com/coqui-ai/TTS/blob/dev/TTS/tts/utils/text/tokenizer.py#L87 - int32_t use_eos_bos = meta_data.use_eos_bos; - int32_t bos_id = meta_data.bos_id; - int32_t eos_id = meta_data.eos_id; - int32_t blank_id = meta_data.blank_id; - int32_t add_blank = meta_data.add_blank; + int32_t use_eos_bos = vits_meta_data.use_eos_bos; + int32_t bos_id = vits_meta_data.bos_id; + int32_t eos_id = vits_meta_data.eos_id; + int32_t blank_id = vits_meta_data.blank_id; + int32_t add_blank = vits_meta_data.add_blank; int32_t comma_id = token2id.at(','); std::vector ans; @@ -189,8 +230,37 @@ static void InitEspeak(const std::string &data_dir) { PiperPhonemizeLexicon::PiperPhonemizeLexicon( const std::string &tokens, const std::string &data_dir, - const OfflineTtsVitsModelMetaData &meta_data) - : meta_data_(meta_data) { + const OfflineTtsVitsModelMetaData &vits_meta_data) + : vits_meta_data_(vits_meta_data) { + { + std::ifstream is(tokens); + token2id_ = ReadTokens(is); + } + + InitEspeak(data_dir); +} + +template +PiperPhonemizeLexicon::PiperPhonemizeLexicon( + Manager *mgr, const std::string &tokens, const std::string &data_dir, + const OfflineTtsVitsModelMetaData &vits_meta_data) + : vits_meta_data_(vits_meta_data) { + { + auto buf = ReadFile(mgr, tokens); + std::istrstream is(buf.data(), buf.size()); + token2id_ = ReadTokens(is); + } + + // We should copy the directory of espeak-ng-data from the asset to + // some internal or external storage and then pass the directory to + // data_dir. + InitEspeak(data_dir); +} + +PiperPhonemizeLexicon::PiperPhonemizeLexicon( + const std::string &tokens, const std::string &data_dir, + const OfflineTtsMatchaModelMetaData &matcha_meta_data) + : matcha_meta_data_(matcha_meta_data), is_matcha_(true) { { std::ifstream is(tokens); token2id_ = ReadTokens(is); @@ -202,8 +272,8 @@ PiperPhonemizeLexicon::PiperPhonemizeLexicon( template PiperPhonemizeLexicon::PiperPhonemizeLexicon( Manager *mgr, const std::string &tokens, const std::string &data_dir, - const OfflineTtsVitsModelMetaData &meta_data) - : meta_data_(meta_data) { + const OfflineTtsMatchaModelMetaData &matcha_meta_data) + : matcha_meta_data_(matcha_meta_data), is_matcha_(true) { { auto buf = ReadFile(mgr, tokens); std::istrstream is(buf.data(), buf.size()); @@ -218,6 +288,15 @@ PiperPhonemizeLexicon::PiperPhonemizeLexicon( std::vector PiperPhonemizeLexicon::ConvertTextToTokenIds( const std::string &text, const std::string &voice /*= ""*/) const { + if (is_matcha_) { + return ConvertTextToTokenIdsMatcha(text, voice); + } else { + return ConvertTextToTokenIdsVits(text, voice); + } +} + +std::vector PiperPhonemizeLexicon::ConvertTextToTokenIdsMatcha( + const std::string &text, const std::string &voice /*= ""*/) const { piper::eSpeakPhonemeConfig config; // ./bin/espeak-ng-bin --path ./install/share/espeak-ng-data/ --voices @@ -226,26 +305,45 @@ std::vector PiperPhonemizeLexicon::ConvertTextToTokenIds( std::vector> phonemes; - static std::mutex espeak_mutex; - { - std::lock_guard lock(espeak_mutex); + CallPhonemizeEspeak(text, config, &phonemes); - // keep multi threads from calling into piper::phonemize_eSpeak - piper::phonemize_eSpeak(text, config, phonemes); + std::vector ans; + + std::vector phoneme_ids; + + for (const auto &p : phonemes) { + phoneme_ids = + PiperPhonemesToIdsMatcha(token2id_, p, matcha_meta_data_.use_eos_bos); + ans.emplace_back(std::move(phoneme_ids)); } + return ans; +} + +std::vector PiperPhonemizeLexicon::ConvertTextToTokenIdsVits( + const std::string &text, const std::string &voice /*= ""*/) const { + piper::eSpeakPhonemeConfig config; + + // ./bin/espeak-ng-bin --path ./install/share/espeak-ng-data/ --voices + // to list available voices + config.voice = voice; // e.g., voice is en-us + + std::vector> phonemes; + + CallPhonemizeEspeak(text, config, &phonemes); + std::vector ans; std::vector phoneme_ids; - if (meta_data_.is_piper || meta_data_.is_icefall) { + if (vits_meta_data_.is_piper || vits_meta_data_.is_icefall) { for (const auto &p : phonemes) { - phoneme_ids = PiperPhonemesToIds(token2id_, p); + phoneme_ids = PiperPhonemesToIdsVits(token2id_, p); ans.emplace_back(std::move(phoneme_ids)); } - } else if (meta_data_.is_coqui) { + } else if (vits_meta_data_.is_coqui) { for (const auto &p : phonemes) { - phoneme_ids = CoquiPhonemesToIds(token2id_, p, meta_data_); + phoneme_ids = CoquiPhonemesToIds(token2id_, p, vits_meta_data_); ans.emplace_back(std::move(phoneme_ids)); } @@ -260,13 +358,18 @@ std::vector PiperPhonemizeLexicon::ConvertTextToTokenIds( #if __ANDROID_API__ >= 9 template PiperPhonemizeLexicon::PiperPhonemizeLexicon( AAssetManager *mgr, const std::string &tokens, const std::string &data_dir, - const OfflineTtsVitsModelMetaData &meta_data); + const OfflineTtsVitsModelMetaData &vits_meta_data); + +template PiperPhonemizeLexicon::PiperPhonemizeLexicon( + AAssetManager *mgr, const std::string &tokens, const std::string &data_dir, + const OfflineTtsMatchaModelMetaData &matcha_meta_data); #endif #if __OHOS__ template PiperPhonemizeLexicon::PiperPhonemizeLexicon( NativeResourceManager *mgr, const std::string &tokens, - const std::string &data_dir, const OfflineTtsVitsModelMetaData &meta_data); + const std::string &data_dir, + const OfflineTtsMatchaModelMetaData &matcha_meta_data); #endif } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/piper-phonemize-lexicon.h b/sherpa-onnx/csrc/piper-phonemize-lexicon.h index ccd790a96..f703f0b87 100644 --- a/sherpa-onnx/csrc/piper-phonemize-lexicon.h +++ b/sherpa-onnx/csrc/piper-phonemize-lexicon.h @@ -10,6 +10,7 @@ #include #include "sherpa-onnx/csrc/offline-tts-frontend.h" +#include "sherpa-onnx/csrc/offline-tts-matcha-model-metadata.h" #include "sherpa-onnx/csrc/offline-tts-vits-model-metadata.h" namespace sherpa_onnx { @@ -17,20 +18,37 @@ namespace sherpa_onnx { class PiperPhonemizeLexicon : public OfflineTtsFrontend { public: PiperPhonemizeLexicon(const std::string &tokens, const std::string &data_dir, - const OfflineTtsVitsModelMetaData &meta_data); + const OfflineTtsVitsModelMetaData &vits_meta_data); + + PiperPhonemizeLexicon(const std::string &tokens, const std::string &data_dir, + const OfflineTtsMatchaModelMetaData &matcha_meta_data); template PiperPhonemizeLexicon(Manager *mgr, const std::string &tokens, const std::string &data_dir, - const OfflineTtsVitsModelMetaData &meta_data); + const OfflineTtsVitsModelMetaData &vits_meta_data); + + template + PiperPhonemizeLexicon(Manager *mgr, const std::string &tokens, + const std::string &data_dir, + const OfflineTtsMatchaModelMetaData &matcha_meta_data); std::vector ConvertTextToTokenIds( const std::string &text, const std::string &voice = "") const override; + private: + std::vector ConvertTextToTokenIdsVits( + const std::string &text, const std::string &voice = "") const; + + std::vector ConvertTextToTokenIdsMatcha( + const std::string &text, const std::string &voice = "") const; + private: // map unicode codepoint to an integer ID std::unordered_map token2id_; - OfflineTtsVitsModelMetaData meta_data_; + OfflineTtsVitsModelMetaData vits_meta_data_; + OfflineTtsMatchaModelMetaData matcha_meta_data_; + bool is_matcha_ = false; }; } // namespace sherpa_onnx From a00d3b482123e65e3544e40ec0360e3d66d47faa Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 2 Jan 2025 15:15:30 +0800 Subject: [PATCH 133/183] Add Java API for Matcha-TTS models. (#1673) --- .github/workflows/run-java-test.yaml | 7 ++ .gitignore | 1 + .../NonStreamingTtsMatchaEn.java | 60 +++++++++ .../NonStreamingTtsMatchaZh.java | 66 ++++++++++ .../run-non-streaming-tts-matcha-en.sh | 45 +++++++ .../run-non-streaming-tts-matcha-zh.sh | 44 +++++++ sherpa-onnx/csrc/piper-phonemize-lexicon.cc | 5 + sherpa-onnx/java-api/Makefile | 1 + .../onnx/OfflineTtsMatchaModelConfig.java | 116 ++++++++++++++++++ .../sherpa/onnx/OfflineTtsModelConfig.java | 12 ++ .../onnx/OfflineTtsVitsModelConfig.java | 4 +- 11 files changed, 359 insertions(+), 2 deletions(-) create mode 100644 java-api-examples/NonStreamingTtsMatchaEn.java create mode 100644 java-api-examples/NonStreamingTtsMatchaZh.java create mode 100755 java-api-examples/run-non-streaming-tts-matcha-en.sh create mode 100755 java-api-examples/run-non-streaming-tts-matcha-zh.sh create mode 100644 sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsMatchaModelConfig.java diff --git a/.github/workflows/run-java-test.yaml b/.github/workflows/run-java-test.yaml index cca94422e..b375f01e5 100644 --- a/.github/workflows/run-java-test.yaml +++ b/.github/workflows/run-java-test.yaml @@ -235,6 +235,13 @@ jobs: shell: bash run: | cd ./java-api-examples + + ./run-non-streaming-tts-matcha-zh.sh + ./run-non-streaming-tts-matcha-en.sh + + rm -rf matcha-icefall-* + rm hifigan_v2.onnx + ./run-non-streaming-tts-piper-en.sh rm -rf vits-piper-* diff --git a/.gitignore b/.gitignore index eeec52d9c..e5226f0e1 100644 --- a/.gitignore +++ b/.gitignore @@ -126,3 +126,4 @@ sherpa-onnx-moonshine-base-en-int8 harmony-os/SherpaOnnxHar/sherpa_onnx/LICENSE harmony-os/SherpaOnnxHar/sherpa_onnx/CHANGELOG.md matcha-icefall-zh-baker +matcha-icefall-en_US-ljspeech diff --git a/java-api-examples/NonStreamingTtsMatchaEn.java b/java-api-examples/NonStreamingTtsMatchaEn.java new file mode 100644 index 000000000..bda41f061 --- /dev/null +++ b/java-api-examples/NonStreamingTtsMatchaEn.java @@ -0,0 +1,60 @@ +// Copyright 2025 Xiaomi Corporation + +// This file shows how to use a matcha English model +// to convert text to speech +import com.k2fsa.sherpa.onnx.*; + +public class NonStreamingTtsMatchaEn { + public static void main(String[] args) { + // please visit + // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker + // to download model files + String acousticModel = "./matcha-icefall-en_US-ljspeech/model-steps-3.onnx"; + String vocoder = "./hifigan_v2.onnx"; + String tokens = "./matcha-icefall-en_US-ljspeech/tokens.txt"; + String dataDir = "./matcha-icefall-en_US-ljspeech/espeak-ng-data"; + String text = + "Today as always, men fall into two groups: slaves and free men. Whoever does not have" + + " two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a" + + " businessman, an official, or a scholar."; + + OfflineTtsMatchaModelConfig matchaModelConfig = + OfflineTtsMatchaModelConfig.builder() + .setAcousticModel(acousticModel) + .setVocoder(vocoder) + .setTokens(tokens) + .setDataDir(dataDir) + .build(); + + OfflineTtsModelConfig modelConfig = + OfflineTtsModelConfig.builder() + .setMatcha(matchaModelConfig) + .setNumThreads(1) + .setDebug(true) + .build(); + + OfflineTtsConfig config = OfflineTtsConfig.builder().setModel(modelConfig).build(); + OfflineTts tts = new OfflineTts(config); + + int sid = 0; + float speed = 1.0f; + long start = System.currentTimeMillis(); + GeneratedAudio audio = tts.generate(text, sid, speed); + long stop = System.currentTimeMillis(); + + float timeElapsedSeconds = (stop - start) / 1000.0f; + + float audioDuration = audio.getSamples().length / (float) audio.getSampleRate(); + float real_time_factor = timeElapsedSeconds / audioDuration; + + String waveFilename = "tts-matcha-en.wav"; + audio.save(waveFilename); + System.out.printf("-- elapsed : %.3f seconds\n", timeElapsedSeconds); + System.out.printf("-- audio duration: %.3f seconds\n", timeElapsedSeconds); + System.out.printf("-- real-time factor (RTF): %.3f\n", real_time_factor); + System.out.printf("-- text: %s\n", text); + System.out.printf("-- Saved to %s\n", waveFilename); + + tts.release(); + } +} diff --git a/java-api-examples/NonStreamingTtsMatchaZh.java b/java-api-examples/NonStreamingTtsMatchaZh.java new file mode 100644 index 000000000..dec24dbb3 --- /dev/null +++ b/java-api-examples/NonStreamingTtsMatchaZh.java @@ -0,0 +1,66 @@ +// Copyright 2025 Xiaomi Corporation + +// This file shows how to use a matcha Chinese TTS model +// to convert text to speech +import com.k2fsa.sherpa.onnx.*; + +public class NonStreamingTtsMatchaZh { + public static void main(String[] args) { + // please visit + // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker + // to download model files + String acousticModel = "./matcha-icefall-zh-baker/model-steps-3.onnx"; + String vocoder = "./hifigan_v2.onnx"; + String tokens = "./matcha-icefall-zh-baker/tokens.txt"; + String lexicon = "./matcha-icefall-zh-baker/lexicon.txt"; + String dictDir = "./matcha-icefall-zh-baker/dict"; + String ruleFsts = + "./matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst"; + String text = + "某某银行的副行长和一些行政领导表示，他们去过长江" + + "和长白山; 经济不断增长。" + + "2024年12月31号，拨打110或者18920240511。" + + "123456块钱。"; + + OfflineTtsMatchaModelConfig matchaModelConfig = + OfflineTtsMatchaModelConfig.builder() + .setAcousticModel(acousticModel) + .setVocoder(vocoder) + .setTokens(tokens) + .setLexicon(lexicon) + .setDictDir(dictDir) + .build(); + + OfflineTtsModelConfig modelConfig = + OfflineTtsModelConfig.builder() + .setMatcha(matchaModelConfig) + .setNumThreads(1) + .setDebug(true) + .build(); + + OfflineTtsConfig config = + OfflineTtsConfig.builder().setModel(modelConfig).setRuleFsts(ruleFsts).build(); + OfflineTts tts = new OfflineTts(config); + + int sid = 0; + float speed = 1.0f; + long start = System.currentTimeMillis(); + GeneratedAudio audio = tts.generate(text, sid, speed); + long stop = System.currentTimeMillis(); + + float timeElapsedSeconds = (stop - start) / 1000.0f; + + float audioDuration = audio.getSamples().length / (float) audio.getSampleRate(); + float real_time_factor = timeElapsedSeconds / audioDuration; + + String waveFilename = "tts-matcha-zh.wav"; + audio.save(waveFilename); + System.out.printf("-- elapsed : %.3f seconds\n", timeElapsedSeconds); + System.out.printf("-- audio duration: %.3f seconds\n", timeElapsedSeconds); + System.out.printf("-- real-time factor (RTF): %.3f\n", real_time_factor); + System.out.printf("-- text: %s\n", text); + System.out.printf("-- Saved to %s\n", waveFilename); + + tts.release(); + } +} diff --git a/java-api-examples/run-non-streaming-tts-matcha-en.sh b/java-api-examples/run-non-streaming-tts-matcha-en.sh new file mode 100755 index 000000000..ce0289fc9 --- /dev/null +++ b/java-api-examples/run-non-streaming-tts-matcha-en.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -ex + +if [[ ! -f ../build/lib/libsherpa-onnx-jni.dylib && ! -f ../build/lib/libsherpa-onnx-jni.so ]]; then + mkdir -p ../build + pushd ../build + cmake \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + -DSHERPA_ONNX_ENABLE_JNI=ON \ + .. + + make -j4 + ls -lh lib + popd +fi + +if [ ! -f ../sherpa-onnx/java-api/build/sherpa-onnx.jar ]; then + pushd ../sherpa-onnx/java-api + make + popd +fi + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker +# matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker +# to download more models +if [ ! -f ./matcha-icefall-en_US-ljspeech/model-steps-3.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 + tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 + rm matcha-icefall-en_US-ljspeech.tar.bz2 +fi + +if [ ! -f ./hifigan_v2.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx +fi + +java \ + -Djava.library.path=$PWD/../build/lib \ + -cp ../sherpa-onnx/java-api/build/sherpa-onnx.jar \ + NonStreamingTtsMatchaEn.java diff --git a/java-api-examples/run-non-streaming-tts-matcha-zh.sh b/java-api-examples/run-non-streaming-tts-matcha-zh.sh new file mode 100755 index 000000000..a339e298a --- /dev/null +++ b/java-api-examples/run-non-streaming-tts-matcha-zh.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +set -ex + +if [[ ! -f ../build/lib/libsherpa-onnx-jni.dylib && ! -f ../build/lib/libsherpa-onnx-jni.so ]]; then + mkdir -p ../build + pushd ../build + cmake \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + -DSHERPA_ONNX_ENABLE_JNI=ON \ + .. + + make -j4 + ls -lh lib + popd +fi + +if [ ! -f ../sherpa-onnx/java-api/build/sherpa-onnx.jar ]; then + pushd ../sherpa-onnx/java-api + make + popd +fi + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker +# to download more models +if [ ! -f ./matcha-icefall-zh-baker/model-steps-3.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 + tar xvf matcha-icefall-zh-baker.tar.bz2 + rm matcha-icefall-zh-baker.tar.bz2 +fi + +if [ ! -f ./hifigan_v2.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx +fi + +java \ + -Djava.library.path=$PWD/../build/lib \ + -cp ../sherpa-onnx/java-api/build/sherpa-onnx.jar \ + NonStreamingTtsMatchaZh.java diff --git a/sherpa-onnx/csrc/piper-phonemize-lexicon.cc b/sherpa-onnx/csrc/piper-phonemize-lexicon.cc index 9bc93ce98..982260003 100644 --- a/sherpa-onnx/csrc/piper-phonemize-lexicon.cc +++ b/sherpa-onnx/csrc/piper-phonemize-lexicon.cc @@ -366,6 +366,11 @@ template PiperPhonemizeLexicon::PiperPhonemizeLexicon( #endif #if __OHOS__ +template PiperPhonemizeLexicon::PiperPhonemizeLexicon( + NativeResourceManager *mgr, const std::string &tokens, + const std::string &data_dir, + const OfflineTtsVitsModelMetaData &vits_meta_data); + template PiperPhonemizeLexicon::PiperPhonemizeLexicon( NativeResourceManager *mgr, const std::string &tokens, const std::string &data_dir, diff --git a/sherpa-onnx/java-api/Makefile b/sherpa-onnx/java-api/Makefile index 8b9278fc0..816d2139f 100644 --- a/sherpa-onnx/java-api/Makefile +++ b/sherpa-onnx/java-api/Makefile @@ -35,6 +35,7 @@ java_files += OfflineRecognizerResult.java java_files += OfflineStream.java java_files += OfflineRecognizer.java +java_files += OfflineTtsMatchaModelConfig.java java_files += OfflineTtsVitsModelConfig.java java_files += OfflineTtsModelConfig.java java_files += OfflineTtsConfig.java diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsMatchaModelConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsMatchaModelConfig.java new file mode 100644 index 000000000..8a95aea75 --- /dev/null +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsMatchaModelConfig.java @@ -0,0 +1,116 @@ +// Copyright 2025 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class OfflineTtsMatchaModelConfig { + private final String acousticModel; + private final String vocoder; + private final String lexicon; + private final String tokens; + private final String dataDir; + private final String dictDir; + private final float noiseScale; + private final float lengthScale; + + private OfflineTtsMatchaModelConfig(Builder builder) { + this.acousticModel = builder.acousticModel; + this.vocoder = builder.vocoder; + this.lexicon = builder.lexicon; + this.tokens = builder.tokens; + this.dataDir = builder.dataDir; + this.dictDir = builder.dictDir; + this.noiseScale = builder.noiseScale; + this.lengthScale = builder.lengthScale; + } + + public static Builder builder() { + return new Builder(); + } + + public String getAcousticModel() { + return acousticModel; + } + + public String getVocoder() { + return vocoder; + } + + public String getLexicon() { + return lexicon; + } + + public String getTokens() { + return tokens; + } + + public String getDataDir() { + return dataDir; + } + + public String getDictDir() { + return dictDir; + } + + public float getLengthScale() { + return lengthScale; + } + + public float getNoiseScale() { + return noiseScale; + } + + public static class Builder { + private String acousticModel = ""; + private String vocoder = ""; + private String lexicon = ""; + private String tokens = ""; + private String dataDir = ""; + private String dictDir = ""; + private float noiseScale = 1.0f; + private float lengthScale = 1.0f; + + public OfflineTtsMatchaModelConfig build() { + return new OfflineTtsMatchaModelConfig(this); + } + + public Builder setAcousticModel(String acousticModel) { + this.acousticModel = acousticModel; + return this; + } + + public Builder setVocoder(String vocoder) { + this.vocoder = vocoder; + return this; + } + + public Builder setTokens(String tokens) { + this.tokens = tokens; + return this; + } + + public Builder setLexicon(String lexicon) { + this.lexicon = lexicon; + return this; + } + + public Builder setDataDir(String dataDir) { + this.dataDir = dataDir; + return this; + } + + public Builder setDictDir(String dictDir) { + this.dictDir = dictDir; + return this; + } + + public Builder setNoiseScale(float noiseScale) { + this.noiseScale = noiseScale; + return this; + } + + public Builder setLengthScale(float lengthScale) { + this.lengthScale = lengthScale; + return this; + } + } +} diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsModelConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsModelConfig.java index 52960217c..ff3589b13 100644 --- a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsModelConfig.java +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsModelConfig.java @@ -4,12 +4,14 @@ public class OfflineTtsModelConfig { private final OfflineTtsVitsModelConfig vits; + private final OfflineTtsMatchaModelConfig matcha; private final int numThreads; private final boolean debug; private final String provider; private OfflineTtsModelConfig(Builder builder) { this.vits = builder.vits; + this.matcha = builder.matcha; this.numThreads = builder.numThreads; this.debug = builder.debug; this.provider = builder.provider; @@ -23,8 +25,13 @@ public OfflineTtsVitsModelConfig getVits() { return vits; } + public OfflineTtsMatchaModelConfig getMatcha() { + return matcha; + } + public static class Builder { private OfflineTtsVitsModelConfig vits = OfflineTtsVitsModelConfig.builder().build(); + private OfflineTtsMatchaModelConfig matcha = OfflineTtsMatchaModelConfig.builder().build(); private int numThreads = 1; private boolean debug = true; private String provider = "cpu"; @@ -38,6 +45,11 @@ public Builder setVits(OfflineTtsVitsModelConfig vits) { return this; } + public Builder setMatcha(OfflineTtsMatchaModelConfig matcha) { + this.matcha = matcha; + return this; + } + public Builder setNumThreads(int numThreads) { this.numThreads = numThreads; return this; diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsVitsModelConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsVitsModelConfig.java index 4cfc9eebd..35bfd41c6 100644 --- a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsVitsModelConfig.java +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsVitsModelConfig.java @@ -60,9 +60,9 @@ public float getNoiseScaleW() { } public static class Builder { - private String model; + private String model = ""; private String lexicon = ""; - private String tokens; + private String tokens = ""; private String dataDir = ""; private String dictDir = ""; private float noiseScale = 0.667f; From a4365dad82dc856c73ab9eb90ecf6997451ba5ec Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 3 Jan 2025 10:37:39 +0800 Subject: [PATCH 134/183] Avoid adding tail padding for VAD in generate-subtitles.py (#1674) --- python-api-examples/generate-subtitles.py | 26 +++++++++-------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/python-api-examples/generate-subtitles.py b/python-api-examples/generate-subtitles.py index bf51f7627..1e36dd070 100755 --- a/python-api-examples/generate-subtitles.py +++ b/python-api-examples/generate-subtitles.py @@ -516,28 +516,22 @@ def main(): is_eof = False # TODO(fangjun): Support multithreads - while True: + while not is_eof: # *2 because int16_t has two bytes data = process.stdout.read(frames_per_read * 2) if not data: - if is_eof: - break + vad.flush() is_eof = True - # pad 1 second at the end of the file for the VAD - data = np.zeros(1 * args.sample_rate, dtype=np.int16) - - samples = np.frombuffer(data, dtype=np.int16) - samples = samples.astype(np.float32) / 32768 + else: + samples = np.frombuffer(data, dtype=np.int16) + samples = samples.astype(np.float32) / 32768 - num_processed_samples += samples.shape[0] + num_processed_samples += samples.shape[0] - buffer = np.concatenate([buffer, samples]) - while len(buffer) > window_size: - vad.accept_waveform(buffer[:window_size]) - buffer = buffer[window_size:] - - if is_eof: - vad.flush() + buffer = np.concatenate([buffer, samples]) + while len(buffer) > window_size: + vad.accept_waveform(buffer[:window_size]) + buffer = buffer[window_size:] streams = [] segments = [] From 9aa4897a9e31cf78c75c76b30ae10198d893b512 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 3 Jan 2025 12:17:26 +0800 Subject: [PATCH 135/183] Add C API for MatchaTTS models (#1675) --- .github/workflows/c-api.yaml | 45 ++++++++++++++ c-api-examples/CMakeLists.txt | 6 ++ c-api-examples/matcha-tts-en-c-api.c | 87 ++++++++++++++++++++++++++++ c-api-examples/matcha-tts-zh-c-api.c | 87 ++++++++++++++++++++++++++++ sherpa-onnx/c-api/c-api.cc | 21 ++++++- sherpa-onnx/c-api/c-api.h | 15 ++++- sherpa-onnx/csrc/offline-tts.h | 2 +- 7 files changed, 260 insertions(+), 3 deletions(-) create mode 100644 c-api-examples/matcha-tts-en-c-api.c create mode 100644 c-api-examples/matcha-tts-zh-c-api.c diff --git a/.github/workflows/c-api.yaml b/.github/workflows/c-api.yaml index 049240d77..18f7c257d 100644 --- a/.github/workflows/c-api.yaml +++ b/.github/workflows/c-api.yaml @@ -81,6 +81,51 @@ jobs: otool -L ./install/lib/libsherpa-onnx-c-api.dylib fi + - name: Test Matcha TTS (zh) + shell: bash + run: | + gcc -o matcha-tts-zh-c-api ./c-api-examples/matcha-tts-zh-c-api.c \ + -I ./build/install/include \ + -L ./build/install/lib/ \ + -l sherpa-onnx-c-api \ + -l onnxruntime + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 + tar xvf matcha-icefall-zh-baker.tar.bz2 + rm matcha-icefall-zh-baker.tar.bz2 + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH + export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH + + ./matcha-tts-zh-c-api + + - name: Test Matcha TTS (en) + shell: bash + run: | + gcc -o matcha-tts-en-c-api ./c-api-examples/matcha-tts-en-c-api.c \ + -I ./build/install/include \ + -L ./build/install/lib/ \ + -l sherpa-onnx-c-api \ + -l onnxruntime + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 + tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 + rm matcha-icefall-en_US-ljspeech.tar.bz2 + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH + export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH + + ./matcha-tts-en-c-api + + - uses: actions/upload-artifact@v4 + with: + name: matcha-tts-${{ matrix.os }} + path: ./generated-matcha-*.wav + - name: Test vad + Whisper tiny.en shell: bash run: | diff --git a/c-api-examples/CMakeLists.txt b/c-api-examples/CMakeLists.txt index c7db2bc27..36d7b4e42 100644 --- a/c-api-examples/CMakeLists.txt +++ b/c-api-examples/CMakeLists.txt @@ -7,6 +7,12 @@ target_link_libraries(decode-file-c-api sherpa-onnx-c-api cargs) if(SHERPA_ONNX_ENABLE_TTS) add_executable(offline-tts-c-api offline-tts-c-api.c) target_link_libraries(offline-tts-c-api sherpa-onnx-c-api cargs) + + add_executable(matcha-tts-zh-c-api matcha-tts-zh-c-api.c) + target_link_libraries(matcha-tts-zh-c-api sherpa-onnx-c-api) + + add_executable(matcha-tts-en-c-api matcha-tts-en-c-api.c) + target_link_libraries(matcha-tts-en-c-api sherpa-onnx-c-api) endif() if(SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION) diff --git a/c-api-examples/matcha-tts-en-c-api.c b/c-api-examples/matcha-tts-en-c-api.c new file mode 100644 index 000000000..103ecd523 --- /dev/null +++ b/c-api-examples/matcha-tts-en-c-api.c @@ -0,0 +1,87 @@ +// c-api-examples/matcha-tts-en-c-api.c +// +// Copyright (c) 2025 Xiaomi Corporation + +// This file shows how to use sherpa-onnx C API +// for English TTS with MatchaTTS. +// +// clang-format off +/* +Usage + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 +tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 +rm matcha-icefall-en_US-ljspeech.tar.bz2 + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +./matcha-tts-en-c-api + + */ +// clang-format on + +#include +#include +#include + +#include "sherpa-onnx/c-api/c-api.h" + +static int32_t ProgressCallback(const float *samples, int32_t num_samples, + float progress) { + fprintf(stderr, "Progress: %.3f%%\n", progress * 100); + // return 1 to continue generating + // return 0 to stop generating + return 1; +} + +int32_t main(int32_t argc, char *argv[]) { + SherpaOnnxOfflineTtsConfig config; + memset(&config, 0, sizeof(config)); + config.model.matcha.acoustic_model = + "./matcha-icefall-en_US-ljspeech/model-steps-3.onnx"; + + config.model.matcha.vocoder = "./hifigan_v2.onnx"; + + config.model.matcha.tokens = "./matcha-icefall-en_US-ljspeech/tokens.txt"; + + config.model.matcha.data_dir = + "./matcha-icefall-en_US-ljspeech/espeak-ng-data"; + + config.model.num_threads = 1; + + // If you don't want to see debug messages, please set it to 0 + config.model.debug = 1; + + const char *filename = "./generated-matcha-en.wav"; + const char *text = + "Today as always, men fall into two groups: slaves and free men. Whoever " + "does not have two-thirds of his day for himself, is a slave, whatever " + "he may be: a statesman, a businessman, an official, or a scholar. " + "Friends fell out often because life was changing so fast. The easiest " + "thing in the world was to lose touch with someone."; + + SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTts(&config); + int32_t sid = 0; + float speed = 1.0; // larger -> faster in speech speed + +#if 0 + // If you don't want to use a callback, then please enable this branch + const SherpaOnnxGeneratedAudio *audio = + SherpaOnnxOfflineTtsGenerate(tts, text, sid, speed); +#else + const SherpaOnnxGeneratedAudio *audio = + SherpaOnnxOfflineTtsGenerateWithProgressCallback(tts, text, sid, speed, + ProgressCallback); +#endif + + SherpaOnnxWriteWave(audio->samples, audio->n, audio->sample_rate, filename); + + SherpaOnnxDestroyOfflineTtsGeneratedAudio(audio); + SherpaOnnxDestroyOfflineTts(tts); + + fprintf(stderr, "Input text is: %s\n", text); + fprintf(stderr, "Speaker ID is is: %d\n", sid); + fprintf(stderr, "Saved to: %s\n", filename); + + return 0; +} diff --git a/c-api-examples/matcha-tts-zh-c-api.c b/c-api-examples/matcha-tts-zh-c-api.c new file mode 100644 index 000000000..c7667f0cb --- /dev/null +++ b/c-api-examples/matcha-tts-zh-c-api.c @@ -0,0 +1,87 @@ +// c-api-examples/matcha-tts-zh-c-api.c +// +// Copyright (c) 2025 Xiaomi Corporation + +// This file shows how to use sherpa-onnx C API +// for Chinese TTS with MatchaTTS. +// +// clang-format off +/* +Usage + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 +tar xvf matcha-icefall-zh-baker.tar.bz2 +rm matcha-icefall-zh-baker.tar.bz2 + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +./matcha-tts-zh-c-api + + */ +// clang-format on + +#include +#include +#include + +#include "sherpa-onnx/c-api/c-api.h" + +static int32_t ProgressCallback(const float *samples, int32_t num_samples, + float progress) { + fprintf(stderr, "Progress: %.3f%%\n", progress * 100); + // return 1 to continue generating + // return 0 to stop generating + return 1; +} + +int32_t main(int32_t argc, char *argv[]) { + SherpaOnnxOfflineTtsConfig config; + memset(&config, 0, sizeof(config)); + config.model.matcha.acoustic_model = + "./matcha-icefall-zh-baker/model-steps-3.onnx"; + config.model.matcha.vocoder = "./hifigan_v2.onnx"; + config.model.matcha.lexicon = "./matcha-icefall-zh-baker/lexicon.txt"; + config.model.matcha.tokens = "./matcha-icefall-zh-baker/tokens.txt"; + config.model.matcha.dict_dir = "./matcha-icefall-zh-baker/dict"; + config.model.num_threads = 1; + + // If you don't want to see debug messages, please set it to 0 + config.model.debug = 1; + + // clang-format off + config.rule_fsts = "./matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst"; + // clang-format on + + const char *filename = "./generated-matcha-zh.wav"; + const char *text = + "当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转，思念如" + "涟漪荡漾，梦境如画卷展开，我与自然融为一体，沉静在这片宁静的美丽之中，感" + "受着生命的奇迹与温柔." + "某某银行的副行长和一些行政领导表示，他们去过长江和长白山; " + "经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。"; + + SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTts(&config); + int32_t sid = 0; + float speed = 1.0; // larger -> faster in speech speed + +#if 0 + // If you don't want to use a callback, then please enable this branch + const SherpaOnnxGeneratedAudio *audio = + SherpaOnnxOfflineTtsGenerate(tts, text, sid, speed); +#else + const SherpaOnnxGeneratedAudio *audio = + SherpaOnnxOfflineTtsGenerateWithProgressCallback(tts, text, sid, speed, + ProgressCallback); +#endif + + SherpaOnnxWriteWave(audio->samples, audio->n, audio->sample_rate, filename); + + SherpaOnnxDestroyOfflineTtsGeneratedAudio(audio); + SherpaOnnxDestroyOfflineTts(tts); + + fprintf(stderr, "Input text is: %s\n", text); + fprintf(stderr, "Speaker ID is is: %d\n", sid); + fprintf(stderr, "Saved to: %s\n", filename); + + return 0; +} diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index 703380730..6afc1bf62 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -1058,6 +1058,7 @@ static sherpa_onnx::OfflineTtsConfig GetOfflineTtsConfig( const SherpaOnnxOfflineTtsConfig *config) { sherpa_onnx::OfflineTtsConfig tts_config; + // vits tts_config.model.vits.model = SHERPA_ONNX_OR(config->model.vits.model, ""); tts_config.model.vits.lexicon = SHERPA_ONNX_OR(config->model.vits.lexicon, ""); @@ -1073,6 +1074,24 @@ static sherpa_onnx::OfflineTtsConfig GetOfflineTtsConfig( tts_config.model.vits.dict_dir = SHERPA_ONNX_OR(config->model.vits.dict_dir, ""); + // matcha + tts_config.model.matcha.acoustic_model = + SHERPA_ONNX_OR(config->model.matcha.acoustic_model, ""); + tts_config.model.matcha.vocoder = + SHERPA_ONNX_OR(config->model.matcha.vocoder, ""); + tts_config.model.matcha.lexicon = + SHERPA_ONNX_OR(config->model.matcha.lexicon, ""); + tts_config.model.matcha.tokens = + SHERPA_ONNX_OR(config->model.matcha.tokens, ""); + tts_config.model.matcha.data_dir = + SHERPA_ONNX_OR(config->model.matcha.data_dir, ""); + tts_config.model.matcha.noise_scale = + SHERPA_ONNX_OR(config->model.matcha.noise_scale, 0.667); + tts_config.model.matcha.length_scale = + SHERPA_ONNX_OR(config->model.matcha.length_scale, 1.0); + tts_config.model.matcha.dict_dir = + SHERPA_ONNX_OR(config->model.matcha.dict_dir, ""); + tts_config.model.num_threads = SHERPA_ONNX_OR(config->model.num_threads, 1); tts_config.model.debug = config->model.debug; tts_config.model.provider = SHERPA_ONNX_OR(config->model.provider, "cpu"); @@ -1082,7 +1101,7 @@ static sherpa_onnx::OfflineTtsConfig GetOfflineTtsConfig( tts_config.rule_fsts = SHERPA_ONNX_OR(config->rule_fsts, ""); tts_config.rule_fars = SHERPA_ONNX_OR(config->rule_fars, ""); - tts_config.max_num_sentences = SHERPA_ONNX_OR(config->max_num_sentences, 2); + tts_config.max_num_sentences = SHERPA_ONNX_OR(config->max_num_sentences, 1); if (tts_config.model.debug) { #if __OHOS__ diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index 167051cd9..e79d951e2 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -894,15 +894,28 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOfflineTtsVitsModelConfig { float noise_scale; float noise_scale_w; - float length_scale; // < 1, faster in speed; > 1, slower in speed + float length_scale; // < 1, faster in speech speed; > 1, slower in speed const char *dict_dir; } SherpaOnnxOfflineTtsVitsModelConfig; +SHERPA_ONNX_API typedef struct SherpaOnnxOfflineTtsMatchaModelConfig { + const char *acoustic_model; + const char *vocoder; + const char *lexicon; + const char *tokens; + const char *data_dir; + + float noise_scale; + float length_scale; // < 1, faster in speech speed; > 1, slower in speed + const char *dict_dir; +} SherpaOnnxOfflineTtsMatchaModelConfig; + SHERPA_ONNX_API typedef struct SherpaOnnxOfflineTtsModelConfig { SherpaOnnxOfflineTtsVitsModelConfig vits; int32_t num_threads; int32_t debug; const char *provider; + SherpaOnnxOfflineTtsMatchaModelConfig matcha; } SherpaOnnxOfflineTtsModelConfig; SHERPA_ONNX_API typedef struct SherpaOnnxOfflineTtsConfig { diff --git a/sherpa-onnx/csrc/offline-tts.h b/sherpa-onnx/csrc/offline-tts.h index 8399443c0..884173e7b 100644 --- a/sherpa-onnx/csrc/offline-tts.h +++ b/sherpa-onnx/csrc/offline-tts.h @@ -30,7 +30,7 @@ struct OfflineTtsConfig { // Maximum number of sentences that we process at a time. // This is to avoid OOM for very long input text. // If you set it to -1, then we process all sentences in a single batch. - int32_t max_num_sentences = 2; + int32_t max_num_sentences = 1; OfflineTtsConfig() = default; OfflineTtsConfig(const OfflineTtsModelConfig &model, From 648903834bf70ab0754522f7e541bacaf95fe752 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 3 Jan 2025 14:16:36 +0800 Subject: [PATCH 136/183] Add CXX API for MatchaTTS models (#1676) --- .github/workflows/c-api.yaml | 8 ++ .github/workflows/cxx-api.yaml | 55 ++++++++++++++ c-api-examples/matcha-tts-en-c-api.c | 2 +- c-api-examples/matcha-tts-zh-c-api.c | 2 +- c-api-examples/offline-tts-c-api.c | 2 +- cxx-api-examples/CMakeLists.txt | 8 ++ cxx-api-examples/matcha-tts-en-cxx-api.cc | 80 ++++++++++++++++++++ cxx-api-examples/matcha-tts-zh-cxx-api.cc | 79 ++++++++++++++++++++ sherpa-onnx/c-api/c-api.cc | 6 +- sherpa-onnx/c-api/c-api.h | 6 +- sherpa-onnx/c-api/cxx-api.cc | 74 +++++++++++++++++++ sherpa-onnx/c-api/cxx-api.h | 89 +++++++++++++++++++++++ 12 files changed, 403 insertions(+), 8 deletions(-) create mode 100644 cxx-api-examples/matcha-tts-en-cxx-api.cc create mode 100644 cxx-api-examples/matcha-tts-zh-cxx-api.cc diff --git a/.github/workflows/c-api.yaml b/.github/workflows/c-api.yaml index 18f7c257d..1549b484d 100644 --- a/.github/workflows/c-api.yaml +++ b/.github/workflows/c-api.yaml @@ -101,6 +101,10 @@ jobs: ./matcha-tts-zh-c-api + rm ./matcha-tts-zh-c-api + rm -rf matcha-icefall-* + rm hifigan_v2.onnx + - name: Test Matcha TTS (en) shell: bash run: | @@ -121,6 +125,10 @@ jobs: ./matcha-tts-en-c-api + rm ./matcha-tts-en-c-api + rm -rf matcha-icefall-* + rm hifigan_v2.onnx + - uses: actions/upload-artifact@v4 with: name: matcha-tts-${{ matrix.os }} diff --git a/.github/workflows/cxx-api.yaml b/.github/workflows/cxx-api.yaml index 8779011a9..2fdc56313 100644 --- a/.github/workflows/cxx-api.yaml +++ b/.github/workflows/cxx-api.yaml @@ -83,6 +83,61 @@ jobs: otool -L ./install/lib/libsherpa-onnx-cxx-api.dylib fi + - name: Test Matcha TTS (zh) + shell: bash + run: | + g++ -std=c++17 -o matcha-tts-zh-cxx-api ./cxx-api-examples/matcha-tts-zh-cxx-api.cc \ + -I ./build/install/include \ + -L ./build/install/lib/ \ + -l sherpa-onnx-cxx-api \ + -l sherpa-onnx-c-api \ + -l onnxruntime + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 + tar xvf matcha-icefall-zh-baker.tar.bz2 + rm matcha-icefall-zh-baker.tar.bz2 + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH + export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH + + ./matcha-tts-zh-cxx-api + + rm -rf matcha-icefall-* + rm hifigan_v2.onnx + rm matcha-tts-zh-cxx-api + + - name: Test Matcha TTS (en) + shell: bash + run: | + g++ -std=c++17 -o matcha-tts-en-cxx-api ./cxx-api-examples/matcha-tts-en-cxx-api.cc \ + -I ./build/install/include \ + -L ./build/install/lib/ \ + -l sherpa-onnx-cxx-api \ + -l sherpa-onnx-c-api \ + -l onnxruntime + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 + tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 + rm matcha-icefall-en_US-ljspeech.tar.bz2 + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH + export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH + + ./matcha-tts-en-cxx-api + + rm matcha-tts-en-cxx-api + rm -rf matcha-icefall-* + rm hifigan_v2.onnx + + - uses: actions/upload-artifact@v4 + with: + name: matcha-tts-${{ matrix.os }} + path: ./generated-matcha-*.wav + - name: Test Moonshine tiny shell: bash run: | diff --git a/c-api-examples/matcha-tts-en-c-api.c b/c-api-examples/matcha-tts-en-c-api.c index 103ecd523..99b0a9742 100644 --- a/c-api-examples/matcha-tts-en-c-api.c +++ b/c-api-examples/matcha-tts-en-c-api.c @@ -60,7 +60,7 @@ int32_t main(int32_t argc, char *argv[]) { "Friends fell out often because life was changing so fast. The easiest " "thing in the world was to lose touch with someone."; - SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTts(&config); + const SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTts(&config); int32_t sid = 0; float speed = 1.0; // larger -> faster in speech speed diff --git a/c-api-examples/matcha-tts-zh-c-api.c b/c-api-examples/matcha-tts-zh-c-api.c index c7667f0cb..9fb9f4597 100644 --- a/c-api-examples/matcha-tts-zh-c-api.c +++ b/c-api-examples/matcha-tts-zh-c-api.c @@ -60,7 +60,7 @@ int32_t main(int32_t argc, char *argv[]) { "某某银行的副行长和一些行政领导表示，他们去过长江和长白山; " "经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。"; - SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTts(&config); + const SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTts(&config); int32_t sid = 0; float speed = 1.0; // larger -> faster in speech speed diff --git a/c-api-examples/offline-tts-c-api.c b/c-api-examples/offline-tts-c-api.c index 7fbdb004c..eaa25af39 100644 --- a/c-api-examples/offline-tts-c-api.c +++ b/c-api-examples/offline-tts-c-api.c @@ -229,7 +229,7 @@ int32_t main(int32_t argc, char *argv[]) { ShowUsage(); } - SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTts(&config); + const SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTts(&config); const SherpaOnnxGeneratedAudio *audio = SherpaOnnxOfflineTtsGenerate(tts, text, sid, 1.0); diff --git a/cxx-api-examples/CMakeLists.txt b/cxx-api-examples/CMakeLists.txt index cc4082e87..dd61d3294 100644 --- a/cxx-api-examples/CMakeLists.txt +++ b/cxx-api-examples/CMakeLists.txt @@ -14,3 +14,11 @@ target_link_libraries(moonshine-cxx-api sherpa-onnx-cxx-api) add_executable(sense-voice-cxx-api ./sense-voice-cxx-api.cc) target_link_libraries(sense-voice-cxx-api sherpa-onnx-cxx-api) + +if(SHERPA_ONNX_ENABLE_TTS) + add_executable(matcha-tts-zh-cxx-api ./matcha-tts-zh-cxx-api.cc) + target_link_libraries(matcha-tts-zh-cxx-api sherpa-onnx-cxx-api) + + add_executable(matcha-tts-en-cxx-api ./matcha-tts-en-cxx-api.cc) + target_link_libraries(matcha-tts-en-cxx-api sherpa-onnx-cxx-api) +endif() diff --git a/cxx-api-examples/matcha-tts-en-cxx-api.cc b/cxx-api-examples/matcha-tts-en-cxx-api.cc new file mode 100644 index 000000000..ef4187d06 --- /dev/null +++ b/cxx-api-examples/matcha-tts-en-cxx-api.cc @@ -0,0 +1,80 @@ +// cxx-api-examples/matcha-tts-en-cxx-api.c +// +// Copyright (c) 2025 Xiaomi Corporation + +// This file shows how to use sherpa-onnx CXX API +// for Chinese TTS with MatchaTTS. +// +// clang-format off +/* +Usage + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 +tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 +rm matcha-icefall-en_US-ljspeech.tar.bz2 + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +./matcha-tts-en-cxx-api + + */ +// clang-format on + +#include + +#include "sherpa-onnx/c-api/cxx-api.h" + +static int32_t ProgressCallback(const float *samples, int32_t num_samples, + float progress, void *arg) { + fprintf(stderr, "Progress: %.3f%%\n", progress * 100); + // return 1 to continue generating + // return 0 to stop generating + return 1; +} + +int32_t main(int32_t argc, char *argv[]) { + using namespace sherpa_onnx::cxx; // NOLINT + OfflineTtsConfig config; + + config.model.matcha.acoustic_model = + "./matcha-icefall-en_US-ljspeech/model-steps-3.onnx"; + + config.model.matcha.vocoder = "./hifigan_v2.onnx"; + + config.model.matcha.tokens = "./matcha-icefall-en_US-ljspeech/tokens.txt"; + + config.model.matcha.data_dir = + "./matcha-icefall-en_US-ljspeech/espeak-ng-data"; + + config.model.num_threads = 1; + + // If you don't want to see debug messages, please set it to 0 + config.model.debug = 1; + + std::string filename = "./generated-matcha-en-cxx.wav"; + std::string text = + "Today as always, men fall into two groups: slaves and free men. Whoever " + "does not have two-thirds of his day for himself, is a slave, whatever " + "he may be: a statesman, a businessman, an official, or a scholar. " + "Friends fell out often because life was changing so fast. The easiest " + "thing in the world was to lose touch with someone."; + + auto tts = OfflineTts::Create(config); + int32_t sid = 0; + float speed = 1.0; // larger -> faster in speech speed + +#if 0 + // If you don't want to use a callback, then please enable this branch + GeneratedAudio audio = tts.Generate(text, sid, speed); +#else + GeneratedAudio audio = tts.Generate(text, sid, speed, ProgressCallback); +#endif + + WriteWave(filename, {audio.samples, audio.sample_rate}); + + fprintf(stderr, "Input text is: %s\n", text.c_str()); + fprintf(stderr, "Speaker ID is is: %d\n", sid); + fprintf(stderr, "Saved to: %s\n", filename.c_str()); + + return 0; +} diff --git a/cxx-api-examples/matcha-tts-zh-cxx-api.cc b/cxx-api-examples/matcha-tts-zh-cxx-api.cc new file mode 100644 index 000000000..f63065994 --- /dev/null +++ b/cxx-api-examples/matcha-tts-zh-cxx-api.cc @@ -0,0 +1,79 @@ +// cxx-api-examples/matcha-tts-zh-cxx-api.c +// +// Copyright (c) 2025 Xiaomi Corporation + +// This file shows how to use sherpa-onnx CXX API +// for Chinese TTS with MatchaTTS. +// +// clang-format off +/* +Usage + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 +tar xvf matcha-icefall-zh-baker.tar.bz2 +rm matcha-icefall-zh-baker.tar.bz2 + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +./matcha-tts-zh-cxx-api + + */ +// clang-format on + +#include + +#include "sherpa-onnx/c-api/cxx-api.h" + +static int32_t ProgressCallback(const float *samples, int32_t num_samples, + float progress, void *arg) { + fprintf(stderr, "Progress: %.3f%%\n", progress * 100); + // return 1 to continue generating + // return 0 to stop generating + return 1; +} + +int32_t main(int32_t argc, char *argv[]) { + using namespace sherpa_onnx::cxx; // NOLINT + OfflineTtsConfig config; + config.model.matcha.acoustic_model = + "./matcha-icefall-zh-baker/model-steps-3.onnx"; + config.model.matcha.vocoder = "./hifigan_v2.onnx"; + config.model.matcha.lexicon = "./matcha-icefall-zh-baker/lexicon.txt"; + config.model.matcha.tokens = "./matcha-icefall-zh-baker/tokens.txt"; + config.model.matcha.dict_dir = "./matcha-icefall-zh-baker/dict"; + config.model.num_threads = 1; + + // If you don't want to see debug messages, please set it to 0 + config.model.debug = 1; + + // clang-format off + config.rule_fsts = "./matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst"; // NOLINT + // clang-format on + + std::string filename = "./generated-matcha-zh-cxx.wav"; + std::string text = + "当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转，思念如" + "涟漪荡漾，梦境如画卷展开，我与自然融为一体，沉静在这片宁静的美丽之中，感" + "受着生命的奇迹与温柔." + "某某银行的副行长和一些行政领导表示，他们去过长江和长白山; " + "经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。"; + + auto tts = OfflineTts::Create(config); + int32_t sid = 0; + float speed = 1.0; // larger -> faster in speech speed + +#if 0 + // If you don't want to use a callback, then please enable this branch + GeneratedAudio audio = tts.Generate(text, sid, speed); +#else + GeneratedAudio audio = tts.Generate(text, sid, speed, ProgressCallback); +#endif + + WriteWave(filename, {audio.samples, audio.sample_rate}); + + fprintf(stderr, "Input text is: %s\n", text.c_str()); + fprintf(stderr, "Speaker ID is is: %d\n", sid); + fprintf(stderr, "Saved to: %s\n", filename.c_str()); + + return 0; +} diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index 6afc1bf62..584f93321 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -1114,7 +1114,7 @@ static sherpa_onnx::OfflineTtsConfig GetOfflineTtsConfig( return tts_config; } -SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTts( +const SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTts( const SherpaOnnxOfflineTtsConfig *config) { auto tts_config = GetOfflineTtsConfig(config); @@ -1130,7 +1130,9 @@ SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTts( return tts; } -void SherpaOnnxDestroyOfflineTts(SherpaOnnxOfflineTts *tts) { delete tts; } +void SherpaOnnxDestroyOfflineTts(const SherpaOnnxOfflineTts *tts) { + delete tts; +} int32_t SherpaOnnxOfflineTtsSampleRate(const SherpaOnnxOfflineTts *tts) { return tts->impl->SampleRate(); diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index e79d951e2..691b92a3b 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -950,11 +950,12 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOfflineTts SherpaOnnxOfflineTts; // Create an instance of offline TTS. The user has to use DestroyOfflineTts() // to free the returned pointer to avoid memory leak. -SHERPA_ONNX_API SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTts( +SHERPA_ONNX_API const SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTts( const SherpaOnnxOfflineTtsConfig *config); // Free the pointer returned by SherpaOnnxCreateOfflineTts() -SHERPA_ONNX_API void SherpaOnnxDestroyOfflineTts(SherpaOnnxOfflineTts *tts); +SHERPA_ONNX_API void SherpaOnnxDestroyOfflineTts( + const SherpaOnnxOfflineTts *tts); // Return the sample rate of the current TTS object SHERPA_ONNX_API int32_t @@ -984,7 +985,6 @@ SHERPA_ONNX_API const SherpaOnnxGeneratedAudio * SherpaOnnxOfflineTtsGenerateWithProgressCallback( const SherpaOnnxOfflineTts *tts, const char *text, int32_t sid, float speed, - SherpaOnnxGeneratedAudioProgressCallback callback); SHERPA_ONNX_API diff --git a/sherpa-onnx/c-api/cxx-api.cc b/sherpa-onnx/c-api/cxx-api.cc index c66221f0e..0b19e112e 100644 --- a/sherpa-onnx/c-api/cxx-api.cc +++ b/sherpa-onnx/c-api/cxx-api.cc @@ -24,6 +24,11 @@ Wave ReadWave(const std::string &filename) { return ans; } +bool WriteWave(const std::string &filename, const Wave &wave) { + return SherpaOnnxWriteWave(wave.samples.data(), wave.samples.size(), + wave.sample_rate, filename.c_str()); +} + OnlineStream::OnlineStream(const SherpaOnnxOnlineStream *p) : MoveOnly(p) {} @@ -311,4 +316,73 @@ OfflineRecognizerResult OfflineRecognizer::GetResult( return ans; } +OfflineTts OfflineTts::Create(const OfflineTtsConfig &config) { + struct SherpaOnnxOfflineTtsConfig c; + memset(&c, 0, sizeof(c)); + + c.model.vits.model = config.model.vits.model.c_str(); + c.model.vits.lexicon = config.model.vits.lexicon.c_str(); + c.model.vits.tokens = config.model.vits.tokens.c_str(); + c.model.vits.data_dir = config.model.vits.data_dir.c_str(); + c.model.vits.noise_scale = config.model.vits.noise_scale; + c.model.vits.noise_scale_w = config.model.vits.noise_scale_w; + c.model.vits.length_scale = config.model.vits.length_scale; + c.model.vits.dict_dir = config.model.vits.dict_dir.c_str(); + + c.model.matcha.acoustic_model = config.model.matcha.acoustic_model.c_str(); + c.model.matcha.vocoder = config.model.matcha.vocoder.c_str(); + c.model.matcha.lexicon = config.model.matcha.lexicon.c_str(); + c.model.matcha.tokens = config.model.matcha.tokens.c_str(); + c.model.matcha.data_dir = config.model.matcha.data_dir.c_str(); + c.model.matcha.noise_scale = config.model.matcha.noise_scale; + c.model.matcha.length_scale = config.model.matcha.length_scale; + c.model.matcha.dict_dir = config.model.matcha.dict_dir.c_str(); + + c.model.num_threads = config.model.num_threads; + c.model.debug = config.model.debug; + c.model.provider = config.model.provider.c_str(); + + c.rule_fsts = config.rule_fsts.c_str(); + c.max_num_sentences = config.max_num_sentences; + c.rule_fars = config.rule_fars.c_str(); + + auto p = SherpaOnnxCreateOfflineTts(&c); + return OfflineTts(p); +} + +OfflineTts::OfflineTts(const SherpaOnnxOfflineTts *p) + : MoveOnly(p) {} + +void OfflineTts::Destroy(const SherpaOnnxOfflineTts *p) const { + SherpaOnnxDestroyOfflineTts(p); +} + +int32_t OfflineTts::SampleRate() const { + return SherpaOnnxOfflineTtsSampleRate(p_); +} + +int32_t OfflineTts::NumSpeakers() const { + return SherpaOnnxOfflineTtsNumSpeakers(p_); +} + +GeneratedAudio OfflineTts::Generate(const std::string &text, + int32_t sid /*= 0*/, float speed /*= 1.0*/, + OfflineTtsCallback callback /*= nullptr*/, + void *arg /*= nullptr*/) const { + const SherpaOnnxGeneratedAudio *audio; + if (!callback) { + audio = SherpaOnnxOfflineTtsGenerate(p_, text.c_str(), sid, speed); + } else { + audio = SherpaOnnxOfflineTtsGenerateWithProgressCallbackWithArg( + p_, text.c_str(), sid, speed, callback, arg); + } + + GeneratedAudio ans; + ans.samples = std::vector{audio->samples, audio->samples + audio->n}; + ans.sample_rate = audio->sample_rate; + + SherpaOnnxDestroyOfflineTtsGeneratedAudio(audio); + return ans; +} + } // namespace sherpa_onnx::cxx diff --git a/sherpa-onnx/c-api/cxx-api.h b/sherpa-onnx/c-api/cxx-api.h index 2a476efa1..12932f3f2 100644 --- a/sherpa-onnx/c-api/cxx-api.h +++ b/sherpa-onnx/c-api/cxx-api.h @@ -97,6 +97,10 @@ struct Wave { SHERPA_ONNX_API Wave ReadWave(const std::string &filename); +// Return true on success; +// Return false on failure +SHERPA_ONNX_API bool WriteWave(const std::string &filename, const Wave &wave); + template class SHERPA_ONNX_API MoveOnly { public: @@ -307,6 +311,91 @@ class SHERPA_ONNX_API OfflineRecognizer explicit OfflineRecognizer(const SherpaOnnxOfflineRecognizer *p); }; +// ============================================================================ +// Non-streaming TTS +// ============================================================================ +struct OfflineTtsVitsModelConfig { + std::string model; + std::string lexicon; + std::string tokens; + std::string data_dir; + std::string dict_dir; + + float noise_scale = 0.667; + float noise_scale_w = 0.8; + float length_scale = 1.0; // < 1, faster in speed; > 1, slower in speed +}; + +struct OfflineTtsMatchaModelConfig { + std::string acoustic_model; + std::string vocoder; + std::string lexicon; + std::string tokens; + std::string data_dir; + std::string dict_dir; + + float noise_scale = 0.667; + float length_scale = 1.0; // < 1, faster in speed; > 1, slower in speed +}; + +struct OfflineTtsModelConfig { + OfflineTtsVitsModelConfig vits; + OfflineTtsMatchaModelConfig matcha; + int32_t num_threads = 1; + bool debug = false; + std::string provider = "cpu"; +}; + +struct OfflineTtsConfig { + OfflineTtsModelConfig model; + std::string rule_fsts; + std::string rule_fars; + int32_t max_num_sentences = 1; +}; + +struct GeneratedAudio { + std::vector samples; // in the range [-1, 1] + int32_t sample_rate; +}; + +// Return 1 to continue generating +// Return 0 to stop generating +using OfflineTtsCallback = int32_t (*)(const float *samples, + int32_t num_samples, float progress, + void *arg); + +class SHERPA_ONNX_API OfflineTts + : public MoveOnly { + public: + static OfflineTts Create(const OfflineTtsConfig &config); + + void Destroy(const SherpaOnnxOfflineTts *p) const; + + // Return the sample rate of the generated audio + int32_t SampleRate() const; + + // Number of supported speakers. + // If it supports only a single speaker, then it return 0 or 1. + int32_t NumSpeakers() const; + + // @param text A string containing words separated by spaces + // @param sid Speaker ID. Used only for multi-speaker models, e.g., models + // trained using the VCTK dataset. It is not used for + // single-speaker models, e.g., models trained using the ljspeech + // dataset. + // @param speed The speed for the generated speech. E.g., 2 means 2x faster. + // @param callback If not NULL, it is called whenever config.max_num_sentences + // sentences have been processed. The callback is called in + // the current thread. + GeneratedAudio Generate(const std::string &text, int32_t sid = 0, + float speed = 1.0, + OfflineTtsCallback callback = nullptr, + void *arg = nullptr) const; + + private: + explicit OfflineTts(const SherpaOnnxOfflineTts *p); +}; + } // namespace sherpa_onnx::cxx #endif // SHERPA_ONNX_C_API_CXX_API_H_ From 0e299f30f5b18099abe466321581a99936295731 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 3 Jan 2025 15:14:28 +0800 Subject: [PATCH 137/183] Add JavaScript API (node-addon-api) for MatchaTTS models. (#1677) --- .github/scripts/test-nodejs-addon-npm.sh | 19 +++++++ .../src/main/cpp/non-streaming-tts.cc | 55 ++++++++++++++++++- nodejs-addon-examples/README.md | 24 ++++++++ ...est_tts_non_streaming_matcha_icefall_en.js | 48 ++++++++++++++++ ...est_tts_non_streaming_matcha_icefall_zh.js | 50 +++++++++++++++++ scripts/node-addon-api/package.json | 2 +- 6 files changed, 194 insertions(+), 4 deletions(-) create mode 100644 nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_en.js create mode 100644 nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_zh.js diff --git a/.github/scripts/test-nodejs-addon-npm.sh b/.github/scripts/test-nodejs-addon-npm.sh index 755cde74b..d3e85f687 100755 --- a/.github/scripts/test-nodejs-addon-npm.sh +++ b/.github/scripts/test-nodejs-addon-npm.sh @@ -85,6 +85,25 @@ fi echo "----------tts----------" +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 +tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 +rm matcha-icefall-en_US-ljspeech.tar.bz2 +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +node ./test_tts_non_streaming_matcha_icefall_en.js +rm hifigan_v2.onnx +rm -rf matcha-icefall-en_US-ljspeech + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 +tar xvf matcha-icefall-zh-baker.tar.bz2 +rm matcha-icefall-zh-baker.tar.bz2 +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +node ./test_tts_non_streaming_matcha_icefall_zh.js +rm hifigan_v2.onnx +rm -rf matcha-icefall-zh-baker +ls -lh *.wav + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_GB-cori-medium.tar.bz2 tar xf vits-piper-en_GB-cori-medium.tar.bz2 rm vits-piper-en_GB-cori-medium.tar.bz2 diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc index 05e27846d..7baf3ce8b 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc @@ -31,6 +31,28 @@ static SherpaOnnxOfflineTtsVitsModelConfig GetOfflineTtsVitsModelConfig( return c; } +static SherpaOnnxOfflineTtsMatchaModelConfig GetOfflineTtsMatchaModelConfig( + Napi::Object obj) { + SherpaOnnxOfflineTtsMatchaModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("matcha") || !obj.Get("matcha").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("matcha").As(); + SHERPA_ONNX_ASSIGN_ATTR_STR(acoustic_model, acousticModel); + SHERPA_ONNX_ASSIGN_ATTR_STR(vocoder, vocoder); + SHERPA_ONNX_ASSIGN_ATTR_STR(lexicon, lexicon); + SHERPA_ONNX_ASSIGN_ATTR_STR(tokens, tokens); + SHERPA_ONNX_ASSIGN_ATTR_STR(data_dir, dataDir); + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(noise_scale, noiseScale); + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(length_scale, lengthScale); + SHERPA_ONNX_ASSIGN_ATTR_STR(dict_dir, dictDir); + + return c; +} + static SherpaOnnxOfflineTtsModelConfig GetOfflineTtsModelConfig( Napi::Object obj) { SherpaOnnxOfflineTtsModelConfig c; @@ -43,6 +65,7 @@ static SherpaOnnxOfflineTtsModelConfig GetOfflineTtsModelConfig( Napi::Object o = obj.Get("model").As(); c.vits = GetOfflineTtsVitsModelConfig(o); + c.matcha = GetOfflineTtsMatchaModelConfig(o); SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); @@ -107,9 +130,10 @@ static Napi::External CreateOfflineTtsWrapper( decltype(&OH_ResourceManager_ReleaseNativeResourceManager)> mgr(OH_ResourceManager_InitNativeResourceManager(env, info[1]), &OH_ResourceManager_ReleaseNativeResourceManager); - SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTtsOHOS(&c, mgr.get()); + const SherpaOnnxOfflineTts *tts = + SherpaOnnxCreateOfflineTtsOHOS(&c, mgr.get()); #else - SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTts(&c); + const SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTts(&c); #endif if (c.model.vits.model) { @@ -132,6 +156,30 @@ static Napi::External CreateOfflineTtsWrapper( delete[] c.model.vits.dict_dir; } + if (c.model.matcha.acoustic_model) { + delete[] c.model.matcha.acoustic_model; + } + + if (c.model.matcha.vocoder) { + delete[] c.model.matcha.vocoder; + } + + if (c.model.matcha.lexicon) { + delete[] c.model.matcha.lexicon; + } + + if (c.model.matcha.tokens) { + delete[] c.model.matcha.tokens; + } + + if (c.model.matcha.data_dir) { + delete[] c.model.matcha.data_dir; + } + + if (c.model.matcha.dict_dir) { + delete[] c.model.matcha.dict_dir; + } + if (c.model.provider) { delete[] c.model.provider; } @@ -152,7 +200,8 @@ static Napi::External CreateOfflineTtsWrapper( } return Napi::External::New( - env, tts, [](Napi::Env env, SherpaOnnxOfflineTts *tts) { + env, const_cast(tts), + [](Napi::Env env, SherpaOnnxOfflineTts *tts) { SherpaOnnxDestroyOfflineTts(tts); }); } diff --git a/nodejs-addon-examples/README.md b/nodejs-addon-examples/README.md index f436bcae2..ec2f23da2 100644 --- a/nodejs-addon-examples/README.md +++ b/nodejs-addon-examples/README.md @@ -133,6 +133,8 @@ The following tables list the examples in this folder. |File| Description| |---|---| +|[./test_tts_non_streaming_matcha_icefall_en.js](./test_tts_non_streaming_matcha_icefall_en.js)| Text-to-speech with a [MatchaTTS English Model](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker)| +|[./test_tts_non_streaming_matcha_icefall_zhjs](./test_tts_non_streaming_matcha_icefall_zh.js)| Text-to-speech with a [MatchaTTS Chinese Model](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker)| |[./test_tts_non_streaming_vits_piper_en.js](./test_tts_non_streaming_vits_piper_en.js)| Text-to-speech with a [piper](https://github.com/rhasspy/piper) English model| |[./test_tts_non_streaming_vits_coqui_de.js](./test_tts_non_streaming_vits_coqui_de.js)| Text-to-speech with a [coqui](https://github.com/coqui-ai/TTS) German model| |[./test_tts_non_streaming_vits_zh_ll.js](./test_tts_non_streaming_vits_zh_ll.js)| Text-to-speech with a Chinese model using [cppjieba](https://github.com/yanyiwu/cppjieba)| @@ -345,6 +347,28 @@ npm install naudiodon2 node ./test_vad_asr_non_streaming_sense_voice_microphone.js ``` +### Text-to-speech with MatchaTTS models (English TTS) +```bash +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 +tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 +rm matcha-icefall-en_US-ljspeech.tar.bz2 + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +node ./test_tts_non_streaming_matcha_icefall_en.js +``` + +### Text-to-speech with MatchaTTS models (Chinese TTS) +```bash +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 +tar xvf matcha-icefall-zh-baker.tar.bz2 +rm matcha-icefall-zh-baker.tar.bz2 + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +node ./test_tts_non_streaming_matcha_icefall_zh.js +``` + ### Text-to-speech with piper VITS models (TTS) ```bash diff --git a/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_en.js b/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_en.js new file mode 100644 index 000000000..8c45d1dd3 --- /dev/null +++ b/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_en.js @@ -0,0 +1,48 @@ +// Copyright (c) 2025 Xiaomi Corporation +const sherpa_onnx = require('sherpa-onnx-node'); + +// please refer to +// https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker +// to download model files +function createOfflineTts() { + const config = { + model: { + matcha: { + acousticModel: './matcha-icefall-en_US-ljspeech/model-steps-3.onnx', + vocoder: './hifigan_v2.onnx', + lexicon: './matcha-icefall-en_US-ljspeech/lexicon.txt', + tokens: './matcha-icefall-en_US-ljspeech/tokens.txt', + dataDir: './matcha-icefall-en_US-ljspeech/espeak-ng-data', + }, + debug: true, + numThreads: 1, + provider: 'cpu', + }, + maxNumSentences: 1, + }; + return new sherpa_onnx.OfflineTts(config); +} + +const tts = createOfflineTts(); + +const text = + 'Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.' + + +let start = Date.now(); +const audio = tts.generate({text: text, sid: 0, speed: 1.0}); +let stop = Date.now(); +const elapsed_seconds = (stop - start) / 1000; +const duration = audio.samples.length / audio.sampleRate; +const real_time_factor = elapsed_seconds / duration; +console.log('Wave duration', duration.toFixed(3), 'secodns') +console.log('Elapsed', elapsed_seconds.toFixed(3), 'secodns') +console.log( + `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, + real_time_factor.toFixed(3)) + +const filename = 'test-matcha-en.wav'; +sherpa_onnx.writeWave( + filename, {samples: audio.samples, sampleRate: audio.sampleRate}); + +console.log(`Saved to ${filename}`); diff --git a/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_zh.js b/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_zh.js new file mode 100644 index 000000000..1f667e3d2 --- /dev/null +++ b/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_zh.js @@ -0,0 +1,50 @@ +// Copyright (c) 2025 Xiaomi Corporation +const sherpa_onnx = require('sherpa-onnx-node'); + +// please refer to +// https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker +// to download model files +function createOfflineTts() { + const config = { + model: { + matcha: { + acousticModel: './matcha-icefall-zh-baker/model-steps-3.onnx', + vocoder: './hifigan_v2.onnx', + lexicon: './matcha-icefall-zh-baker/lexicon.txt', + tokens: './matcha-icefall-zh-baker/tokens.txt', + dictDir: './matcha-icefall-zh-baker/dict', + }, + debug: true, + numThreads: 1, + provider: 'cpu', + }, + maxNumSentences: 1, + ruleFsts: + './matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst', + }; + return new sherpa_onnx.OfflineTts(config); +} + +const tts = createOfflineTts(); + +const text = + '当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转，思念如涟漪荡漾，梦境如画卷展开，我与自然融为一体，沉静在这片宁静的美丽之中，感受着生命的奇迹与温柔. 某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。' + + +let start = Date.now(); +const audio = tts.generate({text: text, sid: 0, speed: 1.0}); +let stop = Date.now(); +const elapsed_seconds = (stop - start) / 1000; +const duration = audio.samples.length / audio.sampleRate; +const real_time_factor = elapsed_seconds / duration; +console.log('Wave duration', duration.toFixed(3), 'secodns') +console.log('Elapsed', elapsed_seconds.toFixed(3), 'secodns') +console.log( + `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, + real_time_factor.toFixed(3)) + +const filename = 'test-matcha-zh.wav'; +sherpa_onnx.writeWave( + filename, {samples: audio.samples, sampleRate: audio.sampleRate}); + +console.log(`Saved to ${filename}`); diff --git a/scripts/node-addon-api/package.json b/scripts/node-addon-api/package.json index f0bb57d0d..201cb77de 100644 --- a/scripts/node-addon-api/package.json +++ b/scripts/node-addon-api/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "description": "Speech-to-text, text-to-speech, and speaker diarization using Next-gen Kaldi without internet connection", "dependencies": { - "cmake-js": "^7.0.0", + "cmake-js": "^7.3.0", "node-addon-api": "^8.3.0", "perf_hooks": "*" }, From bf3330c90667fcd4ae677138935f3074ad18860e Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 3 Jan 2025 17:09:29 +0800 Subject: [PATCH 138/183] Add HarmonyOS examples for MatchaTTS. (#1678) --- .../sherpa_onnx/BuildProfile.ets | 2 +- .../SherpaOnnxHar/sherpa_onnx/Index.ets | 10 +-- .../main/ets/components/NonStreamingTts.ets | 12 ++++ .../ets/workers/NonStreamingTtsWorker.ets | 53 ++++++++++++++- sherpa-onnx/c-api/c-api.cc | 2 +- sherpa-onnx/c-api/c-api.h | 2 +- sherpa-onnx/csrc/jieba-lexicon.cc | 64 +++++++++++++++++++ sherpa-onnx/csrc/jieba-lexicon.h | 6 ++ sherpa-onnx/csrc/offline-tts-matcha-impl.h | 5 +- 9 files changed, 141 insertions(+), 15 deletions(-) diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets index 8eb22c9a7..4cc33f4e9 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets @@ -1,7 +1,7 @@ /** * Use these variables when you tailor your ArkTS code. They must be of the const type. */ -export const HAR_VERSION = '1.10.35'; +export const HAR_VERSION = '1.10.37'; export const BUILD_MODE_NAME = 'debug'; export const DEBUG = true; export const TARGET_NAME = 'default'; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets index 16c6279e1..56deee0ce 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets @@ -1,11 +1,6 @@ export { listRawfileDir, readWave, readWaveFromBinary, } from "libsherpa_onnx.so"; -export { CircularBuffer, - SileroVadConfig, - SpeechSegment, - Vad, - VadConfig, -} from './src/main/ets/components/Vad'; +export { CircularBuffer, SileroVadConfig, SpeechSegment, Vad, VadConfig, } from './src/main/ets/components/Vad'; export { Samples, @@ -36,7 +31,8 @@ export { OnlineStream, OnlineRecognizer, } from './src/main/ets/components/StreamingAsr'; -export { OfflineTtsVitsModelConfig, +export { OfflineTtsMatchaModelConfig, + OfflineTtsVitsModelConfig, OfflineTtsModelConfig, OfflineTtsConfig, OfflineTts, diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets index 556877489..814d6e267 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets @@ -17,8 +17,20 @@ export class OfflineTtsVitsModelConfig { public lengthScale: number = 1.0; } +export class OfflineTtsMatchaModelConfig { + public acousticModel: string = ''; + public vocoder: string = ''; + public lexicon: string = ''; + public tokens: string = ''; + public dataDir: string = ''; + public dictDir: String = ''; + public noiseScale: number = 0.667; + public lengthScale: number = 1.0; +} + export class OfflineTtsModelConfig { public vits: OfflineTtsVitsModelConfig = new OfflineTtsVitsModelConfig(); + public matcha: OfflineTtsMatchaModelConfig = new OfflineTtsMatchaModelConfig(); public numThreads: number = 1; public debug: boolean = false; public provider: string = 'cpu'; diff --git a/harmony-os/SherpaOnnxTts/entry/src/main/ets/workers/NonStreamingTtsWorker.ets b/harmony-os/SherpaOnnxTts/entry/src/main/ets/workers/NonStreamingTtsWorker.ets index bd5c7a5b8..cf841cbe1 100644 --- a/harmony-os/SherpaOnnxTts/entry/src/main/ets/workers/NonStreamingTtsWorker.ets +++ b/harmony-os/SherpaOnnxTts/entry/src/main/ets/workers/NonStreamingTtsWorker.ets @@ -73,7 +73,16 @@ function initTts(context: Context): OfflineTts { // for details let modelDir = ''; + + // for VITS begin let modelName = ''; + // for VITS end + + // for Matcha begin + let acousticModelName = ''; + let vocoder = ''; + // for Matcha end + let ruleFsts = ''; let ruleFars = ''; let lexicon = ''; @@ -134,15 +143,47 @@ function initTts(context: Context): OfflineTts { // dictDir = 'dict'; // ruleFsts = `date.fst,phone.fst,number.fst`; + // Example 8 + // https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models + // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker + // modelDir = 'matcha-icefall-zh-baker' + // acousticModelName = 'model-steps-3.onnx' + // vocoder = 'hifigan_v2.onnx' + // lexicon = 'lexicon.txt' + // dictDir = 'dict'; + // ruleFsts = `date.fst,phone.fst,number.fst`; + + // Example 9 + // https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models + // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker + // modelDir = 'matcha-icefall-en_US-ljspeech' + // acousticModelName = 'model-steps-3.onnx' + // vocoder = 'hifigan_v2.onnx' + // dataDir = 'espeak-ng-data'; + // ============================================================ // Please don't change the remaining part of this function // ============================================================ - if (modelName == '') { + if (modelName == '' && acousticModelName == '' && vocoder == '') { throw new Error('You are supposed to select a model by changing the code before you run the app'); } - modelName = modelDir + '/' + modelName; + if (modelName != '' && acousticModelName != '') { + throw new Error('Please select either VITS or Matcha, not both'); + } + + if (acousticModelName != '' && vocoder == '') { + throw new Error('Please provider vocoder for matcha tts models'); + } + + if (modelName != '') { + modelName = modelDir + '/' + modelName; + } + + if (acousticModelName != '') { + acousticModelName = modelDir + '/' + acousticModelName; + } if (ruleFsts != '') { let fsts = ruleFsts.split(',') @@ -186,6 +227,14 @@ function initTts(context: Context): OfflineTts { config.model.vits.tokens = tokens; config.model.vits.dataDir = dataDir; config.model.vits.dictDir = dictDir; + + config.model.matcha.acousticModel = acousticModelName; + config.model.matcha.vocoder = vocoder; + config.model.matcha.lexicon = lexicon; + config.model.matcha.tokens = tokens; + config.model.matcha.dataDir = dataDir; + config.model.matcha.dictDir = dictDir; + config.model.numThreads = 2; config.model.debug = true; config.ruleFsts = ruleFsts; diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index 584f93321..30c87823a 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -2098,7 +2098,7 @@ SherpaOnnxCreateSpeakerEmbeddingExtractorOHOS( } #if SHERPA_ONNX_ENABLE_TTS == 1 -SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTtsOHOS( +const SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTtsOHOS( const SherpaOnnxOfflineTtsConfig *config, NativeResourceManager *mgr) { if (!mgr) { return SherpaOnnxCreateOfflineTts(config); diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index 691b92a3b..a669b50dc 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -1618,7 +1618,7 @@ SherpaOnnxCreateVoiceActivityDetectorOHOS( const SherpaOnnxVadModelConfig *config, float buffer_size_in_seconds, NativeResourceManager *mgr); -SHERPA_ONNX_API SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTtsOHOS( +SHERPA_ONNX_API const SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTtsOHOS( const SherpaOnnxOfflineTtsConfig *config, NativeResourceManager *mgr); SHERPA_ONNX_API const SherpaOnnxSpeakerEmbeddingExtractor * diff --git a/sherpa-onnx/csrc/jieba-lexicon.cc b/sherpa-onnx/csrc/jieba-lexicon.cc index 57b77666b..189520c4d 100644 --- a/sherpa-onnx/csrc/jieba-lexicon.cc +++ b/sherpa-onnx/csrc/jieba-lexicon.cc @@ -6,12 +6,23 @@ #include #include // NOLINT +#include #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "cppjieba/Jieba.hpp" #include "sherpa-onnx/csrc/file-utils.h" #include "sherpa-onnx/csrc/macros.h" +#include "sherpa-onnx/csrc/onnx-utils.h" #include "sherpa-onnx/csrc/symbol-table.h" #include "sherpa-onnx/csrc/text-utils.h" @@ -56,6 +67,39 @@ class JiebaLexicon::Impl { } } + template + Impl(Manager *mgr, const std::string &lexicon, const std::string &tokens, + const std::string &dict_dir, bool debug) + : debug_(debug) { + std::string dict = dict_dir + "/jieba.dict.utf8"; + std::string hmm = dict_dir + "/hmm_model.utf8"; + std::string user_dict = dict_dir + "/user.dict.utf8"; + std::string idf = dict_dir + "/idf.utf8"; + std::string stop_word = dict_dir + "/stop_words.utf8"; + + AssertFileExists(dict); + AssertFileExists(hmm); + AssertFileExists(user_dict); + AssertFileExists(idf); + AssertFileExists(stop_word); + + jieba_ = + std::make_unique(dict, hmm, user_dict, idf, stop_word); + + { + auto buf = ReadFile(mgr, tokens); + std::istrstream is(buf.data(), buf.size()); + + InitTokens(is); + } + + { + auto buf = ReadFile(mgr, lexicon); + std::istrstream is(buf.data(), buf.size()); + InitLexicon(is); + } + } + std::vector ConvertTextToTokenIds(const std::string &text) const { // see // https://github.com/Plachtaa/VITS-fast-fine-tuning/blob/main/text/mandarin.py#L244 @@ -279,9 +323,29 @@ JiebaLexicon::JiebaLexicon(const std::string &lexicon, const std::string &dict_dir, bool debug) : impl_(std::make_unique(lexicon, tokens, dict_dir, debug)) {} +template +JiebaLexicon::JiebaLexicon(Manager *mgr, const std::string &lexicon, + const std::string &tokens, + const std::string &dict_dir, bool debug) + : impl_(std::make_unique(mgr, lexicon, tokens, dict_dir, debug)) {} + std::vector JiebaLexicon::ConvertTextToTokenIds( const std::string &text, const std::string & /*unused_voice = ""*/) const { return impl_->ConvertTextToTokenIds(text); } +#if __ANDROID_API__ >= 9 +template JiebaLexicon::JiebaLexicon(AAssetManager *mgr, + const std::string &lexicon, + const std::string &tokens, + const std::string &dict_dir, bool debug); +#endif + +#if __OHOS__ +template JiebaLexicon::JiebaLexicon(NativeResourceManager *mgr, + const std::string &lexicon, + const std::string &tokens, + const std::string &dict_dir, bool debug); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/jieba-lexicon.h b/sherpa-onnx/csrc/jieba-lexicon.h index 9de104357..b810a084c 100644 --- a/sherpa-onnx/csrc/jieba-lexicon.h +++ b/sherpa-onnx/csrc/jieba-lexicon.h @@ -17,9 +17,15 @@ namespace sherpa_onnx { class JiebaLexicon : public OfflineTtsFrontend { public: ~JiebaLexicon() override; + JiebaLexicon(const std::string &lexicon, const std::string &tokens, const std::string &dict_dir, bool debug); + template + JiebaLexicon(Manager *mgr, const std::string &lexicon, + const std::string &tokens, const std::string &dict_dir, + bool debug); + std::vector ConvertTextToTokenIds( const std::string &text, const std::string &unused_voice = "") const override; diff --git a/sherpa-onnx/csrc/offline-tts-matcha-impl.h b/sherpa-onnx/csrc/offline-tts-matcha-impl.h index a4f47fadb..7bd45fede 100644 --- a/sherpa-onnx/csrc/offline-tts-matcha-impl.h +++ b/sherpa-onnx/csrc/offline-tts-matcha-impl.h @@ -327,13 +327,12 @@ class OfflineTtsMatchaImpl : public OfflineTtsImpl { // from assets to disk // // for jieba - // we require that you copy tokens.txt, lexicon.txt and dict - // from assets to disk + // we require that you copy dict from assets to disk const auto &meta_data = model_->GetMetaData(); if (meta_data.jieba && !meta_data.has_espeak) { frontend_ = std::make_unique( - config_.model.matcha.lexicon, config_.model.matcha.tokens, + mgr, config_.model.matcha.lexicon, config_.model.matcha.tokens, config_.model.matcha.dict_dir, config_.model.debug); } else if (meta_data.has_espeak && !meta_data.jieba) { frontend_ = std::make_unique( From 8a60985363b537abdab9df8d32b59bb3d4c9ae8f Mon Sep 17 00:00:00 2001 From: Michael Lamothe Date: Sat, 4 Jan 2025 19:39:06 +1100 Subject: [PATCH 139/183] Upgraded to .NET 8 and made code style a little more internally consistent. (#1680) --- dotnet-examples/Common/Common.csproj | 2 +- dotnet-examples/Common/WaveHeader.cs | 281 +++++++++--------- .../keyword-spotting-from-files/Program.cs | 12 +- .../keyword-spotting-from-files.csproj | 2 +- .../Program.cs | 27 +- .../keyword-spotting-from-microphone.csproj | 2 +- .../offline-decode-files/Program.cs | 71 +++-- .../offline-decode-files.csproj | 2 +- .../offline-punctuation/Program.cs | 6 +- .../offline-punctuation.csproj | 2 +- .../offline-speaker-diarization/Program.cs | 13 +- .../offline-speaker-diarization.csproj | 2 +- dotnet-examples/offline-tts-play/Program.cs | 47 ++- .../offline-tts-play/offline-tts-play.csproj | 2 +- dotnet-examples/offline-tts/Program.cs | 37 ++- .../offline-tts/offline-tts.csproj | 2 +- .../online-decode-files/Program.cs | 44 ++- .../online-decode-files.csproj | 2 +- dotnet-examples/sherpa-onnx.sln | 8 +- .../speaker-identification/Program.cs | 26 +- .../speaker-identification.csproj | 2 +- .../Program.cs | 44 ++- .../speech-recognition-from-microphone.csproj | 2 +- .../spoken-language-identification/Program.cs | 5 +- .../spoken-language-identification.csproj | 2 +- .../streaming-hlg-decoding/Program.cs | 17 +- .../streaming-hlg-decoding.csproj | 2 +- .../Program.cs | 54 ++-- .../vad-non-streaming-asr-paraformer.csproj | 2 +- 29 files changed, 335 insertions(+), 385 deletions(-) diff --git a/dotnet-examples/Common/Common.csproj b/dotnet-examples/Common/Common.csproj index a9630614f..57c0ff743 100644 --- a/dotnet-examples/Common/Common.csproj +++ b/dotnet-examples/Common/Common.csproj @@ -1,7 +1,7 @@  - net6.0 + net8.0 true diff --git a/dotnet-examples/Common/WaveHeader.cs b/dotnet-examples/Common/WaveHeader.cs index 7d13b3553..0a6ca5284 100644 --- a/dotnet-examples/Common/WaveHeader.cs +++ b/dotnet-examples/Common/WaveHeader.cs @@ -4,171 +4,166 @@ using System.Runtime.InteropServices; -namespace SherpaOnnx -{ +namespace SherpaOnnx; - [StructLayout(LayoutKind.Sequential)] - public struct WaveHeader +[StructLayout(LayoutKind.Sequential)] +public struct WaveHeader +{ + public int ChunkID; + public int ChunkSize; + public int Format; + public int SubChunk1ID; + public int SubChunk1Size; + public short AudioFormat; + public short NumChannels; + public int SampleRate; + public int ByteRate; + public short BlockAlign; + public short BitsPerSample; + public int SubChunk2ID; + public int SubChunk2Size; + + public bool Validate() { - public Int32 ChunkID; - public Int32 ChunkSize; - public Int32 Format; - public Int32 SubChunk1ID; - public Int32 SubChunk1Size; - public Int16 AudioFormat; - public Int16 NumChannels; - public Int32 SampleRate; - public Int32 ByteRate; - public Int16 BlockAlign; - public Int16 BitsPerSample; - public Int32 SubChunk2ID; - public Int32 SubChunk2Size; - - public bool Validate() + if (ChunkID != 0x46464952) + { + Console.WriteLine($"Invalid chunk ID: 0x{ChunkID:X}. Expect 0x46464952"); + return false; + } + + // E V A W + if (Format != 0x45564157) + { + Console.WriteLine($"Invalid format: 0x{Format:X}. Expect 0x45564157"); + return false; + } + + // t m f + if (SubChunk1ID != 0x20746d66) + { + Console.WriteLine($"Invalid SubChunk1ID: 0x{SubChunk1ID:X}. Expect 0x20746d66"); + return false; + } + + if (SubChunk1Size != 16) + { + Console.WriteLine($"Invalid SubChunk1Size: {SubChunk1Size}. Expect 16"); + return false; + } + + if (AudioFormat != 1) + { + Console.WriteLine($"Invalid AudioFormat: {AudioFormat}. Expect 1"); + return false; + } + + if (NumChannels != 1) + { + Console.WriteLine($"Invalid NumChannels: {NumChannels}. Expect 1"); + return false; + } + + if (ByteRate != (SampleRate * NumChannels * BitsPerSample / 8)) + { + Console.WriteLine($"Invalid byte rate: {ByteRate}."); + return false; + } + + if (BlockAlign != (NumChannels * BitsPerSample / 8)) { - if (ChunkID != 0x46464952) - { - Console.WriteLine($"Invalid chunk ID: 0x{ChunkID:X}. Expect 0x46464952"); - return false; - } - - // E V A W - if (Format != 0x45564157) - { - Console.WriteLine($"Invalid format: 0x{Format:X}. Expect 0x45564157"); - return false; - } - - // t m f - if (SubChunk1ID != 0x20746d66) - { - Console.WriteLine($"Invalid SubChunk1ID: 0x{SubChunk1ID:X}. Expect 0x20746d66"); - return false; - } - - if (SubChunk1Size != 16) - { - Console.WriteLine($"Invalid SubChunk1Size: {SubChunk1Size}. Expect 16"); - return false; - } - - if (AudioFormat != 1) - { - Console.WriteLine($"Invalid AudioFormat: {AudioFormat}. Expect 1"); - return false; - } - - if (NumChannels != 1) - { - Console.WriteLine($"Invalid NumChannels: {NumChannels}. Expect 1"); - return false; - } - - if (ByteRate != (SampleRate * NumChannels * BitsPerSample / 8)) - { - Console.WriteLine($"Invalid byte rate: {ByteRate}."); - return false; - } - - if (BlockAlign != (NumChannels * BitsPerSample / 8)) - { - Console.WriteLine($"Invalid block align: {ByteRate}."); - return false; - } - - if (BitsPerSample != 16) - { // we support only 16 bits per sample - Console.WriteLine($"Invalid bits per sample: {BitsPerSample}. Expect 16"); - return false; - } - - return true; + Console.WriteLine($"Invalid block align: {ByteRate}."); + return false; } + + if (BitsPerSample != 16) + { // we support only 16 bits per sample + Console.WriteLine($"Invalid bits per sample: {BitsPerSample}. Expect 16"); + return false; + } + + return true; } +} - // It supports only 16-bit, single channel WAVE format. - // The sample rate can be any value. - public class WaveReader +// It supports only 16-bit, single channel WAVE format. +// The sample rate can be any value. +public class WaveReader +{ + public WaveReader(string fileName) { - public WaveReader(String fileName) + if (!File.Exists(fileName)) { - if (!File.Exists(fileName)) - { - throw new ApplicationException($"{fileName} does not exist!"); - } - - using (var stream = File.Open(fileName, FileMode.Open)) - { - using (var reader = new BinaryReader(stream)) - { - _header = ReadHeader(reader); - - if (!_header.Validate()) - { - throw new ApplicationException($"Invalid wave file ${fileName}"); - } - - SkipMetaData(reader); - - // now read samples - // _header.SubChunk2Size contains number of bytes in total. - // we assume each sample is of type int16 - byte[] buffer = reader.ReadBytes(_header.SubChunk2Size); - short[] samples_int16 = new short[_header.SubChunk2Size / 2]; - Buffer.BlockCopy(buffer, 0, samples_int16, 0, buffer.Length); - - _samples = new float[samples_int16.Length]; - - for (var i = 0; i < samples_int16.Length; ++i) - { - _samples[i] = samples_int16[i] / 32768.0F; - } - } - } + throw new ApplicationException($"{fileName} does not exist!"); } - private static WaveHeader ReadHeader(BinaryReader reader) - { - byte[] bytes = reader.ReadBytes(Marshal.SizeOf(typeof(WaveHeader))); + using var stream = File.Open(fileName, FileMode.Open); + using var reader = new BinaryReader(stream); - GCHandle handle = GCHandle.Alloc(bytes, GCHandleType.Pinned); - WaveHeader header = (WaveHeader)Marshal.PtrToStructure(handle.AddrOfPinnedObject(), typeof(WaveHeader))!; - handle.Free(); + _header = ReadHeader(reader); - return header; + if (!_header.Validate()) + { + throw new ApplicationException($"Invalid wave file ${fileName}"); } - private void SkipMetaData(BinaryReader reader) + SkipMetaData(reader); + + // now read samples + // _header.SubChunk2Size contains number of bytes in total. + // we assume each sample is of type int16 + var buffer = reader.ReadBytes(_header.SubChunk2Size); + var samples_int16 = new short[_header.SubChunk2Size / 2]; + Buffer.BlockCopy(buffer, 0, samples_int16, 0, buffer.Length); + + _samples = new float[samples_int16.Length]; + + for (var i = 0; i < samples_int16.Length; ++i) { - var bs = reader.BaseStream; - - Int32 subChunk2ID = _header.SubChunk2ID; - Int32 subChunk2Size = _header.SubChunk2Size; - - while (bs.Position != bs.Length && subChunk2ID != 0x61746164) - { - bs.Seek(subChunk2Size, SeekOrigin.Current); - subChunk2ID = reader.ReadInt32(); - subChunk2Size = reader.ReadInt32(); - } - _header.SubChunk2ID = subChunk2ID; - _header.SubChunk2Size = subChunk2Size; + _samples[i] = samples_int16[i] / 32768.0F; } + } - private WaveHeader _header; + private static WaveHeader ReadHeader(BinaryReader reader) + { + var bytes = reader.ReadBytes(Marshal.SizeOf(typeof(WaveHeader))); + + GCHandle handle = GCHandle.Alloc(bytes, GCHandleType.Pinned); + WaveHeader header = (WaveHeader)Marshal.PtrToStructure(handle.AddrOfPinnedObject(), typeof(WaveHeader))!; + handle.Free(); + + return header; + } - // Samples are normalized to the range [-1, 1] - private float[] _samples; + private void SkipMetaData(BinaryReader reader) + { + var bs = reader.BaseStream; - public int SampleRate => _header.SampleRate; - public float[] Samples => _samples; + var subChunk2ID = _header.SubChunk2ID; + var subChunk2Size = _header.SubChunk2Size; - public static void Test(String fileName) + while (bs.Position != bs.Length && subChunk2ID != 0x61746164) { - WaveReader reader = new WaveReader(fileName); - Console.WriteLine($"samples length: {reader.Samples.Length}"); - Console.WriteLine($"samples rate: {reader.SampleRate}"); + bs.Seek(subChunk2Size, SeekOrigin.Current); + subChunk2ID = reader.ReadInt32(); + subChunk2Size = reader.ReadInt32(); } + _header.SubChunk2ID = subChunk2ID; + _header.SubChunk2Size = subChunk2Size; } + private WaveHeader _header; + + // Samples are normalized to the range [-1, 1] + private float[] _samples; + + public int SampleRate => _header.SampleRate; + + public float[] Samples => _samples; + + public static void Test(string fileName) + { + WaveReader reader = new WaveReader(fileName); + Console.WriteLine($"samples length: {reader.Samples.Length}"); + Console.WriteLine($"samples rate: {reader.SampleRate}"); + } } diff --git a/dotnet-examples/keyword-spotting-from-files/Program.cs b/dotnet-examples/keyword-spotting-from-files/Program.cs index 2fea260d1..00ba3777a 100644 --- a/dotnet-examples/keyword-spotting-from-files/Program.cs +++ b/dotnet-examples/keyword-spotting-from-files/Program.cs @@ -13,8 +13,6 @@ // dotnet run using SherpaOnnx; -using System.Collections.Generic; -using System; class KeywordSpotterDemo { @@ -38,11 +36,11 @@ static void Main(string[] args) var filename = "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/test_wavs/3.wav"; - WaveReader waveReader = new WaveReader(filename); + var waveReader = new WaveReader(filename); Console.WriteLine("----------Use pre-defined keywords----------"); - OnlineStream s = kws.CreateStream(); + var s = kws.CreateStream(); s.AcceptWaveform(waveReader.SampleRate, waveReader.Samples); float[] tailPadding = new float[(int)(waveReader.SampleRate * 0.3)]; @@ -53,7 +51,7 @@ static void Main(string[] args) { kws.Decode(s); var result = kws.GetResult(s); - if (result.Keyword != "") + if (result.Keyword != string.Empty) { Console.WriteLine("Detected: {0}", result.Keyword); } @@ -70,7 +68,7 @@ static void Main(string[] args) { kws.Decode(s); var result = kws.GetResult(s); - if (result.Keyword != "") + if (result.Keyword != string.Empty) { Console.WriteLine("Detected: {0}", result.Keyword); } @@ -89,7 +87,7 @@ static void Main(string[] args) { kws.Decode(s); var result = kws.GetResult(s); - if (result.Keyword != "") + if (result.Keyword != string.Empty) { Console.WriteLine("Detected: {0}", result.Keyword); } diff --git a/dotnet-examples/keyword-spotting-from-files/keyword-spotting-from-files.csproj b/dotnet-examples/keyword-spotting-from-files/keyword-spotting-from-files.csproj index 992f8e0e3..21b9d3ea5 100644 --- a/dotnet-examples/keyword-spotting-from-files/keyword-spotting-from-files.csproj +++ b/dotnet-examples/keyword-spotting-from-files/keyword-spotting-from-files.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 keyword_spotting_from_files enable enable diff --git a/dotnet-examples/keyword-spotting-from-microphone/Program.cs b/dotnet-examples/keyword-spotting-from-microphone/Program.cs index cb0c922f4..05d22aee0 100644 --- a/dotnet-examples/keyword-spotting-from-microphone/Program.cs +++ b/dotnet-examples/keyword-spotting-from-microphone/Program.cs @@ -12,12 +12,9 @@ // // dotnet run +using PortAudioSharp; using SherpaOnnx; -using System.Collections.Generic; using System.Runtime.InteropServices; -using System; - -using PortAudioSharp; class KeywordSpotterDemo { @@ -41,11 +38,11 @@ static void Main(string[] args) var filename = "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/test_wavs/3.wav"; - WaveReader waveReader = new WaveReader(filename); + var waveReader = new WaveReader(filename); Console.WriteLine("----------Use pre-defined keywords----------"); - OnlineStream s = kws.CreateStream(); + var s = kws.CreateStream(); Console.WriteLine(PortAudio.VersionInfo.versionText); PortAudio.Initialize(); @@ -54,7 +51,7 @@ static void Main(string[] args) for (int i = 0; i != PortAudio.DeviceCount; ++i) { Console.WriteLine($" Device {i}"); - DeviceInfo deviceInfo = PortAudio.GetDeviceInfo(i); + var deviceInfo = PortAudio.GetDeviceInfo(i); Console.WriteLine($" Name: {deviceInfo.name}"); Console.WriteLine($" Max input channels: {deviceInfo.maxInputChannels}"); Console.WriteLine($" Default sample rate: {deviceInfo.defaultSampleRate}"); @@ -66,12 +63,12 @@ static void Main(string[] args) Environment.Exit(1); } - DeviceInfo info = PortAudio.GetDeviceInfo(deviceIndex); + var info = PortAudio.GetDeviceInfo(deviceIndex); Console.WriteLine(); Console.WriteLine($"Use default device {deviceIndex} ({info.name})"); - StreamParameters param = new StreamParameters(); + var param = new StreamParameters(); param.device = deviceIndex; param.channelCount = 1; param.sampleFormat = SampleFormat.Float32; @@ -79,21 +76,21 @@ static void Main(string[] args) param.hostApiSpecificStreamInfo = IntPtr.Zero; PortAudioSharp.Stream.Callback callback = (IntPtr input, IntPtr output, - UInt32 frameCount, + uint frameCount, ref StreamCallbackTimeInfo timeInfo, StreamCallbackFlags statusFlags, IntPtr userData ) => { - float[] samples = new float[frameCount]; - Marshal.Copy(input, samples, 0, (Int32)frameCount); + var samples = new float[frameCount]; + Marshal.Copy(input, samples, 0, (int)frameCount); s.AcceptWaveform(config.FeatConfig.SampleRate, samples); return StreamCallbackResult.Continue; }; - PortAudioSharp.Stream stream = new PortAudioSharp.Stream(inParams: param, outParams: null, sampleRate: config.FeatConfig.SampleRate, + var stream = new PortAudioSharp.Stream(inParams: param, outParams: null, sampleRate: config.FeatConfig.SampleRate, framesPerBuffer: 0, streamFlags: StreamFlags.ClipOff, callback: callback, @@ -113,15 +110,13 @@ IntPtr userData } var result = kws.GetResult(s); - if (result.Keyword != "") + if (result.Keyword != string.Empty) { Console.WriteLine("Detected: {0}", result.Keyword); } Thread.Sleep(200); // ms } - - PortAudio.Terminate(); } } diff --git a/dotnet-examples/keyword-spotting-from-microphone/keyword-spotting-from-microphone.csproj b/dotnet-examples/keyword-spotting-from-microphone/keyword-spotting-from-microphone.csproj index b3afae784..12415b81b 100644 --- a/dotnet-examples/keyword-spotting-from-microphone/keyword-spotting-from-microphone.csproj +++ b/dotnet-examples/keyword-spotting-from-microphone/keyword-spotting-from-microphone.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 keyword_spotting_from_microphone enable enable diff --git a/dotnet-examples/offline-decode-files/Program.cs b/dotnet-examples/offline-decode-files/Program.cs index d855da6f8..0d944e5a3 100644 --- a/dotnet-examples/offline-decode-files/Program.cs +++ b/dotnet-examples/offline-decode-files/Program.cs @@ -5,17 +5,14 @@ // Please refer to // https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html // to download non-streaming models -using CommandLine.Text; using CommandLine; +using CommandLine.Text; using SherpaOnnx; -using System.Collections.Generic; -using System; class OfflineDecodeFiles { class Options { - [Option("sample-rate", Required = false, Default = 16000, HelpText = "Sample rate of the data used to train the model")] public int SampleRate { get; set; } = 16000; @@ -23,58 +20,58 @@ class Options public int FeatureDim { get; set; } = 80; [Option(Required = false, HelpText = "Path to tokens.txt")] - public string Tokens { get; set; } = ""; + public string Tokens { get; set; } = string.Empty; [Option(Required = false, Default = "", HelpText = "Path to transducer encoder.onnx. Used only for transducer models")] - public string Encoder { get; set; } = ""; + public string Encoder { get; set; } = string.Empty; [Option(Required = false, Default = "", HelpText = "Path to transducer decoder.onnx. Used only for transducer models")] - public string Decoder { get; set; } = ""; + public string Decoder { get; set; } = string.Empty; [Option(Required = false, Default = "", HelpText = "Path to transducer joiner.onnx. Used only for transducer models")] - public string Joiner { get; set; } = ""; + public string Joiner { get; set; } = string.Empty; [Option("model-type", Required = false, Default = "", HelpText = "model type")] - public string ModelType { get; set; } = ""; + public string ModelType { get; set; } = string.Empty; [Option("whisper-encoder", Required = false, Default = "", HelpText = "Path to whisper encoder.onnx. Used only for whisper models")] - public string WhisperEncoder { get; set; } = ""; + public string WhisperEncoder { get; set; } = string.Empty; [Option("whisper-decoder", Required = false, Default = "", HelpText = "Path to whisper decoder.onnx. Used only for whisper models")] - public string WhisperDecoder { get; set; } = ""; + public string WhisperDecoder { get; set; } = string.Empty; [Option("whisper-language", Required = false, Default = "", HelpText = "Language of the input file. Can be empty")] - public string WhisperLanguage { get; set; } = ""; + public string WhisperLanguage { get; set; } = string.Empty; [Option("whisper-task", Required = false, Default = "transcribe", HelpText = "transcribe or translate")] public string WhisperTask { get; set; } = "transcribe"; [Option("moonshine-preprocessor", Required = false, Default = "", HelpText = "Path to preprocess.onnx. Used only for Moonshine models")] - public string MoonshinePreprocessor { get; set; } = ""; + public string MoonshinePreprocessor { get; set; } = string.Empty; [Option("moonshine-encoder", Required = false, Default = "", HelpText = "Path to encode.onnx. Used only for Moonshine models")] - public string MoonshineEncoder { get; set; } = ""; + public string MoonshineEncoder { get; set; } = string.Empty; [Option("moonshine-uncached-decoder", Required = false, Default = "", HelpText = "Path to uncached_decode.onnx. Used only for Moonshine models")] - public string MoonshineUncachedDecoder { get; set; } = ""; + public string MoonshineUncachedDecoder { get; set; } = string.Empty; [Option("moonshine-cached-decoder", Required = false, Default = "", HelpText = "Path to cached_decode.onnx. Used only for Moonshine models")] - public string MoonshineCachedDecoder { get; set; } = ""; + public string MoonshineCachedDecoder { get; set; } = string.Empty; [Option("tdnn-model", Required = false, Default = "", HelpText = "Path to tdnn yesno model")] - public string TdnnModel { get; set; } = ""; + public string TdnnModel { get; set; } = string.Empty; [Option(Required = false, HelpText = "Path to model.onnx. Used only for paraformer models")] - public string Paraformer { get; set; } = ""; + public string Paraformer { get; set; } = string.Empty; [Option("nemo-ctc", Required = false, HelpText = "Path to model.onnx. Used only for NeMo CTC models")] - public string NeMoCtc { get; set; } = ""; + public string NeMoCtc { get; set; } = string.Empty; [Option("telespeech-ctc", Required = false, HelpText = "Path to model.onnx. Used only for TeleSpeech CTC models")] - public string TeleSpeechCtc { get; set; } = ""; + public string TeleSpeechCtc { get; set; } = string.Empty; [Option("sense-voice-model", Required = false, HelpText = "Path to model.onnx. Used only for SenseVoice CTC models")] - public string SenseVoiceModel { get; set; } = ""; + public string SenseVoiceModel { get; set; } = string.Empty; [Option("sense-voice-use-itn", Required = false, HelpText = "1 to use inverse text normalization for sense voice.")] public int SenseVoiceUseItn { get; set; } = 1; @@ -88,7 +85,7 @@ class Options [Option("rule-fsts", Required = false, Default = "", HelpText = "If not empty, path to rule fst for inverse text normalization")] - public string RuleFsts { get; set; } = ""; + public string RuleFsts { get; set; } = string.Empty; [Option("max-active-paths", Required = false, Default = 4, HelpText = @"Used only when --decoding--method is modified_beam_search. @@ -96,7 +93,7 @@ class Options public int MaxActivePaths { get; set; } = 4; [Option("hotwords-file", Required = false, Default = "", HelpText = "Path to hotwords.txt")] - public string HotwordsFile { get; set; } = ""; + public string HotwordsFile { get; set; } = string.Empty; [Option("hotwords-score", Required = false, Default = 1.5F, HelpText = "hotwords score")] public float HotwordsScore { get; set; } = 1.5F; @@ -117,7 +114,7 @@ static void Main(string[] args) private static void DisplayHelp(ParserResult result, IEnumerable errs) { - string usage = @" + var usage = @" # Zipformer dotnet run \ @@ -213,42 +210,42 @@ private static void Run(Options options) config.ModelConfig.Tokens = options.Tokens; - if (!String.IsNullOrEmpty(options.Encoder)) + if (!string.IsNullOrEmpty(options.Encoder)) { // this is a transducer model config.ModelConfig.Transducer.Encoder = options.Encoder; config.ModelConfig.Transducer.Decoder = options.Decoder; config.ModelConfig.Transducer.Joiner = options.Joiner; } - else if (!String.IsNullOrEmpty(options.Paraformer)) + else if (!string.IsNullOrEmpty(options.Paraformer)) { config.ModelConfig.Paraformer.Model = options.Paraformer; } - else if (!String.IsNullOrEmpty(options.NeMoCtc)) + else if (!string.IsNullOrEmpty(options.NeMoCtc)) { config.ModelConfig.NeMoCtc.Model = options.NeMoCtc; } - else if (!String.IsNullOrEmpty(options.TeleSpeechCtc)) + else if (!string.IsNullOrEmpty(options.TeleSpeechCtc)) { config.ModelConfig.TeleSpeechCtc = options.TeleSpeechCtc; } - else if (!String.IsNullOrEmpty(options.WhisperEncoder)) + else if (!string.IsNullOrEmpty(options.WhisperEncoder)) { config.ModelConfig.Whisper.Encoder = options.WhisperEncoder; config.ModelConfig.Whisper.Decoder = options.WhisperDecoder; config.ModelConfig.Whisper.Language = options.WhisperLanguage; config.ModelConfig.Whisper.Task = options.WhisperTask; } - else if (!String.IsNullOrEmpty(options.TdnnModel)) + else if (!string.IsNullOrEmpty(options.TdnnModel)) { config.ModelConfig.Tdnn.Model = options.TdnnModel; } - else if (!String.IsNullOrEmpty(options.SenseVoiceModel)) + else if (!string.IsNullOrEmpty(options.SenseVoiceModel)) { config.ModelConfig.SenseVoice.Model = options.SenseVoiceModel; config.ModelConfig.SenseVoice.UseInverseTextNormalization = options.SenseVoiceUseItn; } - else if (!String.IsNullOrEmpty(options.MoonshinePreprocessor)) + else if (!string.IsNullOrEmpty(options.MoonshinePreprocessor)) { config.ModelConfig.Moonshine.Preprocessor = options.MoonshinePreprocessor; config.ModelConfig.Moonshine.Encoder = options.MoonshineEncoder; @@ -270,17 +267,17 @@ private static void Run(Options options) config.ModelConfig.Debug = 0; - OfflineRecognizer recognizer = new OfflineRecognizer(config); + var recognizer = new OfflineRecognizer(config); - string[] files = options.Files.ToArray(); + var files = options.Files.ToArray(); // We create a separate stream for each file - List streams = new List(); + var streams = new List(); streams.EnsureCapacity(files.Length); for (int i = 0; i != files.Length; ++i) { - OfflineStream s = recognizer.CreateStream(); + var s = recognizer.CreateStream(); WaveReader waveReader = new WaveReader(files[i]); s.AcceptWaveform(waveReader.SampleRate, waveReader.Samples); @@ -299,7 +296,7 @@ private static void Run(Options options) Console.WriteLine("Tokens: [{0}]", string.Join(", ", r.Tokens)); if (r.Timestamps != null && r.Timestamps.Length > 0) { Console.Write("Timestamps: ["); - var sep = ""; + var sep = string.Empty; for (int k = 0; k != r.Timestamps.Length; ++k) { Console.Write("{0}{1}", sep, r.Timestamps[k].ToString("0.00")); diff --git a/dotnet-examples/offline-decode-files/offline-decode-files.csproj b/dotnet-examples/offline-decode-files/offline-decode-files.csproj index ffdfb6ace..5b28d48b7 100644 --- a/dotnet-examples/offline-decode-files/offline-decode-files.csproj +++ b/dotnet-examples/offline-decode-files/offline-decode-files.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 offline_decode_files enable enable diff --git a/dotnet-examples/offline-punctuation/Program.cs b/dotnet-examples/offline-punctuation/Program.cs index d299f8abc..6f85237b6 100644 --- a/dotnet-examples/offline-punctuation/Program.cs +++ b/dotnet-examples/offline-punctuation/Program.cs @@ -12,8 +12,6 @@ // dotnet run using SherpaOnnx; -using System.Collections.Generic; -using System; class OfflinePunctuationDemo { @@ -25,14 +23,14 @@ static void Main(string[] args) config.Model.NumThreads = 1; var punct = new OfflinePunctuation(config); - string[] textList = new string[] { + var textList = new string[] { "这是一个测试你好吗How are you我很好thank you are you ok谢谢你", "我们都是木头人不会说话不会动", "The African blogosphere is rapidly expanding bringing more voices online in the form of commentaries opinions analyses rants and poetry", }; Console.WriteLine("---------"); - foreach (string text in textList) + foreach (var text in textList) { string textWithPunct = punct.AddPunct(text); Console.WriteLine("Input text: {0}", text); diff --git a/dotnet-examples/offline-punctuation/offline-punctuation.csproj b/dotnet-examples/offline-punctuation/offline-punctuation.csproj index 2d94fcb38..0e3ee42a9 100644 --- a/dotnet-examples/offline-punctuation/offline-punctuation.csproj +++ b/dotnet-examples/offline-punctuation/offline-punctuation.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 offline_punctuation enable enable diff --git a/dotnet-examples/offline-speaker-diarization/Program.cs b/dotnet-examples/offline-speaker-diarization/Program.cs index 45316fe75..4d8d91b0e 100644 --- a/dotnet-examples/offline-speaker-diarization/Program.cs +++ b/dotnet-examples/offline-speaker-diarization/Program.cs @@ -34,7 +34,6 @@ dotnet run */ using SherpaOnnx; -using System; class OfflineSpeakerDiarizationDemo { @@ -54,7 +53,7 @@ static void Main(string[] args) var sd = new OfflineSpeakerDiarization(config); var testWaveFile = "./0-four-speakers-zh.wav"; - WaveReader waveReader = new WaveReader(testWaveFile); + var waveReader = new WaveReader(testWaveFile); if (sd.SampleRate != waveReader.SampleRate) { Console.WriteLine($"Expected sample rate: {sd.SampleRate}. Given: {waveReader.SampleRate}"); @@ -65,19 +64,19 @@ static void Main(string[] args) // var segments = sd.Process(waveReader.Samples); // this one is also ok - var MyProgressCallback = (int numProcessedChunks, int numTotalChunks, IntPtr arg) => + var progressCallback = (int numProcessedChunks, int numTotalChunks, IntPtr arg) => { - float progress = 100.0F * numProcessedChunks / numTotalChunks; - Console.WriteLine("Progress {0}%", String.Format("{0:0.00}", progress)); + var progress = 100.0F * numProcessedChunks / numTotalChunks; + Console.WriteLine("Progress {0}%", string.Format("{0:0.00}", progress)); return 0; }; - var callback = new OfflineSpeakerDiarizationProgressCallback(MyProgressCallback); + var callback = new OfflineSpeakerDiarizationProgressCallback(progressCallback); var segments = sd.ProcessWithCallback(waveReader.Samples, callback, IntPtr.Zero); foreach (var s in segments) { - Console.WriteLine("{0} -- {1} speaker_{2}", String.Format("{0:0.00}", s.Start), String.Format("{0:0.00}", s.End), s.Speaker); + Console.WriteLine("{0} -- {1} speaker_{2}", string.Format("{0:0.00}", s.Start), string.Format("{0:0.00}", s.End), s.Speaker); } } } diff --git a/dotnet-examples/offline-speaker-diarization/offline-speaker-diarization.csproj b/dotnet-examples/offline-speaker-diarization/offline-speaker-diarization.csproj index 3374dbca1..c7b15faa5 100644 --- a/dotnet-examples/offline-speaker-diarization/offline-speaker-diarization.csproj +++ b/dotnet-examples/offline-speaker-diarization/offline-speaker-diarization.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 offline_speaker_diarization enable enable diff --git a/dotnet-examples/offline-tts-play/Program.cs b/dotnet-examples/offline-tts-play/Program.cs index a142c127e..65eb22bf4 100644 --- a/dotnet-examples/offline-tts-play/Program.cs +++ b/dotnet-examples/offline-tts-play/Program.cs @@ -10,15 +10,12 @@ // Note that you need a speaker to run this file since it will play // the generated audio as it is generating. -using CommandLine.Text; using CommandLine; +using CommandLine.Text; using PortAudioSharp; using SherpaOnnx; using System.Collections.Concurrent; -using System.Collections.Generic; using System.Runtime.InteropServices; -using System.Threading; -using System; class OfflineTtsPlayDemo { @@ -26,13 +23,13 @@ class Options { [Option("tts-rule-fsts", Required = false, Default = "", HelpText = "path to rule.fst")] - public string RuleFsts { get; set; } + public string? RuleFsts { get; set; } [Option("vits-dict-dir", Required = false, Default = "", HelpText = "Path to the directory containing dict for jieba.")] - public string DictDir { get; set; } + public string? DictDir { get; set; } [Option("vits-data-dir", Required = false, Default = "", HelpText = "Path to the directory containing dict for espeak-ng.")] - public string DataDir { get; set; } + public string? DataDir { get; set; } [Option("vits-length-scale", Required = false, Default = 1, HelpText = "speech speed. Larger->Slower; Smaller->faster")] public float LengthScale { get; set; } @@ -44,10 +41,10 @@ class Options public float NoiseScaleW { get; set; } [Option("vits-lexicon", Required = false, Default = "", HelpText = "Path to lexicon.txt")] - public string Lexicon { get; set; } + public string? Lexicon { get; set; } [Option("vits-tokens", Required = false, Default = "", HelpText = "Path to tokens.txt")] - public string Tokens { get; set; } + public string? Tokens { get; set; } [Option("tts-max-num-sentences", Required = false, Default = 1, HelpText = "Maximum number of sentences that we process at a time.")] public int MaxNumSentences { get; set; } @@ -56,16 +53,16 @@ class Options public int Debug { get; set; } [Option("vits-model", Required = true, HelpText = "Path to VITS model")] - public string Model { get; set; } + public string? Model { get; set; } [Option("sid", Required = false, Default = 0, HelpText = "Speaker ID")] public int SpeakerId { get; set; } [Option("text", Required = true, HelpText = "Text to synthesize")] - public string Text { get; set; } + public string? Text { get; set; } [Option("output-filename", Required = true, Default = "./generated.wav", HelpText = "Path to save the generated audio")] - public string OutputFilename { get; set; } + public string? OutputFilename { get; set; } } static void Main(string[] args) @@ -124,10 +121,9 @@ to download more models. Console.WriteLine(helpText); } - private static void Run(Options options) { - OfflineTtsConfig config = new OfflineTtsConfig(); + var config = new OfflineTtsConfig(); config.Model.Vits.Model = options.Model; config.Model.Vits.Lexicon = options.Lexicon; config.Model.Vits.Tokens = options.Tokens; @@ -142,10 +138,9 @@ private static void Run(Options options) config.RuleFsts = options.RuleFsts; config.MaxNumSentences = options.MaxNumSentences; - OfflineTts tts = new OfflineTts(config); - float speed = 1.0f / options.LengthScale; - int sid = options.SpeakerId; - + var tts = new OfflineTts(config); + var speed = 1.0f / options.LengthScale; + var sid = options.SpeakerId; Console.WriteLine(PortAudio.VersionInfo.versionText); PortAudio.Initialize(); @@ -166,11 +161,11 @@ private static void Run(Options options) Environment.Exit(1); } - DeviceInfo info = PortAudio.GetDeviceInfo(deviceIndex); + var info = PortAudio.GetDeviceInfo(deviceIndex); Console.WriteLine(); Console.WriteLine($"Use output default device {deviceIndex} ({info.name})"); - StreamParameters param = new StreamParameters(); + var param = new StreamParameters(); param.device = deviceIndex; param.channelCount = 1; param.sampleFormat = SampleFormat.Float32; @@ -178,7 +173,7 @@ private static void Run(Options options) param.hostApiSpecificStreamInfo = IntPtr.Zero; // https://learn.microsoft.com/en-us/dotnet/standard/collections/thread-safe/blockingcollection-overview - BlockingCollection dataItems = new BlockingCollection(); + var dataItems = new BlockingCollection(); var MyCallback = (IntPtr samples, int n) => { @@ -193,9 +188,9 @@ private static void Run(Options options) return 1; }; - bool playFinished = false; + var playFinished = false; - float[] lastSampleArray = null; + float[]? lastSampleArray = null; int lastIndex = 0; // not played PortAudioSharp.Stream.Callback playCallback = (IntPtr input, IntPtr output, @@ -270,10 +265,10 @@ IntPtr userData stream.Start(); - OfflineTtsCallback callback = new OfflineTtsCallback(MyCallback); + var callback = new OfflineTtsCallback(MyCallback); - OfflineTtsGeneratedAudio audio = tts.GenerateWithCallback(options.Text, speed, sid, callback); - bool ok = audio.SaveToWaveFile(options.OutputFilename); + var audio = tts.GenerateWithCallback(options.Text, speed, sid, callback); + var ok = audio.SaveToWaveFile(options.OutputFilename); if (ok) { diff --git a/dotnet-examples/offline-tts-play/offline-tts-play.csproj b/dotnet-examples/offline-tts-play/offline-tts-play.csproj index d28ae62c8..b777bcafe 100644 --- a/dotnet-examples/offline-tts-play/offline-tts-play.csproj +++ b/dotnet-examples/offline-tts-play/offline-tts-play.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 offline_tts_play enable enable diff --git a/dotnet-examples/offline-tts/Program.cs b/dotnet-examples/offline-tts/Program.cs index 6216095f4..f434ebf19 100644 --- a/dotnet-examples/offline-tts/Program.cs +++ b/dotnet-examples/offline-tts/Program.cs @@ -6,28 +6,25 @@ // and // https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models // to download pre-trained models -using CommandLine.Text; using CommandLine; +using CommandLine.Text; using SherpaOnnx; -using System.Collections.Generic; -using System; class OfflineTtsDemo { class Options { - [Option("tts-rule-fsts", Required = false, Default = "", HelpText = "path to rule.fst")] - public string RuleFsts { get; set; } = ""; + public string RuleFsts { get; set; } = string.Empty; [Option("tts-rule-fars", Required = false, Default = "", HelpText = "path to rule.far")] - public string RuleFars { get; set; } = ""; + public string RuleFars { get; set; } = string.Empty; [Option("vits-dict-dir", Required = false, Default = "", HelpText = "Path to the directory containing dict for jieba.")] - public string DictDir { get; set; } = ""; + public string DictDir { get; set; } = string.Empty; [Option("vits-data-dir", Required = false, Default = "", HelpText = "Path to the directory containing dict for espeak-ng.")] - public string DataDir { get; set; } = ""; + public string DataDir { get; set; } = string.Empty; [Option("vits-length-scale", Required = false, Default = 1, HelpText = "speech speed. Larger->Slower; Smaller->faster")] public float LengthScale { get; set; } = 1; @@ -39,10 +36,10 @@ class Options public float NoiseScaleW { get; set; } = 0.8F; [Option("vits-lexicon", Required = false, Default = "", HelpText = "Path to lexicon.txt")] - public string Lexicon { get; set; } = ""; + public string Lexicon { get; set; } = string.Empty; [Option("vits-tokens", Required = false, Default = "", HelpText = "Path to tokens.txt")] - public string Tokens { get; set; } = ""; + public string Tokens { get; set; } = string.Empty; [Option("tts-max-num-sentences", Required = false, Default = 1, HelpText = "Maximum number of sentences that we process at a time.")] public int MaxNumSentences { get; set; } = 1; @@ -51,13 +48,13 @@ class Options public int Debug { get; set; } = 0; [Option("vits-model", Required = true, HelpText = "Path to VITS model")] - public string Model { get; set; } = ""; + public string Model { get; set; } = string.Empty; [Option("sid", Required = false, Default = 0, HelpText = "Speaker ID")] public int SpeakerId { get; set; } = 0; [Option("text", Required = true, HelpText = "Text to synthesize")] - public string Text { get; set; } = ""; + public string Text { get; set; } = string.Empty; [Option("output-filename", Required = true, Default = "./generated.wav", HelpText = "Path to save the generated audio")] public string OutputFilename { get; set; } = "./generated.wav"; @@ -65,7 +62,7 @@ class Options static void Main(string[] args) { - var parser = new CommandLine.Parser(with => with.HelpWriter = null); + var parser = new Parser(with => with.HelpWriter = null); var parserResult = parser.ParseArguments(args); parserResult @@ -75,7 +72,7 @@ static void Main(string[] args) private static void DisplayHelp(ParserResult result, IEnumerable errs) { - string usage = @" + var usage = @" # vits-aishell3 curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-icefall-zh-aishell3.tar.bz2 @@ -122,7 +119,7 @@ to download more models. private static void Run(Options options) { - OfflineTtsConfig config = new OfflineTtsConfig(); + var config = new OfflineTtsConfig(); config.Model.Vits.Model = options.Model; config.Model.Vits.Lexicon = options.Lexicon; config.Model.Vits.Tokens = options.Tokens; @@ -138,11 +135,11 @@ private static void Run(Options options) config.RuleFars = options.RuleFars; config.MaxNumSentences = options.MaxNumSentences; - OfflineTts tts = new OfflineTts(config); - float speed = 1.0f / options.LengthScale; - int sid = options.SpeakerId; - OfflineTtsGeneratedAudio audio = tts.Generate(options.Text, speed, sid); - bool ok = audio.SaveToWaveFile(options.OutputFilename); + var tts = new OfflineTts(config); + var speed = 1.0f / options.LengthScale; + var sid = options.SpeakerId; + var audio = tts.Generate(options.Text, speed, sid); + var ok = audio.SaveToWaveFile(options.OutputFilename); if (ok) { diff --git a/dotnet-examples/offline-tts/offline-tts.csproj b/dotnet-examples/offline-tts/offline-tts.csproj index 48548fc4c..20b048f19 100644 --- a/dotnet-examples/offline-tts/offline-tts.csproj +++ b/dotnet-examples/offline-tts/offline-tts.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 offline_tts enable enable diff --git a/dotnet-examples/online-decode-files/Program.cs b/dotnet-examples/online-decode-files/Program.cs index ad53624de..a1f01be57 100644 --- a/dotnet-examples/online-decode-files/Program.cs +++ b/dotnet-examples/online-decode-files/Program.cs @@ -6,40 +6,37 @@ // https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html // to download streaming models -using CommandLine.Text; using CommandLine; +using CommandLine.Text; using SherpaOnnx; -using System.Collections.Generic; -using System.Linq; -using System; class OnlineDecodeFiles { class Options { [Option(Required = true, HelpText = "Path to tokens.txt")] - public string Tokens { get; set; } = ""; + public string Tokens { get; set; } = string.Empty; [Option(Required = false, Default = "cpu", HelpText = "Provider, e.g., cpu, coreml")] - public string Provider { get; set; } = ""; + public string Provider { get; set; } = string.Empty; [Option(Required = false, HelpText = "Path to transducer encoder.onnx")] - public string Encoder { get; set; } = ""; + public string Encoder { get; set; } = string.Empty; [Option(Required = false, HelpText = "Path to transducer decoder.onnx")] - public string Decoder { get; set; } = ""; + public string Decoder { get; set; } = string.Empty; [Option(Required = false, HelpText = "Path to transducer joiner.onnx")] - public string Joiner { get; set; } = ""; + public string Joiner { get; set; } = string.Empty; [Option("paraformer-encoder", Required = false, HelpText = "Path to paraformer encoder.onnx")] - public string ParaformerEncoder { get; set; } = ""; + public string ParaformerEncoder { get; set; } = string.Empty; [Option("paraformer-decoder", Required = false, HelpText = "Path to paraformer decoder.onnx")] - public string ParaformerDecoder { get; set; } = ""; + public string ParaformerDecoder { get; set; } = string.Empty; [Option("zipformer2-ctc", Required = false, HelpText = "Path to zipformer2 CTC onnx model")] - public string Zipformer2Ctc { get; set; } = ""; + public string Zipformer2Ctc { get; set; } = string.Empty; [Option("num-threads", Required = false, Default = 1, HelpText = "Number of threads for computation")] public int NumThreads { get; set; } = 1; @@ -80,15 +77,14 @@ larger than this value after something that is not blank has been decoded. Used public float Rule3MinUtteranceLength { get; set; } = 20.0F; [Option("hotwords-file", Required = false, Default = "", HelpText = "Path to hotwords.txt")] - public string HotwordsFile { get; set; } = ""; + public string HotwordsFile { get; set; } = string.Empty; [Option("hotwords-score", Required = false, Default = 1.5F, HelpText = "hotwords score")] public float HotwordsScore { get; set; } = 1.5F; [Option("rule-fsts", Required = false, Default = "", HelpText = "If not empty, path to rule fst for inverse text normalization")] - public string RuleFsts { get; set; } = ""; - + public string RuleFsts { get; set; } = string.Empty; [Option("files", Required = true, HelpText = "Audio files for decoding")] public IEnumerable Files { get; set; } = new string[] {}; @@ -162,7 +158,7 @@ to download pre-trained streaming models. private static void Run(Options options) { - OnlineRecognizerConfig config = new OnlineRecognizerConfig(); + var config = new OnlineRecognizerConfig(); config.FeatConfig.SampleRate = options.SampleRate; // All models from icefall using feature dim 80. @@ -194,22 +190,22 @@ private static void Run(Options options) config.HotwordsScore = options.HotwordsScore; config.RuleFsts = options.RuleFsts; - OnlineRecognizer recognizer = new OnlineRecognizer(config); + var recognizer = new OnlineRecognizer(config); - string[] files = options.Files.ToArray(); + var files = options.Files.ToArray(); // We create a separate stream for each file - List streams = new List(); + var streams = new List(); streams.EnsureCapacity(files.Length); for (int i = 0; i != files.Length; ++i) { - OnlineStream s = recognizer.CreateStream(); + var s = recognizer.CreateStream(); - WaveReader waveReader = new WaveReader(files[i]); + var waveReader = new WaveReader(files[i]); s.AcceptWaveform(waveReader.SampleRate, waveReader.Samples); - float[] tailPadding = new float[(int)(waveReader.SampleRate * 0.3)]; + var tailPadding = new float[(int)(waveReader.SampleRate * 0.3)]; s.AcceptWaveform(waveReader.SampleRate, tailPadding); s.InputFinished(); @@ -230,7 +226,7 @@ private static void Run(Options options) // display results for (int i = 0; i != files.Length; ++i) { - OnlineRecognizerResult r = recognizer.GetResult(streams[i]); + var r = recognizer.GetResult(streams[i]); var text = r.Text; var tokens = r.Tokens; Console.WriteLine("--------------------"); @@ -238,7 +234,7 @@ private static void Run(Options options) Console.WriteLine("text: {0}", text); Console.WriteLine("tokens: [{0}]", string.Join(", ", tokens)); Console.Write("timestamps: ["); - r.Timestamps.ToList().ForEach(i => Console.Write(String.Format("{0:0.00}", i) + ", ")); + r.Timestamps.ToList().ForEach(i => Console.Write(string.Format("{0:0.00}", i) + ", ")); Console.WriteLine("]"); } Console.WriteLine("--------------------"); diff --git a/dotnet-examples/online-decode-files/online-decode-files.csproj b/dotnet-examples/online-decode-files/online-decode-files.csproj index 0ff581102..f1cc3baa7 100644 --- a/dotnet-examples/online-decode-files/online-decode-files.csproj +++ b/dotnet-examples/online-decode-files/online-decode-files.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 online_decode_files enable enable diff --git a/dotnet-examples/sherpa-onnx.sln b/dotnet-examples/sherpa-onnx.sln index 0bff03f5c..1ebcdf464 100644 --- a/dotnet-examples/sherpa-onnx.sln +++ b/dotnet-examples/sherpa-onnx.sln @@ -29,9 +29,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "keyword-spotting-from-files EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "keyword-spotting-from-microphone", "keyword-spotting-from-microphone\keyword-spotting-from-microphone.csproj", "{AEE0ED2B-C86F-4952-863C-EAD3219CB4EC}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TTS", "TTS\TTS.csproj", "{DACE4A18-4FC8-4437-92BF-5A90BA81286C}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "offline-speaker-diarization", "offline-speaker-diarization\offline-speaker-diarization.csproj", "{D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "offline-speaker-diarization", "offline-speaker-diarization\offline-speaker-diarization.csproj", "{D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -91,10 +89,6 @@ Global {AEE0ED2B-C86F-4952-863C-EAD3219CB4EC}.Debug|Any CPU.Build.0 = Debug|Any CPU {AEE0ED2B-C86F-4952-863C-EAD3219CB4EC}.Release|Any CPU.ActiveCfg = Release|Any CPU {AEE0ED2B-C86F-4952-863C-EAD3219CB4EC}.Release|Any CPU.Build.0 = Release|Any CPU - {DACE4A18-4FC8-4437-92BF-5A90BA81286C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {DACE4A18-4FC8-4437-92BF-5A90BA81286C}.Debug|Any CPU.Build.0 = Debug|Any CPU - {DACE4A18-4FC8-4437-92BF-5A90BA81286C}.Release|Any CPU.ActiveCfg = Release|Any CPU - {DACE4A18-4FC8-4437-92BF-5A90BA81286C}.Release|Any CPU.Build.0 = Release|Any CPU {D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}.Debug|Any CPU.Build.0 = Debug|Any CPU {D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/dotnet-examples/speaker-identification/Program.cs b/dotnet-examples/speaker-identification/Program.cs index aef53e851..20ac70390 100644 --- a/dotnet-examples/speaker-identification/Program.cs +++ b/dotnet-examples/speaker-identification/Program.cs @@ -16,20 +16,18 @@ // dotnet run using SherpaOnnx; -using System.Collections.Generic; -using System; class SpeakerIdentificationDemo { - public static float[] ComputeEmbedding(SpeakerEmbeddingExtractor extractor, String filename) + public static float[] ComputeEmbedding(SpeakerEmbeddingExtractor extractor, string filename) { - WaveReader reader = new WaveReader(filename); + var reader = new WaveReader(filename); - OnlineStream stream = extractor.CreateStream(); + var stream = extractor.CreateStream(); stream.AcceptWaveform(reader.SampleRate, reader.Samples); stream.InputFinished(); - float[] embedding = extractor.Compute(stream); + var embedding = extractor.Compute(stream); return embedding; } @@ -43,25 +41,25 @@ static void Main(string[] args) var manager = new SpeakerEmbeddingManager(extractor.Dim); - string[] spk1Files = + var spk1Files = new string[] { "./sr-data/enroll/fangjun-sr-1.wav", "./sr-data/enroll/fangjun-sr-2.wav", "./sr-data/enroll/fangjun-sr-3.wav", }; - float[][] spk1Vec = new float[spk1Files.Length][]; + var spk1Vec = new float[spk1Files.Length][]; for (int i = 0; i < spk1Files.Length; ++i) { spk1Vec[i] = ComputeEmbedding(extractor, spk1Files[i]); } - string[] spk2Files = + var spk2Files = new string[] { "./sr-data/enroll/leijun-sr-1.wav", "./sr-data/enroll/leijun-sr-2.wav", }; - float[][] spk2Vec = new float[spk2Files.Length][]; + var spk2Vec = new float[spk2Files.Length][]; for (int i = 0; i < spk2Files.Length; ++i) { @@ -100,14 +98,14 @@ static void Main(string[] args) Console.WriteLine("---All speakers---"); - string[] allSpeakers = manager.GetAllSpeakers(); + var allSpeakers = manager.GetAllSpeakers(); foreach (var s in allSpeakers) { Console.WriteLine(s); } Console.WriteLine("------------"); - string[] testFiles = + var testFiles = new string[] { "./sr-data/test/fangjun-test-sr-1.wav", "./sr-data/test/leijun-test-sr-1.wav", @@ -117,9 +115,9 @@ static void Main(string[] args) float threshold = 0.6f; foreach (var file in testFiles) { - float[] embedding = ComputeEmbedding(extractor, file); + var embedding = ComputeEmbedding(extractor, file); - String name = manager.Search(embedding, threshold); + var name = manager.Search(embedding, threshold); if (name == "") { name = ""; diff --git a/dotnet-examples/speaker-identification/speaker-identification.csproj b/dotnet-examples/speaker-identification/speaker-identification.csproj index 7c857fa54..45a42f49e 100644 --- a/dotnet-examples/speaker-identification/speaker-identification.csproj +++ b/dotnet-examples/speaker-identification/speaker-identification.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 speaker_identification enable enable diff --git a/dotnet-examples/speech-recognition-from-microphone/Program.cs b/dotnet-examples/speech-recognition-from-microphone/Program.cs index 586e3b162..aa0e7803f 100644 --- a/dotnet-examples/speech-recognition-from-microphone/Program.cs +++ b/dotnet-examples/speech-recognition-from-microphone/Program.cs @@ -6,47 +6,43 @@ // https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html // to download streaming models -using CommandLine.Text; using CommandLine; +using CommandLine.Text; using PortAudioSharp; -using System.Threading; using SherpaOnnx; -using System.Collections.Generic; using System.Runtime.InteropServices; -using System; - class SpeechRecognitionFromMicrophone { class Options { [Option(Required = true, HelpText = "Path to tokens.txt")] - public string Tokens { get; set; } + public string? Tokens { get; set; } [Option(Required = false, Default = "cpu", HelpText = "Provider, e.g., cpu, coreml")] - public string Provider { get; set; } + public string? Provider { get; set; } [Option(Required = false, HelpText = "Path to transducer encoder.onnx")] - public string Encoder { get; set; } + public string? Encoder { get; set; } [Option(Required = false, HelpText = "Path to transducer decoder.onnx")] - public string Decoder { get; set; } + public string? Decoder { get; set; } [Option(Required = false, HelpText = "Path to transducer joiner.onnx")] - public string Joiner { get; set; } + public string? Joiner { get; set; } [Option("paraformer-encoder", Required = false, HelpText = "Path to paraformer encoder.onnx")] - public string ParaformerEncoder { get; set; } + public string? ParaformerEncoder { get; set; } [Option("paraformer-decoder", Required = false, HelpText = "Path to paraformer decoder.onnx")] - public string ParaformerDecoder { get; set; } + public string? ParaformerDecoder { get; set; } [Option("num-threads", Required = false, Default = 1, HelpText = "Number of threads for computation")] public int NumThreads { get; set; } [Option("decoding-method", Required = false, Default = "greedy_search", HelpText = "Valid decoding methods are: greedy_search, modified_beam_search")] - public string DecodingMethod { get; set; } + public string? DecodingMethod { get; set; } [Option(Required = false, Default = false, HelpText = "True to show model info during loading")] public bool Debug { get; set; } @@ -126,7 +122,7 @@ to download pre-trained streaming models. private static void Run(Options options) { - OnlineRecognizerConfig config = new OnlineRecognizerConfig(); + var config = new OnlineRecognizerConfig(); config.FeatConfig.SampleRate = options.SampleRate; // All models from icefall using feature dim 80. @@ -153,9 +149,9 @@ private static void Run(Options options) config.Rule2MinTrailingSilence = options.Rule2MinTrailingSilence; config.Rule3MinUtteranceLength = options.Rule3MinUtteranceLength; - OnlineRecognizer recognizer = new OnlineRecognizer(config); + var recognizer = new OnlineRecognizer(config); - OnlineStream s = recognizer.CreateStream(); + var s = recognizer.CreateStream(); Console.WriteLine(PortAudio.VersionInfo.versionText); PortAudio.Initialize(); @@ -176,12 +172,12 @@ private static void Run(Options options) Environment.Exit(1); } - DeviceInfo info = PortAudio.GetDeviceInfo(deviceIndex); + var info = PortAudio.GetDeviceInfo(deviceIndex); Console.WriteLine(); Console.WriteLine($"Use default device {deviceIndex} ({info.name})"); - StreamParameters param = new StreamParameters(); + var param = new StreamParameters(); param.device = deviceIndex; param.channelCount = 1; param.sampleFormat = SampleFormat.Float32; @@ -189,14 +185,14 @@ private static void Run(Options options) param.hostApiSpecificStreamInfo = IntPtr.Zero; PortAudioSharp.Stream.Callback callback = (IntPtr input, IntPtr output, - UInt32 frameCount, + uint frameCount, ref StreamCallbackTimeInfo timeInfo, StreamCallbackFlags statusFlags, IntPtr userData ) => { - float[] samples = new float[frameCount]; - Marshal.Copy(input, samples, 0, (Int32)frameCount); + var samples = new float[frameCount]; + Marshal.Copy(input, samples, 0, (int)frameCount); s.AcceptWaveform(options.SampleRate, samples); @@ -215,7 +211,7 @@ IntPtr userData stream.Start(); - String lastText = ""; + var lastText = string.Empty; int segmentIndex = 0; while (true) @@ -245,9 +241,5 @@ IntPtr userData Thread.Sleep(200); // ms } - - PortAudio.Terminate(); - - } } diff --git a/dotnet-examples/speech-recognition-from-microphone/speech-recognition-from-microphone.csproj b/dotnet-examples/speech-recognition-from-microphone/speech-recognition-from-microphone.csproj index 901c8a158..72b7b6c91 100644 --- a/dotnet-examples/speech-recognition-from-microphone/speech-recognition-from-microphone.csproj +++ b/dotnet-examples/speech-recognition-from-microphone/speech-recognition-from-microphone.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 speech_recognition_from_microphone enable enable diff --git a/dotnet-examples/spoken-language-identification/Program.cs b/dotnet-examples/spoken-language-identification/Program.cs index 05a785d7c..d2f210e85 100644 --- a/dotnet-examples/spoken-language-identification/Program.cs +++ b/dotnet-examples/spoken-language-identification/Program.cs @@ -15,12 +15,9 @@ // dotnet run using SherpaOnnx; -using System.Collections.Generic; -using System; class SpokenLanguageIdentificationDemo { - static void Main(string[] args) { var config = new SpokenLanguageIdentificationConfig(); @@ -30,7 +27,7 @@ static void Main(string[] args) var slid = new SpokenLanguageIdentification(config); var filename = "./sherpa-onnx-whisper-tiny/test_wavs/0.wav"; - WaveReader waveReader = new WaveReader(filename); + var waveReader = new WaveReader(filename); var s = slid.CreateStream(); s.AcceptWaveform(waveReader.SampleRate, waveReader.Samples); diff --git a/dotnet-examples/spoken-language-identification/spoken-language-identification.csproj b/dotnet-examples/spoken-language-identification/spoken-language-identification.csproj index b8b431a48..e424b2d57 100644 --- a/dotnet-examples/spoken-language-identification/spoken-language-identification.csproj +++ b/dotnet-examples/spoken-language-identification/spoken-language-identification.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 spoken_language_identification enable enable diff --git a/dotnet-examples/streaming-hlg-decoding/Program.cs b/dotnet-examples/streaming-hlg-decoding/Program.cs index 6ac7c8c94..e522b8164 100644 --- a/dotnet-examples/streaming-hlg-decoding/Program.cs +++ b/dotnet-examples/streaming-hlg-decoding/Program.cs @@ -13,12 +13,9 @@ // dotnet run using SherpaOnnx; -using System.Collections.Generic; -using System; class StreamingHlgDecodingDemo { - static void Main(string[] args) { var config = new OnlineRecognizerConfig(); @@ -32,15 +29,15 @@ static void Main(string[] args) config.ModelConfig.Debug = 0; config.CtcFstDecoderConfig.Graph = "./sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18/HLG.fst"; - OnlineRecognizer recognizer = new OnlineRecognizer(config); + var recognizer = new OnlineRecognizer(config); var filename = "./sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18/test_wavs/8k.wav"; - WaveReader waveReader = new WaveReader(filename); - OnlineStream s = recognizer.CreateStream(); + var waveReader = new WaveReader(filename); + var s = recognizer.CreateStream(); s.AcceptWaveform(waveReader.SampleRate, waveReader.Samples); - float[] tailPadding = new float[(int)(waveReader.SampleRate * 0.3)]; + var tailPadding = new float[(int)(waveReader.SampleRate * 0.3)]; s.AcceptWaveform(waveReader.SampleRate, tailPadding); s.InputFinished(); @@ -49,7 +46,7 @@ static void Main(string[] args) recognizer.Decode(s); } - OnlineRecognizerResult r = recognizer.GetResult(s); + var r = recognizer.GetResult(s); var text = r.Text; var tokens = r.Tokens; Console.WriteLine("--------------------"); @@ -57,10 +54,8 @@ static void Main(string[] args) Console.WriteLine("text: {0}", text); Console.WriteLine("tokens: [{0}]", string.Join(", ", tokens)); Console.Write("timestamps: ["); - r.Timestamps.ToList().ForEach(i => Console.Write(String.Format("{0:0.00}", i) + ", ")); + r.Timestamps.ToList().ForEach(i => Console.Write(string.Format("{0:0.00}", i) + ", ")); Console.WriteLine("]"); Console.WriteLine("--------------------"); } } - - diff --git a/dotnet-examples/streaming-hlg-decoding/streaming-hlg-decoding.csproj b/dotnet-examples/streaming-hlg-decoding/streaming-hlg-decoding.csproj index 66e0401f1..6ed8fc699 100644 --- a/dotnet-examples/streaming-hlg-decoding/streaming-hlg-decoding.csproj +++ b/dotnet-examples/streaming-hlg-decoding/streaming-hlg-decoding.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 streaming_hlg_decoding enable enable diff --git a/dotnet-examples/vad-non-streaming-asr-paraformer/Program.cs b/dotnet-examples/vad-non-streaming-asr-paraformer/Program.cs index abc080b88..e8dfbe6fa 100644 --- a/dotnet-examples/vad-non-streaming-asr-paraformer/Program.cs +++ b/dotnet-examples/vad-non-streaming-asr-paraformer/Program.cs @@ -3,8 +3,6 @@ // This file shows how to use a silero_vad model with a non-streaming Paraformer // for speech recognition. using SherpaOnnx; -using System.Collections.Generic; -using System; class VadNonStreamingAsrParaformer { @@ -12,45 +10,49 @@ static void Main(string[] args) { // please download model files from // https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models - OfflineRecognizerConfig config = new OfflineRecognizerConfig(); + var config = new OfflineRecognizerConfig(); config.ModelConfig.Paraformer.Model = "./sherpa-onnx-paraformer-zh-2023-09-14/model.int8.onnx"; config.ModelConfig.Tokens = "./sherpa-onnx-paraformer-zh-2023-09-14/tokens.txt"; config.ModelConfig.Debug = 0; - OfflineRecognizer recognizer = new OfflineRecognizer(config); + var recognizer = new OfflineRecognizer(config); - VadModelConfig vadModelConfig = new VadModelConfig(); + var vadModelConfig = new VadModelConfig(); vadModelConfig.SileroVad.Model = "./silero_vad.onnx"; vadModelConfig.Debug = 0; - VoiceActivityDetector vad = new VoiceActivityDetector(vadModelConfig, 60); + var vad = new VoiceActivityDetector(vadModelConfig, 60); - string testWaveFilename = "./lei-jun-test.wav"; - WaveReader reader = new WaveReader(testWaveFilename); + var testWaveFilename = "./lei-jun-test.wav"; + var reader = new WaveReader(testWaveFilename); int numSamples = reader.Samples.Length; int windowSize = vadModelConfig.SileroVad.WindowSize; int sampleRate = vadModelConfig.SampleRate; int numIter = numSamples / windowSize; - for (int i = 0; i != numIter; ++i) { + for (int i = 0; i != numIter; ++i) + { int start = i * windowSize; - float[] samples = new float[windowSize]; + var samples = new float[windowSize]; Array.Copy(reader.Samples, start, samples, 0, windowSize); vad.AcceptWaveform(samples); - if (vad.IsSpeechDetected()) { - while (!vad.IsEmpty()) { + if (vad.IsSpeechDetected()) + { + while (!vad.IsEmpty()) + { SpeechSegment segment = vad.Front(); - float startTime = segment.Start / (float)sampleRate; - float duration = segment.Samples.Length / (float)sampleRate; + var startTime = segment.Start / (float)sampleRate; + var duration = segment.Samples.Length / (float)sampleRate; OfflineStream stream = recognizer.CreateStream(); stream.AcceptWaveform(sampleRate, segment.Samples); recognizer.Decode(stream); - String text = stream.Result.Text; + var text = stream.Result.Text; - if (!String.IsNullOrEmpty(text)) { - Console.WriteLine("{0}--{1}: {2}", String.Format("{0:0.00}", startTime), - String.Format("{0:0.00}", startTime+duration), text); + if (!string.IsNullOrEmpty(text)) + { + Console.WriteLine("{0}--{1}: {2}", string.Format("{0:0.00}", startTime), + string.Format("{0:0.00}", startTime + duration), text); } vad.Pop(); @@ -60,19 +62,21 @@ static void Main(string[] args) vad.Flush(); - while (!vad.IsEmpty()) { - SpeechSegment segment = vad.Front(); + while (!vad.IsEmpty()) + { + var segment = vad.Front(); float startTime = segment.Start / (float)sampleRate; float duration = segment.Samples.Length / (float)sampleRate; - OfflineStream stream = recognizer.CreateStream(); + var stream = recognizer.CreateStream(); stream.AcceptWaveform(sampleRate, segment.Samples); recognizer.Decode(stream); - String text = stream.Result.Text; + var text = stream.Result.Text; - if (!String.IsNullOrEmpty(text)) { - Console.WriteLine("{0}--{1}: {2}", String.Format("{0:0.00}", startTime), - String.Format("{0:0.00}", startTime+duration), text); + if (!string.IsNullOrEmpty(text)) + { + Console.WriteLine("{0}--{1}: {2}", string.Format("{0:0.00}", startTime), + string.Format("{0:0.00}", startTime + duration), text); } vad.Pop(); diff --git a/dotnet-examples/vad-non-streaming-asr-paraformer/vad-non-streaming-asr-paraformer.csproj b/dotnet-examples/vad-non-streaming-asr-paraformer/vad-non-streaming-asr-paraformer.csproj index a5c5f1022..1736869a8 100644 --- a/dotnet-examples/vad-non-streaming-asr-paraformer/vad-non-streaming-asr-paraformer.csproj +++ b/dotnet-examples/vad-non-streaming-asr-paraformer/vad-non-streaming-asr-paraformer.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 vad_non_streaming_asr_paraformer enable enable From 1ef9e5ee3a20adb52821e14da542ad180b433bb8 Mon Sep 17 00:00:00 2001 From: Michael Lamothe Date: Sat, 4 Jan 2025 22:54:49 +1100 Subject: [PATCH 140/183] Update workflows to use .NET 8.0 also. (#1681) --- .github/workflows/dot-net.yaml | 4 +--- .github/workflows/test-dot-net-nuget.yaml | 4 ++-- .github/workflows/test-dot-net.yaml | 3 +-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dot-net.yaml b/.github/workflows/dot-net.yaml index 4c6e44133..899cb9995 100644 --- a/.github/workflows/dot-net.yaml +++ b/.github/workflows/dot-net.yaml @@ -125,9 +125,7 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v4 with: - dotnet-version: | - 6.0.x - 7.0.x + dotnet-version: 8.0.x - name: Install Python dependencies shell: bash diff --git a/.github/workflows/test-dot-net-nuget.yaml b/.github/workflows/test-dot-net-nuget.yaml index d32582441..b89781be5 100644 --- a/.github/workflows/test-dot-net-nuget.yaml +++ b/.github/workflows/test-dot-net-nuget.yaml @@ -75,10 +75,10 @@ jobs: run: | df -h - - name: Setup .NET 6.0 + - name: Setup .NET 8.0 uses: actions/setup-dotnet@v4 with: - dotnet-version: 6.0.x + dotnet-version: 8.0.x - name: Check dotnet run: dotnet --info diff --git a/.github/workflows/test-dot-net.yaml b/.github/workflows/test-dot-net.yaml index d046542ba..cc20c9ab5 100644 --- a/.github/workflows/test-dot-net.yaml +++ b/.github/workflows/test-dot-net.yaml @@ -115,8 +115,7 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v4 with: - dotnet-version: | - 6.0.x + dotnet-version: 8.0.x - name: Check dotnet run: dotnet --info From 3eced3e7ee0ba5a51a3cba9fd819a5476940353f Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sun, 5 Jan 2025 15:08:19 +0800 Subject: [PATCH 141/183] Add C# and JavaScript (wasm) API for MatchaTTS models (#1682) --- .github/scripts/test-dot-net.sh | 32 +++-- .github/scripts/test-nodejs-npm.sh | 54 ++++++-- .github/workflows/test-dot-net.yaml | 44 +++++++ dotnet-examples/offline-tts-play/Program.cs | 109 ++++++++++++---- .../offline-tts-play/run-hf-fanchen.sh | 4 +- .../offline-tts-play/run-matcha-en.sh | 26 ++++ .../offline-tts-play/run-matcha-zh.sh | 27 ++++ dotnet-examples/offline-tts-play/run-piper.sh | 4 +- dotnet-examples/offline-tts/Program.cs | 74 +++++++++-- dotnet-examples/offline-tts/run-aishell3.sh | 4 +- dotnet-examples/offline-tts/run-hf-fanchen.sh | 6 +- dotnet-examples/offline-tts/run-matcha-en.sh | 26 ++++ dotnet-examples/offline-tts/run-matcha-zh.sh | 27 ++++ dotnet-examples/offline-tts/run-piper.sh | 4 +- nodejs-examples/README.md | 48 ++++++- nodejs-examples/test-offline-tts-matcha-en.js | 40 ++++++ nodejs-examples/test-offline-tts-matcha-zh.js | 41 ++++++ ...-tts-en.js => test-offline-tts-vits-en.js} | 4 +- ...-tts-zh.js => test-offline-tts-vits-zh.js} | 4 +- scripts/dotnet/OfflineTtsMatchaModelConfig.cs | 44 +++++++ scripts/dotnet/OfflineTtsModelConfig.cs | 5 +- scripts/dotnet/examples/Common.csproj | 2 +- scripts/dotnet/sherpa-onnx.csproj.in | 2 +- scripts/dotnet/sherpa-onnx.csproj.runtime.in | 2 +- wasm/tts/sherpa-onnx-tts.js | 117 +++++++++++++++++- wasm/tts/sherpa-onnx-wasm-main-tts.cc | 15 ++- 26 files changed, 677 insertions(+), 88 deletions(-) create mode 100755 dotnet-examples/offline-tts-play/run-matcha-en.sh create mode 100755 dotnet-examples/offline-tts-play/run-matcha-zh.sh create mode 100755 dotnet-examples/offline-tts/run-matcha-en.sh create mode 100755 dotnet-examples/offline-tts/run-matcha-zh.sh create mode 100644 nodejs-examples/test-offline-tts-matcha-en.js create mode 100644 nodejs-examples/test-offline-tts-matcha-zh.js rename nodejs-examples/{test-offline-tts-en.js => test-offline-tts-vits-en.js} (92%) rename nodejs-examples/{test-offline-tts-zh.js => test-offline-tts-vits-zh.js} (92%) create mode 100644 scripts/dotnet/OfflineTtsMatchaModelConfig.cs diff --git a/.github/scripts/test-dot-net.sh b/.github/scripts/test-dot-net.sh index f4bfc66c5..7c339e157 100755 --- a/.github/scripts/test-dot-net.sh +++ b/.github/scripts/test-dot-net.sh @@ -2,7 +2,27 @@ cd dotnet-examples/ -cd ./offline-speaker-diarization +cd ./offline-tts +./run-matcha-zh.sh +ls -lh *.wav +./run-matcha-en.sh +ls -lh *.wav +./run-aishell3.sh +ls -lh *.wav +./run-piper.sh +ls -lh *.wav +./run-hf-fanchen.sh +ls -lh *.wav +ls -lh + +pushd ../.. + +mkdir tts + +cp dotnet-examples/offline-tts/*.wav ./tts +popd + +cd ../offline-speaker-diarization ./run.sh rm -rfv *.onnx rm -fv *.wav @@ -76,14 +96,4 @@ cd ../spoken-language-identification ./run.sh rm -rf sherpa-onnx-* -cd ../offline-tts -./run-aishell3.sh -./run-piper.sh -./run-hf-fanchen.sh -ls -lh -cd ../.. - -mkdir tts - -cp dotnet-examples/offline-tts/*.wav ./tts diff --git a/.github/scripts/test-nodejs-npm.sh b/.github/scripts/test-nodejs-npm.sh index 518d173b6..967944843 100755 --- a/.github/scripts/test-nodejs-npm.sh +++ b/.github/scripts/test-nodejs-npm.sh @@ -9,6 +9,48 @@ git status ls -lh ls -lh node_modules +# offline tts +# +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 +tar xvf matcha-icefall-zh-baker.tar.bz2 +rm matcha-icefall-zh-baker.tar.bz2 + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +node ./test-offline-tts-matcha-zh.js + +rm -rf matcha-icefall-zh-baker +rm hifigan_v2.onnx + +echo "---" + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 +tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 +rm matcha-icefall-en_US-ljspeech.tar.bz2 + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +node ./test-offline-tts-matcha-en.js + +rm -rf matcha-icefall-en_US-ljspeech +rm hifigan_v2.onnx + +echo "---" + +curl -LS -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 +tar xf vits-piper-en_US-amy-low.tar.bz2 +node ./test-offline-tts-vits-en.js +rm -rf vits-piper-en_US-amy-low* + +echo "---" + +curl -LS -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-icefall-zh-aishell3.tar.bz2 +tar xvf vits-icefall-zh-aishell3.tar.bz2 +node ./test-offline-tts-vits-zh.js +rm -rf vits-icefall-zh-aishell3* + +ls -lh *.wav + echo '-----speaker diarization----------' curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 @@ -147,15 +189,3 @@ tar xvf sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18.tar.bz2 rm sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18.tar.bz2 node ./test-online-zipformer2-ctc-hlg.js rm -rf sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18 - -# offline tts - -curl -LS -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 -tar xf vits-piper-en_US-amy-low.tar.bz2 -node ./test-offline-tts-en.js -rm -rf vits-piper-en_US-amy-low* - -curl -LS -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-icefall-zh-aishell3.tar.bz2 -tar xvf vits-icefall-zh-aishell3.tar.bz2 -node ./test-offline-tts-zh.js -rm -rf vits-icefall-zh-aishell3* diff --git a/.github/workflows/test-dot-net.yaml b/.github/workflows/test-dot-net.yaml index cc20c9ab5..8ca19439f 100644 --- a/.github/workflows/test-dot-net.yaml +++ b/.github/workflows/test-dot-net.yaml @@ -92,6 +92,50 @@ jobs: python-version: ["3.8"] steps: + - name: Check space + shell: bash + run: | + df -h + + - name: Free space + shell: bash + run: | + df -h + rm -rf /opt/hostedtoolcache + df -h + + - name: Free more space + shell: bash + run: | + # https://github.com/orgs/community/discussions/25678 + cd /opt + find . -maxdepth 1 -mindepth 1 '!' -path ./containerd '!' -path ./actionarchivecache '!' -path ./runner '!' -path ./runner-cache -exec rm -rf '{}' ';' + + sudo rm -rf /usr/share/dotnet + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: false + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: false + haskell: true + large-packages: true + docker-images: false + swap-storage: true + + - name: Check space + shell: bash + run: | + df -h + - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/dotnet-examples/offline-tts-play/Program.cs b/dotnet-examples/offline-tts-play/Program.cs index 65eb22bf4..543a50cdd 100644 --- a/dotnet-examples/offline-tts-play/Program.cs +++ b/dotnet-examples/offline-tts-play/Program.cs @@ -21,48 +21,56 @@ class OfflineTtsPlayDemo { class Options { - [Option("tts-rule-fsts", Required = false, Default = "", HelpText = "path to rule.fst")] - public string? RuleFsts { get; set; } + public string RuleFsts { get; set; } = string.Empty; + + [Option("tts-rule-fars", Required = false, Default = "", HelpText = "path to rule.far")] + public string RuleFars { get; set; } = string.Empty; - [Option("vits-dict-dir", Required = false, Default = "", HelpText = "Path to the directory containing dict for jieba.")] - public string? DictDir { get; set; } + [Option("dict-dir", Required = false, Default = "", HelpText = "Path to the directory containing dict for jieba.")] + public string DictDir { get; set; } = string.Empty; - [Option("vits-data-dir", Required = false, Default = "", HelpText = "Path to the directory containing dict for espeak-ng.")] - public string? DataDir { get; set; } + [Option("data-dir", Required = false, Default = "", HelpText = "Path to the directory containing dict for espeak-ng.")] + public string DataDir { get; set; } = string.Empty; - [Option("vits-length-scale", Required = false, Default = 1, HelpText = "speech speed. Larger->Slower; Smaller->faster")] - public float LengthScale { get; set; } + [Option("length-scale", Required = false, Default = 1, HelpText = "speech speed. Larger->Slower; Smaller->faster")] + public float LengthScale { get; set; } = 1; - [Option("vits-noise-scale", Required = false, Default = 0.667f, HelpText = "noise_scale for VITS models")] - public float NoiseScale { get; set; } + [Option("noise-scale", Required = false, Default = 0.667f, HelpText = "noise_scale for VITS or Matcha models")] + public float NoiseScale { get; set; } = 0.667F; - [Option("vits-noise-scale-w", Required = false, Default = 0.8f, HelpText = "noise_scale_w for VITS models")] - public float NoiseScaleW { get; set; } + [Option("vits-noise-scale-w", Required = false, Default = 0.8F, HelpText = "noise_scale_w for VITS models")] + public float NoiseScaleW { get; set; } = 0.8F; - [Option("vits-lexicon", Required = false, Default = "", HelpText = "Path to lexicon.txt")] - public string? Lexicon { get; set; } + [Option("lexicon", Required = false, Default = "", HelpText = "Path to lexicon.txt")] + public string Lexicon { get; set; } = string.Empty; - [Option("vits-tokens", Required = false, Default = "", HelpText = "Path to tokens.txt")] - public string? Tokens { get; set; } + [Option("tokens", Required = true, Default = "", HelpText = "Path to tokens.txt")] + public string Tokens { get; set; } = string.Empty; [Option("tts-max-num-sentences", Required = false, Default = 1, HelpText = "Maximum number of sentences that we process at a time.")] - public int MaxNumSentences { get; set; } + public int MaxNumSentences { get; set; } = 1; [Option(Required = false, Default = 0, HelpText = "1 to show debug messages.")] - public int Debug { get; set; } + public int Debug { get; set; } = 0; + + [Option("vits-model", Required = false, HelpText = "Path to VITS model")] + public string Model { get; set; } = string.Empty; - [Option("vits-model", Required = true, HelpText = "Path to VITS model")] - public string? Model { get; set; } + [Option("matcha-acoustic-model", Required = false, HelpText = "Path to the acoustic model of Matcha")] + public string AcousticModel { get; set; } = ""; + + [Option("matcha-vocoder", Required = false, HelpText = "Path to the vocoder model of Matcha")] + public string Vocoder { get; set; } = ""; [Option("sid", Required = false, Default = 0, HelpText = "Speaker ID")] - public int SpeakerId { get; set; } + public int SpeakerId { get; set; } = 0; [Option("text", Required = true, HelpText = "Text to synthesize")] - public string? Text { get; set; } + public string Text { get; set; } = string.Empty; [Option("output-filename", Required = true, Default = "./generated.wav", HelpText = "Path to save the generated audio")] - public string? OutputFilename { get; set; } + public string OutputFilename { get; set; } = "./generated.wav"; } static void Main(string[] args) @@ -78,6 +86,42 @@ static void Main(string[] args) private static void DisplayHelp(ParserResult result, IEnumerable errs) { string usage = @" +# matcha-icefall-zh-baker + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 +tar xvf matcha-icefall-zh-baker.tar.bz2 +rm matcha-icefall-zh-baker.tar.bz2 + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +dotnet run \ + --matcha-acoustic-model=./matcha-icefall-zh-baker/model-steps-3.onnx \ + --matcha-vocoder=./hifigan_v2.onnx \ + --lexicon=./matcha-icefall-zh-baker/lexicon.txt \ + --tokens=./matcha-icefall-zh-baker/tokens.txt \ + --dict-dir=./matcha-icefall-zh-baker/dict \ + --tts-rule-fsts=./matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst \ + --debug=1 \ + --output-filename=./matcha-zh.wav \ + --text='某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。' + +# matcha-icefall-en_US-ljspeech + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 +tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 +rm matcha-icefall-en_US-ljspeech.tar.bz2 + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +dotnet run \ + --matcha-acoustic-model=./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ + --matcha-vocoder=./hifigan_v2.onnx \ + --tokens=./matcha-icefall-zh-baker/tokens.txt \ + --data-dir=./matcha-icefall-en_US-ljspeech/espeak-ng-data \ + --debug=1 \ + --output-filename=./matcha-zh.wav \ + --text='Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.' + # vits-aishell3 wget -qq https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-zh-aishell3.tar.bz2 @@ -85,8 +129,8 @@ tar xf vits-zh-aishell3.tar.bz2 dotnet run \ --vits-model=./vits-zh-aishell3/vits-aishell3.onnx \ - --vits-tokens=./vits-zh-aishell3/tokens.txt \ - --vits-lexicon=./vits-zh-aishell3/lexicon.txt \ + --tokens=./vits-zh-aishell3/tokens.txt \ + --lexicon=./vits-zh-aishell3/lexicon.txt \ --tts-rule-fsts=./vits-zh-aishell3/rule.fst \ --sid=66 \ --debug=1 \ @@ -100,8 +144,8 @@ tar xf vits-piper-en_US-amy-low.tar.bz2 dotnet run \ --vits-model=./vits-piper-en_US-amy-low/en_US-amy-low.onnx \ - --vits-tokens=./vits-piper-en_US-amy-low/tokens.txt \ - --vits-data-dir=./vits-piper-en_US-amy-low/espeak-ng-data \ + ---tokens=./vits-piper-en_US-amy-low/tokens.txt \ + --data-dir=./vits-piper-en_US-amy-low/espeak-ng-data \ --debug=1 \ --output-filename=./amy.wav \ --text='This is a text to speech application in dotnet with Next Generation Kaldi' @@ -124,6 +168,7 @@ to download more models. private static void Run(Options options) { var config = new OfflineTtsConfig(); + config.Model.Vits.Model = options.Model; config.Model.Vits.Lexicon = options.Lexicon; config.Model.Vits.Tokens = options.Tokens; @@ -132,6 +177,16 @@ private static void Run(Options options) config.Model.Vits.NoiseScale = options.NoiseScale; config.Model.Vits.NoiseScaleW = options.NoiseScaleW; config.Model.Vits.LengthScale = options.LengthScale; + + config.Model.Matcha.AcousticModel = options.AcousticModel; + config.Model.Matcha.Vocoder = options.Vocoder; + config.Model.Matcha.Lexicon = options.Lexicon; + config.Model.Matcha.Tokens = options.Tokens; + config.Model.Matcha.DataDir = options.DataDir; + config.Model.Matcha.DictDir = options.DictDir; + config.Model.Matcha.NoiseScale = options.NoiseScale; + config.Model.Matcha.LengthScale = options.LengthScale; + config.Model.NumThreads = 1; config.Model.Debug = options.Debug; config.Model.Provider = "cpu"; diff --git a/dotnet-examples/offline-tts-play/run-hf-fanchen.sh b/dotnet-examples/offline-tts-play/run-hf-fanchen.sh index b16a3ca68..84e668578 100755 --- a/dotnet-examples/offline-tts-play/run-hf-fanchen.sh +++ b/dotnet-examples/offline-tts-play/run-hf-fanchen.sh @@ -8,8 +8,8 @@ fi dotnet run \ --vits-model=./vits-zh-hf-fanchen-C/vits-zh-hf-fanchen-C.onnx \ - --vits-tokens=./vits-zh-hf-fanchen-C/tokens.txt \ - --vits-lexicon=./vits-zh-hf-fanchen-C/lexicon.txt \ + --tokens=./vits-zh-hf-fanchen-C/tokens.txt \ + --lexicon=./vits-zh-hf-fanchen-C/lexicon.txt \ --tts-rule-fsts=./vits-zh-hf-fanchen-C/phone.fst,./vits-zh-hf-fanchen-C/date.fst,./vits-zh-hf-fanchen-C/number.fst \ --vits-dict-dir=./vits-zh-hf-fanchen-C/dict \ --sid=100 \ diff --git a/dotnet-examples/offline-tts-play/run-matcha-en.sh b/dotnet-examples/offline-tts-play/run-matcha-en.sh new file mode 100755 index 000000000..0f7caa215 --- /dev/null +++ b/dotnet-examples/offline-tts-play/run-matcha-en.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -ex + + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker +# matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker +# to download more models +if [ ! -f ./matcha-icefall-en_US-ljspeech/model-steps-3.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 + tar xf matcha-icefall-en_US-ljspeech.tar.bz2 + rm matcha-icefall-en_US-ljspeech.tar.bz2 +fi + +if [ ! -f ./hifigan_v2.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx +fi + +dotnet run \ + --matcha-acoustic-model=./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ + --matcha-vocoder=./hifigan_v2.onnx \ + --tokens=./matcha-icefall-en_US-ljspeech/tokens.txt \ + --data-dir=./matcha-icefall-en_US-ljspeech/espeak-ng-data \ + --debug=1 \ + --output-filename=./matcha-en.wav \ + --text='Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.' diff --git a/dotnet-examples/offline-tts-play/run-matcha-zh.sh b/dotnet-examples/offline-tts-play/run-matcha-zh.sh new file mode 100755 index 000000000..e3b34268c --- /dev/null +++ b/dotnet-examples/offline-tts-play/run-matcha-zh.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -ex + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker +# to download more models +if [ ! -f ./matcha-icefall-zh-baker/model-steps-3.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 + tar xvf matcha-icefall-zh-baker.tar.bz2 + rm matcha-icefall-zh-baker.tar.bz2 +fi + +if [ ! -f ./hifigan_v2.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx +fi + + +dotnet run \ + --matcha-acoustic-model=./matcha-icefall-zh-baker/model-steps-3.onnx \ + --matcha-vocoder=./hifigan_v2.onnx \ + --lexicon=./matcha-icefall-zh-baker/lexicon.txt \ + --tokens=./matcha-icefall-zh-baker/tokens.txt \ + --dict-dir=./matcha-icefall-zh-baker/dict \ + --tts-rule-fsts=./matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst \ + --debug=1 \ + --output-filename=./matcha-zh.wav \ + --text="某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。" diff --git a/dotnet-examples/offline-tts-play/run-piper.sh b/dotnet-examples/offline-tts-play/run-piper.sh index 7c97498d2..1a4d10806 100755 --- a/dotnet-examples/offline-tts-play/run-piper.sh +++ b/dotnet-examples/offline-tts-play/run-piper.sh @@ -9,8 +9,8 @@ fi dotnet run \ --vits-model=./vits-piper-en_US-amy-low/en_US-amy-low.onnx \ - --vits-tokens=./vits-piper-en_US-amy-low/tokens.txt \ - --vits-data-dir=./vits-piper-en_US-amy-low/espeak-ng-data \ + --tokens=./vits-piper-en_US-amy-low/tokens.txt \ + --data-dir=./vits-piper-en_US-amy-low/espeak-ng-data \ --debug=1 \ --output-filename=./amy.wav \ --text="This is a text to speech application in dotnet with Next Generation Kaldi" diff --git a/dotnet-examples/offline-tts/Program.cs b/dotnet-examples/offline-tts/Program.cs index f434ebf19..21f90c525 100644 --- a/dotnet-examples/offline-tts/Program.cs +++ b/dotnet-examples/offline-tts/Program.cs @@ -20,25 +20,25 @@ class Options [Option("tts-rule-fars", Required = false, Default = "", HelpText = "path to rule.far")] public string RuleFars { get; set; } = string.Empty; - [Option("vits-dict-dir", Required = false, Default = "", HelpText = "Path to the directory containing dict for jieba.")] + [Option("dict-dir", Required = false, Default = "", HelpText = "Path to the directory containing dict for jieba.")] public string DictDir { get; set; } = string.Empty; - [Option("vits-data-dir", Required = false, Default = "", HelpText = "Path to the directory containing dict for espeak-ng.")] + [Option("data-dir", Required = false, Default = "", HelpText = "Path to the directory containing dict for espeak-ng.")] public string DataDir { get; set; } = string.Empty; - [Option("vits-length-scale", Required = false, Default = 1, HelpText = "speech speed. Larger->Slower; Smaller->faster")] + [Option("length-scale", Required = false, Default = 1, HelpText = "speech speed. Larger->Slower; Smaller->faster")] public float LengthScale { get; set; } = 1; - [Option("vits-noise-scale", Required = false, Default = 0.667f, HelpText = "noise_scale for VITS models")] + [Option("noise-scale", Required = false, Default = 0.667f, HelpText = "noise_scale for VITS or Matcha models")] public float NoiseScale { get; set; } = 0.667F; [Option("vits-noise-scale-w", Required = false, Default = 0.8F, HelpText = "noise_scale_w for VITS models")] public float NoiseScaleW { get; set; } = 0.8F; - [Option("vits-lexicon", Required = false, Default = "", HelpText = "Path to lexicon.txt")] + [Option("lexicon", Required = false, Default = "", HelpText = "Path to lexicon.txt")] public string Lexicon { get; set; } = string.Empty; - [Option("vits-tokens", Required = false, Default = "", HelpText = "Path to tokens.txt")] + [Option("tokens", Required = true, Default = "", HelpText = "Path to tokens.txt")] public string Tokens { get; set; } = string.Empty; [Option("tts-max-num-sentences", Required = false, Default = 1, HelpText = "Maximum number of sentences that we process at a time.")] @@ -47,9 +47,15 @@ class Options [Option(Required = false, Default = 0, HelpText = "1 to show debug messages.")] public int Debug { get; set; } = 0; - [Option("vits-model", Required = true, HelpText = "Path to VITS model")] + [Option("vits-model", Required = false, HelpText = "Path to VITS model")] public string Model { get; set; } = string.Empty; + [Option("matcha-acoustic-model", Required = false, HelpText = "Path to the acoustic model of Matcha")] + public string AcousticModel { get; set; } = ""; + + [Option("matcha-vocoder", Required = false, HelpText = "Path to the vocoder model of Matcha")] + public string Vocoder { get; set; } = ""; + [Option("sid", Required = false, Default = 0, HelpText = "Speaker ID")] public int SpeakerId { get; set; } = 0; @@ -73,6 +79,42 @@ static void Main(string[] args) private static void DisplayHelp(ParserResult result, IEnumerable errs) { var usage = @" +# matcha-icefall-zh-baker + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 +tar xvf matcha-icefall-zh-baker.tar.bz2 +rm matcha-icefall-zh-baker.tar.bz2 + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +dotnet run \ + --matcha-acoustic-model=./matcha-icefall-zh-baker/model-steps-3.onnx \ + --matcha-vocoder=./hifigan_v2.onnx \ + --lexicon=./matcha-icefall-zh-baker/lexicon.txt \ + --tokens=./matcha-icefall-zh-baker/tokens.txt \ + --dict-dir=./matcha-icefall-zh-baker/dict \ + --tts-rule-fsts=./matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst \ + --debug=1 \ + --output-filename=./matcha-zh.wav \ + --text='某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。' + +# matcha-icefall-en_US-ljspeech + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 +tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 +rm matcha-icefall-en_US-ljspeech.tar.bz2 + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +dotnet run \ + --matcha-acoustic-model=./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ + --matcha-vocoder=./hifigan_v2.onnx \ + --tokens=./matcha-icefall-zh-baker/tokens.txt \ + --data-dir=./matcha-icefall-en_US-ljspeech/espeak-ng-data \ + --debug=1 \ + --output-filename=./matcha-zh.wav \ + --text='Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.' + # vits-aishell3 curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-icefall-zh-aishell3.tar.bz2 @@ -80,8 +122,8 @@ tar xvf vits-icefall-zh-aishell3.tar.bz2 dotnet run \ --vits-model=./vits-icefall-zh-aishell3/model.onnx \ - --vits-tokens=./vits-icefall-zh-aishell3/tokens.txt \ - --vits-lexicon=./vits-icefall-zh-aishell3/lexicon.txt \ + --tokens=./vits-icefall-zh-aishell3/tokens.txt \ + --lexicon=./vits-icefall-zh-aishell3/lexicon.txt \ --tts-rule-fsts=./vits-icefall-zh-aishell3/phone.fst,./vits-icefall-zh-aishell3/date.fst,./vits-icefall-zh-aishell3/number.fst \ --tts-rule-fars=./vits-icefall-zh-aishell3/rule.far \ --sid=66 \ @@ -96,8 +138,8 @@ tar xf vits-piper-en_US-amy-low.tar.bz2 dotnet run \ --vits-model=./vits-piper-en_US-amy-low/en_US-amy-low.onnx \ - --vits-tokens=./vits-piper-en_US-amy-low/tokens.txt \ - --vits-data-dir=./vits-piper-en_US-amy-low/espeak-ng-data \ + --tokens=./vits-piper-en_US-amy-low/tokens.txt \ + --data-dir=./vits-piper-en_US-amy-low/espeak-ng-data \ --debug=1 \ --output-filename=./amy.wav \ --text='This is a text to speech application in dotnet with Next Generation Kaldi' @@ -128,6 +170,16 @@ private static void Run(Options options) config.Model.Vits.NoiseScale = options.NoiseScale; config.Model.Vits.NoiseScaleW = options.NoiseScaleW; config.Model.Vits.LengthScale = options.LengthScale; + + config.Model.Matcha.AcousticModel = options.AcousticModel; + config.Model.Matcha.Vocoder = options.Vocoder; + config.Model.Matcha.Lexicon = options.Lexicon; + config.Model.Matcha.Tokens = options.Tokens; + config.Model.Matcha.DataDir = options.DataDir; + config.Model.Matcha.DictDir = options.DictDir; + config.Model.Matcha.NoiseScale = options.NoiseScale; + config.Model.Matcha.LengthScale = options.LengthScale; + config.Model.NumThreads = 1; config.Model.Debug = options.Debug; config.Model.Provider = "cpu"; diff --git a/dotnet-examples/offline-tts/run-aishell3.sh b/dotnet-examples/offline-tts/run-aishell3.sh index 02380f07c..9a54df349 100755 --- a/dotnet-examples/offline-tts/run-aishell3.sh +++ b/dotnet-examples/offline-tts/run-aishell3.sh @@ -8,8 +8,8 @@ fi dotnet run \ --vits-model=./vits-icefall-zh-aishell3/model.onnx \ - --vits-tokens=./vits-icefall-zh-aishell3/tokens.txt \ - --vits-lexicon=./vits-icefall-zh-aishell3/lexicon.txt \ + --tokens=./vits-icefall-zh-aishell3/tokens.txt \ + --lexicon=./vits-icefall-zh-aishell3/lexicon.txt \ --tts-rule-fsts=./vits-icefall-zh-aishell3/phone.fst,./vits-icefall-zh-aishell3/date.fst,./vits-icefall-zh-aishell3/number.fst \ --tts-rule-fars=./vits-icefall-zh-aishell3/rule.far \ --sid=66 \ diff --git a/dotnet-examples/offline-tts/run-hf-fanchen.sh b/dotnet-examples/offline-tts/run-hf-fanchen.sh index b16a3ca68..a7a52e733 100755 --- a/dotnet-examples/offline-tts/run-hf-fanchen.sh +++ b/dotnet-examples/offline-tts/run-hf-fanchen.sh @@ -8,10 +8,10 @@ fi dotnet run \ --vits-model=./vits-zh-hf-fanchen-C/vits-zh-hf-fanchen-C.onnx \ - --vits-tokens=./vits-zh-hf-fanchen-C/tokens.txt \ - --vits-lexicon=./vits-zh-hf-fanchen-C/lexicon.txt \ + --tokens=./vits-zh-hf-fanchen-C/tokens.txt \ + --lexicon=./vits-zh-hf-fanchen-C/lexicon.txt \ --tts-rule-fsts=./vits-zh-hf-fanchen-C/phone.fst,./vits-zh-hf-fanchen-C/date.fst,./vits-zh-hf-fanchen-C/number.fst \ - --vits-dict-dir=./vits-zh-hf-fanchen-C/dict \ + --dict-dir=./vits-zh-hf-fanchen-C/dict \ --sid=100 \ --debug=1 \ --output-filename=./fanchen-100.wav \ diff --git a/dotnet-examples/offline-tts/run-matcha-en.sh b/dotnet-examples/offline-tts/run-matcha-en.sh new file mode 100755 index 000000000..0f7caa215 --- /dev/null +++ b/dotnet-examples/offline-tts/run-matcha-en.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -ex + + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker +# matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker +# to download more models +if [ ! -f ./matcha-icefall-en_US-ljspeech/model-steps-3.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 + tar xf matcha-icefall-en_US-ljspeech.tar.bz2 + rm matcha-icefall-en_US-ljspeech.tar.bz2 +fi + +if [ ! -f ./hifigan_v2.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx +fi + +dotnet run \ + --matcha-acoustic-model=./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ + --matcha-vocoder=./hifigan_v2.onnx \ + --tokens=./matcha-icefall-en_US-ljspeech/tokens.txt \ + --data-dir=./matcha-icefall-en_US-ljspeech/espeak-ng-data \ + --debug=1 \ + --output-filename=./matcha-en.wav \ + --text='Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.' diff --git a/dotnet-examples/offline-tts/run-matcha-zh.sh b/dotnet-examples/offline-tts/run-matcha-zh.sh new file mode 100755 index 000000000..e3b34268c --- /dev/null +++ b/dotnet-examples/offline-tts/run-matcha-zh.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -ex + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker +# to download more models +if [ ! -f ./matcha-icefall-zh-baker/model-steps-3.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 + tar xvf matcha-icefall-zh-baker.tar.bz2 + rm matcha-icefall-zh-baker.tar.bz2 +fi + +if [ ! -f ./hifigan_v2.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx +fi + + +dotnet run \ + --matcha-acoustic-model=./matcha-icefall-zh-baker/model-steps-3.onnx \ + --matcha-vocoder=./hifigan_v2.onnx \ + --lexicon=./matcha-icefall-zh-baker/lexicon.txt \ + --tokens=./matcha-icefall-zh-baker/tokens.txt \ + --dict-dir=./matcha-icefall-zh-baker/dict \ + --tts-rule-fsts=./matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst \ + --debug=1 \ + --output-filename=./matcha-zh.wav \ + --text="某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。" diff --git a/dotnet-examples/offline-tts/run-piper.sh b/dotnet-examples/offline-tts/run-piper.sh index ff639c570..273799bb3 100755 --- a/dotnet-examples/offline-tts/run-piper.sh +++ b/dotnet-examples/offline-tts/run-piper.sh @@ -10,8 +10,8 @@ fi dotnet run \ --vits-model=./vits-piper-en_US-amy-low/en_US-amy-low.onnx \ - --vits-tokens=./vits-piper-en_US-amy-low/tokens.txt \ - --vits-data-dir=./vits-piper-en_US-amy-low/espeak-ng-data \ + --tokens=./vits-piper-en_US-amy-low/tokens.txt \ + --data-dir=./vits-piper-en_US-amy-low/espeak-ng-data \ --debug=1 \ --output-filename=./amy.wav \ --text="This is a text to speech application in dotnet with Next Generation Kaldi" diff --git a/nodejs-examples/README.md b/nodejs-examples/README.md index a953573b6..0c59b7bcc 100644 --- a/nodejs-examples/README.md +++ b/nodejs-examples/README.md @@ -42,9 +42,45 @@ node ./test-offline-speaker-diarization.js In the following, we demonstrate how to run text-to-speech. -## ./test-offline-tts-en.js +## ./test-offline-tts-matcha-zh.js -[./test-offline-tts-en.js](./test-offline-tts-en.js) shows how to use +[./test-offline-tts-matcha-zh.js](./test-offline-tts-matcha-zh.js) shows how to use +[matcha-icefall-zh-baker](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker) +for text-to-speech. + +You can use the following command to run it: + +```bash +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 +tar xvf matcha-icefall-zh-baker.tar.bz2 +rm matcha-icefall-zh-baker.tar.bz2 + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +node ./test-offline-tts-matcha-zh.js +``` + +## ./test-offline-tts-matcha-en.js + +[./test-offline-tts-matcha-en.js](./test-offline-tts-matcha-en.js) shows how to use +[matcha-icefall-en_US-ljspeech](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker) +for text-to-speech. + +You can use the following command to run it: + +```bash +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 +tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 +rm matcha-icefall-en_US-ljspeech.tar.bz2 + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx + +node ./test-offline-tts-matcha-en.js +``` + +## ./test-offline-tts-vits-en.js + +[./test-offline-tts-vits-en.js](./test-offline-tts-vits-en.js) shows how to use [vits-piper-en_US-amy-low.tar.bz2](https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2) for text-to-speech. @@ -53,12 +89,12 @@ You can use the following command to run it: ```bash wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 tar xvf vits-piper-en_US-amy-low.tar.bz2 -node ./test-offline-tts-en.js +node ./test-offline-tts-vits-en.js ``` -## ./test-offline-tts-zh.js +## ./test-offline-tts-vits-zh.js -[./test-offline-tts-zh.js](./test-offline-tts-zh.js) shows how to use +[./test-offline-tts-vits-zh.js](./test-offline-tts-vits-zh.js) shows how to use a VITS pretrained model [aishell3](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/vits.html#vits-model-aishell3) for text-to-speech. @@ -68,7 +104,7 @@ You can use the following command to run it: ```bash wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-icefall-zh-aishell3.tar.bz2 tar xvf vits-icefall-zh-aishell3.tar.bz2 -node ./test-offline-tts-zh.js +node ./test-offline-tts-vits-zh.js ``` # Speech-to-text diff --git a/nodejs-examples/test-offline-tts-matcha-en.js b/nodejs-examples/test-offline-tts-matcha-en.js new file mode 100644 index 000000000..c2f982043 --- /dev/null +++ b/nodejs-examples/test-offline-tts-matcha-en.js @@ -0,0 +1,40 @@ +// Copyright (c) 2025 Xiaomi Corporation (authors: Fangjun Kuang) + +const sherpa_onnx = require('sherpa-onnx'); + +function createOfflineTts() { + let offlineTtsMatchaModelConfig = { + acousticModel: './matcha-icefall-en_US-ljspeech/model-steps-3.onnx', + vocoder: './hifigan_v2.onnx', + lexicon: './matcha-icefall-en_US-ljspeech/lexicon.txt', + tokens: './matcha-icefall-en_US-ljspeech/tokens.txt', + dataDir: './matcha-icefall-en_US-ljspeech/espeak-ng-data', + + noiseScale: 0.667, + lengthScale: 1.0, + }; + let offlineTtsModelConfig = { + offlineTtsMatchaModelConfig: offlineTtsMatchaModelConfig, + numThreads: 1, + debug: 1, + provider: 'cpu', + }; + + let offlineTtsConfig = { + offlineTtsModelConfig: offlineTtsModelConfig, + maxNumSentences: 1, + }; + + return sherpa_onnx.createOfflineTts(offlineTtsConfig); +} + +const tts = createOfflineTts(); +const speakerId = 0; +const speed = 1.0; +const text = + 'Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.' + +const audio = tts.generate({text: text, sid: speakerId, speed: speed}); +tts.save('./test-matcha-en.wav', audio); +console.log('Saved to test-matcha-en.wav successfully.'); +tts.free(); diff --git a/nodejs-examples/test-offline-tts-matcha-zh.js b/nodejs-examples/test-offline-tts-matcha-zh.js new file mode 100644 index 000000000..21c6a0875 --- /dev/null +++ b/nodejs-examples/test-offline-tts-matcha-zh.js @@ -0,0 +1,41 @@ +// Copyright (c) 2025 Xiaomi Corporation (authors: Fangjun Kuang) + +const sherpa_onnx = require('sherpa-onnx'); + +function createOfflineTts() { + let offlineTtsMatchaModelConfig = { + acousticModel: './matcha-icefall-zh-baker/model-steps-3.onnx', + vocoder: './hifigan_v2.onnx', + lexicon: './matcha-icefall-zh-baker/lexicon.txt', + tokens: './matcha-icefall-zh-baker/tokens.txt', + dictDir: './matcha-icefall-zh-baker/dict', + noiseScale: 0.667, + lengthScale: 1.0, + }; + let offlineTtsModelConfig = { + offlineTtsMatchaModelConfig: offlineTtsMatchaModelConfig, + numThreads: 1, + debug: 1, + provider: 'cpu', + }; + + let offlineTtsConfig = { + offlineTtsModelConfig: offlineTtsModelConfig, + maxNumSentences: 1, + ruleFsts: + './matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst', + }; + + return sherpa_onnx.createOfflineTts(offlineTtsConfig); +} + +const tts = createOfflineTts(); +const speakerId = 0; +const speed = 1.0; +const text = + '当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转，思念如涟漪荡漾，梦境如画卷展开，我与自然融为一体，沉静在这片宁静的美丽之中，感受着生命的奇迹与温柔. 某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。' + +const audio = tts.generate({text: text, sid: speakerId, speed: speed}); +tts.save('./test-matcha-zh.wav', audio); +console.log('Saved to test-matcha-zh.wav successfully.'); +tts.free(); diff --git a/nodejs-examples/test-offline-tts-en.js b/nodejs-examples/test-offline-tts-vits-en.js similarity index 92% rename from nodejs-examples/test-offline-tts-en.js rename to nodejs-examples/test-offline-tts-vits-en.js index 61c23f5eb..9e6c8da58 100644 --- a/nodejs-examples/test-offline-tts-en.js +++ b/nodejs-examples/test-offline-tts-vits-en.js @@ -37,7 +37,7 @@ const audio = tts.generate({ speed: speed }); -tts.save('./test-en.wav', audio); -console.log('Saved to test-en.wav successfully.'); +tts.save('./test-vits-en.wav', audio); +console.log('Saved to test-vits-en.wav successfully.'); tts.free(); diff --git a/nodejs-examples/test-offline-tts-zh.js b/nodejs-examples/test-offline-tts-vits-zh.js similarity index 92% rename from nodejs-examples/test-offline-tts-zh.js rename to nodejs-examples/test-offline-tts-vits-zh.js index 7be148862..78c81ead3 100644 --- a/nodejs-examples/test-offline-tts-zh.js +++ b/nodejs-examples/test-offline-tts-vits-zh.js @@ -34,6 +34,6 @@ const speakerId = 66; const speed = 1.0; const audio = tts.generate( {text: '3年前中国总人口是1411778724人', sid: speakerId, speed: speed}); -tts.save('./test-zh.wav', audio); -console.log('Saved to test-zh.wav successfully.'); +tts.save('./test-vits-zh.wav', audio); +console.log('Saved to test-vits-zh.wav successfully.'); tts.free(); diff --git a/scripts/dotnet/OfflineTtsMatchaModelConfig.cs b/scripts/dotnet/OfflineTtsMatchaModelConfig.cs new file mode 100644 index 000000000..8743e1223 --- /dev/null +++ b/scripts/dotnet/OfflineTtsMatchaModelConfig.cs @@ -0,0 +1,44 @@ +/// Copyright (c) 2025 Xiaomi Corporation (authors: Fangjun Kuang) + +using System.Runtime.InteropServices; + +namespace SherpaOnnx +{ + [StructLayout(LayoutKind.Sequential)] + public struct OfflineTtsMatchaModelConfig + { + public OfflineTtsMatchaModelConfig() + { + AcousticModel = ""; + Vocoder = ""; + Lexicon = ""; + Tokens = ""; + DataDir = ""; + + NoiseScale = 0.667F; + LengthScale = 1.0F; + + DictDir = ""; + } + [MarshalAs(UnmanagedType.LPStr)] + public string AcousticModel; + + [MarshalAs(UnmanagedType.LPStr)] + public string Vocoder; + + [MarshalAs(UnmanagedType.LPStr)] + public string Lexicon; + + [MarshalAs(UnmanagedType.LPStr)] + public string Tokens; + + [MarshalAs(UnmanagedType.LPStr)] + public string DataDir; + + public float NoiseScale; + public float LengthScale; + + [MarshalAs(UnmanagedType.LPStr)] + public string DictDir; + } +} diff --git a/scripts/dotnet/OfflineTtsModelConfig.cs b/scripts/dotnet/OfflineTtsModelConfig.cs index 40aa63912..e5caa1173 100644 --- a/scripts/dotnet/OfflineTtsModelConfig.cs +++ b/scripts/dotnet/OfflineTtsModelConfig.cs @@ -11,6 +11,7 @@ public struct OfflineTtsModelConfig public OfflineTtsModelConfig() { Vits = new OfflineTtsVitsModelConfig(); + Matcha = new OfflineTtsMatchaModelConfig(); NumThreads = 1; Debug = 0; Provider = "cpu"; @@ -21,5 +22,7 @@ public OfflineTtsModelConfig() public int Debug; [MarshalAs(UnmanagedType.LPStr)] public string Provider; + + public OfflineTtsMatchaModelConfig Matcha; } -} \ No newline at end of file +} diff --git a/scripts/dotnet/examples/Common.csproj b/scripts/dotnet/examples/Common.csproj index 868c1470f..6925e0381 100644 --- a/scripts/dotnet/examples/Common.csproj +++ b/scripts/dotnet/examples/Common.csproj @@ -1,7 +1,7 @@  - .net6 + net8.0 /tmp/packages;$(RestoreSources);https://api.nuget.org/v3/index.json diff --git a/scripts/dotnet/sherpa-onnx.csproj.in b/scripts/dotnet/sherpa-onnx.csproj.in index 0cbe7efab..d81a883e5 100644 --- a/scripts/dotnet/sherpa-onnx.csproj.in +++ b/scripts/dotnet/sherpa-onnx.csproj.in @@ -4,7 +4,7 @@ README.md Library 10.0 - net6.0;net45;net40;net35;net20;netstandard2.0 + net8.0;net7.0;net6.0;net45;net40;net35;net20;netstandard2.0 linux-x64;linux-arm64;osx-x64;osx-arm64;win-x64;win-x86;win-arm64 true sherpa-onnx diff --git a/scripts/dotnet/sherpa-onnx.csproj.runtime.in b/scripts/dotnet/sherpa-onnx.csproj.runtime.in index 2a387ccea..f21c3da37 100644 --- a/scripts/dotnet/sherpa-onnx.csproj.runtime.in +++ b/scripts/dotnet/sherpa-onnx.csproj.runtime.in @@ -3,7 +3,7 @@ Apache-2.0 README.md Library - net6.0;net45;net40;net35;net20;netstandard2.0 + net8.0;net7.0;net6.0;net45;net40;net35;net20;netstandard2.0 {{ dotnet_rid }} sherpa-onnx {{ version }} diff --git a/wasm/tts/sherpa-onnx-tts.js b/wasm/tts/sherpa-onnx-tts.js index 4d68b854f..59158ae76 100644 --- a/wasm/tts/sherpa-onnx-tts.js +++ b/wasm/tts/sherpa-onnx-tts.js @@ -8,6 +8,10 @@ function freeConfig(config, Module) { freeConfig(config.config, Module) } + if ('config2' in config) { + freeConfig(config.config2, Module) + } + Module._free(config.ptr); } @@ -66,11 +70,103 @@ function initSherpaOnnxOfflineTtsVitsModelConfig(config, Module) { } } +function initSherpaOnnxOfflineTtsMatchaModelConfig(config, Module) { + const acousticModelLen = Module.lengthBytesUTF8(config.acousticModel) + 1; + const vocoderLen = Module.lengthBytesUTF8(config.vocoder) + 1; + const lexiconLen = Module.lengthBytesUTF8(config.lexicon || '') + 1; + const tokensLen = Module.lengthBytesUTF8(config.tokens || '') + 1; + const dataDirLen = Module.lengthBytesUTF8(config.dataDir || '') + 1; + const dictDirLen = Module.lengthBytesUTF8(config.dictDir || '') + 1; + + const n = acousticModelLen + vocoderLen + lexiconLen + tokensLen + + dataDirLen + dictDirLen; + + const buffer = Module._malloc(n); + + const len = 8 * 4; + const ptr = Module._malloc(len); + + let offset = 0; + Module.stringToUTF8( + config.acousticModel || '', buffer + offset, acousticModelLen); + offset += acousticModelLen; + + Module.stringToUTF8(config.vocoder || '', buffer + offset, vocoderLen); + offset += vocoderLen; + + Module.stringToUTF8(config.lexicon || '', buffer + offset, lexiconLen); + offset += lexiconLen; + + Module.stringToUTF8(config.tokens || '', buffer + offset, tokensLen); + offset += tokensLen; + + Module.stringToUTF8(config.dataDir || '', buffer + offset, dataDirLen); + offset += dataDirLen; + + Module.stringToUTF8(config.dictDir || '', buffer + offset, dictDirLen); + offset += dictDirLen; + + offset = 0; + Module.setValue(ptr, buffer + offset, 'i8*'); + offset += acousticModelLen; + + Module.setValue(ptr + 4, buffer + offset, 'i8*'); + offset += vocoderLen; + + Module.setValue(ptr + 8, buffer + offset, 'i8*'); + offset += lexiconLen; + + Module.setValue(ptr + 12, buffer + offset, 'i8*'); + offset += tokensLen; + + Module.setValue(ptr + 16, buffer + offset, 'i8*'); + offset += dataDirLen; + + Module.setValue(ptr + 20, config.noiseScale || 0.667, 'float'); + Module.setValue(ptr + 24, config.lengthScale || 1.0, 'float'); + Module.setValue(ptr + 28, buffer + offset, 'i8*'); + offset += dictDirLen; + + return { + buffer: buffer, ptr: ptr, len: len, + } +} + function initSherpaOnnxOfflineTtsModelConfig(config, Module) { + if (!('offlineTtsVitsModelConfig' in config)) { + config.offlineTtsVitsModelConfig = { + model: '', + lexicon: '', + tokens: '', + noiseScale: 0.667, + noiseScaleW: 0.8, + lengthScale: 1.0, + dataDir: '', + dictDir: '', + }; + } + + if (!('offlineTtsMatchaModelConfig' in config)) { + config.offlineTtsMatchaModelConfig = { + acousticModel: '', + vocoder: '', + lexicon: '', + tokens: '', + noiseScale: 0.667, + lengthScale: 1.0, + dataDir: '', + dictDir: '', + }; + } + + const vitsModelConfig = initSherpaOnnxOfflineTtsVitsModelConfig( config.offlineTtsVitsModelConfig, Module); - const len = vitsModelConfig.len + 3 * 4; + const matchaModelConfig = initSherpaOnnxOfflineTtsMatchaModelConfig( + config.offlineTtsMatchaModelConfig, Module); + + const len = vitsModelConfig.len + matchaModelConfig.len + 3 * 4; const ptr = Module._malloc(len); let offset = 0; @@ -87,9 +183,14 @@ function initSherpaOnnxOfflineTtsModelConfig(config, Module) { const buffer = Module._malloc(providerLen); Module.stringToUTF8(config.provider, buffer, providerLen); Module.setValue(ptr + offset, buffer, 'i8*'); + offset += 4; + + Module._CopyHeap(matchaModelConfig.ptr, matchaModelConfig.len, ptr + offset); + offset += matchaModelConfig.len; return { buffer: buffer, ptr: ptr, len: len, config: vitsModelConfig, + config2: matchaModelConfig } } @@ -195,12 +296,26 @@ function createOfflineTts(Module, myConfig) { noiseScaleW: 0.8, lengthScale: 1.0, }; + + const offlineTtsMatchaModelConfig = { + acousticModel: '', + vocoder: '', + lexicon: '', + tokens: '', + dataDir: '', + dictDir: '', + noiseScale: 0.667, + lengthScale: 1.0, + }; + const offlineTtsModelConfig = { offlineTtsVitsModelConfig: offlineTtsVitsModelConfig, + offlineTtsMatchaModelConfig: offlineTtsMatchaModelConfig, numThreads: 1, debug: 1, provider: 'cpu', }; + let offlineTtsConfig = { offlineTtsModelConfig: offlineTtsModelConfig, ruleFsts: '', diff --git a/wasm/tts/sherpa-onnx-wasm-main-tts.cc b/wasm/tts/sherpa-onnx-wasm-main-tts.cc index 872a1c853..3508b860d 100644 --- a/wasm/tts/sherpa-onnx-wasm-main-tts.cc +++ b/wasm/tts/sherpa-onnx-wasm-main-tts.cc @@ -14,8 +14,10 @@ extern "C" { static_assert(sizeof(SherpaOnnxOfflineTtsVitsModelConfig) == 8 * 4, ""); +static_assert(sizeof(SherpaOnnxOfflineTtsMatchaModelConfig) == 8 * 4, ""); static_assert(sizeof(SherpaOnnxOfflineTtsModelConfig) == - sizeof(SherpaOnnxOfflineTtsVitsModelConfig) + 3 * 4, + sizeof(SherpaOnnxOfflineTtsVitsModelConfig) + + sizeof(SherpaOnnxOfflineTtsMatchaModelConfig) + 3 * 4, ""); static_assert(sizeof(SherpaOnnxOfflineTtsConfig) == sizeof(SherpaOnnxOfflineTtsModelConfig) + 3 * 4, @@ -24,6 +26,7 @@ static_assert(sizeof(SherpaOnnxOfflineTtsConfig) == void MyPrint(SherpaOnnxOfflineTtsConfig *tts_config) { auto tts_model_config = &tts_config->model; auto vits_model_config = &tts_model_config->vits; + auto matcha_model_config = &tts_model_config->matcha; fprintf(stdout, "----------vits model config----------\n"); fprintf(stdout, "model: %s\n", vits_model_config->model); fprintf(stdout, "lexicon: %s\n", vits_model_config->lexicon); @@ -34,6 +37,16 @@ void MyPrint(SherpaOnnxOfflineTtsConfig *tts_config) { fprintf(stdout, "length scale: %.3f\n", vits_model_config->length_scale); fprintf(stdout, "dict_dir: %s\n", vits_model_config->dict_dir); + fprintf(stdout, "----------matcha model config----------\n"); + fprintf(stdout, "acoustic_model: %s\n", matcha_model_config->acoustic_model); + fprintf(stdout, "vocoder: %s\n", matcha_model_config->vocoder); + fprintf(stdout, "lexicon: %s\n", matcha_model_config->lexicon); + fprintf(stdout, "tokens: %s\n", matcha_model_config->tokens); + fprintf(stdout, "data_dir: %s\n", matcha_model_config->data_dir); + fprintf(stdout, "noise scale: %.3f\n", matcha_model_config->noise_scale); + fprintf(stdout, "length scale: %.3f\n", matcha_model_config->length_scale); + fprintf(stdout, "dict_dir: %s\n", matcha_model_config->dict_dir); + fprintf(stdout, "----------tts model config----------\n"); fprintf(stdout, "num threads: %d\n", tts_model_config->num_threads); fprintf(stdout, "debug: %d\n", tts_model_config->debug); From 1fe5fe495fc29cbef663c7d9bd4faddfaf40f024 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 6 Jan 2025 06:44:09 +0800 Subject: [PATCH 142/183] Add Android demo for MatchaTTS models. (#1683) --- .github/workflows/apk-tts-engine.yaml | 1 + .../com/k2fsa/sherpa/onnx/MainActivity.kt | 56 +++++++++++++---- .../sherpa/onnx/tts/engine/MainActivity.kt | 10 +-- .../k2fsa/sherpa/onnx/tts/engine/TtsEngine.kt | 61 +++++++++++++++---- scripts/apk/build-apk-tts-engine.sh.in | 20 +++++- scripts/apk/build-apk-tts.sh.in | 23 ++++++- scripts/apk/generate-tts-apk-script.py | 36 ++++++++++- sherpa-onnx/csrc/offline-tts-vits-impl.h | 4 ++ sherpa-onnx/kotlin-api/Tts.kt | 49 ++++++++++++--- 9 files changed, 222 insertions(+), 38 deletions(-) diff --git a/.github/workflows/apk-tts-engine.yaml b/.github/workflows/apk-tts-engine.yaml index b8614cb76..68fdaa05d 100644 --- a/.github/workflows/apk-tts-engine.yaml +++ b/.github/workflows/apk-tts-engine.yaml @@ -26,6 +26,7 @@ jobs: total: ["40"] index: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39"] + steps: - uses: actions/checkout@v4 with: diff --git a/android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt b/android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt index b95ad7d78..5119a50b2 100644 --- a/android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt +++ b/android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt @@ -183,6 +183,8 @@ class MainActivity : AppCompatActivity() { private fun initTts() { var modelDir: String? var modelName: String? + var acousticModelName: String? + var vocoder: String? var ruleFsts: String? var ruleFars: String? var lexicon: String? @@ -193,8 +195,18 @@ class MainActivity : AppCompatActivity() { // The purpose of such a design is to make the CI test easier // Please see // https://github.com/k2-fsa/sherpa-onnx/blob/master/scripts/apk/generate-tts-apk-script.py - modelDir = null + + // VITS -- begin modelName = null + // VITS -- end + + // Matcha -- begin + acousticModelName = null + vocoder = null + // Matcha -- end + + + modelDir = null ruleFsts = null ruleFars = null lexicon = null @@ -217,7 +229,6 @@ class MainActivity : AppCompatActivity() { // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-icefall-zh-aishell3.tar.bz2 // modelDir = "vits-icefall-zh-aishell3" // modelName = "model.onnx" - // ruleFsts = "vits-icefall-zh-aishell3/phone.fst,vits-icefall-zh-aishell3/date.fst,vits-icefall-zh-aishell3/number.fst,vits-icefall-zh-aishell3/new_heteronym.fst" // ruleFars = "vits-icefall-zh-aishell3/rule.far" // lexicon = "lexicon.txt" @@ -233,24 +244,47 @@ class MainActivity : AppCompatActivity() { // modelDir = "vits-coqui-de-css10" // modelName = "model.onnx" + // Example 6 + // vits-melo-tts-zh_en + // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/vits.html#vits-melo-tts-zh-en-chinese-english-1-speaker + // modelDir = "vits-melo-tts-zh_en" + // modelName = "model.onnx" + // lexicon = "lexicon.txt" + // dictDir = "vits-melo-tts-zh_en/dict" + + // Example 7 + // matcha-icefall-zh-baker + // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker + // modelDir = "matcha-icefall-zh-baker" + // acousticModelName = "model-steps-3.onnx" + // vocoder = "hifigan_v2.onnx" + // lexicon = "lexicon.txt" + // dictDir = "matcha-icefall-zh-baker/dict" + + // Example 8 + // matcha-icefall-en_US-ljspeech + // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker + // modelDir = "matcha-icefall-en_US-ljspeech" + // acousticModelName = "model-steps-3.onnx" + // vocoder = "hifigan_v2.onnx" + // dataDir = "matcha-icefall-en_US-ljspeech/espeak-ng-data" + if (dataDir != null) { - val newDir = copyDataDir(modelDir!!) - modelDir = newDir + "/" + modelDir - dataDir = newDir + "/" + dataDir - assets = null + val newDir = copyDataDir(dataDir!!) + dataDir = "$newDir/$dataDir" } if (dictDir != null) { - val newDir = copyDataDir(modelDir!!) - modelDir = newDir + "/" + modelDir - dictDir = modelDir + "/" + "dict" + val newDir = copyDataDir(dictDir!!) + dictDir = "$newDir/$dictDir" ruleFsts = "$modelDir/phone.fst,$modelDir/date.fst,$modelDir/number.fst" - assets = null } val config = getOfflineTtsConfig( modelDir = modelDir!!, - modelName = modelName!!, + modelName = modelName ?: "", + acousticModelName = acousticModelName ?: "", + vocoder = vocoder ?: "", lexicon = lexicon ?: "", dataDir = dataDir ?: "", dictDir = dictDir ?: "", diff --git a/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/MainActivity.kt b/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/MainActivity.kt index 9a6bd47ab..f7e34c5dd 100644 --- a/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/MainActivity.kt +++ b/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/MainActivity.kt @@ -57,7 +57,7 @@ class MainActivity : ComponentActivity() { color = MaterialTheme.colorScheme.background ) { Scaffold(topBar = { - TopAppBar(title = { Text("Next-gen Kaldi: TTS") }) + TopAppBar(title = { Text("Next-gen Kaldi: TTS Engine") }) }) { Box(modifier = Modifier.padding(it)) { Column(modifier = Modifier.padding(16.dp)) { @@ -65,8 +65,8 @@ class MainActivity : ComponentActivity() { Text("Speed " + String.format("%.1f", TtsEngine.speed)) Slider( value = TtsEngine.speedState.value, - onValueChange = { - TtsEngine.speed = it + onValueChange = { + TtsEngine.speed = it preferenceHelper.setSpeed(it) }, valueRange = 0.2F..3.0F, @@ -138,7 +138,9 @@ class MainActivity : ComponentActivity() { val filename = application.filesDir.absolutePath + "/generated.wav" val ok = - audio.samples.isNotEmpty() && audio.save(filename) + audio.samples.isNotEmpty() && audio.save( + filename + ) if (ok) { stopMediaPlayer() diff --git a/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsEngine.kt b/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsEngine.kt index 480f8a384..cec07ffd5 100644 --- a/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsEngine.kt +++ b/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsEngine.kt @@ -1,5 +1,6 @@ package com.k2fsa.sherpa.onnx.tts.engine +import PreferenceHelper import android.content.Context import android.content.res.AssetManager import android.util.Log @@ -11,7 +12,6 @@ import com.k2fsa.sherpa.onnx.getOfflineTtsConfig import java.io.File import java.io.FileOutputStream import java.io.IOException -import PreferenceHelper object TtsEngine { var tts: OfflineTts? = null @@ -41,6 +41,8 @@ object TtsEngine { private var modelDir: String? = null private var modelName: String? = null + private var acousticModelName: String? = null + private var vocoder: String? = null private var ruleFsts: String? = null private var ruleFars: String? = null private var lexicon: String? = null @@ -52,8 +54,17 @@ object TtsEngine { // The purpose of such a design is to make the CI test easier // Please see // https://github.com/k2-fsa/sherpa-onnx/blob/master/scripts/apk/generate-tts-apk-script.py - modelDir = null + // + // For VITS -- begin modelName = null + // For VITS -- end + + // For Matcha -- begin + acousticModelName = null + vocoder = null + // For Matcha -- end + + modelDir = null ruleFsts = null ruleFars = null lexicon = null @@ -82,7 +93,6 @@ object TtsEngine { // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-icefall-zh-aishell3.tar.bz2 // modelDir = "vits-icefall-zh-aishell3" // modelName = "model.onnx" - // ruleFsts = "vits-icefall-zh-aishell3/phone.fst,vits-icefall-zh-aishell3/date.fst,vits-icefall-zh-aishell3/number.fst,vits-icefall-zh-aishell3/new_heteronym.fst" // ruleFars = "vits-icefall-zh-aishell3/rule.far" // lexicon = "lexicon.txt" // lang = "zho" @@ -101,8 +111,35 @@ object TtsEngine { // modelDir = "vits-coqui-de-css10" // modelName = "model.onnx" // lang = "deu" - } + // Example 6 + // vits-melo-tts-zh_en + // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/vits.html#vits-melo-tts-zh-en-chinese-english-1-speaker + // modelDir = "vits-melo-tts-zh_en" + // modelName = "model.onnx" + // lexicon = "lexicon.txt" + // dictDir = "vits-melo-tts-zh_en/dict" + // lang = "zho" + + // Example 7 + // matcha-icefall-zh-baker + // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker + // modelDir = "matcha-icefall-zh-baker" + // acousticModelName = "model-steps-3.onnx" + // vocoder = "hifigan_v2.onnx" + // lexicon = "lexicon.txt" + // dictDir = "matcha-icefall-zh-baker/dict" + // lang = "zho" + + // Example 8 + // matcha-icefall-en_US-ljspeech + // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker + // modelDir = "matcha-icefall-en_US-ljspeech" + // acousticModelName = "model-steps-3.onnx" + // vocoder = "hifigan_v2.onnx" + // dataDir = "matcha-icefall-en_US-ljspeech/espeak-ng-data" + // lang = "eng" + } fun createTts(context: Context) { Log.i(TAG, "Init Next-gen Kaldi TTS") @@ -115,22 +152,22 @@ object TtsEngine { assets = context.assets if (dataDir != null) { - val newDir = copyDataDir(context, modelDir!!) - modelDir = "$newDir/$modelDir" + val newDir = copyDataDir(context, dataDir!!) dataDir = "$newDir/$dataDir" - assets = null } if (dictDir != null) { - val newDir = copyDataDir(context, modelDir!!) - modelDir = "$newDir/$modelDir" - dictDir = "$modelDir/dict" + val newDir = copyDataDir(context, dictDir!!) + dictDir = "$newDir/$dictDir" ruleFsts = "$modelDir/phone.fst,$modelDir/date.fst,$modelDir/number.fst" - assets = null } val config = getOfflineTtsConfig( - modelDir = modelDir!!, modelName = modelName!!, lexicon = lexicon ?: "", + modelDir = modelDir!!, + modelName = modelName ?: "", + acousticModelName = acousticModelName ?: "", + vocoder = vocoder ?: "", + lexicon = lexicon ?: "", dataDir = dataDir ?: "", dictDir = dictDir ?: "", ruleFsts = ruleFsts ?: "", diff --git a/scripts/apk/build-apk-tts-engine.sh.in b/scripts/apk/build-apk-tts-engine.sh.in index c611c061b..69933d2fc 100644 --- a/scripts/apk/build-apk-tts-engine.sh.in +++ b/scripts/apk/build-apk-tts-engine.sh.in @@ -37,6 +37,8 @@ mkdir -p apks pushd ./android/SherpaOnnxTtsEngine/app/src/main/assets/ model_dir={{ tts_model.model_dir }} model_name={{ tts_model.model_name }} +acoustic_model_name={{ tts_model.acoustic_model_name }} +vocoder={{ tts_model.vocoder }} lang={{ tts_model.lang }} lang_iso_639_3={{ tts_model.lang_iso_639_3 }} @@ -44,15 +46,30 @@ wget -qq https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/$mod tar xf $model_dir.tar.bz2 rm $model_dir.tar.bz2 +{% if tts_model.vocoder %} + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/$vocoder +{% endif %} + popd # Now we are at the project root directory git checkout . pushd android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine sed -i.bak s/"modelDir = null"/"modelDir = \"$model_dir\""/ ./TtsEngine.kt -sed -i.bak s/"modelName = null"/"modelName = \"$model_name\""/ ./TtsEngine.kt sed -i.bak s/"lang = null"/"lang = \"$lang_iso_639_3\""/ ./TtsEngine.kt +{% if tts_model.model_name %} + sed -i.bak s/"modelName = null"/"modelName = \"$model_name\""/ ./TtsEngine.kt +{% endif %} + +{% if tts_model.model_name %} + sed -i.bak s/"acousticModelName = null"/"acousticModelName = \"$acoustic_model_name\""/ ./TtsEngine.kt +{% endif %} + +{% if tts_model.vocoder %} + sed -i.bak s/"vocoder = null"/"vocoder = \"$vocoder\""/ ./TtsEngine.kt +{% endif %} + {% if tts_model.rule_fsts %} rule_fsts={{ tts_model.rule_fsts }} sed -i.bak s%"ruleFsts = null"%"ruleFsts = \"$rule_fsts\""% ./TtsEngine.kt @@ -109,6 +126,7 @@ for arch in arm64-v8a armeabi-v7a x86_64 x86; do done rm -rf ./android/SherpaOnnxTtsEngine/app/src/main/assets/$model_dir +rm -fv ./android/SherpaOnnxTtsEngine/app/src/main/assets/*.onnx {% endfor %} git checkout . diff --git a/scripts/apk/build-apk-tts.sh.in b/scripts/apk/build-apk-tts.sh.in index 2e62ad636..34135f1a1 100644 --- a/scripts/apk/build-apk-tts.sh.in +++ b/scripts/apk/build-apk-tts.sh.in @@ -37,19 +37,38 @@ mkdir -p apks pushd ./android/SherpaOnnxTts/app/src/main/assets/ model_dir={{ tts_model.model_dir }} model_name={{ tts_model.model_name }} +acoustic_model_name={{ tts_model.acoustic_model_name }} +vocoder={{ tts_model.vocoder }} lang={{ tts_model.lang }} wget -qq https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/$model_dir.tar.bz2 tar xf $model_dir.tar.bz2 rm $model_dir.tar.bz2 +{% if tts_model.vocoder %} + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/$vocoder +{% endif %} + popd # Now we are at the project root directory git checkout . pushd android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx sed -i.bak s/"modelDir = null"/"modelDir = \"$model_dir\""/ ./MainActivity.kt -sed -i.bak s/"modelName = null"/"modelName = \"$model_name\""/ ./MainActivity.kt + + +{% if tts_model.model_name %} + sed -i.bak s/"modelName = null"/"modelName = \"$model_name\""/ ./MainActivity.kt +{% endif %} + +{% if tts_model.acoustic_model_name %} + sed -i.bak s/"acousticModelName = null"/"acousticModelName = \"$acoustic_model_name\""/ ./MainActivity.kt +{% endif %} + +{% if tts_model.vocoder %} + sed -i.bak s/"vocoder = null"/"vocoder = \"$vocoder\""/ ./MainActivity.kt +{% endif %} + {% if tts_model.rule_fsts %} rule_fsts={{ tts_model.rule_fsts }} @@ -107,6 +126,8 @@ for arch in arm64-v8a armeabi-v7a x86_64 x86; do done rm -rf ./android/SherpaOnnxTts/app/src/main/assets/$model_dir +rm -fv ./android/SherpaOnnxTts/app/src/main/assets/*.onnx + {% endfor %} git checkout . diff --git a/scripts/apk/generate-tts-apk-script.py b/scripts/apk/generate-tts-apk-script.py index 1aa034945..1d804ecf9 100755 --- a/scripts/apk/generate-tts-apk-script.py +++ b/scripts/apk/generate-tts-apk-script.py @@ -30,7 +30,9 @@ def get_args(): @dataclass class TtsModel: model_dir: str - model_name: str = "" + model_name: str = "" # for vits + acoustic_model_name: str = "" # for matcha + vocoder: str = "" # for matcha lang: str = "" # en, zh, fr, de, etc. rule_fsts: Optional[List[str]] = None rule_fars: Optional[List[str]] = None @@ -378,6 +380,35 @@ def get_vits_models() -> List[TtsModel]: return all_models +def get_matcha_models() -> List[TtsModel]: + chinese_models = [ + TtsModel( + model_dir="matcha-icefall-zh-baker", + acoustic_model_name="model-steps-3.onnx", + lang="zh", + ) + ] + rule_fsts = ["phone.fst", "date.fst", "number.fst"] + for m in chinese_models: + s = [f"{m.model_dir}/{r}" for r in rule_fsts] + m.rule_fsts = ",".join(s) + m.dict_dir = m.model_dir + "/dict" + m.vocoder = "hifigan_v2.onnx" + + english_models = [ + TtsModel( + model_dir="matcha-icefall-en_US-ljspeech", + acoustic_model_name="model-steps-3.onnx", + lang="en", + ) + ] + for m in english_models: + m.data_dir = f"{m.model_dir}/espeak-ng-data" + m.vocoder = "hifigan_v2.onnx" + + return chinese_models + english_models + + def main(): args = get_args() index = args.index @@ -389,7 +420,10 @@ def main(): all_model_list += get_piper_models() all_model_list += get_mimic3_models() all_model_list += get_coqui_models() + all_model_list += get_matcha_models() + convert_lang_to_iso_639_3(all_model_list) + print(all_model_list) num_models = len(all_model_list) diff --git a/sherpa-onnx/csrc/offline-tts-vits-impl.h b/sherpa-onnx/csrc/offline-tts-vits-impl.h index 1cc8d5f95..72146b02c 100644 --- a/sherpa-onnx/csrc/offline-tts-vits-impl.h +++ b/sherpa-onnx/csrc/offline-tts-vits-impl.h @@ -348,6 +348,10 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { mgr, config_.model.vits.lexicon, config_.model.vits.tokens, config_.model.vits.dict_dir, model_->GetMetaData(), config_.model.debug); + } else if (meta_data.jieba && !config_.model.vits.dict_dir.empty()) { + frontend_ = std::make_unique( + mgr, config_.model.vits.lexicon, config_.model.vits.tokens, + config_.model.vits.dict_dir, config_.model.debug); } else if (meta_data.is_melo_tts && meta_data.language == "English") { frontend_ = std::make_unique( mgr, config_.model.vits.lexicon, config_.model.vits.tokens, diff --git a/sherpa-onnx/kotlin-api/Tts.kt b/sherpa-onnx/kotlin-api/Tts.kt index 231b87d81..98efe6644 100644 --- a/sherpa-onnx/kotlin-api/Tts.kt +++ b/sherpa-onnx/kotlin-api/Tts.kt @@ -173,22 +173,55 @@ class OfflineTts( // to download models fun getOfflineTtsConfig( modelDir: String, - modelName: String, + modelName: String, // for VITS + acousticModelName: String, // for Matcha + vocoder: String, // for Matcha lexicon: String, dataDir: String, dictDir: String, ruleFsts: String, ruleFars: String ): OfflineTtsConfig { + if (modelName.isEmpty() && acousticModelName.isEmpty()) { + throw IllegalArgumentException("Please specify a TTS model") + } + + if (modelName.isNotEmpty() && acousticModelName.isNotEmpty()) { + throw IllegalArgumentException("Please specify either a VITS or a Matcha model, but not both") + } + + if (acousticModelName.isNotEmpty() && vocoder.isEmpty()) { + throw IllegalArgumentException("Please provide vocoder for Matcha TTS") + } + val vits = if (modelName.isNotEmpty()) { + OfflineTtsVitsModelConfig( + model = "$modelDir/$modelName", + lexicon = "$modelDir/$lexicon", + tokens = "$modelDir/tokens.txt", + dataDir = dataDir, + dictDir = dictDir, + ) + } else { + OfflineTtsVitsModelConfig() + } + + val matcha = if (acousticModelName.isNotEmpty()) { + OfflineTtsMatchaModelConfig( + acousticModel = "$modelDir/$acousticModelName", + vocoder = vocoder, + lexicon = "$modelDir/$lexicon", + tokens = "$modelDir/tokens.txt", + dictDir = dictDir, + dataDir = dataDir, + ) + } else { + OfflineTtsMatchaModelConfig() + } + return OfflineTtsConfig( model = OfflineTtsModelConfig( - vits = OfflineTtsVitsModelConfig( - model = "$modelDir/$modelName", - lexicon = "$modelDir/$lexicon", - tokens = "$modelDir/tokens.txt", - dataDir = dataDir, - dictDir = dictDir, - ), + vits = vits, + matcha = matcha, numThreads = 2, debug = true, provider = "cpu", From 6f085babcc51dd05a411b6949318c401a5adbf3f Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 6 Jan 2025 07:23:45 +0800 Subject: [PATCH 143/183] Add Swift API for MatchaTTS models. (#1684) --- .github/scripts/test-swift.sh | 18 ++++- .../run-non-streaming-tts-matcha-en.sh | 2 +- nodejs-addon-examples/README.md | 2 +- nodejs-examples/README.md | 2 +- swift-api-examples/.gitignore | 4 +- swift-api-examples/SherpaOnnx.swift | 33 +++++++-- swift-api-examples/run-tts-matcha-en.sh | 42 ++++++++++++ swift-api-examples/run-tts-matcha-zh.sh | 41 +++++++++++ .../{run-tts.sh => run-tts-vits.sh} | 10 +-- swift-api-examples/tts-matcha-en.swift | 65 ++++++++++++++++++ swift-api-examples/tts-matcha-zh.swift | 68 +++++++++++++++++++ .../{tts.swift => tts-vits.swift} | 2 +- 12 files changed, 271 insertions(+), 18 deletions(-) create mode 100755 swift-api-examples/run-tts-matcha-en.sh create mode 100755 swift-api-examples/run-tts-matcha-zh.sh rename swift-api-examples/{run-tts.sh => run-tts-vits.sh} (86%) create mode 100644 swift-api-examples/tts-matcha-en.swift create mode 100644 swift-api-examples/tts-matcha-zh.swift rename swift-api-examples/{tts.swift => tts-vits.swift} (98%) diff --git a/.github/scripts/test-swift.sh b/.github/scripts/test-swift.sh index 0da23eb24..f333bc0a9 100755 --- a/.github/scripts/test-swift.sh +++ b/.github/scripts/test-swift.sh @@ -7,6 +7,18 @@ echo "pwd: $PWD" cd swift-api-examples ls -lh +./run-tts-vits.sh +ls -lh +rm -rf vits-piper-* + +./run-tts-matcha-zh.sh +ls -lh +rm -rf matcha-icefall-* + +./run-tts-matcha-en.sh +ls -lh +rm -rf matcha-icefall-* + ./run-speaker-diarization.sh rm -rf *.onnx rm -rf sherpa-onnx-pyannote-segmentation-3-0 @@ -38,8 +50,9 @@ popd ls -lh /Users/fangjun/Desktop cat /Users/fangjun/Desktop/Obama.srt -./run-tts.sh -ls -lh +rm -rf sherpa-onnx-whisper* +rm -f *.onnx +rm /Users/fangjun/Desktop/Obama.wav ./run-decode-file.sh rm decode-file @@ -48,5 +61,4 @@ sed -i.bak '20d' ./decode-file.swift ./run-decode-file-non-streaming.sh - ls -lh diff --git a/java-api-examples/run-non-streaming-tts-matcha-en.sh b/java-api-examples/run-non-streaming-tts-matcha-en.sh index ce0289fc9..ba03beaf2 100755 --- a/java-api-examples/run-non-streaming-tts-matcha-en.sh +++ b/java-api-examples/run-non-streaming-tts-matcha-en.sh @@ -31,7 +31,7 @@ fi # to download more models if [ ! -f ./matcha-icefall-en_US-ljspeech/model-steps-3.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 - tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 + tar xf matcha-icefall-en_US-ljspeech.tar.bz2 rm matcha-icefall-en_US-ljspeech.tar.bz2 fi diff --git a/nodejs-addon-examples/README.md b/nodejs-addon-examples/README.md index ec2f23da2..2de8a2143 100644 --- a/nodejs-addon-examples/README.md +++ b/nodejs-addon-examples/README.md @@ -350,7 +350,7 @@ node ./test_vad_asr_non_streaming_sense_voice_microphone.js ### Text-to-speech with MatchaTTS models (English TTS) ```bash wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 -tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 +tar xf matcha-icefall-en_US-ljspeech.tar.bz2 rm matcha-icefall-en_US-ljspeech.tar.bz2 wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx diff --git a/nodejs-examples/README.md b/nodejs-examples/README.md index 0c59b7bcc..3db3a2952 100644 --- a/nodejs-examples/README.md +++ b/nodejs-examples/README.md @@ -70,7 +70,7 @@ You can use the following command to run it: ```bash wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 -tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 +tar xf matcha-icefall-en_US-ljspeech.tar.bz2 rm matcha-icefall-en_US-ljspeech.tar.bz2 wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx diff --git a/swift-api-examples/.gitignore b/swift-api-examples/.gitignore index 97b559df4..1a90488aa 100644 --- a/swift-api-examples/.gitignore +++ b/swift-api-examples/.gitignore @@ -2,7 +2,7 @@ decode-file decode-file-non-streaming generate-subtitles spoken-language-identification -tts +tts-vits vits-vctk sherpa-onnx-paraformer-zh-2023-09-14 !*.sh @@ -10,3 +10,5 @@ sherpa-onnx-paraformer-zh-2023-09-14 streaming-hlg-decode-file keyword-spotting-from-file add-punctuations +tts-matcha-zh +tts-matcha-en diff --git a/swift-api-examples/SherpaOnnx.swift b/swift-api-examples/SherpaOnnx.swift index b100ef408..6d11a0011 100644 --- a/swift-api-examples/SherpaOnnx.swift +++ b/swift-api-examples/SherpaOnnx.swift @@ -719,9 +719,9 @@ class SherpaOnnxVoiceActivityDetectorWrapper { // offline tts func sherpaOnnxOfflineTtsVitsModelConfig( - model: String, - lexicon: String, - tokens: String, + model: String = "", + lexicon: String = "", + tokens: String = "", dataDir: String = "", noiseScale: Float = 0.667, noiseScaleW: Float = 0.8, @@ -739,8 +739,30 @@ func sherpaOnnxOfflineTtsVitsModelConfig( dict_dir: toCPointer(dictDir)) } +func sherpaOnnxOfflineTtsMatchaModelConfig( + acousticModel: String = "", + vocoder: String = "", + lexicon: String = "", + tokens: String = "", + dataDir: String = "", + noiseScale: Float = 0.667, + lengthScale: Float = 1.0, + dictDir: String = "" +) -> SherpaOnnxOfflineTtsMatchaModelConfig { + return SherpaOnnxOfflineTtsMatchaModelConfig( + acoustic_model: toCPointer(acousticModel), + vocoder: toCPointer(vocoder), + lexicon: toCPointer(lexicon), + tokens: toCPointer(tokens), + data_dir: toCPointer(dataDir), + noise_scale: noiseScale, + length_scale: lengthScale, + dict_dir: toCPointer(dictDir)) +} + func sherpaOnnxOfflineTtsModelConfig( - vits: SherpaOnnxOfflineTtsVitsModelConfig, + vits: SherpaOnnxOfflineTtsVitsModelConfig = sherpaOnnxOfflineTtsVitsModelConfig(), + matcha: SherpaOnnxOfflineTtsMatchaModelConfig = sherpaOnnxOfflineTtsMatchaModelConfig(), numThreads: Int = 1, debug: Int = 0, provider: String = "cpu" @@ -749,7 +771,8 @@ func sherpaOnnxOfflineTtsModelConfig( vits: vits, num_threads: Int32(numThreads), debug: Int32(debug), - provider: toCPointer(provider) + provider: toCPointer(provider), + matcha: matcha ) } diff --git a/swift-api-examples/run-tts-matcha-en.sh b/swift-api-examples/run-tts-matcha-en.sh new file mode 100755 index 000000000..1f23f56ec --- /dev/null +++ b/swift-api-examples/run-tts-matcha-en.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -ex + +if [ ! -d ../build-swift-macos ]; then + echo "Please run ../build-swift-macos.sh first!" + exit 1 +fi + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker +# matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker +# to download more models +if [ ! -f ./matcha-icefall-en_US-ljspeech/model-steps-3.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 + tar xf matcha-icefall-en_US-ljspeech.tar.bz2 + rm matcha-icefall-en_US-ljspeech.tar.bz2 +fi + +if [ ! -f ./hifigan_v2.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx +fi + +if [ ! -e ./tts ]; then + # Note: We use -lc++ to link against libc++ instead of libstdc++ + swiftc \ + -lc++ \ + -I ../build-swift-macos/install/include \ + -import-objc-header ./SherpaOnnx-Bridging-Header.h \ + ./tts-matcha-en.swift ./SherpaOnnx.swift \ + -L ../build-swift-macos/install/lib/ \ + -l sherpa-onnx \ + -l onnxruntime \ + -o tts-matcha-en + + strip tts-matcha-en +else + echo "./tts-matcha-en exists - skip building" +fi + +export DYLD_LIBRARY_PATH=$PWD/../build-swift-macos/install/lib:$DYLD_LIBRARY_PATH +./tts-matcha-en diff --git a/swift-api-examples/run-tts-matcha-zh.sh b/swift-api-examples/run-tts-matcha-zh.sh new file mode 100755 index 000000000..decbbde4a --- /dev/null +++ b/swift-api-examples/run-tts-matcha-zh.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +set -ex + +if [ ! -d ../build-swift-macos ]; then + echo "Please run ../build-swift-macos.sh first!" + exit 1 +fi + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker +# to download more models +if [ ! -f ./matcha-icefall-zh-baker/model-steps-3.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 + tar xvf matcha-icefall-zh-baker.tar.bz2 + rm matcha-icefall-zh-baker.tar.bz2 +fi + +if [ ! -f ./hifigan_v2.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx +fi + +if [ ! -e ./tts ]; then + # Note: We use -lc++ to link against libc++ instead of libstdc++ + swiftc \ + -lc++ \ + -I ../build-swift-macos/install/include \ + -import-objc-header ./SherpaOnnx-Bridging-Header.h \ + ./tts-matcha-zh.swift ./SherpaOnnx.swift \ + -L ../build-swift-macos/install/lib/ \ + -l sherpa-onnx \ + -l onnxruntime \ + -o tts-matcha-zh + + strip tts-matcha-zh +else + echo "./tts-matcha-zh exists - skip building" +fi + +export DYLD_LIBRARY_PATH=$PWD/../build-swift-macos/install/lib:$DYLD_LIBRARY_PATH +./tts-matcha-zh diff --git a/swift-api-examples/run-tts.sh b/swift-api-examples/run-tts-vits.sh similarity index 86% rename from swift-api-examples/run-tts.sh rename to swift-api-examples/run-tts-vits.sh index 5604a43a8..4f385bd72 100755 --- a/swift-api-examples/run-tts.sh +++ b/swift-api-examples/run-tts-vits.sh @@ -21,16 +21,16 @@ if [ ! -e ./tts ]; then -lc++ \ -I ../build-swift-macos/install/include \ -import-objc-header ./SherpaOnnx-Bridging-Header.h \ - ./tts.swift ./SherpaOnnx.swift \ + ./tts-vits.swift ./SherpaOnnx.swift \ -L ../build-swift-macos/install/lib/ \ -l sherpa-onnx \ -l onnxruntime \ - -o tts + -o tts-vits - strip tts + strip tts-vits else - echo "./tts exists - skip building" + echo "./tts-vits exists - skip building" fi export DYLD_LIBRARY_PATH=$PWD/../build-swift-macos/install/lib:$DYLD_LIBRARY_PATH -./tts +./tts-vits diff --git a/swift-api-examples/tts-matcha-en.swift b/swift-api-examples/tts-matcha-en.swift new file mode 100644 index 000000000..ec55f72d2 --- /dev/null +++ b/swift-api-examples/tts-matcha-en.swift @@ -0,0 +1,65 @@ +class MyClass { + func playSamples(samples: [Float]) { + print("Play \(samples.count) samples") + } +} + +func run() { + let acousticModel = "./matcha-icefall-en_US-ljspeech/model-steps-3.onnx" + let vocoder = "./hifigan_v2.onnx" + let tokens = "./matcha-icefall-en_US-ljspeech/tokens.txt" + let dataDir = "./matcha-icefall-en_US-ljspeech/espeak-ng-data" + let matcha = sherpaOnnxOfflineTtsMatchaModelConfig( + acousticModel: acousticModel, + vocoder: vocoder, + tokens: tokens, + dataDir: dataDir + ) + let modelConfig = sherpaOnnxOfflineTtsModelConfig(matcha: matcha, debug: 0) + var ttsConfig = sherpaOnnxOfflineTtsConfig(model: modelConfig) + + let myClass = MyClass() + + // We use Unretained here so myClass must be kept alive as the callback is invoked + // + // See also + // https://medium.com/codex/swift-c-callback-interoperability-6d57da6c8ee6 + let arg = Unmanaged.passUnretained(myClass).toOpaque() + + let callback: TtsCallbackWithArg = { samples, n, arg in + let o = Unmanaged.fromOpaque(arg!).takeUnretainedValue() + var savedSamples: [Float] = [] + for index in 0...passUnretained(myClass).toOpaque() + + let callback: TtsCallbackWithArg = { samples, n, arg in + let o = Unmanaged.fromOpaque(arg!).takeUnretainedValue() + var savedSamples: [Float] = [] + for index in 0.. Date: Mon, 6 Jan 2025 08:03:03 +0800 Subject: [PATCH 144/183] Add Go API for MatchaTTS models (#1685) --- .github/workflows/test-go-package.yaml | 27 ++++++++++++++ .github/workflows/test-go.yaml | 9 +++++ go-api-examples/non-streaming-tts/main.go | 11 ++++++ .../non-streaming-tts/run-matcha-en.sh | 31 ++++++++++++++++ .../non-streaming-tts/run-matcha-zh.sh | 31 ++++++++++++++++ .../run-vits-piper-en_US-lessac-medium.sh | 2 +- .../non-streaming-tts/run-matcha-en.sh | 1 + .../non-streaming-tts/run-matcha-zh.sh | 1 + scripts/go/sherpa_onnx.go | 37 ++++++++++++++++++- 9 files changed, 148 insertions(+), 2 deletions(-) create mode 100755 go-api-examples/non-streaming-tts/run-matcha-en.sh create mode 100755 go-api-examples/non-streaming-tts/run-matcha-zh.sh create mode 120000 scripts/go/_internal/non-streaming-tts/run-matcha-en.sh create mode 120000 scripts/go/_internal/non-streaming-tts/run-matcha-zh.sh diff --git a/.github/workflows/test-go-package.yaml b/.github/workflows/test-go-package.yaml index 649735699..95c868b8f 100644 --- a/.github/workflows/test-go-package.yaml +++ b/.github/workflows/test-go-package.yaml @@ -209,6 +209,15 @@ jobs: go build ls -lh + echo "Test matcha zh" + ./run-matcha-zh.sh + rm -rf matcha-icefall-* + + echo "Test matcha en" + ./run-matcha-en.sh + rm -rf matcha-icefall-* + ls -lh *.wav + echo "Test vits-ljs" ./run-vits-ljs.sh rm -rf vits-ljs @@ -246,6 +255,15 @@ jobs: cp -v /C/Users/runneradmin/go/pkg/mod/github.com/k2-fsa/sherpa-onnx-go-windows*/lib/x86_64-pc-windows-gnu/*.dll . ls -lh + echo "Test matcha zh" + ./run-matcha-zh.sh + rm -rf matcha-icefall-* + + echo "Test matcha en" + ./run-matcha-en.sh + rm -rf matcha-icefall-* + ls -lh *.wav + echo "Test vits-ljs" ./run-vits-ljs.sh rm -rf vits-ljs @@ -291,6 +309,15 @@ jobs: cp -v /C/Users/runneradmin/go/pkg/mod/github.com/k2-fsa/sherpa-onnx-go-windows*/lib/i686-pc-windows-gnu/*.dll . ls -lh + echo "Test matcha zh" + ./run-matcha-zh.sh + rm -rf matcha-icefall-* + + echo "Test matcha en" + ./run-matcha-en.sh + rm -rf matcha-icefall-* + ls -lh *.wav + echo "Test vits-ljs" ./run-vits-ljs.sh rm -rf vits-ljs diff --git a/.github/workflows/test-go.yaml b/.github/workflows/test-go.yaml index eaf561818..440402939 100644 --- a/.github/workflows/test-go.yaml +++ b/.github/workflows/test-go.yaml @@ -226,6 +226,15 @@ jobs: go build ls -lh + echo "Test matcha zh" + ./run-matcha-zh.sh + rm -rf matcha-icefall-* + + echo "Test matcha en" + ./run-matcha-en.sh + rm -rf matcha-icefall-* + ls -lh *.wav + echo "Test vits-ljs" ./run-vits-ljs.sh rm -rf vits-ljs diff --git a/go-api-examples/non-streaming-tts/main.go b/go-api-examples/non-streaming-tts/main.go index 0ddeb8fe4..73638d8f4 100644 --- a/go-api-examples/non-streaming-tts/main.go +++ b/go-api-examples/non-streaming-tts/main.go @@ -17,11 +17,22 @@ func main() { flag.StringVar(&config.Model.Vits.Lexicon, "vits-lexicon", "", "Path to lexicon.txt") flag.StringVar(&config.Model.Vits.Tokens, "vits-tokens", "", "Path to tokens.txt") flag.StringVar(&config.Model.Vits.DataDir, "vits-data-dir", "", "Path to espeak-ng-data") + flag.StringVar(&config.Model.Matcha.DictDir, "vits-dict-dir", "", "Path to dict for jieba") flag.Float32Var(&config.Model.Vits.NoiseScale, "vits-noise-scale", 0.667, "noise_scale for VITS") flag.Float32Var(&config.Model.Vits.NoiseScaleW, "vits-noise-scale-w", 0.8, "noise_scale_w for VITS") flag.Float32Var(&config.Model.Vits.LengthScale, "vits-length-scale", 1.0, "length_scale for VITS. small -> faster in speech speed; large -> slower") + flag.StringVar(&config.Model.Matcha.AcousticModel, "matcha-acoustic-model", "", "Path to the matcha acoustic model") + flag.StringVar(&config.Model.Matcha.Vocoder, "matcha-vocoder", "", "Path to the matcha vocoder model") + flag.StringVar(&config.Model.Matcha.Lexicon, "matcha-lexicon", "", "Path to lexicon.txt") + flag.StringVar(&config.Model.Matcha.Tokens, "matcha-tokens", "", "Path to tokens.txt") + flag.StringVar(&config.Model.Matcha.DataDir, "matcha-data-dir", "", "Path to espeak-ng-data") + flag.StringVar(&config.Model.Matcha.DictDir, "matcha-dict-dir", "", "Path to dict for jieba") + + flag.Float32Var(&config.Model.Matcha.NoiseScale, "matcha-noise-scale", 0.667, "noise_scale for Matcha") + flag.Float32Var(&config.Model.Matcha.LengthScale, "matcha-length-scale", 1.0, "length_scale for Matcha. small -> faster in speech speed; large -> slower") + flag.IntVar(&config.Model.NumThreads, "num-threads", 1, "Number of threads for computing") flag.IntVar(&config.Model.Debug, "debug", 0, "Whether to show debug message") flag.StringVar(&config.Model.Provider, "provider", "cpu", "Provider to use") diff --git a/go-api-examples/non-streaming-tts/run-matcha-en.sh b/go-api-examples/non-streaming-tts/run-matcha-en.sh new file mode 100755 index 000000000..f0932da56 --- /dev/null +++ b/go-api-examples/non-streaming-tts/run-matcha-en.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -ex + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker +# matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker +# to download more models +if [ ! -f ./matcha-icefall-en_US-ljspeech/model-steps-3.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 + tar xf matcha-icefall-en_US-ljspeech.tar.bz2 + rm matcha-icefall-en_US-ljspeech.tar.bz2 +fi + +if [ ! -f ./hifigan_v2.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx +fi + +go mod tidy +go build + +./non-streaming-tts \ + --matcha-acoustic-model=./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ + --matcha-vocoder=./hifigan_v2.onnx \ + --matcha-tokens=./matcha-icefall-en_US-ljspeech/tokens.txt \ + --matcha-data-dir=./matcha-icefall-en_US-ljspeech/espeak-ng-data \ + --debug=1 \ + --output-filename=./test-matcha-en.wav \ + "Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone." + + diff --git a/go-api-examples/non-streaming-tts/run-matcha-zh.sh b/go-api-examples/non-streaming-tts/run-matcha-zh.sh new file mode 100755 index 000000000..ef4165d04 --- /dev/null +++ b/go-api-examples/non-streaming-tts/run-matcha-zh.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -ex + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker +# to download more models +if [ ! -f ./matcha-icefall-zh-baker/model-steps-3.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 + tar xvf matcha-icefall-zh-baker.tar.bz2 + rm matcha-icefall-zh-baker.tar.bz2 +fi + +if [ ! -f ./hifigan_v2.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx +fi + +go mod tidy +go build + +./non-streaming-tts \ + --matcha-acoustic-model=./matcha-icefall-zh-baker/model-steps-3.onnx \ + --matcha-vocoder=./hifigan_v2.onnx \ + --matcha-lexicon=./matcha-icefall-zh-baker/lexicon.txt \ + --matcha-tokens=./matcha-icefall-zh-baker/tokens.txt \ + --matcha-dict-dir=./matcha-icefall-zh-baker/dict \ + --debug=1 \ + --tts-rule-fsts=./matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst \ + --output-filename=./test-matcha-zh.wav \ + "某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。" + diff --git a/go-api-examples/non-streaming-tts/run-vits-piper-en_US-lessac-medium.sh b/go-api-examples/non-streaming-tts/run-vits-piper-en_US-lessac-medium.sh index 15e4f1dbd..6f8c98e80 100755 --- a/go-api-examples/non-streaming-tts/run-vits-piper-en_US-lessac-medium.sh +++ b/go-api-examples/non-streaming-tts/run-vits-piper-en_US-lessac-medium.sh @@ -4,7 +4,7 @@ set -ex if [ ! -d vits-piper-en_US-lessac-medium ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-lessac-medium.tar.bz2 - tar xvf vits-piper-en_US-lessac-medium.tar.bz2 + tar xf vits-piper-en_US-lessac-medium.tar.bz2 rm vits-piper-en_US-lessac-medium.tar.bz2 fi diff --git a/scripts/go/_internal/non-streaming-tts/run-matcha-en.sh b/scripts/go/_internal/non-streaming-tts/run-matcha-en.sh new file mode 120000 index 000000000..013e5bacc --- /dev/null +++ b/scripts/go/_internal/non-streaming-tts/run-matcha-en.sh @@ -0,0 +1 @@ +../../../../go-api-examples/non-streaming-tts/run-matcha-en.sh \ No newline at end of file diff --git a/scripts/go/_internal/non-streaming-tts/run-matcha-zh.sh b/scripts/go/_internal/non-streaming-tts/run-matcha-zh.sh new file mode 120000 index 000000000..73ef170ee --- /dev/null +++ b/scripts/go/_internal/non-streaming-tts/run-matcha-zh.sh @@ -0,0 +1 @@ +../../../../go-api-examples/non-streaming-tts/run-matcha-zh.sh \ No newline at end of file diff --git a/scripts/go/sherpa_onnx.go b/scripts/go/sherpa_onnx.go index 17afd32f2..763752840 100644 --- a/scripts/go/sherpa_onnx.go +++ b/scripts/go/sherpa_onnx.go @@ -671,8 +671,20 @@ type OfflineTtsVitsModelConfig struct { DictDir string // Path to dict directory for jieba (used only in Chinese tts) } +type OfflineTtsMatchaModelConfig struct { + AcousticModel string // Path to the acoustic model for MatchaTTS + Vocoder string // Path to the vocoder model for MatchaTTS + Lexicon string // Path to lexicon.txt + Tokens string // Path to tokens.txt + DataDir string // Path to espeak-ng-data directory + NoiseScale float32 // noise scale for vits models. Please use 0.667 in general + LengthScale float32 // Please use 1.0 in general. Smaller -> Faster speech speed. Larger -> Slower speech speed + DictDir string // Path to dict directory for jieba (used only in Chinese tts) +} + type OfflineTtsModelConfig struct { - Vits OfflineTtsVitsModelConfig + Vits OfflineTtsVitsModelConfig + Matcha OfflineTtsMatchaModelConfig // Number of threads to use for neural network computation NumThreads int @@ -722,6 +734,7 @@ func NewOfflineTts(config *OfflineTtsConfig) *OfflineTts { c.max_num_sentences = C.int(config.MaxNumSentences) + // vits c.model.vits.model = C.CString(config.Model.Vits.Model) defer C.free(unsafe.Pointer(c.model.vits.model)) @@ -741,6 +754,28 @@ func NewOfflineTts(config *OfflineTtsConfig) *OfflineTts { c.model.vits.dict_dir = C.CString(config.Model.Vits.DictDir) defer C.free(unsafe.Pointer(c.model.vits.dict_dir)) + // matcha + c.model.matcha.acoustic_model = C.CString(config.Model.Matcha.AcousticModel) + defer C.free(unsafe.Pointer(c.model.matcha.acoustic_model)) + + c.model.matcha.vocoder = C.CString(config.Model.Matcha.Vocoder) + defer C.free(unsafe.Pointer(c.model.matcha.vocoder)) + + c.model.matcha.lexicon = C.CString(config.Model.Matcha.Lexicon) + defer C.free(unsafe.Pointer(c.model.matcha.lexicon)) + + c.model.matcha.tokens = C.CString(config.Model.Matcha.Tokens) + defer C.free(unsafe.Pointer(c.model.matcha.tokens)) + + c.model.matcha.data_dir = C.CString(config.Model.Matcha.DataDir) + defer C.free(unsafe.Pointer(c.model.matcha.data_dir)) + + c.model.matcha.noise_scale = C.float(config.Model.Matcha.NoiseScale) + c.model.matcha.length_scale = C.float(config.Model.Matcha.LengthScale) + + c.model.matcha.dict_dir = C.CString(config.Model.Matcha.DictDir) + defer C.free(unsafe.Pointer(c.model.matcha.dict_dir)) + c.model.num_threads = C.int(config.Model.NumThreads) c.model.debug = C.int(config.Model.Debug) From c6fcd32552d754a21721045b0937696fb9c38da1 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 6 Jan 2025 10:04:35 +0800 Subject: [PATCH 145/183] Add Pascal API for MatchaTTS models. (#1686) --- .github/workflows/pascal.yaml | 13 + pascal-api-examples/tts/.gitignore | 4 + .../tts/matcha-en-playback.pas | 239 +++++++++++++++++ pascal-api-examples/tts/matcha-en.pas | 55 ++++ .../tts/matcha-zh-playback.pas | 241 ++++++++++++++++++ pascal-api-examples/tts/matcha-zh.pas | 57 +++++ pascal-api-examples/tts/piper-playback.pas | 2 +- .../tts/run-matcha-en-playback.sh | 53 ++++ pascal-api-examples/tts/run-matcha-en.sh | 49 ++++ .../tts/run-matcha-zh-playback.sh | 52 ++++ pascal-api-examples/tts/run-matcha-zh.sh | 48 ++++ sherpa-onnx/pascal-api/sherpa_onnx.pas | 65 ++++- 12 files changed, 875 insertions(+), 3 deletions(-) create mode 100644 pascal-api-examples/tts/matcha-en-playback.pas create mode 100644 pascal-api-examples/tts/matcha-en.pas create mode 100644 pascal-api-examples/tts/matcha-zh-playback.pas create mode 100644 pascal-api-examples/tts/matcha-zh.pas create mode 100755 pascal-api-examples/tts/run-matcha-en-playback.sh create mode 100755 pascal-api-examples/tts/run-matcha-en.sh create mode 100755 pascal-api-examples/tts/run-matcha-zh-playback.sh create mode 100755 pascal-api-examples/tts/run-matcha-zh.sh diff --git a/.github/workflows/pascal.yaml b/.github/workflows/pascal.yaml index 306ae6480..5193241b6 100644 --- a/.github/workflows/pascal.yaml +++ b/.github/workflows/pascal.yaml @@ -152,6 +152,19 @@ jobs: ./run-piper.sh rm -rf vits-piper-* + rm piper + ls -lh + echo "---" + + ./run-matcha-zh.sh + rm -rf matcha-icefall-* + rm matcha-zh + ls -lh + echo "---" + + ./run-matcha-en.sh + rm -rf matcha-icefall-* + rm matcha-en ls -lh echo "---" diff --git a/pascal-api-examples/tts/.gitignore b/pascal-api-examples/tts/.gitignore index b7076ab5c..c7d282825 100644 --- a/pascal-api-examples/tts/.gitignore +++ b/pascal-api-examples/tts/.gitignore @@ -2,3 +2,7 @@ piper piper-playback link*.res +matcha-zh +matcha-en +matcha-zh-playback +matcha-en-playback diff --git a/pascal-api-examples/tts/matcha-en-playback.pas b/pascal-api-examples/tts/matcha-en-playback.pas new file mode 100644 index 000000000..e750099cb --- /dev/null +++ b/pascal-api-examples/tts/matcha-en-playback.pas @@ -0,0 +1,239 @@ +{ Copyright (c) 2025 Xiaomi Corporation } +program matcha_en_playback; +{ +This file shows how to use the text to speech API of sherpa-onnx +with Piper models. + +It generates speech from text and saves it to a wave file. + +Note that it plays the audio back as it is still generating. +} + +{$mode objfpc} + +uses + {$ifdef unix} + cthreads, + {$endif} + SysUtils, + dos, + ctypes, + portaudio, + sherpa_onnx; + +var + CriticalSection: TRTLCriticalSection; + + Tts: TSherpaOnnxOfflineTts; + Audio: TSherpaOnnxGeneratedAudio; + Resampler: TSherpaOnnxLinearResampler; + + Text: AnsiString; + Speed: Single = 1.0; {Use a larger value to speak faster} + SpeakerId: Integer = 0; + Buffer: TSherpaOnnxCircularBuffer; + FinishedGeneration: Boolean = False; + FinishedPlaying: Boolean = False; + + Version: String; + EnvStr: String; + Status: Integer; + NumDevices: Integer; + DeviceIndex: Integer; + DeviceInfo: PPaDeviceInfo; + + { If you get EDivByZero: Division by zero error, please change the sample rate + to the one supported by your microphone. + } + DeviceSampleRate: Integer = 48000; + I: Integer; + Param: TPaStreamParameters; + Stream: PPaStream; + Wave: TSherpaOnnxWave; + +function GenerateCallback( + Samples: pcfloat; N: cint32; + Arg: Pointer): cint; cdecl; +begin + EnterCriticalSection(CriticalSection); + try + if Resampler <> nil then + Buffer.Push(Resampler.Resample(Samples, N, False)) + else + Buffer.Push(Samples, N); + finally + LeaveCriticalSection(CriticalSection); + end; + + { 1 means to continue generating; 0 means to stop generating. } + Result := 1; +end; + +function PlayCallback( + input: Pointer; output: Pointer; + frameCount: culong; + timeInfo: PPaStreamCallbackTimeInfo; + statusFlags: TPaStreamCallbackFlags; + userData: Pointer ): cint; cdecl; +var + Samples: TSherpaOnnxSamplesArray; + I: Integer; +begin + EnterCriticalSection(CriticalSection); + try + if Buffer.Size >= frameCount then + begin + Samples := Buffer.Get(Buffer.Head, FrameCount); + Buffer.Pop(FrameCount); + end + else if Buffer.Size > 0 then + begin + Samples := Buffer.Get(Buffer.Head, Buffer.Size); + Buffer.Pop(Buffer.Size); + SetLength(Samples, frameCount); + end + else + SetLength(Samples, frameCount); + + for I := 0 to frameCount - 1 do + pcfloat(output)[I] := Samples[I]; + + if (Buffer.Size > 0) or (not FinishedGeneration) then + Result := paContinue + else + begin + Result := paComplete; + FinishedPlaying := True; + end; + finally + LeaveCriticalSection(CriticalSection); + end; +end; + +function GetOfflineTts: TSherpaOnnxOfflineTts; +var + Config: TSherpaOnnxOfflineTtsConfig; +begin + Config.Model.Matcha.AcousticModel := './matcha-icefall-en_US-ljspeech/model-steps-3.onnx'; + Config.Model.Matcha.Vocoder := './hifigan_v2.onnx'; + Config.Model.Matcha.Tokens := './matcha-icefall-en_US-ljspeech/tokens.txt'; + Config.Model.Matcha.DataDir := './matcha-icefall-en_US-ljspeech/espeak-ng-data'; + Config.Model.NumThreads := 1; + Config.Model.Debug := False; + Config.MaxNumSentences := 1; + + Result := TSherpaOnnxOfflineTts.Create(Config); +end; + +begin + Tts := GetOfflineTts; + if Tts.GetSampleRate <> DeviceSampleRate then + Resampler := TSherpaOnnxLinearResampler.Create(Tts.GetSampleRate, DeviceSampleRate); + + Version := String(Pa_GetVersionText); + WriteLn('Version is ', Version); + Status := Pa_Initialize; + if Status <> paNoError then + begin + WriteLn('Failed to initialize portaudio, ', Pa_GetErrorText(Status)); + Exit; + end; + + NumDevices := Pa_GetDeviceCount; + WriteLn('Num devices: ', NumDevices); + + DeviceIndex := Pa_GetDefaultOutputDevice; + + if DeviceIndex = paNoDevice then + begin + WriteLn('No default output device found'); + Pa_Terminate; + Exit; + end; + + EnvStr := GetEnv('SHERPA_ONNX_MIC_DEVICE'); + if EnvStr <> '' then + begin + DeviceIndex := StrToIntDef(EnvStr, DeviceIndex); + WriteLn('Use device index from environment variable SHERPA_ONNX_MIC_DEVICE: ', EnvStr); + end; + + for I := 0 to (NumDevices - 1) do + begin + DeviceInfo := Pa_GetDeviceInfo(I); + if I = DeviceIndex then + { WriteLn(Format(' * %d %s', [I, DeviceInfo^.Name])) } + WriteLn(Format(' * %d %s', [I, AnsiString(DeviceInfo^.Name)])) + else + WriteLn(Format(' %d %s', [I, AnsiString(DeviceInfo^.Name)])); + end; + + WriteLn('Use device ', DeviceIndex); + WriteLn(' Name ', Pa_GetDeviceInfo(DeviceIndex)^.Name); + WriteLn(' Max output channels ', Pa_GetDeviceInfo(DeviceIndex)^.MaxOutputChannels); + + Initialize(Param); + Param.Device := DeviceIndex; + Param.ChannelCount := 1; + Param.SampleFormat := paFloat32; + param.SuggestedLatency := Pa_GetDeviceInfo(DeviceIndex)^.DefaultHighOutputLatency; + param.HostApiSpecificStreamInfo := nil; + + Buffer := TSherpaOnnxCircularBuffer.Create(30 * DeviceSampleRate); + + + { Note(fangjun): PortAudio invokes PlayCallback in a separate thread. } + Status := Pa_OpenStream(stream, nil, @Param, DeviceSampleRate, paFramesPerBufferUnspecified, paNoFlag, + PPaStreamCallback(@PlayCallback), nil); + + if Status <> paNoError then + begin + WriteLn('Failed to open stream, ', Pa_GetErrorText(Status)); + Pa_Terminate; + Exit; + end; + + InitCriticalSection(CriticalSection); + + Status := Pa_StartStream(stream); + if Status <> paNoError then + begin + WriteLn('Failed to start stream, ', Pa_GetErrorText(Status)); + Pa_Terminate; + Exit; + end; + + WriteLn('There are ', Tts.GetNumSpeakers, ' speakers'); + + Text := 'Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone.'; + + Audio := Tts.Generate(Text, SpeakerId, Speed, + PSherpaOnnxGeneratedAudioCallbackWithArg(@GenerateCallback), nil); + FinishedGeneration := True; + SherpaOnnxWriteWave('./matcha-zh-playback.wav', Audio.Samples, Audio.SampleRate); + WriteLn('Saved to ./matcha-zh-playback.wav'); + + while not FinishedPlaying do + Pa_Sleep(100); {sleep for 0.1 second } + {TODO(fangjun): Use an event to indicate the play is finished} + + DoneCriticalSection(CriticalSection); + + FreeAndNil(Tts); + FreeAndNil(Resampler); + + Status := Pa_CloseStream(stream); + if Status <> paNoError then + begin + WriteLn('Failed to close stream, ', Pa_GetErrorText(Status)); + Exit; + end; + + Status := Pa_Terminate; + if Status <> paNoError then + begin + WriteLn('Failed to deinitialize portaudio, ', Pa_GetErrorText(Status)); + Exit; + end; +end. + diff --git a/pascal-api-examples/tts/matcha-en.pas b/pascal-api-examples/tts/matcha-en.pas new file mode 100644 index 000000000..7ef34b703 --- /dev/null +++ b/pascal-api-examples/tts/matcha-en.pas @@ -0,0 +1,55 @@ +{ Copyright (c) 2025 Xiaomi Corporation } +program matcha_en; +{ +This file shows how to use the text to speech API of sherpa-onnx +with MatchaTTS models. + +It generates speech from text and saves it to a wave file. + +If you want to play it while it is generating, please see +./matcha-zh-playback.pas +} + +{$mode objfpc} + +uses + SysUtils, + sherpa_onnx; + +function GetOfflineTts: TSherpaOnnxOfflineTts; +var + Config: TSherpaOnnxOfflineTtsConfig; +begin + Config.Model.Matcha.AcousticModel := './matcha-icefall-en_US-ljspeech/model-steps-3.onnx'; + Config.Model.Matcha.Vocoder := './hifigan_v2.onnx'; + Config.Model.Matcha.Tokens := './matcha-icefall-en_US-ljspeech/tokens.txt'; + Config.Model.Matcha.DataDir := './matcha-icefall-en_US-ljspeech/espeak-ng-data'; + Config.Model.NumThreads := 1; + Config.Model.Debug := False; + Config.MaxNumSentences := 1; + + Result := TSherpaOnnxOfflineTts.Create(Config); +end; + +var + Tts: TSherpaOnnxOfflineTts; + Audio: TSherpaOnnxGeneratedAudio; + + Text: AnsiString; + Speed: Single = 1.0; {Use a larger value to speak faster} + SpeakerId: Integer = 0; + +begin + Tts := GetOfflineTts; + + WriteLn('There are ', Tts.GetNumSpeakers, ' speakers'); + + Text := 'Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone.'; + + Audio := Tts.Generate(Text, SpeakerId, Speed); + SherpaOnnxWriteWave('./matcha-en.wav', Audio.Samples, Audio.SampleRate); + WriteLn('Saved to ./matcha-en.wav'); + + FreeAndNil(Tts); +end. + diff --git a/pascal-api-examples/tts/matcha-zh-playback.pas b/pascal-api-examples/tts/matcha-zh-playback.pas new file mode 100644 index 000000000..08b2bbe2d --- /dev/null +++ b/pascal-api-examples/tts/matcha-zh-playback.pas @@ -0,0 +1,241 @@ +{ Copyright (c) 2025 Xiaomi Corporation } +program matcha_zh_playback; +{ +This file shows how to use the text to speech API of sherpa-onnx +with Piper models. + +It generates speech from text and saves it to a wave file. + +Note that it plays the audio back as it is still generating. +} + +{$mode objfpc} + +uses + {$ifdef unix} + cthreads, + {$endif} + SysUtils, + dos, + ctypes, + portaudio, + sherpa_onnx; + +var + CriticalSection: TRTLCriticalSection; + + Tts: TSherpaOnnxOfflineTts; + Audio: TSherpaOnnxGeneratedAudio; + Resampler: TSherpaOnnxLinearResampler; + + Text: AnsiString; + Speed: Single = 1.0; {Use a larger value to speak faster} + SpeakerId: Integer = 0; + Buffer: TSherpaOnnxCircularBuffer; + FinishedGeneration: Boolean = False; + FinishedPlaying: Boolean = False; + + Version: String; + EnvStr: String; + Status: Integer; + NumDevices: Integer; + DeviceIndex: Integer; + DeviceInfo: PPaDeviceInfo; + + { If you get EDivByZero: Division by zero error, please change the sample rate + to the one supported by your microphone. + } + DeviceSampleRate: Integer = 48000; + I: Integer; + Param: TPaStreamParameters; + Stream: PPaStream; + Wave: TSherpaOnnxWave; + +function GenerateCallback( + Samples: pcfloat; N: cint32; + Arg: Pointer): cint; cdecl; +begin + EnterCriticalSection(CriticalSection); + try + if Resampler <> nil then + Buffer.Push(Resampler.Resample(Samples, N, False)) + else + Buffer.Push(Samples, N); + finally + LeaveCriticalSection(CriticalSection); + end; + + { 1 means to continue generating; 0 means to stop generating. } + Result := 1; +end; + +function PlayCallback( + input: Pointer; output: Pointer; + frameCount: culong; + timeInfo: PPaStreamCallbackTimeInfo; + statusFlags: TPaStreamCallbackFlags; + userData: Pointer ): cint; cdecl; +var + Samples: TSherpaOnnxSamplesArray; + I: Integer; +begin + EnterCriticalSection(CriticalSection); + try + if Buffer.Size >= frameCount then + begin + Samples := Buffer.Get(Buffer.Head, FrameCount); + Buffer.Pop(FrameCount); + end + else if Buffer.Size > 0 then + begin + Samples := Buffer.Get(Buffer.Head, Buffer.Size); + Buffer.Pop(Buffer.Size); + SetLength(Samples, frameCount); + end + else + SetLength(Samples, frameCount); + + for I := 0 to frameCount - 1 do + pcfloat(output)[I] := Samples[I]; + + if (Buffer.Size > 0) or (not FinishedGeneration) then + Result := paContinue + else + begin + Result := paComplete; + FinishedPlaying := True; + end; + finally + LeaveCriticalSection(CriticalSection); + end; +end; + +function GetOfflineTts: TSherpaOnnxOfflineTts; +var + Config: TSherpaOnnxOfflineTtsConfig; +begin + Config.Model.Matcha.AcousticModel := './matcha-icefall-zh-baker/model-steps-3.onnx'; + Config.Model.Matcha.Vocoder := './hifigan_v2.onnx'; + Config.Model.Matcha.Lexicon := './matcha-icefall-zh-baker/lexicon.txt'; + Config.Model.Matcha.Tokens := './matcha-icefall-zh-baker/tokens.txt'; + Config.Model.Matcha.DictDir := './matcha-icefall-zh-baker/dict'; + Config.Model.NumThreads := 1; + Config.Model.Debug := False; + Config.RuleFsts := './matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst'; + Config.MaxNumSentences := 1; + + Result := TSherpaOnnxOfflineTts.Create(Config); +end; + +begin + Tts := GetOfflineTts; + if Tts.GetSampleRate <> DeviceSampleRate then + Resampler := TSherpaOnnxLinearResampler.Create(Tts.GetSampleRate, DeviceSampleRate); + + Version := String(Pa_GetVersionText); + WriteLn('Version is ', Version); + Status := Pa_Initialize; + if Status <> paNoError then + begin + WriteLn('Failed to initialize portaudio, ', Pa_GetErrorText(Status)); + Exit; + end; + + NumDevices := Pa_GetDeviceCount; + WriteLn('Num devices: ', NumDevices); + + DeviceIndex := Pa_GetDefaultOutputDevice; + + if DeviceIndex = paNoDevice then + begin + WriteLn('No default output device found'); + Pa_Terminate; + Exit; + end; + + EnvStr := GetEnv('SHERPA_ONNX_MIC_DEVICE'); + if EnvStr <> '' then + begin + DeviceIndex := StrToIntDef(EnvStr, DeviceIndex); + WriteLn('Use device index from environment variable SHERPA_ONNX_MIC_DEVICE: ', EnvStr); + end; + + for I := 0 to (NumDevices - 1) do + begin + DeviceInfo := Pa_GetDeviceInfo(I); + if I = DeviceIndex then + { WriteLn(Format(' * %d %s', [I, DeviceInfo^.Name])) } + WriteLn(Format(' * %d %s', [I, AnsiString(DeviceInfo^.Name)])) + else + WriteLn(Format(' %d %s', [I, AnsiString(DeviceInfo^.Name)])); + end; + + WriteLn('Use device ', DeviceIndex); + WriteLn(' Name ', Pa_GetDeviceInfo(DeviceIndex)^.Name); + WriteLn(' Max output channels ', Pa_GetDeviceInfo(DeviceIndex)^.MaxOutputChannels); + + Initialize(Param); + Param.Device := DeviceIndex; + Param.ChannelCount := 1; + Param.SampleFormat := paFloat32; + param.SuggestedLatency := Pa_GetDeviceInfo(DeviceIndex)^.DefaultHighOutputLatency; + param.HostApiSpecificStreamInfo := nil; + + Buffer := TSherpaOnnxCircularBuffer.Create(30 * DeviceSampleRate); + + + { Note(fangjun): PortAudio invokes PlayCallback in a separate thread. } + Status := Pa_OpenStream(stream, nil, @Param, DeviceSampleRate, paFramesPerBufferUnspecified, paNoFlag, + PPaStreamCallback(@PlayCallback), nil); + + if Status <> paNoError then + begin + WriteLn('Failed to open stream, ', Pa_GetErrorText(Status)); + Pa_Terminate; + Exit; + end; + + InitCriticalSection(CriticalSection); + + Status := Pa_StartStream(stream); + if Status <> paNoError then + begin + WriteLn('Failed to start stream, ', Pa_GetErrorText(Status)); + Pa_Terminate; + Exit; + end; + + WriteLn('There are ', Tts.GetNumSpeakers, ' speakers'); + + Text := '某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。'; + + Audio := Tts.Generate(Text, SpeakerId, Speed, + PSherpaOnnxGeneratedAudioCallbackWithArg(@GenerateCallback), nil); + FinishedGeneration := True; + SherpaOnnxWriteWave('./matcha-zh-playback.wav', Audio.Samples, Audio.SampleRate); + WriteLn('Saved to ./matcha-zh-playback.wav'); + + while not FinishedPlaying do + Pa_Sleep(100); {sleep for 0.1 second } + {TODO(fangjun): Use an event to indicate the play is finished} + + DoneCriticalSection(CriticalSection); + + FreeAndNil(Tts); + FreeAndNil(Resampler); + + Status := Pa_CloseStream(stream); + if Status <> paNoError then + begin + WriteLn('Failed to close stream, ', Pa_GetErrorText(Status)); + Exit; + end; + + Status := Pa_Terminate; + if Status <> paNoError then + begin + WriteLn('Failed to deinitialize portaudio, ', Pa_GetErrorText(Status)); + Exit; + end; +end. + diff --git a/pascal-api-examples/tts/matcha-zh.pas b/pascal-api-examples/tts/matcha-zh.pas new file mode 100644 index 000000000..c94100952 --- /dev/null +++ b/pascal-api-examples/tts/matcha-zh.pas @@ -0,0 +1,57 @@ +{ Copyright (c) 2025 Xiaomi Corporation } +program matcha_zh; +{ +This file shows how to use the text to speech API of sherpa-onnx +with MatchaTTS models. + +It generates speech from text and saves it to a wave file. + +If you want to play it while it is generating, please see +./matcha-zh-playback.pas +} + +{$mode objfpc} + +uses + SysUtils, + sherpa_onnx; + +function GetOfflineTts: TSherpaOnnxOfflineTts; +var + Config: TSherpaOnnxOfflineTtsConfig; +begin + Config.Model.Matcha.AcousticModel := './matcha-icefall-zh-baker/model-steps-3.onnx'; + Config.Model.Matcha.Vocoder := './hifigan_v2.onnx'; + Config.Model.Matcha.Lexicon := './matcha-icefall-zh-baker/lexicon.txt'; + Config.Model.Matcha.Tokens := './matcha-icefall-zh-baker/tokens.txt'; + Config.Model.Matcha.DictDir := './matcha-icefall-zh-baker/dict'; + Config.Model.NumThreads := 1; + Config.Model.Debug := False; + Config.RuleFsts := './matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst'; + Config.MaxNumSentences := 1; + + Result := TSherpaOnnxOfflineTts.Create(Config); +end; + +var + Tts: TSherpaOnnxOfflineTts; + Audio: TSherpaOnnxGeneratedAudio; + + Text: AnsiString; + Speed: Single = 1.0; {Use a larger value to speak faster} + SpeakerId: Integer = 0; + +begin + Tts := GetOfflineTts; + + WriteLn('There are ', Tts.GetNumSpeakers, ' speakers'); + + Text := '某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。'; + + Audio := Tts.Generate(Text, SpeakerId, Speed); + SherpaOnnxWriteWave('./matcha-zh.wav', Audio.Samples, Audio.SampleRate); + WriteLn('Saved to ./matcha-zh.wav'); + + FreeAndNil(Tts); +end. + diff --git a/pascal-api-examples/tts/piper-playback.pas b/pascal-api-examples/tts/piper-playback.pas index b9cd10b71..5e65a34f0 100644 --- a/pascal-api-examples/tts/piper-playback.pas +++ b/pascal-api-examples/tts/piper-playback.pas @@ -1,5 +1,5 @@ { Copyright (c) 2024 Xiaomi Corporation } -program piper; +program piper_playback; { This file shows how to use the text to speech API of sherpa-onnx with Piper models. diff --git a/pascal-api-examples/tts/run-matcha-en-playback.sh b/pascal-api-examples/tts/run-matcha-en-playback.sh new file mode 100755 index 000000000..ffa677e94 --- /dev/null +++ b/pascal-api-examples/tts/run-matcha-en-playback.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +set -ex + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +SHERPA_ONNX_DIR=$(cd $SCRIPT_DIR/../.. && pwd) + +echo "SHERPA_ONNX_DIR: $SHERPA_ONNX_DIR" + +if [[ ! -f ../../build/install/lib/libsherpa-onnx-c-api.dylib && ! -f ../../build/install/lib/libsherpa-onnx-c-api.so && ! -f ../../build/install/lib/sherpa-onnx-c-api.dll ]]; then + mkdir -p ../../build + pushd ../../build + cmake \ + -DCMAKE_INSTALL_PREFIX=./install \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + .. + + cmake --build . --target install --config Release + popd +fi + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker +# matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker +# to download more models +if [ ! -f ./matcha-icefall-en_US-ljspeech/model-steps-3.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 + tar xf matcha-icefall-en_US-ljspeech.tar.bz2 + rm matcha-icefall-en_US-ljspeech.tar.bz2 +fi + +if [ ! -f ./hifigan_v2.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx +fi + +fpc \ + -dSHERPA_ONNX_USE_SHARED_LIBS \ + -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \ + -Fl$SHERPA_ONNX_DIR/build/install/lib \ + -Fl/usr/local/Cellar/portaudio/19.7.0/lib \ + ./matcha-en-playback.pas + +# Please see ../portaudio-test/README.md +# for how to install portaudio on macOS + +export LD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$LD_LIBRARY_PATH +export DYLD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$DYLD_LIBRARY_PATH + +./matcha-en-playback diff --git a/pascal-api-examples/tts/run-matcha-en.sh b/pascal-api-examples/tts/run-matcha-en.sh new file mode 100755 index 000000000..084e672b1 --- /dev/null +++ b/pascal-api-examples/tts/run-matcha-en.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -ex + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +SHERPA_ONNX_DIR=$(cd $SCRIPT_DIR/../.. && pwd) + +echo "SHERPA_ONNX_DIR: $SHERPA_ONNX_DIR" + +if [[ ! -f ../../build/install/lib/libsherpa-onnx-c-api.dylib && ! -f ../../build/install/lib/libsherpa-onnx-c-api.so && ! -f ../../build/install/lib/sherpa-onnx-c-api.dll ]]; then + mkdir -p ../../build + pushd ../../build + cmake \ + -DCMAKE_INSTALL_PREFIX=./install \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + .. + + cmake --build . --target install --config Release + popd +fi + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker +# matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker +# to download more models +if [ ! -f ./matcha-icefall-en_US-ljspeech/model-steps-3.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 + tar xf matcha-icefall-en_US-ljspeech.tar.bz2 + rm matcha-icefall-en_US-ljspeech.tar.bz2 +fi + +if [ ! -f ./hifigan_v2.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx +fi + +fpc \ + -dSHERPA_ONNX_USE_SHARED_LIBS \ + -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \ + -Fl$SHERPA_ONNX_DIR/build/install/lib \ + ./matcha-en.pas + +export LD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$LD_LIBRARY_PATH +export DYLD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$DYLD_LIBRARY_PATH + +./matcha-en diff --git a/pascal-api-examples/tts/run-matcha-zh-playback.sh b/pascal-api-examples/tts/run-matcha-zh-playback.sh new file mode 100755 index 000000000..e12ad22af --- /dev/null +++ b/pascal-api-examples/tts/run-matcha-zh-playback.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +set -ex + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +SHERPA_ONNX_DIR=$(cd $SCRIPT_DIR/../.. && pwd) + +echo "SHERPA_ONNX_DIR: $SHERPA_ONNX_DIR" + +if [[ ! -f ../../build/install/lib/libsherpa-onnx-c-api.dylib && ! -f ../../build/install/lib/libsherpa-onnx-c-api.so && ! -f ../../build/install/lib/sherpa-onnx-c-api.dll ]]; then + mkdir -p ../../build + pushd ../../build + cmake \ + -DCMAKE_INSTALL_PREFIX=./install \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + .. + + cmake --build . --target install --config Release + popd +fi + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker +# to download more models +if [ ! -f ./matcha-icefall-zh-baker/model-steps-3.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 + tar xvf matcha-icefall-zh-baker.tar.bz2 + rm matcha-icefall-zh-baker.tar.bz2 +fi + +if [ ! -f ./hifigan_v2.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx +fi + +fpc \ + -dSHERPA_ONNX_USE_SHARED_LIBS \ + -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \ + -Fl$SHERPA_ONNX_DIR/build/install/lib \ + -Fl/usr/local/Cellar/portaudio/19.7.0/lib \ + ./matcha-zh-playback.pas + +# Please see ../portaudio-test/README.md +# for how to install portaudio on macOS + +export LD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$LD_LIBRARY_PATH +export DYLD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$DYLD_LIBRARY_PATH + +./matcha-zh-playback diff --git a/pascal-api-examples/tts/run-matcha-zh.sh b/pascal-api-examples/tts/run-matcha-zh.sh new file mode 100755 index 000000000..a7d83d379 --- /dev/null +++ b/pascal-api-examples/tts/run-matcha-zh.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +set -ex + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +SHERPA_ONNX_DIR=$(cd $SCRIPT_DIR/../.. && pwd) + +echo "SHERPA_ONNX_DIR: $SHERPA_ONNX_DIR" + +if [[ ! -f ../../build/install/lib/libsherpa-onnx-c-api.dylib && ! -f ../../build/install/lib/libsherpa-onnx-c-api.so && ! -f ../../build/install/lib/sherpa-onnx-c-api.dll ]]; then + mkdir -p ../../build + pushd ../../build + cmake \ + -DCMAKE_INSTALL_PREFIX=./install \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + .. + + cmake --build . --target install --config Release + popd +fi + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker +# to download more models +if [ ! -f ./matcha-icefall-zh-baker/model-steps-3.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 + tar xvf matcha-icefall-zh-baker.tar.bz2 + rm matcha-icefall-zh-baker.tar.bz2 +fi + +if [ ! -f ./hifigan_v2.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx +fi + +fpc \ + -dSHERPA_ONNX_USE_SHARED_LIBS \ + -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \ + -Fl$SHERPA_ONNX_DIR/build/install/lib \ + ./matcha-zh.pas + +export LD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$LD_LIBRARY_PATH +export DYLD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$DYLD_LIBRARY_PATH + +./matcha-zh diff --git a/sherpa-onnx/pascal-api/sherpa_onnx.pas b/sherpa-onnx/pascal-api/sherpa_onnx.pas index cff215759..442c8a504 100644 --- a/sherpa-onnx/pascal-api/sherpa_onnx.pas +++ b/sherpa-onnx/pascal-api/sherpa_onnx.pas @@ -62,11 +62,26 @@ TSherpaOnnxOfflineTtsVitsModelConfig = record class operator Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineTtsVitsModelConfig); end; + TSherpaOnnxOfflineTtsMatchaModelConfig = record + AcousticModel: AnsiString; + Vocoder: AnsiString; + Lexicon: AnsiString; + Tokens: AnsiString; + DataDir: AnsiString; + NoiseScale: Single; + LengthScale: Single; + DictDir: AnsiString; + + function ToString: AnsiString; + class operator Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineTtsMatchaModelConfig); + end; + TSherpaOnnxOfflineTtsModelConfig = record Vits: TSherpaOnnxOfflineTtsVitsModelConfig; NumThreads: Integer; Debug: Boolean; Provider: AnsiString; + Matcha: TSherpaOnnxOfflineTtsMatchaModelConfig; function ToString: AnsiString; class operator Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineTtsModelConfig); @@ -713,11 +728,23 @@ SherpaOnnxOfflineTtsVitsModelConfig = record DictDir: PAnsiChar; end; + SherpaOnnxOfflineTtsMatchaModelConfig = record + AcousticModel: PAnsiChar; + Vocoder: PAnsiChar; + Lexicon: PAnsiChar; + Tokens: PAnsiChar; + DataDir: PAnsiChar; + NoiseScale: cfloat; + LengthScale: cfloat; + DictDir: PAnsiChar; + end; + SherpaOnnxOfflineTtsModelConfig = record Vits: SherpaOnnxOfflineTtsVitsModelConfig; NumThreads: cint32; Debug: cint32; Provider: PAnsiChar; + Matcha: SherpaOnnxOfflineTtsMatchaModelConfig; end; SherpaOnnxOfflineTtsConfig = record @@ -1853,15 +1880,40 @@ function TSherpaOnnxOfflineTtsVitsModelConfig.ToString: AnsiString; Dest.LengthScale := 1.0; end; +function TSherpaOnnxOfflineTtsMatchaModelConfig.ToString: AnsiString; +begin + Result := Format('TSherpaOnnxOfflineTtsMatchaModelConfig(' + + 'AcousticModel := %s, ' + + 'Vocoder := %s, ' + + 'Lexicon := %s, ' + + 'Tokens := %s, ' + + 'DataDir := %s, ' + + 'NoiseScale := %.2f, ' + + 'LengthScale := %.2f, ' + + 'DictDir := %s' + + ')', + [Self.AcousticModel, Self.Vocoder, Self.Lexicon, Self.Tokens, + Self.DataDir, Self.NoiseScale, Self.LengthScale, Self.DictDir + ]); +end; + +class operator TSherpaOnnxOfflineTtsMatchaModelConfig.Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineTtsMatchaModelConfig); +begin + Dest.NoiseScale := 0.667; + Dest.LengthScale := 1.0; +end; + function TSherpaOnnxOfflineTtsModelConfig.ToString: AnsiString; begin Result := Format('TSherpaOnnxOfflineTtsModelConfig(' + 'Vits := %s, ' + 'NumThreads := %d, ' + 'Debug := %s, ' + - 'Provider := %s' + + 'Provider := %s, ' + + 'Matcha := %s' + ')', - [Self.Vits.ToString, Self.NumThreads, Self.Debug.ToString, Self.Provider + [Self.Vits.ToString, Self.NumThreads, Self.Debug.ToString, Self.Provider, + Self.Matcha.ToString ]); end; @@ -1905,6 +1957,15 @@ constructor TSherpaOnnxOfflineTts.Create(Config: TSherpaOnnxOfflineTtsConfig); C.Model.Vits.LengthScale := Config.Model.Vits.LengthScale; C.Model.Vits.DictDir := PAnsiChar(Config.Model.Vits.DictDir); + C.Model.Matcha.AcousticModel := PAnsiChar(Config.Model.Matcha.AcousticModel); + C.Model.Matcha.Vocoder := PAnsiChar(Config.Model.Matcha.Vocoder); + C.Model.Matcha.Lexicon := PAnsiChar(Config.Model.Matcha.Lexicon); + C.Model.Matcha.Tokens := PAnsiChar(Config.Model.Matcha.Tokens); + C.Model.Matcha.DataDir := PAnsiChar(Config.Model.Matcha.DataDir); + C.Model.Matcha.NoiseScale := Config.Model.Matcha.NoiseScale; + C.Model.Matcha.LengthScale := Config.Model.Matcha.LengthScale; + C.Model.Matcha.DictDir := PAnsiChar(Config.Model.Matcha.DictDir); + C.Model.NumThreads := Config.Model.NumThreads; C.Model.Provider := PAnsiChar(Config.Model.Provider); C.Model.Debug := Ord(Config.Model.Debug); From d7c95d33a351d8505335ff7735be68b37cb54f9f Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 6 Jan 2025 11:03:31 +0800 Subject: [PATCH 146/183] Add Dart API for MatchaTTS models (#1687) --- .github/scripts/test-dart.sh | 41 +++++---- .github/workflows/checksum.yaml | 1 + dart-api-examples/tts/bin/matcha-en.dart | 86 ++++++++++++++++++ dart-api-examples/tts/bin/matcha-zh.dart | 90 +++++++++++++++++++ .../tts/bin/{zh.dart => vits-zh.dart} | 0 dart-api-examples/tts/run-matcha-en.sh | 32 +++++++ dart-api-examples/tts/run-matcha-zh.sh | 45 ++++++++++ .../tts/{run-zh.sh => run-vits-zh.sh} | 8 +- .../lib/src/sherpa_onnx_bindings.dart | 17 ++++ flutter/sherpa_onnx/lib/src/tts.dart | 53 ++++++++++- 10 files changed, 349 insertions(+), 24 deletions(-) create mode 100644 dart-api-examples/tts/bin/matcha-en.dart create mode 100644 dart-api-examples/tts/bin/matcha-zh.dart rename dart-api-examples/tts/bin/{zh.dart => vits-zh.dart} (100%) create mode 100755 dart-api-examples/tts/run-matcha-en.sh create mode 100755 dart-api-examples/tts/run-matcha-zh.sh rename dart-api-examples/tts/{run-zh.sh => run-vits-zh.sh} (92%) diff --git a/.github/scripts/test-dart.sh b/.github/scripts/test-dart.sh index 881a4765b..6ba747652 100755 --- a/.github/scripts/test-dart.sh +++ b/.github/scripts/test-dart.sh @@ -4,6 +4,31 @@ set -ex cd dart-api-examples +pushd tts + +echo '----------matcha tts----------' +./run-matcha-zh.sh +./run-matcha-en.sh +ls -lh *.wav +rm -rf matcha-icefall-* +rm *.onnx + +echo '----------piper tts----------' +./run-piper.sh +rm -rf vits-piper-* + +echo '----------coqui tts----------' +./run-coqui.sh +rm -rf vits-coqui-* + +echo '----------zh tts----------' +./run-vits-zh.sh +rm -rf sherpa-onnx-* + +ls -lh *.wav + +popd # tts + pushd speaker-diarization echo '----------speaker diarization----------' ./run.sh @@ -106,22 +131,6 @@ rm -rf sherpa-onnx-* popd # non-streaming-asr -pushd tts - -echo '----------piper tts----------' -./run-piper.sh -rm -rf vits-piper-* - -echo '----------coqui tts----------' -./run-coqui.sh -rm -rf vits-coqui-* - -echo '----------zh tts----------' -./run-zh.sh -rm -rf sherpa-onnx-* - -popd # tts - pushd streaming-asr echo '----------streaming zipformer ctc HLG----------' diff --git a/.github/workflows/checksum.yaml b/.github/workflows/checksum.yaml index 07af3768c..e500209d6 100644 --- a/.github/workflows/checksum.yaml +++ b/.github/workflows/checksum.yaml @@ -7,6 +7,7 @@ on: jobs: checksum: + if: github.repository_owner == 'k2-fsa' runs-on: macos-latest strategy: matrix: diff --git a/dart-api-examples/tts/bin/matcha-en.dart b/dart-api-examples/tts/bin/matcha-en.dart new file mode 100644 index 000000000..fa4c07653 --- /dev/null +++ b/dart-api-examples/tts/bin/matcha-en.dart @@ -0,0 +1,86 @@ +// Copyright (c) 2025 Xiaomi Corporation +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; + +import './init.dart'; + +void main(List arguments) async { + await initSherpaOnnx(); + + final parser = ArgParser() + ..addOption('acoustic-model', help: 'Path to the acoustic model') + ..addOption('vocoder', help: 'Path to the vocoder model') + ..addOption('tokens', help: 'Path to tokens.txt') + ..addOption( + 'data-dir', + help: 'Path to espeak-ng-data directory', + defaultsTo: '', + ) + ..addOption('rule-fsts', help: 'Path to rule fsts', defaultsTo: '') + ..addOption('rule-fars', help: 'Path to rule fars', defaultsTo: '') + ..addOption('text', help: 'Text to generate TTS for') + ..addOption('output-wav', help: 'Filename to save the generated audio') + ..addOption('speed', help: 'Speech speed', defaultsTo: '1.0') + ..addOption( + 'sid', + help: 'Speaker ID to select. Used only for multi-speaker TTS', + defaultsTo: '0', + ); + final res = parser.parse(arguments); + if (res['acoustic-model'] == null || + res['vocoder'] == null || + res['tokens'] == null || + res['data-dir'] == null || + res['output-wav'] == null || + res['text'] == null) { + print(parser.usage); + exit(1); + } + final acousticModel = res['acoustic-model'] as String; + final vocoder = res['vocoder'] as String; + final tokens = res['tokens'] as String; + final dataDir = res['data-dir'] as String; + final ruleFsts = res['rule-fsts'] as String; + final ruleFars = res['rule-fars'] as String; + final text = res['text'] as String; + final outputWav = res['output-wav'] as String; + var speed = double.tryParse(res['speed'] as String) ?? 1.0; + final sid = int.tryParse(res['sid'] as String) ?? 0; + + if (speed == 0) { + speed = 1.0; + } + + final matcha = sherpa_onnx.OfflineTtsMatchaModelConfig( + acousticModel: acousticModel, + vocoder: vocoder, + tokens: tokens, + dataDir: dataDir, + lengthScale: 1 / speed, + ); + + final modelConfig = sherpa_onnx.OfflineTtsModelConfig( + matcha: matcha, + numThreads: 1, + debug: true, + ); + final config = sherpa_onnx.OfflineTtsConfig( + model: modelConfig, + maxNumSenetences: 1, + ruleFsts: ruleFsts, + ruleFars: ruleFars, + ); + + final tts = sherpa_onnx.OfflineTts(config); + final audio = tts.generate(text: text, sid: sid, speed: speed); + tts.free(); + + sherpa_onnx.writeWave( + filename: outputWav, + samples: audio.samples, + sampleRate: audio.sampleRate, + ); + print('Saved to $outputWav'); +} diff --git a/dart-api-examples/tts/bin/matcha-zh.dart b/dart-api-examples/tts/bin/matcha-zh.dart new file mode 100644 index 000000000..d52175e74 --- /dev/null +++ b/dart-api-examples/tts/bin/matcha-zh.dart @@ -0,0 +1,90 @@ +// Copyright (c) 2025 Xiaomi Corporation +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; + +import './init.dart'; + +void main(List arguments) async { + await initSherpaOnnx(); + + final parser = ArgParser() + ..addOption('acoustic-model', help: 'Path to the acoustic model') + ..addOption('vocoder', help: 'Path to the vocoder model') + ..addOption('tokens', help: 'Path to tokens.txt') + ..addOption('lexicon', help: 'Path to lexicon.txt') + ..addOption( + 'dict-dir', + help: 'Path to jieba dict directory', + defaultsTo: '', + ) + ..addOption('rule-fsts', help: 'Path to rule fsts', defaultsTo: '') + ..addOption('rule-fars', help: 'Path to rule fars', defaultsTo: '') + ..addOption('text', help: 'Text to generate TTS for') + ..addOption('output-wav', help: 'Filename to save the generated audio') + ..addOption('speed', help: 'Speech speed', defaultsTo: '1.0') + ..addOption( + 'sid', + help: 'Speaker ID to select. Used only for multi-speaker TTS', + defaultsTo: '0', + ); + final res = parser.parse(arguments); + if (res['acoustic-model'] == null || + res['vocoder'] == null || + res['lexicon'] == null || + res['tokens'] == null || + res['dict-dir'] == null || + res['output-wav'] == null || + res['text'] == null) { + print(parser.usage); + exit(1); + } + final acousticModel = res['acoustic-model'] as String; + final vocoder = res['vocoder'] as String; + final lexicon = res['lexicon'] as String; + final tokens = res['tokens'] as String; + final dictDir = res['dict-dir'] as String; + final ruleFsts = res['rule-fsts'] as String; + final ruleFars = res['rule-fars'] as String; + final text = res['text'] as String; + final outputWav = res['output-wav'] as String; + var speed = double.tryParse(res['speed'] as String) ?? 1.0; + final sid = int.tryParse(res['sid'] as String) ?? 0; + + if (speed == 0) { + speed = 1.0; + } + + final matcha = sherpa_onnx.OfflineTtsMatchaModelConfig( + acousticModel: acousticModel, + vocoder: vocoder, + lexicon: lexicon, + tokens: tokens, + dictDir: dictDir, + lengthScale: 1 / speed, + ); + + final modelConfig = sherpa_onnx.OfflineTtsModelConfig( + matcha: matcha, + numThreads: 1, + debug: true, + ); + final config = sherpa_onnx.OfflineTtsConfig( + model: modelConfig, + maxNumSenetences: 1, + ruleFsts: ruleFsts, + ruleFars: ruleFars, + ); + + final tts = sherpa_onnx.OfflineTts(config); + final audio = tts.generate(text: text, sid: sid, speed: speed); + tts.free(); + + sherpa_onnx.writeWave( + filename: outputWav, + samples: audio.samples, + sampleRate: audio.sampleRate, + ); + print('Saved to $outputWav'); +} diff --git a/dart-api-examples/tts/bin/zh.dart b/dart-api-examples/tts/bin/vits-zh.dart similarity index 100% rename from dart-api-examples/tts/bin/zh.dart rename to dart-api-examples/tts/bin/vits-zh.dart diff --git a/dart-api-examples/tts/run-matcha-en.sh b/dart-api-examples/tts/run-matcha-en.sh new file mode 100755 index 000000000..f727ee5c8 --- /dev/null +++ b/dart-api-examples/tts/run-matcha-en.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -ex + +dart pub get + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker +# matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker +# to download more models +if [ ! -f ./matcha-icefall-en_US-ljspeech/model-steps-3.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 + tar xf matcha-icefall-en_US-ljspeech.tar.bz2 + rm matcha-icefall-en_US-ljspeech.tar.bz2 +fi + +if [ ! -f ./hifigan_v2.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx +fi + +dart run \ + ./bin/matcha-en.dart \ + --acoustic-model ./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ + --vocoder ./hifigan_v2.onnx \ + --tokens ./matcha-icefall-en_US-ljspeech/tokens.txt \ + --data-dir ./matcha-icefall-en_US-ljspeech/espeak-ng-data \ + --sid 0 \ + --speed 1.0 \ + --output-wav matcha-en-1.wav \ + --text "Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone." \ + +ls -lh *.wav diff --git a/dart-api-examples/tts/run-matcha-zh.sh b/dart-api-examples/tts/run-matcha-zh.sh new file mode 100755 index 000000000..be95a827a --- /dev/null +++ b/dart-api-examples/tts/run-matcha-zh.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -ex + +dart pub get + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker +# to download more models +if [ ! -f ./matcha-icefall-zh-baker/model-steps-3.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 + tar xvf matcha-icefall-zh-baker.tar.bz2 + rm matcha-icefall-zh-baker.tar.bz2 +fi + +if [ ! -f ./hifigan_v2.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx +fi + +dart run \ + ./bin/matcha-zh.dart \ + --acoustic-model ./matcha-icefall-zh-baker/model-steps-3.onnx \ + --vocoder ./hifigan_v2.onnx \ + --lexicon ./matcha-icefall-zh-baker/lexicon.txt \ + --tokens ./matcha-icefall-zh-baker/tokens.txt \ + --dict-dir ./matcha-icefall-zh-baker/dict \ + --rule-fsts ./matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst \ + --sid 0 \ + --speed 1.0 \ + --output-wav matcha-zh-1.wav \ + --text "某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。" \ + +dart run \ + ./bin/matcha-zh.dart \ + --acoustic-model ./matcha-icefall-zh-baker/model-steps-3.onnx \ + --vocoder ./hifigan_v2.onnx \ + --lexicon ./matcha-icefall-zh-baker/lexicon.txt \ + --tokens ./matcha-icefall-zh-baker/tokens.txt \ + --dict-dir ./matcha-icefall-zh-baker/dict \ + --sid 0 \ + --speed 1.0 \ + --output-wav matcha-zh-2.wav \ + --text "当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转，思念如涟漪荡漾，梦境如画卷展开，我与自然融为一体，沉静在这片宁静的美丽之中，感受着生命的奇迹与温柔." \ + +ls -lh *.wav diff --git a/dart-api-examples/tts/run-zh.sh b/dart-api-examples/tts/run-vits-zh.sh similarity index 92% rename from dart-api-examples/tts/run-zh.sh rename to dart-api-examples/tts/run-vits-zh.sh index 057260b61..2298f9eb1 100755 --- a/dart-api-examples/tts/run-zh.sh +++ b/dart-api-examples/tts/run-vits-zh.sh @@ -16,7 +16,7 @@ if [[ ! -f ./sherpa-onnx-vits-zh-ll/tokens.txt ]]; then fi dart run \ - ./bin/zh.dart \ + ./bin/vits-zh.dart \ --model ./sherpa-onnx-vits-zh-ll/model.onnx \ --lexicon ./sherpa-onnx-vits-zh-ll/lexicon.txt \ --tokens ./sherpa-onnx-vits-zh-ll/tokens.txt \ @@ -24,10 +24,10 @@ dart run \ --sid 2 \ --speed 1.0 \ --text '当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转，思念如涟漪荡漾，梦境如画卷展开，我与自然融为一体，沉静在这片宁静的美丽之中，感受着生命的奇迹与温柔。' \ - --output-wav zh-jieba-2.wav + --output-wav vits-zh-jieba-2.wav dart run \ - ./bin/zh.dart \ + ./bin/vits-zh.dart \ --model ./sherpa-onnx-vits-zh-ll/model.onnx \ --lexicon ./sherpa-onnx-vits-zh-ll/lexicon.txt \ --tokens ./sherpa-onnx-vits-zh-ll/tokens.txt \ @@ -36,6 +36,6 @@ dart run \ --sid 3 \ --speed 1.0 \ --text '今天是2024年6月15号，13点23分。如果有困难，请拨打110或者18920240511。123456块钱。' \ - --output-wav zh-jieba-3.wav + --output-wav vits-zh-jieba-3.wav ls -lh *.wav diff --git a/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart b/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart index 0d463cf6d..7baf53f26 100644 --- a/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart +++ b/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart @@ -131,6 +131,22 @@ final class SherpaOnnxOfflineTtsVitsModelConfig extends Struct { external Pointer dictDir; } +final class SherpaOnnxOfflineTtsMatchaModelConfig extends Struct { + external Pointer acousticModel; + external Pointer vocoder; + external Pointer lexicon; + external Pointer tokens; + external Pointer dataDir; + + @Float() + external double noiseScale; + + @Float() + external double lengthScale; + + external Pointer dictDir; +} + final class SherpaOnnxOfflineTtsModelConfig extends Struct { external SherpaOnnxOfflineTtsVitsModelConfig vits; @Int32() @@ -140,6 +156,7 @@ final class SherpaOnnxOfflineTtsModelConfig extends Struct { external int debug; external Pointer provider; + external SherpaOnnxOfflineTtsMatchaModelConfig matcha; } final class SherpaOnnxOfflineTtsConfig extends Struct { diff --git a/flutter/sherpa_onnx/lib/src/tts.dart b/flutter/sherpa_onnx/lib/src/tts.dart index 2f550e273..b5dcda48d 100644 --- a/flutter/sherpa_onnx/lib/src/tts.dart +++ b/flutter/sherpa_onnx/lib/src/tts.dart @@ -8,9 +8,9 @@ import './sherpa_onnx_bindings.dart'; class OfflineTtsVitsModelConfig { const OfflineTtsVitsModelConfig({ - required this.model, + this.model = '', this.lexicon = '', - required this.tokens, + this.tokens = '', this.dataDir = '', this.noiseScale = 0.667, this.noiseScaleW = 0.8, @@ -33,9 +33,37 @@ class OfflineTtsVitsModelConfig { final String dictDir; } +class OfflineTtsMatchaModelConfig { + const OfflineTtsMatchaModelConfig({ + this.acousticModel = '', + this.vocoder = '', + this.lexicon = '', + this.tokens = '', + this.dataDir = '', + this.noiseScale = 0.667, + this.lengthScale = 1.0, + this.dictDir = '', + }); + + @override + String toString() { + return 'OfflineTtsMatchaModelConfig(acousticModel: $acousticModel, vocoder: $vocoder, lexicon: $lexicon, tokens: $tokens, dataDir: $dataDir, noiseScale: $noiseScale, lengthScale: $lengthScale, dictDir: $dictDir)'; + } + + final String acousticModel; + final String vocoder; + final String lexicon; + final String tokens; + final String dataDir; + final double noiseScale; + final double lengthScale; + final String dictDir; +} + class OfflineTtsModelConfig { const OfflineTtsModelConfig({ - required this.vits, + this.vits = const OfflineTtsVitsModelConfig(), + this.matcha = const OfflineTtsMatchaModelConfig(), this.numThreads = 1, this.debug = true, this.provider = 'cpu', @@ -43,10 +71,11 @@ class OfflineTtsModelConfig { @override String toString() { - return 'OfflineTtsModelConfig(vits: $vits, numThreads: $numThreads, debug: $debug, provider: $provider)'; + return 'OfflineTtsModelConfig(vits: $vits, matcha: $matcha, numThreads: $numThreads, debug: $debug, provider: $provider)'; } final OfflineTtsVitsModelConfig vits; + final OfflineTtsMatchaModelConfig matcha; final int numThreads; final bool debug; final String provider; @@ -99,6 +128,16 @@ class OfflineTts { c.ref.model.vits.lengthScale = config.model.vits.lengthScale; c.ref.model.vits.dictDir = config.model.vits.dictDir.toNativeUtf8(); + c.ref.model.matcha.acousticModel = + config.model.matcha.acousticModel.toNativeUtf8(); + c.ref.model.matcha.vocoder = config.model.matcha.vocoder.toNativeUtf8(); + c.ref.model.matcha.lexicon = config.model.matcha.lexicon.toNativeUtf8(); + c.ref.model.matcha.tokens = config.model.matcha.tokens.toNativeUtf8(); + c.ref.model.matcha.dataDir = config.model.matcha.dataDir.toNativeUtf8(); + c.ref.model.matcha.noiseScale = config.model.matcha.noiseScale; + c.ref.model.matcha.lengthScale = config.model.matcha.lengthScale; + c.ref.model.matcha.dictDir = config.model.matcha.dictDir.toNativeUtf8(); + c.ref.model.numThreads = config.model.numThreads; c.ref.model.debug = config.model.debug ? 1 : 0; c.ref.model.provider = config.model.provider.toNativeUtf8(); @@ -112,6 +151,12 @@ class OfflineTts { calloc.free(c.ref.ruleFars); calloc.free(c.ref.ruleFsts); calloc.free(c.ref.model.provider); + calloc.free(c.ref.model.matcha.dictDir); + calloc.free(c.ref.model.matcha.dataDir); + calloc.free(c.ref.model.matcha.tokens); + calloc.free(c.ref.model.matcha.lexicon); + calloc.free(c.ref.model.matcha.vocoder); + calloc.free(c.ref.model.matcha.acousticModel); calloc.free(c.ref.model.vits.dictDir); calloc.free(c.ref.model.vits.dataDir); calloc.free(c.ref.model.vits.tokens); From 930986b06ca0dcd3442f93664819fcb946afe3f6 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 6 Jan 2025 11:31:18 +0800 Subject: [PATCH 147/183] Release v1.10.38 (#1688) --- .../workflows/aarch64-linux-gnu-shared.yaml | 2 -- .../workflows/aarch64-linux-gnu-static.yaml | 2 -- .github/workflows/android-static.yaml | 2 -- .github/workflows/android.yaml | 2 -- .github/workflows/arm-linux-gnueabihf.yaml | 2 -- .github/workflows/c-api-from-buffer.yaml | 4 +--- .github/workflows/c-api.yaml | 2 -- .github/workflows/cxx-api.yaml | 2 -- .github/workflows/jni.yaml | 2 -- .github/workflows/lazarus.yaml | 2 -- .github/workflows/linux-gpu.yaml | 2 -- .github/workflows/linux.yaml | 2 -- .github/workflows/macos.yaml | 2 -- .github/workflows/mfc.yaml | 2 -- .github/workflows/pascal.yaml | 2 -- .github/workflows/pkg-config.yaml | 2 -- .github/workflows/riscv64-linux.yaml | 2 -- .github/workflows/run-java-test.yaml | 2 -- .github/workflows/run-python-test-macos.yaml | 2 -- .github/workflows/run-python-test.yaml | 2 -- .github/workflows/swift.yaml | 2 -- .github/workflows/test-build-wheel.yaml | 2 -- .github/workflows/test-dot-net.yaml | 2 -- .github/workflows/test-go.yaml | 2 -- .github/workflows/test-nodejs-addon-api.yaml | 2 -- .../test-nodejs-addon-npm-aarch64.yaml | 2 -- .../test-nodejs-addon-npm-win-x86.yaml | 2 -- .github/workflows/test-nodejs-addon-npm.yaml | 2 -- .github/workflows/test-nodejs.yaml | 2 -- .github/workflows/test-piper-phonemize.yaml | 2 -- .../test-python-offline-websocket-server.yaml | 2 -- .../test-python-online-websocket-server.yaml | 2 -- .github/workflows/windows-arm64.yaml | 2 -- .github/workflows/windows-x64-cuda.yaml | 2 -- .github/workflows/windows-x64-debug.yaml | 2 -- .github/workflows/windows-x64.yaml | 2 -- .github/workflows/windows-x86-debug.yaml | 2 -- .github/workflows/windows-x86.yaml | 2 -- CHANGELOG.md | 22 +++++++++++++++++ CMakeLists.txt | 2 +- android/SherpaOnnxAar/README.md | 6 ++--- android/SherpaOnnxJavaDemo/app/build.gradle | 2 +- build-ios-shared.sh | 2 +- .../add-punctuations/pubspec.yaml | 2 +- dart-api-examples/audio-tagging/pubspec.yaml | 2 +- .../keyword-spotter/pubspec.yaml | 2 +- .../non-streaming-asr/pubspec.yaml | 2 +- .../speaker-diarization/pubspec.yaml | 2 +- .../speaker-identification/pubspec.yaml | 2 +- dart-api-examples/streaming-asr/pubspec.yaml | 2 +- dart-api-examples/tts/pubspec.yaml | 2 +- .../vad-with-non-streaming-asr/pubspec.yaml | 2 +- dart-api-examples/vad/pubspec.yaml | 2 +- flutter-examples/streaming_asr/pubspec.yaml | 4 ++-- flutter-examples/tts/pubspec.yaml | 4 ++-- flutter/sherpa_onnx/pubspec.yaml | 12 +++++----- .../ios/sherpa_onnx_ios.podspec | 2 +- .../macos/sherpa_onnx_macos.podspec | 2 +- .../SherpaOnnxHar/sherpa_onnx/README.md | 2 +- .../sherpa_onnx/oh-package.json5 | 2 +- .../entry/oh-package.json5 | 2 +- .../entry/oh-package.json5 | 2 +- .../entry/oh-package.json5 | 2 +- .../SherpaOnnxTts/entry/oh-package.json5 | 2 +- harmony-os/SherpaOnnxVadAsr/entry/README.md | 2 +- .../SherpaOnnxVadAsr/entry/oh-package.json5 | 2 +- jitpack.yml | 6 ++--- new-release.sh | 24 +++++++++---------- nodejs-addon-examples/package.json | 2 +- pom.xml | 2 +- 70 files changed, 76 insertions(+), 130 deletions(-) diff --git a/.github/workflows/aarch64-linux-gnu-shared.yaml b/.github/workflows/aarch64-linux-gnu-shared.yaml index c347e56b9..185164525 100644 --- a/.github/workflows/aarch64-linux-gnu-shared.yaml +++ b/.github/workflows/aarch64-linux-gnu-shared.yaml @@ -9,7 +9,6 @@ on: - 'v[0-9]+.[0-9]+.[0-9]+*' paths: - '.github/workflows/aarch64-linux-gnu-shared.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' @@ -19,7 +18,6 @@ on: - master paths: - '.github/workflows/aarch64-linux-gnu-shared.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' diff --git a/.github/workflows/aarch64-linux-gnu-static.yaml b/.github/workflows/aarch64-linux-gnu-static.yaml index ae3b9eef2..66ce6ec24 100644 --- a/.github/workflows/aarch64-linux-gnu-static.yaml +++ b/.github/workflows/aarch64-linux-gnu-static.yaml @@ -9,7 +9,6 @@ on: - 'v[0-9]+.[0-9]+.[0-9]+*' paths: - '.github/workflows/aarch64-linux-gnu-static.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' @@ -19,7 +18,6 @@ on: - master paths: - '.github/workflows/aarch64-linux-gnu-static.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' diff --git a/.github/workflows/android-static.yaml b/.github/workflows/android-static.yaml index 84c05d153..7dad8128b 100644 --- a/.github/workflows/android-static.yaml +++ b/.github/workflows/android-static.yaml @@ -9,7 +9,6 @@ on: - android-link-onnxruntime-statically paths: - '.github/workflows/android-static.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/jni/*' @@ -21,7 +20,6 @@ on: - master paths: - '.github/workflows/android-static.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/jni/*' diff --git a/.github/workflows/android.yaml b/.github/workflows/android.yaml index 17b0f6786..1a740005d 100644 --- a/.github/workflows/android.yaml +++ b/.github/workflows/android.yaml @@ -6,7 +6,6 @@ on: - master paths: - '.github/workflows/android.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/jni/*' @@ -18,7 +17,6 @@ on: - master paths: - '.github/workflows/android.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/jni/*' diff --git a/.github/workflows/arm-linux-gnueabihf.yaml b/.github/workflows/arm-linux-gnueabihf.yaml index a70f2b2f4..63a5cf414 100644 --- a/.github/workflows/arm-linux-gnueabihf.yaml +++ b/.github/workflows/arm-linux-gnueabihf.yaml @@ -7,7 +7,6 @@ on: - master paths: - '.github/workflows/arm-linux-gnueabihf.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' @@ -19,7 +18,6 @@ on: - master paths: - '.github/workflows/arm-linux-gnueabihf.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' diff --git a/.github/workflows/c-api-from-buffer.yaml b/.github/workflows/c-api-from-buffer.yaml index 4352cd7ce..5d9bc11db 100644 --- a/.github/workflows/c-api-from-buffer.yaml +++ b/.github/workflows/c-api-from-buffer.yaml @@ -8,7 +8,6 @@ on: - 'v[0-9]+.[0-9]+.[0-9]+*' paths: - '.github/workflows/c-api-from-buffer.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' @@ -19,7 +18,6 @@ on: - master paths: - '.github/workflows/c-api-from-buffer.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' @@ -215,4 +213,4 @@ jobs: ./keywords-spotter-buffered-tokens-keywords-c-api - rm -rf sherpa-onnx-kws-zipformer-* \ No newline at end of file + rm -rf sherpa-onnx-kws-zipformer-* diff --git a/.github/workflows/c-api.yaml b/.github/workflows/c-api.yaml index 1549b484d..3f1776412 100644 --- a/.github/workflows/c-api.yaml +++ b/.github/workflows/c-api.yaml @@ -6,7 +6,6 @@ on: - master paths: - '.github/workflows/c-api.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' @@ -17,7 +16,6 @@ on: - master paths: - '.github/workflows/c-api.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' diff --git a/.github/workflows/cxx-api.yaml b/.github/workflows/cxx-api.yaml index 2fdc56313..e2108ed03 100644 --- a/.github/workflows/cxx-api.yaml +++ b/.github/workflows/cxx-api.yaml @@ -7,7 +7,6 @@ on: - cxx-api-asr-non-streaming paths: - '.github/workflows/cxx-api.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' @@ -17,7 +16,6 @@ on: - master paths: - '.github/workflows/cxx-api.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' diff --git a/.github/workflows/jni.yaml b/.github/workflows/jni.yaml index 0dc775f4c..3bce5cdcd 100644 --- a/.github/workflows/jni.yaml +++ b/.github/workflows/jni.yaml @@ -6,7 +6,6 @@ on: - master paths: - '.github/workflows/jni.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'kotlin-api-examples/**' - 'sherpa-onnx/csrc/*' @@ -16,7 +15,6 @@ on: - master paths: - '.github/workflows/jni.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'kotlin-api-examples/**' - 'sherpa-onnx/csrc/*' diff --git a/.github/workflows/lazarus.yaml b/.github/workflows/lazarus.yaml index 16bfdbba6..72958d639 100644 --- a/.github/workflows/lazarus.yaml +++ b/.github/workflows/lazarus.yaml @@ -7,7 +7,6 @@ on: - lazarus paths: - '.github/workflows/lazarus.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'lazarus-examples/**' - 'sherpa-onnx/csrc/*' @@ -19,7 +18,6 @@ on: - master paths: - '.github/workflows/lazarus.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'lazarus-examples/**' - 'sherpa-onnx/csrc/*' diff --git a/.github/workflows/linux-gpu.yaml b/.github/workflows/linux-gpu.yaml index 2a9d0529d..c1a97aa73 100644 --- a/.github/workflows/linux-gpu.yaml +++ b/.github/workflows/linux-gpu.yaml @@ -14,7 +14,6 @@ on: - '.github/scripts/test-offline-ctc.sh' - '.github/scripts/test-online-ctc.sh' - '.github/scripts/test-offline-tts.sh' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' @@ -31,7 +30,6 @@ on: - '.github/scripts/test-online-ctc.sh' - '.github/scripts/test-online-ctc.sh' - '.github/scripts/test-offline-tts.sh' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml index ea64662b5..ea3bd2b4a 100644 --- a/.github/workflows/linux.yaml +++ b/.github/workflows/linux.yaml @@ -21,7 +21,6 @@ on: - '.github/scripts/test-speaker-diarization.sh' - '.github/scripts/test-c-api.sh' - '.github/scripts/test-cxx-api.sh' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' @@ -44,7 +43,6 @@ on: - '.github/scripts/test-speaker-diarization.sh' - '.github/scripts/test-c-api.sh' - '.github/scripts/test-cxx-api.sh' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml index e6f627e15..813b8fd0e 100644 --- a/.github/workflows/macos.yaml +++ b/.github/workflows/macos.yaml @@ -21,7 +21,6 @@ on: - '.github/scripts/test-speaker-diarization.sh' - '.github/scripts/test-c-api.sh' - '.github/scripts/test-cxx-api.sh' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' @@ -43,7 +42,6 @@ on: - '.github/scripts/test-speaker-diarization.sh' - '.github/scripts/test-c-api.sh' - '.github/scripts/test-cxx-api.sh' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' diff --git a/.github/workflows/mfc.yaml b/.github/workflows/mfc.yaml index e501478a2..1315092c2 100644 --- a/.github/workflows/mfc.yaml +++ b/.github/workflows/mfc.yaml @@ -8,7 +8,6 @@ on: - 'v[0-9]+.[0-9]+.[0-9]+*' paths: - '.github/workflows/mfc.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'mfc-examples/**' - 'sherpa-onnx/csrc/*' @@ -18,7 +17,6 @@ on: - master paths: - '.github/workflows/mfc.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'mfc-examples/**' - 'sherpa-onnx/csrc/*' diff --git a/.github/workflows/pascal.yaml b/.github/workflows/pascal.yaml index 5193241b6..bb04c98b5 100644 --- a/.github/workflows/pascal.yaml +++ b/.github/workflows/pascal.yaml @@ -6,7 +6,6 @@ on: - master paths: - '.github/workflows/pascal.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'pascal-api-examples/**' - 'sherpa-onnx/csrc/*' @@ -17,7 +16,6 @@ on: - master paths: - '.github/workflows/pascal.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'pascal-api-examples/**' - 'sherpa-onnx/csrc/*' diff --git a/.github/workflows/pkg-config.yaml b/.github/workflows/pkg-config.yaml index 57ed8a21a..48ef160ba 100644 --- a/.github/workflows/pkg-config.yaml +++ b/.github/workflows/pkg-config.yaml @@ -10,7 +10,6 @@ on: paths: - '.github/workflows/pkg-config.yaml' - '.github/scripts/test-offline-tts.sh' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' @@ -21,7 +20,6 @@ on: paths: - '.github/workflows/pkg-config.yaml' - '.github/scripts/test-offline-tts.sh' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' diff --git a/.github/workflows/riscv64-linux.yaml b/.github/workflows/riscv64-linux.yaml index e3e5e8b19..f81d5cb2e 100644 --- a/.github/workflows/riscv64-linux.yaml +++ b/.github/workflows/riscv64-linux.yaml @@ -6,7 +6,6 @@ on: - master paths: - '.github/workflows/riscv64-linux.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' @@ -19,7 +18,6 @@ on: - master paths: - '.github/workflows/riscv64-linux.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'toolchains/riscv64-linux-gnu.toolchain.cmake' diff --git a/.github/workflows/run-java-test.yaml b/.github/workflows/run-java-test.yaml index b375f01e5..0976f89e2 100644 --- a/.github/workflows/run-java-test.yaml +++ b/.github/workflows/run-java-test.yaml @@ -6,7 +6,6 @@ on: - master paths: - '.github/workflows/run-java-test.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'java-api-examples/**' - 'sherpa-onnx/csrc/*' @@ -17,7 +16,6 @@ on: - master paths: - '.github/workflows/run-java-test.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'java-api-examples/**' - 'sherpa-onnx/csrc/*' diff --git a/.github/workflows/run-python-test-macos.yaml b/.github/workflows/run-python-test-macos.yaml index a81d6f249..c9fafe68a 100644 --- a/.github/workflows/run-python-test-macos.yaml +++ b/.github/workflows/run-python-test-macos.yaml @@ -7,7 +7,6 @@ on: paths: - '.github/workflows/run-python-test-macos.yaml' - '.github/scripts/test-python.sh' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'python-api-examples/**' @@ -17,7 +16,6 @@ on: paths: - '.github/workflows/run-python-test-macos.yaml' - '.github/scripts/test-python.sh' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'python-api-examples/**' diff --git a/.github/workflows/run-python-test.yaml b/.github/workflows/run-python-test.yaml index 9260775cf..7080420f9 100644 --- a/.github/workflows/run-python-test.yaml +++ b/.github/workflows/run-python-test.yaml @@ -7,7 +7,6 @@ on: paths: - '.github/workflows/run-python-test.yaml' - '.github/scripts/test-python.sh' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'python-api-examples/**' @@ -17,7 +16,6 @@ on: paths: - '.github/workflows/run-python-test.yaml' - '.github/scripts/test-python.sh' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'python-api-examples/**' diff --git a/.github/workflows/swift.yaml b/.github/workflows/swift.yaml index 2143dccde..35bb7ab36 100644 --- a/.github/workflows/swift.yaml +++ b/.github/workflows/swift.yaml @@ -9,7 +9,6 @@ on: paths: - './build-swift-macos.sh' - '.github/workflows/swift.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'swift-api-examples/**' - 'sherpa-onnx/csrc/*' @@ -22,7 +21,6 @@ on: paths: - './build-swift-macos.sh' - '.github/workflows/swift.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'swift-api-examples/**' - 'sherpa-onnx/csrc/*' diff --git a/.github/workflows/test-build-wheel.yaml b/.github/workflows/test-build-wheel.yaml index 371d0712c..d9c863160 100644 --- a/.github/workflows/test-build-wheel.yaml +++ b/.github/workflows/test-build-wheel.yaml @@ -7,7 +7,6 @@ on: paths: - 'setup.py' - '.github/workflows/test-build-wheel.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/python/**' @@ -17,7 +16,6 @@ on: paths: - 'setup.py' - '.github/workflows/test-build-wheel.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/python/**' diff --git a/.github/workflows/test-dot-net.yaml b/.github/workflows/test-dot-net.yaml index 8ca19439f..9b46b64d9 100644 --- a/.github/workflows/test-dot-net.yaml +++ b/.github/workflows/test-dot-net.yaml @@ -6,7 +6,6 @@ on: - master paths: - '.github/workflows/test-dot-net.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'dotnet-examples/**' @@ -17,7 +16,6 @@ on: - master paths: - '.github/workflows/test-dot-net.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'dotnet-examples/**' diff --git a/.github/workflows/test-go.yaml b/.github/workflows/test-go.yaml index 440402939..083f01d6b 100644 --- a/.github/workflows/test-go.yaml +++ b/.github/workflows/test-go.yaml @@ -6,7 +6,6 @@ on: - master paths: - '.github/workflows/test-go.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'go-api-examples/**' @@ -16,7 +15,6 @@ on: - master paths: - '.github/workflows/test-go.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'go-api-examples/**' diff --git a/.github/workflows/test-nodejs-addon-api.yaml b/.github/workflows/test-nodejs-addon-api.yaml index 224fc0f0b..539025c8c 100644 --- a/.github/workflows/test-nodejs-addon-api.yaml +++ b/.github/workflows/test-nodejs-addon-api.yaml @@ -6,7 +6,6 @@ on: - master paths: - '.github/workflows/test-nodejs-addon-api.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' @@ -17,7 +16,6 @@ on: - master paths: - '.github/workflows/test-nodejs-addon-api.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' diff --git a/.github/workflows/test-nodejs-addon-npm-aarch64.yaml b/.github/workflows/test-nodejs-addon-npm-aarch64.yaml index 07ab8d878..232f8fe27 100644 --- a/.github/workflows/test-nodejs-addon-npm-aarch64.yaml +++ b/.github/workflows/test-nodejs-addon-npm-aarch64.yaml @@ -6,7 +6,6 @@ on: - master paths: - '.github/workflows/test-nodejs-addon-npm-aarch64.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' @@ -19,7 +18,6 @@ on: - master paths: - '.github/workflows/test-nodejs-addon-npm-aarch64.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' diff --git a/.github/workflows/test-nodejs-addon-npm-win-x86.yaml b/.github/workflows/test-nodejs-addon-npm-win-x86.yaml index 98cba9dec..0a21630de 100644 --- a/.github/workflows/test-nodejs-addon-npm-win-x86.yaml +++ b/.github/workflows/test-nodejs-addon-npm-win-x86.yaml @@ -6,7 +6,6 @@ on: - master paths: - '.github/workflows/test-nodejs-addon-npm-win-x86.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' @@ -20,7 +19,6 @@ on: - master paths: - '.github/workflows/test-nodejs-addon-npm-win-x86.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' diff --git a/.github/workflows/test-nodejs-addon-npm.yaml b/.github/workflows/test-nodejs-addon-npm.yaml index 27a962357..0e2b9f55f 100644 --- a/.github/workflows/test-nodejs-addon-npm.yaml +++ b/.github/workflows/test-nodejs-addon-npm.yaml @@ -6,7 +6,6 @@ on: - master paths: - '.github/workflows/test-nodejs-addon-npm.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' @@ -19,7 +18,6 @@ on: - master paths: - '.github/workflows/test-nodejs-addon-npm.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' diff --git a/.github/workflows/test-nodejs.yaml b/.github/workflows/test-nodejs.yaml index 25f3c38fd..78788ad04 100644 --- a/.github/workflows/test-nodejs.yaml +++ b/.github/workflows/test-nodejs.yaml @@ -6,7 +6,6 @@ on: - master paths: - '.github/workflows/test-nodejs.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' @@ -18,7 +17,6 @@ on: - master paths: - '.github/workflows/test-nodejs.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/c-api/*' diff --git a/.github/workflows/test-piper-phonemize.yaml b/.github/workflows/test-piper-phonemize.yaml index 1edbae6d2..744095411 100644 --- a/.github/workflows/test-piper-phonemize.yaml +++ b/.github/workflows/test-piper-phonemize.yaml @@ -5,7 +5,6 @@ on: - master paths: - '.github/workflows/test-piper-phonemize.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' pull_request: @@ -13,7 +12,6 @@ on: - master paths: - '.github/workflows/test-piper-phonemize.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' diff --git a/.github/workflows/test-python-offline-websocket-server.yaml b/.github/workflows/test-python-offline-websocket-server.yaml index 52a22ee5a..4fa98464c 100644 --- a/.github/workflows/test-python-offline-websocket-server.yaml +++ b/.github/workflows/test-python-offline-websocket-server.yaml @@ -6,7 +6,6 @@ on: - master paths: - '.github/workflows/test-python-offline-websocket-server.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/python/**' @@ -15,7 +14,6 @@ on: - master paths: - '.github/workflows/test-python-offline-websocket-server.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/python/**' diff --git a/.github/workflows/test-python-online-websocket-server.yaml b/.github/workflows/test-python-online-websocket-server.yaml index badf343a0..d22e93002 100644 --- a/.github/workflows/test-python-online-websocket-server.yaml +++ b/.github/workflows/test-python-online-websocket-server.yaml @@ -6,7 +6,6 @@ on: - master paths: - '.github/workflows/test-python-online-websocket-server.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/python/**' @@ -15,7 +14,6 @@ on: - master paths: - '.github/workflows/test-python-online-websocket-server.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' - 'sherpa-onnx/python/**' diff --git a/.github/workflows/windows-arm64.yaml b/.github/workflows/windows-arm64.yaml index 105260ce2..b6ab5bf7e 100644 --- a/.github/workflows/windows-arm64.yaml +++ b/.github/workflows/windows-arm64.yaml @@ -8,7 +8,6 @@ on: - 'v[0-9]+.[0-9]+.[0-9]+*' paths: - '.github/workflows/windows-arm64.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' pull_request: @@ -16,7 +15,6 @@ on: - master paths: - '.github/workflows/windows-arm64.yaml' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' diff --git a/.github/workflows/windows-x64-cuda.yaml b/.github/workflows/windows-x64-cuda.yaml index fd4570455..0d15af946 100644 --- a/.github/workflows/windows-x64-cuda.yaml +++ b/.github/workflows/windows-x64-cuda.yaml @@ -14,7 +14,6 @@ on: - '.github/scripts/test-offline-ctc.sh' - '.github/scripts/test-online-ctc.sh' - '.github/scripts/test-offline-tts.sh' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' pull_request: @@ -28,7 +27,6 @@ on: - '.github/scripts/test-offline-ctc.sh' - '.github/scripts/test-online-ctc.sh' - '.github/scripts/test-offline-tts.sh' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' diff --git a/.github/workflows/windows-x64-debug.yaml b/.github/workflows/windows-x64-debug.yaml index 09f93fd0d..7abf02285 100644 --- a/.github/workflows/windows-x64-debug.yaml +++ b/.github/workflows/windows-x64-debug.yaml @@ -14,7 +14,6 @@ on: - '.github/scripts/test-offline-ctc.sh' - '.github/scripts/test-online-ctc.sh' - '.github/scripts/test-offline-tts.sh' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' pull_request: @@ -28,7 +27,6 @@ on: - '.github/scripts/test-offline-ctc.sh' - '.github/scripts/test-online-ctc.sh' - '.github/scripts/test-offline-tts.sh' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' diff --git a/.github/workflows/windows-x64.yaml b/.github/workflows/windows-x64.yaml index 50bf014d4..76dd42623 100644 --- a/.github/workflows/windows-x64.yaml +++ b/.github/workflows/windows-x64.yaml @@ -20,7 +20,6 @@ on: - '.github/scripts/test-speaker-diarization.sh' - '.github/scripts/test-c-api.sh' - '.github/scripts/test-cxx-api.sh' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' pull_request: @@ -40,7 +39,6 @@ on: - '.github/scripts/test-speaker-diarization.sh' - '.github/scripts/test-c-api.sh' - '.github/scripts/test-cxx-api.sh' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' diff --git a/.github/workflows/windows-x86-debug.yaml b/.github/workflows/windows-x86-debug.yaml index f72bf2566..59d9ef370 100644 --- a/.github/workflows/windows-x86-debug.yaml +++ b/.github/workflows/windows-x86-debug.yaml @@ -14,7 +14,6 @@ on: - '.github/scripts/test-offline-ctc.sh' - '.github/scripts/test-offline-tts.sh' - '.github/scripts/test-online-ctc.sh' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' pull_request: @@ -28,7 +27,6 @@ on: - '.github/scripts/test-offline-ctc.sh' - '.github/scripts/test-offline-tts.sh' - '.github/scripts/test-online-ctc.sh' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' diff --git a/.github/workflows/windows-x86.yaml b/.github/workflows/windows-x86.yaml index 8a1370959..f1498c0c0 100644 --- a/.github/workflows/windows-x86.yaml +++ b/.github/workflows/windows-x86.yaml @@ -20,7 +20,6 @@ on: - '.github/scripts/test-speaker-diarization.sh' - '.github/scripts/test-c-api.sh' - '.github/scripts/test-cxx-api.sh' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' pull_request: @@ -40,7 +39,6 @@ on: - '.github/scripts/test-speaker-diarization.sh' - '.github/scripts/test-c-api.sh' - '.github/scripts/test-cxx-api.sh' - - 'CMakeLists.txt' - 'cmake/**' - 'sherpa-onnx/csrc/*' diff --git a/CHANGELOG.md b/CHANGELOG.md index b64604100..e9db4c24e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,25 @@ +## 1.10.38 + +* Fix initializing TTS in Python. (#1664) +* Remove spaces after punctuations for TTS (#1666) +* Add constructor fromPtr() for all flutter class with factory ctor. (#1667) +* Add Kotlin API for Matcha-TTS models. (#1668) +* Support Matcha-TTS models using espeak-ng (#1672) +* Add Java API for Matcha-TTS models. (#1673) +* Avoid adding tail padding for VAD in generate-subtitles.py (#1674) +* Add C API for MatchaTTS models (#1675) +* Add CXX API for MatchaTTS models (#1676) +* Add JavaScript API (node-addon-api) for MatchaTTS models. (#1677) +* Add HarmonyOS examples for MatchaTTS. (#1678) +* Upgraded to .NET 8 and made code style a little more internally consistent. (#1680) +* Update workflows to use .NET 8.0 also. (#1681) +* Add C# and JavaScript (wasm) API for MatchaTTS models (#1682) +* Add Android demo for MatchaTTS models. (#1683) +* Add Swift API for MatchaTTS models. (#1684) +* Add Go API for MatchaTTS models (#1685) +* Add Pascal API for MatchaTTS models. (#1686) +* Add Dart API for MatchaTTS models (#1687) + ## 1.10.37 * Add new tts models for Latvia and Persian+English (#1644) diff --git a/CMakeLists.txt b/CMakeLists.txt index b287dc9d7..7f0fe2990 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ project(sherpa-onnx) # Remember to update # ./CHANGELOG.md # ./new-release.sh -set(SHERPA_ONNX_VERSION "1.10.37") +set(SHERPA_ONNX_VERSION "1.10.38") # Disable warning about # diff --git a/android/SherpaOnnxAar/README.md b/android/SherpaOnnxAar/README.md index 432777ab3..caea152ff 100644 --- a/android/SherpaOnnxAar/README.md +++ b/android/SherpaOnnxAar/README.md @@ -4,8 +4,8 @@ git clone https://github.com/k2-fsa/sherpa-onnx cd sherpa-onnx -wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.37/sherpa-onnx-v1.10.37-android.tar.bz2 -tar xvf sherpa-onnx-v1.10.37-android.tar.bz2 +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.38/sherpa-onnx-v1.10.38-android.tar.bz2 +tar xvf sherpa-onnx-v1.10.38-android.tar.bz2 cp -v jniLibs/arm64-v8a/* android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/arm64-v8a/ cp -v jniLibs/armeabi-v7a/* android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/armeabi-v7a/ @@ -16,5 +16,5 @@ cd android/SherpaOnnxAar ./gradlew :sherpa_onnx:assembleRelease ls -lh ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar -cp ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar ../../sherpa-onnx-1.10.37.aar +cp ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar ../../sherpa-onnx-1.10.38.aar ``` diff --git a/android/SherpaOnnxJavaDemo/app/build.gradle b/android/SherpaOnnxJavaDemo/app/build.gradle index 7ddf31c10..e04feafe4 100644 --- a/android/SherpaOnnxJavaDemo/app/build.gradle +++ b/android/SherpaOnnxJavaDemo/app/build.gradle @@ -34,5 +34,5 @@ dependencies { implementation 'pub.devrel:easypermissions:3.0.0' implementation 'androidx.core:core-ktx:1.7.0' // implementation files('/Users/fangjun/open-source/sherpa-onnx/android/SherpaOnnxAar/sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar') - implementation 'com.github.k2-fsa:sherpa-onnx:v1.10.37' + implementation 'com.github.k2-fsa:sherpa-onnx:v1.10.38' } diff --git a/build-ios-shared.sh b/build-ios-shared.sh index 5fb90c691..4cdeae190 100755 --- a/build-ios-shared.sh +++ b/build-ios-shared.sh @@ -242,7 +242,7 @@ for d in ios-arm64_x86_64-simulator ios-arm64; do CFBundlePackageType FMWK CFBundleShortVersionString - 1.10.37 + 1.10.38 CFBundleSupportedPlatforms iPhoneOS diff --git a/dart-api-examples/add-punctuations/pubspec.yaml b/dart-api-examples/add-punctuations/pubspec.yaml index 78b65ca03..952f5421b 100644 --- a/dart-api-examples/add-punctuations/pubspec.yaml +++ b/dart-api-examples/add-punctuations/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.37 + sherpa_onnx: ^1.10.38 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/audio-tagging/pubspec.yaml b/dart-api-examples/audio-tagging/pubspec.yaml index 0115353ed..1fe474327 100644 --- a/dart-api-examples/audio-tagging/pubspec.yaml +++ b/dart-api-examples/audio-tagging/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.37 + sherpa_onnx: ^1.10.38 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/keyword-spotter/pubspec.yaml b/dart-api-examples/keyword-spotter/pubspec.yaml index 13c34aaa7..cff5ac008 100644 --- a/dart-api-examples/keyword-spotter/pubspec.yaml +++ b/dart-api-examples/keyword-spotter/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.37 + sherpa_onnx: ^1.10.38 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/non-streaming-asr/pubspec.yaml b/dart-api-examples/non-streaming-asr/pubspec.yaml index 75523f007..90b9308cf 100644 --- a/dart-api-examples/non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.37 + sherpa_onnx: ^1.10.38 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/speaker-diarization/pubspec.yaml b/dart-api-examples/speaker-diarization/pubspec.yaml index 9b9544cf0..648005fc2 100644 --- a/dart-api-examples/speaker-diarization/pubspec.yaml +++ b/dart-api-examples/speaker-diarization/pubspec.yaml @@ -8,7 +8,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.37 + sherpa_onnx: ^1.10.38 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/speaker-identification/pubspec.yaml b/dart-api-examples/speaker-identification/pubspec.yaml index 368c1dde5..c1f690ca4 100644 --- a/dart-api-examples/speaker-identification/pubspec.yaml +++ b/dart-api-examples/speaker-identification/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.37 + sherpa_onnx: ^1.10.38 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/streaming-asr/pubspec.yaml b/dart-api-examples/streaming-asr/pubspec.yaml index 451e92163..70812eacc 100644 --- a/dart-api-examples/streaming-asr/pubspec.yaml +++ b/dart-api-examples/streaming-asr/pubspec.yaml @@ -11,7 +11,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.37 + sherpa_onnx: ^1.10.38 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/tts/pubspec.yaml b/dart-api-examples/tts/pubspec.yaml index b64d7ef83..35f593dd7 100644 --- a/dart-api-examples/tts/pubspec.yaml +++ b/dart-api-examples/tts/pubspec.yaml @@ -8,7 +8,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.37 + sherpa_onnx: ^1.10.38 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml index aedcc1f5a..ed4e28eed 100644 --- a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.37 + sherpa_onnx: ^1.10.38 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad/pubspec.yaml b/dart-api-examples/vad/pubspec.yaml index 3e16d5b92..ab17a871e 100644 --- a/dart-api-examples/vad/pubspec.yaml +++ b/dart-api-examples/vad/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.37 + sherpa_onnx: ^1.10.38 path: ^1.9.0 args: ^2.5.0 diff --git a/flutter-examples/streaming_asr/pubspec.yaml b/flutter-examples/streaming_asr/pubspec.yaml index 78748432b..807319289 100644 --- a/flutter-examples/streaming_asr/pubspec.yaml +++ b/flutter-examples/streaming_asr/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' -version: 1.10.37 +version: 1.10.38 topics: - speech-recognition @@ -31,7 +31,7 @@ dependencies: record: ^5.1.0 url_launcher: ^6.2.6 - sherpa_onnx: ^1.10.37 + sherpa_onnx: ^1.10.38 # sherpa_onnx: # path: ../../flutter/sherpa_onnx diff --git a/flutter-examples/tts/pubspec.yaml b/flutter-examples/tts/pubspec.yaml index 854492f80..40ec36075 100644 --- a/flutter-examples/tts/pubspec.yaml +++ b/flutter-examples/tts/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' # Remove this line if you wish to publish to pub.dev -version: 1.10.37 +version: 1.10.38 environment: sdk: ">=2.17.0 <4.0.0" @@ -18,7 +18,7 @@ dependencies: cupertino_icons: ^1.0.6 path_provider: ^2.1.3 path: ^1.9.0 - sherpa_onnx: ^1.10.37 + sherpa_onnx: ^1.10.38 # sherpa_onnx: # path: ../../flutter/sherpa_onnx url_launcher: 6.2.6 diff --git a/flutter/sherpa_onnx/pubspec.yaml b/flutter/sherpa_onnx/pubspec.yaml index dd69368c6..da08299b2 100644 --- a/flutter/sherpa_onnx/pubspec.yaml +++ b/flutter/sherpa_onnx/pubspec.yaml @@ -17,7 +17,7 @@ topics: - voice-activity-detection # remember to change the version in ../sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec -version: 1.10.37 +version: 1.10.38 homepage: https://github.com/k2-fsa/sherpa-onnx @@ -30,23 +30,23 @@ dependencies: flutter: sdk: flutter - sherpa_onnx_android: ^1.10.37 + sherpa_onnx_android: ^1.10.38 # sherpa_onnx_android: # path: ../sherpa_onnx_android - sherpa_onnx_macos: ^1.10.37 + sherpa_onnx_macos: ^1.10.38 # sherpa_onnx_macos: # path: ../sherpa_onnx_macos - sherpa_onnx_linux: ^1.10.37 + sherpa_onnx_linux: ^1.10.38 # sherpa_onnx_linux: # path: ../sherpa_onnx_linux - sherpa_onnx_windows: ^1.10.37 + sherpa_onnx_windows: ^1.10.38 # sherpa_onnx_windows: # path: ../sherpa_onnx_windows - sherpa_onnx_ios: ^1.10.37 + sherpa_onnx_ios: ^1.10.38 # sherpa_onnx_ios: # path: ../sherpa_onnx_ios diff --git a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec index dc0b24ecb..528c2021e 100644 --- a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec +++ b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec @@ -7,7 +7,7 @@ # https://groups.google.com/g/dart-ffi/c/nUATMBy7r0c Pod::Spec.new do |s| s.name = 'sherpa_onnx_ios' - s.version = '1.10.37' + s.version = '1.10.38' s.summary = 'A new Flutter FFI plugin project.' s.description = <<-DESC A new Flutter FFI plugin project. diff --git a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec index 849ead253..319c6eae4 100644 --- a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec +++ b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec @@ -4,7 +4,7 @@ # Pod::Spec.new do |s| s.name = 'sherpa_onnx_macos' - s.version = '1.10.37' + s.version = '1.10.38' s.summary = 'sherpa-onnx Flutter FFI plugin project.' s.description = <<-DESC sherpa-onnx Flutter FFI plugin project. diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md b/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md index 76e44a733..a7e3e5317 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md @@ -23,7 +23,7 @@ or update your `oh-package.json5` to include the following: ``` "dependencies": { - "sherpa_onnx": "1.10.37", + "sherpa_onnx": "1.10.38", }, ``` diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 index 3c365ced4..f64630c3b 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 @@ -1,6 +1,6 @@ { "name": "sherpa_onnx", - "version": "1.10.37", + "version": "1.10.38", "description": "On-device speech-to-text, text-to-speech, and speaker diarization using Next-gen Kaldi without Internet connection", "main": "Index.ets", "author": "The next-gen Kaldi team", diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 b/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 index 68c0d2d8f..c625a902c 100644 --- a/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.37" + "sherpa_onnx": "1.10.38" } } diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 b/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 index ebcc97a46..802435184 100644 --- a/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.37", + "sherpa_onnx": "1.10.38", } } diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 index ebcc97a46..802435184 100644 --- a/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.37", + "sherpa_onnx": "1.10.38", } } diff --git a/harmony-os/SherpaOnnxTts/entry/oh-package.json5 b/harmony-os/SherpaOnnxTts/entry/oh-package.json5 index ebcc97a46..802435184 100644 --- a/harmony-os/SherpaOnnxTts/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxTts/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.37", + "sherpa_onnx": "1.10.38", } } diff --git a/harmony-os/SherpaOnnxVadAsr/entry/README.md b/harmony-os/SherpaOnnxVadAsr/entry/README.md index 7b5390da4..59053f756 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/README.md +++ b/harmony-os/SherpaOnnxVadAsr/entry/README.md @@ -1,6 +1,6 @@ # Introduction -Please download ./sherpa_onnx-v1.10.37.har +Please download ./sherpa_onnx-v1.10.38.har from Hint: For users who have no access to huggingface, please use diff --git a/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 b/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 index 223a04081..55d35f104 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 @@ -7,7 +7,7 @@ "license": "", "dependencies": { // please see https://ohpm.openharmony.cn/#/cn/detail/sherpa_onnx - "sherpa_onnx": "1.10.37", + "sherpa_onnx": "1.10.38", } } diff --git a/jitpack.yml b/jitpack.yml index 45d46be6e..6c1ecb597 100644 --- a/jitpack.yml +++ b/jitpack.yml @@ -2,8 +2,8 @@ jdk: - openjdk17 before_install: - - wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.37/sherpa-onnx-1.10.37.aar + - wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.38/sherpa-onnx-1.10.38.aar install: - - FILE="-Dfile=sherpa-onnx-1.10.37.aar" - - mvn install:install-file $FILE -DgroupId=com.k2fsa.sherpa.onnx -DartifactId=sherpa-onnx -Dversion=1.10.37 -Dpackaging=aar -DgeneratePom=true + - FILE="-Dfile=sherpa-onnx-1.10.38.aar" + - mvn install:install-file $FILE -DgroupId=com.k2fsa.sherpa.onnx -DartifactId=sherpa-onnx -Dversion=1.10.38 -Dpackaging=aar -DgeneratePom=true diff --git a/new-release.sh b/new-release.sh index abb13fb8a..1a6993071 100755 --- a/new-release.sh +++ b/new-release.sh @@ -2,18 +2,18 @@ set -ex -sed -i.bak 's/1\.10\.36/1\.10\.37/g' ./build-ios-shared.sh -sed -i.bak 's/1\.10\.36/1\.10\.37/g' ./pom.xml -sed -i.bak 's/1\.10\.36/1\.10\.37/g' ./jitpack.yml -sed -i.bak 's/1\.10\.36/1\.10\.37/g' ./android/SherpaOnnxAar/README.md +sed -i.bak 's/1\.10\.37/1\.10\.38/g' ./build-ios-shared.sh +sed -i.bak 's/1\.10\.37/1\.10\.38/g' ./pom.xml +sed -i.bak 's/1\.10\.37/1\.10\.38/g' ./jitpack.yml +sed -i.bak 's/1\.10\.37/1\.10\.38/g' ./android/SherpaOnnxAar/README.md -find android -name build.gradle -type f -exec sed -i.bak 's/sherpa-onnx:v1\.10\.36/sherpa-onnx:v1\.10\.37/g' {} \; +find android -name build.gradle -type f -exec sed -i.bak 's/sherpa-onnx:v1\.10\.37/sherpa-onnx:v1\.10\.38/g' {} \; -find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.36/1\.10\.37/g' {} \; -find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.36/1\.10\.37/g' {} \; -find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.36/1\.10\.37/g' {} \; -find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.36/1\.10\.37/g' {} \; -find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.36/1\.10\.37/g' {} \; +find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.37/1\.10\.38/g' {} \; +find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.37/1\.10\.38/g' {} \; +find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.37/1\.10\.38/g' {} \; +find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.37/1\.10\.38/g' {} \; +find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.37/1\.10\.38/g' {} \; -find harmony-os -name "README.md" -type f -exec sed -i.bak 's/1\.10\.36/1\.10\.37/g' {} \; -find harmony-os -name oh-package.json5 -type f -exec sed -i.bak 's/1\.10\.36/1\.10\.37/g' {} \; +find harmony-os -name "README.md" -type f -exec sed -i.bak 's/1\.10\.37/1\.10\.38/g' {} \; +find harmony-os -name oh-package.json5 -type f -exec sed -i.bak 's/1\.10\.37/1\.10\.38/g' {} \; diff --git a/nodejs-addon-examples/package.json b/nodejs-addon-examples/package.json index 592f1a7ab..0cdfc5912 100644 --- a/nodejs-addon-examples/package.json +++ b/nodejs-addon-examples/package.json @@ -1,5 +1,5 @@ { "dependencies": { - "sherpa-onnx-node": "^1.10.37" + "sherpa-onnx-node": "^1.10.38" } } diff --git a/pom.xml b/pom.xml index 05f74237c..f412806ec 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.k2fsa.sherpa.onnx sherpa-onnx-android - 1.10.37 + 1.10.38 https://github.com/k2-fsa/sherpa-onnx pom First Android Library From 6d18430dbf522c037fe946242452fe67ab87319f Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 7 Jan 2025 09:59:20 +0800 Subject: [PATCH 148/183] Fix building without TTS (#1691) --- .../NonStreamingTextToSpeechDlg.h | 2 +- sherpa-onnx/c-api/c-api.cc | 173 +++++++++++++++++- 2 files changed, 173 insertions(+), 2 deletions(-) diff --git a/mfc-examples/NonStreamingTextToSpeech/NonStreamingTextToSpeechDlg.h b/mfc-examples/NonStreamingTextToSpeech/NonStreamingTextToSpeechDlg.h index 0f938bd61..f2db39e9c 100644 --- a/mfc-examples/NonStreamingTextToSpeech/NonStreamingTextToSpeechDlg.h +++ b/mfc-examples/NonStreamingTextToSpeech/NonStreamingTextToSpeechDlg.h @@ -53,7 +53,7 @@ class CNonStreamingTextToSpeechDlg : public CDialogEx CButton generate_btn_; afx_msg void OnBnClickedOk(); - SherpaOnnxOfflineTts *tts_ = nullptr; + const SherpaOnnxOfflineTts *tts_ = nullptr; CEdit my_text_; CEdit output_filename_; diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index 30c87823a..625f608a9 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -1220,6 +1220,69 @@ void SherpaOnnxDestroyOfflineTtsGeneratedAudio( delete p; } } +#else +const SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTts( + const SherpaOnnxOfflineTtsConfig *config) { + SHERPA_ONNX_LOGE("TTS is not enabled. Please rebuild sherpa-onnx"); + return nullptr; +} + +void SherpaOnnxDestroyOfflineTts(const SherpaOnnxOfflineTts *tts) { + SHERPA_ONNX_LOGE("TTS is not enabled. Please rebuild sherpa-onnx"); +} + +int32_t SherpaOnnxOfflineTtsSampleRate(const SherpaOnnxOfflineTts *tts) { + SHERPA_ONNX_LOGE("TTS is not enabled. Please rebuild sherpa-onnx"); + return 0; +} + +int32_t SherpaOnnxOfflineTtsNumSpeakers(const SherpaOnnxOfflineTts *tts) { + SHERPA_ONNX_LOGE("TTS is not enabled. Please rebuild sherpa-onnx"); + return 0; +} + +const SherpaOnnxGeneratedAudio *SherpaOnnxOfflineTtsGenerate( + const SherpaOnnxOfflineTts *tts, const char *text, int32_t sid, + float speed) { + SHERPA_ONNX_LOGE("TTS is not enabled. Please rebuild sherpa-onnx"); + return nullptr; +} + +const SherpaOnnxGeneratedAudio *SherpaOnnxOfflineTtsGenerateWithCallback( + const SherpaOnnxOfflineTts *tts, const char *text, int32_t sid, float speed, + SherpaOnnxGeneratedAudioCallback callback) { + SHERPA_ONNX_LOGE("TTS is not enabled. Please rebuild sherpa-onnx"); + return nullptr; +} + +const SherpaOnnxGeneratedAudio * +SherpaOnnxOfflineTtsGenerateWithProgressCallback( + const SherpaOnnxOfflineTts *tts, const char *text, int32_t sid, float speed, + SherpaOnnxGeneratedAudioProgressCallback callback) { + SHERPA_ONNX_LOGE("TTS is not enabled. Please rebuild sherpa-onnx"); + return nullptr; +} + +const SherpaOnnxGeneratedAudio * +SherpaOnnxOfflineTtsGenerateWithProgressCallbackWithArg( + const SherpaOnnxOfflineTts *tts, const char *text, int32_t sid, float speed, + SherpaOnnxGeneratedAudioProgressCallbackWithArg callback, void *arg) { + SHERPA_ONNX_LOGE("TTS is not enabled. Please rebuild sherpa-onnx"); + return nullptr; +} + +const SherpaOnnxGeneratedAudio *SherpaOnnxOfflineTtsGenerateWithCallbackWithArg( + const SherpaOnnxOfflineTts *tts, const char *text, int32_t sid, float speed, + SherpaOnnxGeneratedAudioCallbackWithArg callback, void *arg) { + SHERPA_ONNX_LOGE("TTS is not enabled. Please rebuild sherpa-onnx"); + return nullptr; +} + +void SherpaOnnxDestroyOfflineTtsGeneratedAudio( + const SherpaOnnxGeneratedAudio *p) { + SHERPA_ONNX_LOGE("TTS is not enabled. Please rebuild sherpa-onnx"); +} + #endif // SHERPA_ONNX_ENABLE_TTS == 1 int32_t SherpaOnnxWriteWave(const float *samples, int32_t n, @@ -2027,6 +2090,99 @@ SherpaOnnxOfflineSpeakerDiarizationProcessWithCallbackNoArg( return ans; } +#else + +const SherpaOnnxOfflineSpeakerDiarization * +SherpaOnnxCreateOfflineSpeakerDiarization( + const SherpaOnnxOfflineSpeakerDiarizationConfig *config) { + SHERPA_ONNX_LOGE( + "Speaker diarization is not enabled. Please rebuild sherpa-onnx"); + return nullptr; +} + +void SherpaOnnxDestroyOfflineSpeakerDiarization( + const SherpaOnnxOfflineSpeakerDiarization *sd) { + SHERPA_ONNX_LOGE( + "Speaker diarization is not enabled. Please rebuild sherpa-onnx"); +} + +int32_t SherpaOnnxOfflineSpeakerDiarizationGetSampleRate( + const SherpaOnnxOfflineSpeakerDiarization *sd) { + SHERPA_ONNX_LOGE( + "Speaker diarization is not enabled. Please rebuild sherpa-onnx"); + return 0; +} + +void SherpaOnnxOfflineSpeakerDiarizationSetConfig( + const SherpaOnnxOfflineSpeakerDiarization *sd, + const SherpaOnnxOfflineSpeakerDiarizationConfig *config) { + SHERPA_ONNX_LOGE( + "Speaker diarization is not enabled. Please rebuild sherpa-onnx"); +} + +int32_t SherpaOnnxOfflineSpeakerDiarizationResultGetNumSpeakers( + const SherpaOnnxOfflineSpeakerDiarizationResult *r) { + SHERPA_ONNX_LOGE( + "Speaker diarization is not enabled. Please rebuild sherpa-onnx"); + return 0; +} + +int32_t SherpaOnnxOfflineSpeakerDiarizationResultGetNumSegments( + const SherpaOnnxOfflineSpeakerDiarizationResult *r) { + SHERPA_ONNX_LOGE( + "Speaker diarization is not enabled. Please rebuild sherpa-onnx"); + return 0; +} + +const SherpaOnnxOfflineSpeakerDiarizationSegment * +SherpaOnnxOfflineSpeakerDiarizationResultSortByStartTime( + const SherpaOnnxOfflineSpeakerDiarizationResult *r) { + SHERPA_ONNX_LOGE( + "Speaker diarization is not enabled. Please rebuild sherpa-onnx"); + return nullptr; +} + +void SherpaOnnxOfflineSpeakerDiarizationDestroySegment( + const SherpaOnnxOfflineSpeakerDiarizationSegment *s) { + SHERPA_ONNX_LOGE( + "Speaker diarization is not enabled. Please rebuild sherpa-onnx"); +} + +const SherpaOnnxOfflineSpeakerDiarizationResult * +SherpaOnnxOfflineSpeakerDiarizationProcess( + const SherpaOnnxOfflineSpeakerDiarization *sd, const float *samples, + int32_t n) { + SHERPA_ONNX_LOGE( + "Speaker diarization is not enabled. Please rebuild sherpa-onnx"); + return nullptr; +} + +const SherpaOnnxOfflineSpeakerDiarizationResult * +SherpaOnnxOfflineSpeakerDiarizationProcessWithCallback( + const SherpaOnnxOfflineSpeakerDiarization *sd, const float *samples, + int32_t n, SherpaOnnxOfflineSpeakerDiarizationProgressCallback callback, + void *arg) { + SHERPA_ONNX_LOGE( + "Speaker diarization is not enabled. Please rebuild sherpa-onnx"); + return nullptr; +} + +const SherpaOnnxOfflineSpeakerDiarizationResult * +SherpaOnnxOfflineSpeakerDiarizationProcessWithCallbackNoArg( + const SherpaOnnxOfflineSpeakerDiarization *sd, const float *samples, + int32_t n, + SherpaOnnxOfflineSpeakerDiarizationProgressCallbackNoArg callback) { + SHERPA_ONNX_LOGE( + "Speaker diarization is not enabled. Please rebuild sherpa-onnx"); + return nullptr; +} + +void SherpaOnnxOfflineSpeakerDiarizationDestroyResult( + const SherpaOnnxOfflineSpeakerDiarizationResult *r) { + SHERPA_ONNX_LOGE( + "Speaker diarization is not enabled. Please rebuild sherpa-onnx"); +} + #endif #ifdef __OHOS__ @@ -2112,7 +2268,12 @@ const SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTtsOHOS( return tts; } - +#else +const SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTtsOHOS( + const SherpaOnnxOfflineTtsConfig *config, NativeResourceManager *mgr) { + SHERPA_ONNX_LOGE("TTS is not enabled. Please rebuild sherpa-onnx"); + return nullptr; +} #endif // #if SHERPA_ONNX_ENABLE_TTS == 1 // #if SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION == 1 @@ -2134,6 +2295,16 @@ SherpaOnnxCreateOfflineSpeakerDiarizationOHOS( return sd; } +#else + +const SherpaOnnxOfflineSpeakerDiarization * +SherpaOnnxCreateOfflineSpeakerDiarizationOHOS( + const SherpaOnnxOfflineSpeakerDiarizationConfig *config, + NativeResourceManager *mgr) { + SHERPA_ONNX_LOGE( + "Speaker diarization is not enabled. Please rebuild sherpa-onnx"); + return nullptr; +} #endif // #if SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION == 1 From 0cb2db385afc8fd9bc2a06b561d245b34bbe4552 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 7 Jan 2025 11:04:10 +0800 Subject: [PATCH 149/183] Add README for android libs. (#1693) --- .github/scripts/test-nodejs-addon-npm.sh | 8 ++++---- .github/workflows/android.yaml | 4 ++++ .github/workflows/lazarus.yaml | 1 + build-android-arm64-v8a.sh | 16 ++++++++++++++++ build-android-armv7-eabi.sh | 17 +++++++++++++++++ build-android-x86-64.sh | 17 +++++++++++++++++ build-android-x86.sh | 17 +++++++++++++++++ 7 files changed, 76 insertions(+), 4 deletions(-) diff --git a/.github/scripts/test-nodejs-addon-npm.sh b/.github/scripts/test-nodejs-addon-npm.sh index d3e85f687..e2d8487be 100755 --- a/.github/scripts/test-nodejs-addon-npm.sh +++ b/.github/scripts/test-nodejs-addon-npm.sh @@ -85,19 +85,19 @@ fi echo "----------tts----------" -wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 rm matcha-icefall-en_US-ljspeech.tar.bz2 -wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx node ./test_tts_non_streaming_matcha_icefall_en.js rm hifigan_v2.onnx rm -rf matcha-icefall-en_US-ljspeech -wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 tar xvf matcha-icefall-zh-baker.tar.bz2 rm matcha-icefall-zh-baker.tar.bz2 -wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx node ./test_tts_non_streaming_matcha_icefall_zh.js rm hifigan_v2.onnx diff --git a/.github/workflows/android.yaml b/.github/workflows/android.yaml index 1a740005d..b7da9b8a6 100644 --- a/.github/workflows/android.yaml +++ b/.github/workflows/android.yaml @@ -64,6 +64,7 @@ jobs: ./build-android-arm64-v8a.sh mkdir -p jniLibs/arm64-v8a/ cp -v ./build-android-arm64-v8a/install/lib/*.so ./jniLibs/arm64-v8a/ + cp -v ./build-android-arm64-v8a/install/lib/README.md ./jniLibs/arm64-v8a/ rm -rf ./build-android-arm64-v8a/ - name: build android armv7-eabi @@ -77,6 +78,7 @@ jobs: ./build-android-armv7-eabi.sh mkdir -p ./jniLibs/armeabi-v7a/ cp -v ./build-android-armv7-eabi/install/lib/*.so ./jniLibs/armeabi-v7a/ + cp -v ./build-android-armv7-eabi/install/lib/README.md ./jniLibs/armeabi-v7a/ rm -rf ./build-android-armv7-eabi - name: build android x86_64 @@ -90,6 +92,7 @@ jobs: ./build-android-x86-64.sh mkdir -p ./jniLibs/x86_64 cp -v ./build-android-x86-64/install/lib/*.so ./jniLibs/x86_64 + cp -v ./build-android-x86-64/install/lib/README.md ./jniLibs/x86_64 rm -rf ./build-android-x86-64 - name: build android x86 @@ -103,6 +106,7 @@ jobs: ./build-android-x86.sh mkdir -p ./jniLibs/x86 cp -v ./build-android-x86/install/lib/*.so ./jniLibs/x86 + cp -v ./build-android-x86/install/lib/README.md ./jniLibs/x86 rm -rf ./build-android-x86 - name: Copy files diff --git a/.github/workflows/lazarus.yaml b/.github/workflows/lazarus.yaml index 72958d639..d28b7cba4 100644 --- a/.github/workflows/lazarus.yaml +++ b/.github/workflows/lazarus.yaml @@ -355,6 +355,7 @@ jobs: git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-bin huggingface cd huggingface + git remote set-url origin https://csukuangfj:$HF_TOKEN@huggingface.co/sherpa-onnx-bin git fetch git pull git merge -m "merge remote" --ff origin main diff --git a/build-android-arm64-v8a.sh b/build-android-arm64-v8a.sh index 3d53cc23d..88ba09ef0 100755 --- a/build-android-arm64-v8a.sh +++ b/build-android-arm64-v8a.sh @@ -150,6 +150,22 @@ cp -fv $onnxruntime_version/jni/arm64-v8a/libonnxruntime.so install/lib 2>/dev/n rm -rf install/share rm -rf install/lib/pkgconfig rm -rf install/lib/lib*.a +if [ -f install/lib/libsherpa-onnx-c-api.so ]; then + cat >install/lib/README.md </dev rm -rf install/share rm -rf install/lib/pkgconfig rm -rf install/lib/lib*.a + +if [ -f install/lib/libsherpa-onnx-c-api.so ]; then + cat >install/lib/README.md </dev/null rm -rf install/share rm -rf install/lib/pkgconfig rm -rf install/lib/lib*.a + +if [ -f install/lib/libsherpa-onnx-c-api.so ]; then + cat >install/lib/README.md <install/lib/README.md < Date: Fri, 10 Jan 2025 19:26:36 +0800 Subject: [PATCH 150/183] Fix: export-onnx.py(expected all tensors to be on the same device) (#1699) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 由于SenseVoiceSmall.from_pretrained() 调用的funasr.auto.auto_model.AutoModel.build_model()默认device是cuda (在cuda available的环境中) ```py device = kwargs.get("device", "cuda") if not torch.cuda.is_available() or kwargs.get("ngpu", 1) == 0: device = "cpu" kwargs["batch_size"] = 1 kwargs["device"] = device ``` 而export-onnx.py里的tensor默认都是cpu, 导致 RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu 所以直接在加载model的时候指定cpu --- scripts/sense-voice/export-onnx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sense-voice/export-onnx.py b/scripts/sense-voice/export-onnx.py index 97d9a506e..48b686369 100755 --- a/scripts/sense-voice/export-onnx.py +++ b/scripts/sense-voice/export-onnx.py @@ -119,7 +119,7 @@ def display_params(params): def main(): - model, params = SenseVoiceSmall.from_pretrained(model="iic/SenseVoiceSmall") + model, params = SenseVoiceSmall.from_pretrained(model="iic/SenseVoiceSmall", device="cpu") display_params(params) generate_tokens(params) From 0d20558b5e5c7c44284edbb8eaef147cf5c95334 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 13 Jan 2025 10:17:04 +0800 Subject: [PATCH 151/183] Fix passing strings from C# to C. (#1701) See also https://github.com/k2-fsa/sherpa-onnx/issues/1695#issuecomment-2585725190 We need to place a 0 at the end of the buffer. --- scripts/dotnet/KeywordSpotter.cs | 5 ++++- scripts/dotnet/OfflinePunctuation.cs | 5 ++++- scripts/dotnet/OfflineTts.cs | 10 +++++++-- scripts/dotnet/OfflineTtsGeneratedAudio.cs | 5 ++++- scripts/dotnet/SpeakerEmbeddingManager.cs | 25 +++++++++++++++++----- 5 files changed, 40 insertions(+), 10 deletions(-) diff --git a/scripts/dotnet/KeywordSpotter.cs b/scripts/dotnet/KeywordSpotter.cs index 9c76acefc..6f19fc945 100644 --- a/scripts/dotnet/KeywordSpotter.cs +++ b/scripts/dotnet/KeywordSpotter.cs @@ -26,7 +26,10 @@ public OnlineStream CreateStream() public OnlineStream CreateStream(string keywords) { byte[] utf8Bytes = Encoding.UTF8.GetBytes(keywords); - IntPtr p = SherpaOnnxCreateKeywordStreamWithKeywords(_handle.Handle, utf8Bytes); + byte[] utf8BytesWithNull = new byte[utf8Bytes.Length + 1]; // +1 for null terminator + Array.Copy(utf8Bytes, utf8BytesWithNull, utf8Bytes.Length); + utf8BytesWithNull[utf8Bytes.Length] = 0; // Null terminator + IntPtr p = SherpaOnnxCreateKeywordStreamWithKeywords(_handle.Handle, utf8BytesWithNull); return new OnlineStream(p); } diff --git a/scripts/dotnet/OfflinePunctuation.cs b/scripts/dotnet/OfflinePunctuation.cs index 4b39c3d32..4730bb6a0 100644 --- a/scripts/dotnet/OfflinePunctuation.cs +++ b/scripts/dotnet/OfflinePunctuation.cs @@ -17,8 +17,11 @@ public OfflinePunctuation(OfflinePunctuationConfig config) public String AddPunct(String text) { byte[] utf8Bytes = Encoding.UTF8.GetBytes(text); + byte[] utf8BytesWithNull = new byte[utf8Bytes.Length + 1]; // +1 for null terminator + Array.Copy(utf8Bytes, utf8BytesWithNull, utf8Bytes.Length); + utf8BytesWithNull[utf8Bytes.Length] = 0; // Null terminator - IntPtr p = SherpaOfflinePunctuationAddPunct(_handle.Handle, utf8Bytes); + IntPtr p = SherpaOfflinePunctuationAddPunct(_handle.Handle, utf8BytesWithNull); string s = ""; int length = 0; diff --git a/scripts/dotnet/OfflineTts.cs b/scripts/dotnet/OfflineTts.cs index 6e36d816f..e4d29733c 100644 --- a/scripts/dotnet/OfflineTts.cs +++ b/scripts/dotnet/OfflineTts.cs @@ -19,14 +19,20 @@ public OfflineTts(OfflineTtsConfig config) public OfflineTtsGeneratedAudio Generate(String text, float speed, int speakerId) { byte[] utf8Bytes = Encoding.UTF8.GetBytes(text); - IntPtr p = SherpaOnnxOfflineTtsGenerate(_handle.Handle, utf8Bytes, speakerId, speed); + byte[] utf8BytesWithNull = new byte[utf8Bytes.Length + 1]; // +1 for null terminator + Array.Copy(utf8Bytes, utf8BytesWithNull, utf8Bytes.Length); + utf8BytesWithNull[utf8Bytes.Length] = 0; // Null terminator + IntPtr p = SherpaOnnxOfflineTtsGenerate(_handle.Handle, utf8BytesWithNull, speakerId, speed); return new OfflineTtsGeneratedAudio(p); } public OfflineTtsGeneratedAudio GenerateWithCallback(String text, float speed, int speakerId, OfflineTtsCallback callback) { byte[] utf8Bytes = Encoding.UTF8.GetBytes(text); - IntPtr p = SherpaOnnxOfflineTtsGenerateWithCallback(_handle.Handle, utf8Bytes, speakerId, speed, callback); + byte[] utf8BytesWithNull = new byte[utf8Bytes.Length + 1]; // +1 for null terminator + Array.Copy(utf8Bytes, utf8BytesWithNull, utf8Bytes.Length); + utf8BytesWithNull[utf8Bytes.Length] = 0; // Null terminator + IntPtr p = SherpaOnnxOfflineTtsGenerateWithCallback(_handle.Handle, utf8BytesWithNull, speakerId, speed, callback); return new OfflineTtsGeneratedAudio(p); } diff --git a/scripts/dotnet/OfflineTtsGeneratedAudio.cs b/scripts/dotnet/OfflineTtsGeneratedAudio.cs index 2b521649c..282820790 100644 --- a/scripts/dotnet/OfflineTtsGeneratedAudio.cs +++ b/scripts/dotnet/OfflineTtsGeneratedAudio.cs @@ -16,7 +16,10 @@ public bool SaveToWaveFile(String filename) { Impl impl = (Impl)Marshal.PtrToStructure(Handle, typeof(Impl)); byte[] utf8Filename = Encoding.UTF8.GetBytes(filename); - int status = SherpaOnnxWriteWave(impl.Samples, impl.NumSamples, impl.SampleRate, utf8Filename); + byte[] utf8FilenameWithNull = new byte[utf8Filename.Length + 1]; // +1 for null terminator + Array.Copy(utf8Filename, utf8FilenameWithNull, utf8Filename.Length); + utf8FilenameWithNull[utf8Filename.Length] = 0; // Null terminator + int status = SherpaOnnxWriteWave(impl.Samples, impl.NumSamples, impl.SampleRate, utf8FilenameWithNull); return status == 1; } diff --git a/scripts/dotnet/SpeakerEmbeddingManager.cs b/scripts/dotnet/SpeakerEmbeddingManager.cs index 85b4812a7..d9c4215c3 100644 --- a/scripts/dotnet/SpeakerEmbeddingManager.cs +++ b/scripts/dotnet/SpeakerEmbeddingManager.cs @@ -18,7 +18,10 @@ public SpeakerEmbeddingManager(int dim) public bool Add(string name, float[] v) { byte[] utf8Name = Encoding.UTF8.GetBytes(name); - return SherpaOnnxSpeakerEmbeddingManagerAdd(_handle.Handle, utf8Name, v) == 1; + byte[] utf8NameWithNull = new byte[utf8Name.Length + 1]; // +1 for null terminator + Array.Copy(utf8Name, utf8NameWithNull, utf8Name.Length); + utf8NameWithNull[utf8Name.Length] = 0; // Null terminator + return SherpaOnnxSpeakerEmbeddingManagerAdd(_handle.Handle, utf8NameWithNull, v) == 1; } public bool Add(string name, ICollection v_list) @@ -33,13 +36,19 @@ public bool Add(string name, ICollection v_list) } byte[] utf8Name = Encoding.UTF8.GetBytes(name); - return SherpaOnnxSpeakerEmbeddingManagerAddListFlattened(_handle.Handle, utf8Name, v, n) == 1; + byte[] utf8NameWithNull = new byte[utf8Name.Length + 1]; // +1 for null terminator + Array.Copy(utf8Name, utf8NameWithNull, utf8Name.Length); + utf8NameWithNull[utf8Name.Length] = 0; // Null terminator + return SherpaOnnxSpeakerEmbeddingManagerAddListFlattened(_handle.Handle, utf8NameWithNull, v, n) == 1; } public bool Remove(string name) { byte[] utf8Name = Encoding.UTF8.GetBytes(name); - return SherpaOnnxSpeakerEmbeddingManagerRemove(_handle.Handle, utf8Name) == 1; + byte[] utf8NameWithNull = new byte[utf8Name.Length + 1]; // +1 for null terminator + Array.Copy(utf8Name, utf8NameWithNull, utf8Name.Length); + utf8NameWithNull[utf8Name.Length] = 0; // Null terminator + return SherpaOnnxSpeakerEmbeddingManagerRemove(_handle.Handle, utf8NameWithNull) == 1; } public string Search(float[] v, float threshold) @@ -77,13 +86,19 @@ public string Search(float[] v, float threshold) public bool Verify(string name, float[] v, float threshold) { byte[] utf8Name = Encoding.UTF8.GetBytes(name); - return SherpaOnnxSpeakerEmbeddingManagerVerify(_handle.Handle, utf8Name, v, threshold) == 1; + byte[] utf8NameWithNull = new byte[utf8Name.Length + 1]; // +1 for null terminator + Array.Copy(utf8Name, utf8NameWithNull, utf8Name.Length); + utf8NameWithNull[utf8Name.Length] = 0; // Null terminator + return SherpaOnnxSpeakerEmbeddingManagerVerify(_handle.Handle, utf8NameWithNull, v, threshold) == 1; } public bool Contains(string name) { byte[] utf8Name = Encoding.UTF8.GetBytes(name); - return SherpaOnnxSpeakerEmbeddingManagerContains(_handle.Handle, utf8Name) == 1; + byte[] utf8NameWithNull = new byte[utf8Name.Length + 1]; // +1 for null terminator + Array.Copy(utf8Name, utf8NameWithNull, utf8Name.Length); + utf8NameWithNull[utf8Name.Length] = 0; // Null terminator + return SherpaOnnxSpeakerEmbeddingManagerContains(_handle.Handle, utf8NameWithNull) == 1; } public string[] GetAllSpeakers() From cbe07ac1b6a933910bd6d5c44282116027199135 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 13 Jan 2025 10:28:05 +0800 Subject: [PATCH 152/183] Release v1.10.39 (#1702) --- CHANGELOG.md | 7 ++++++ CMakeLists.txt | 2 +- android/SherpaOnnxAar/README.md | 6 ++--- android/SherpaOnnxJavaDemo/app/build.gradle | 2 +- build-ios-shared.sh | 2 +- .../add-punctuations/pubspec.yaml | 2 +- dart-api-examples/audio-tagging/pubspec.yaml | 2 +- .../keyword-spotter/pubspec.yaml | 2 +- .../non-streaming-asr/pubspec.yaml | 2 +- .../speaker-diarization/pubspec.yaml | 2 +- .../speaker-identification/pubspec.yaml | 2 +- dart-api-examples/streaming-asr/pubspec.yaml | 2 +- dart-api-examples/tts/pubspec.yaml | 2 +- .../vad-with-non-streaming-asr/pubspec.yaml | 2 +- dart-api-examples/vad/pubspec.yaml | 2 +- flutter-examples/streaming_asr/pubspec.yaml | 4 ++-- flutter-examples/tts/pubspec.yaml | 4 ++-- flutter/sherpa_onnx/pubspec.yaml | 12 +++++----- .../ios/sherpa_onnx_ios.podspec | 2 +- .../macos/sherpa_onnx_macos.podspec | 2 +- .../SherpaOnnxHar/sherpa_onnx/README.md | 2 +- .../sherpa_onnx/oh-package.json5 | 2 +- .../entry/oh-package.json5 | 2 +- .../entry/oh-package.json5 | 2 +- .../entry/oh-package.json5 | 2 +- .../SherpaOnnxTts/entry/oh-package.json5 | 2 +- harmony-os/SherpaOnnxVadAsr/entry/README.md | 2 +- .../SherpaOnnxVadAsr/entry/oh-package.json5 | 2 +- jitpack.yml | 6 ++--- new-release.sh | 24 +++++++++---------- nodejs-addon-examples/package.json | 2 +- pom.xml | 2 +- .../csrc/offline-whisper-model-config.cc | 2 +- 33 files changed, 61 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9db4c24e..b7a29ecb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## 1.10.39 + +* Fix building without TTS (#1691) +* Add README for android libs. (#1693) +* Fix: export-onnx.py(expected all tensors to be on the same device) (#1699) +* Fix passing strings from C# to C. (#1701) + ## 1.10.38 * Fix initializing TTS in Python. (#1664) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7f0fe2990..e9f427666 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ project(sherpa-onnx) # Remember to update # ./CHANGELOG.md # ./new-release.sh -set(SHERPA_ONNX_VERSION "1.10.38") +set(SHERPA_ONNX_VERSION "1.10.39") # Disable warning about # diff --git a/android/SherpaOnnxAar/README.md b/android/SherpaOnnxAar/README.md index caea152ff..ec50ebf40 100644 --- a/android/SherpaOnnxAar/README.md +++ b/android/SherpaOnnxAar/README.md @@ -4,8 +4,8 @@ git clone https://github.com/k2-fsa/sherpa-onnx cd sherpa-onnx -wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.38/sherpa-onnx-v1.10.38-android.tar.bz2 -tar xvf sherpa-onnx-v1.10.38-android.tar.bz2 +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.39/sherpa-onnx-v1.10.39-android.tar.bz2 +tar xvf sherpa-onnx-v1.10.39-android.tar.bz2 cp -v jniLibs/arm64-v8a/* android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/arm64-v8a/ cp -v jniLibs/armeabi-v7a/* android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/armeabi-v7a/ @@ -16,5 +16,5 @@ cd android/SherpaOnnxAar ./gradlew :sherpa_onnx:assembleRelease ls -lh ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar -cp ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar ../../sherpa-onnx-1.10.38.aar +cp ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar ../../sherpa-onnx-1.10.39.aar ``` diff --git a/android/SherpaOnnxJavaDemo/app/build.gradle b/android/SherpaOnnxJavaDemo/app/build.gradle index e04feafe4..8e66f4355 100644 --- a/android/SherpaOnnxJavaDemo/app/build.gradle +++ b/android/SherpaOnnxJavaDemo/app/build.gradle @@ -34,5 +34,5 @@ dependencies { implementation 'pub.devrel:easypermissions:3.0.0' implementation 'androidx.core:core-ktx:1.7.0' // implementation files('/Users/fangjun/open-source/sherpa-onnx/android/SherpaOnnxAar/sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar') - implementation 'com.github.k2-fsa:sherpa-onnx:v1.10.38' + implementation 'com.github.k2-fsa:sherpa-onnx:v1.10.39' } diff --git a/build-ios-shared.sh b/build-ios-shared.sh index 4cdeae190..39e5c0f2f 100755 --- a/build-ios-shared.sh +++ b/build-ios-shared.sh @@ -242,7 +242,7 @@ for d in ios-arm64_x86_64-simulator ios-arm64; do CFBundlePackageType FMWK CFBundleShortVersionString - 1.10.38 + 1.10.39 CFBundleSupportedPlatforms iPhoneOS diff --git a/dart-api-examples/add-punctuations/pubspec.yaml b/dart-api-examples/add-punctuations/pubspec.yaml index 952f5421b..ca01d32df 100644 --- a/dart-api-examples/add-punctuations/pubspec.yaml +++ b/dart-api-examples/add-punctuations/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.38 + sherpa_onnx: ^1.10.39 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/audio-tagging/pubspec.yaml b/dart-api-examples/audio-tagging/pubspec.yaml index 1fe474327..d47d93bfb 100644 --- a/dart-api-examples/audio-tagging/pubspec.yaml +++ b/dart-api-examples/audio-tagging/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.38 + sherpa_onnx: ^1.10.39 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/keyword-spotter/pubspec.yaml b/dart-api-examples/keyword-spotter/pubspec.yaml index cff5ac008..4210cbfb4 100644 --- a/dart-api-examples/keyword-spotter/pubspec.yaml +++ b/dart-api-examples/keyword-spotter/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.38 + sherpa_onnx: ^1.10.39 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/non-streaming-asr/pubspec.yaml b/dart-api-examples/non-streaming-asr/pubspec.yaml index 90b9308cf..93a0ad3a2 100644 --- a/dart-api-examples/non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.38 + sherpa_onnx: ^1.10.39 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/speaker-diarization/pubspec.yaml b/dart-api-examples/speaker-diarization/pubspec.yaml index 648005fc2..df44568d3 100644 --- a/dart-api-examples/speaker-diarization/pubspec.yaml +++ b/dart-api-examples/speaker-diarization/pubspec.yaml @@ -8,7 +8,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.38 + sherpa_onnx: ^1.10.39 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/speaker-identification/pubspec.yaml b/dart-api-examples/speaker-identification/pubspec.yaml index c1f690ca4..bc70976d9 100644 --- a/dart-api-examples/speaker-identification/pubspec.yaml +++ b/dart-api-examples/speaker-identification/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.38 + sherpa_onnx: ^1.10.39 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/streaming-asr/pubspec.yaml b/dart-api-examples/streaming-asr/pubspec.yaml index 70812eacc..ee79c40dd 100644 --- a/dart-api-examples/streaming-asr/pubspec.yaml +++ b/dart-api-examples/streaming-asr/pubspec.yaml @@ -11,7 +11,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.38 + sherpa_onnx: ^1.10.39 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/tts/pubspec.yaml b/dart-api-examples/tts/pubspec.yaml index 35f593dd7..718c4201a 100644 --- a/dart-api-examples/tts/pubspec.yaml +++ b/dart-api-examples/tts/pubspec.yaml @@ -8,7 +8,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.38 + sherpa_onnx: ^1.10.39 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml index ed4e28eed..250c45d9a 100644 --- a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.38 + sherpa_onnx: ^1.10.39 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad/pubspec.yaml b/dart-api-examples/vad/pubspec.yaml index ab17a871e..66ad030ac 100644 --- a/dart-api-examples/vad/pubspec.yaml +++ b/dart-api-examples/vad/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.38 + sherpa_onnx: ^1.10.39 path: ^1.9.0 args: ^2.5.0 diff --git a/flutter-examples/streaming_asr/pubspec.yaml b/flutter-examples/streaming_asr/pubspec.yaml index 807319289..12116dd6e 100644 --- a/flutter-examples/streaming_asr/pubspec.yaml +++ b/flutter-examples/streaming_asr/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' -version: 1.10.38 +version: 1.10.39 topics: - speech-recognition @@ -31,7 +31,7 @@ dependencies: record: ^5.1.0 url_launcher: ^6.2.6 - sherpa_onnx: ^1.10.38 + sherpa_onnx: ^1.10.39 # sherpa_onnx: # path: ../../flutter/sherpa_onnx diff --git a/flutter-examples/tts/pubspec.yaml b/flutter-examples/tts/pubspec.yaml index 40ec36075..976a03e5d 100644 --- a/flutter-examples/tts/pubspec.yaml +++ b/flutter-examples/tts/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' # Remove this line if you wish to publish to pub.dev -version: 1.10.38 +version: 1.10.39 environment: sdk: ">=2.17.0 <4.0.0" @@ -18,7 +18,7 @@ dependencies: cupertino_icons: ^1.0.6 path_provider: ^2.1.3 path: ^1.9.0 - sherpa_onnx: ^1.10.38 + sherpa_onnx: ^1.10.39 # sherpa_onnx: # path: ../../flutter/sherpa_onnx url_launcher: 6.2.6 diff --git a/flutter/sherpa_onnx/pubspec.yaml b/flutter/sherpa_onnx/pubspec.yaml index da08299b2..8dc0f43b4 100644 --- a/flutter/sherpa_onnx/pubspec.yaml +++ b/flutter/sherpa_onnx/pubspec.yaml @@ -17,7 +17,7 @@ topics: - voice-activity-detection # remember to change the version in ../sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec -version: 1.10.38 +version: 1.10.39 homepage: https://github.com/k2-fsa/sherpa-onnx @@ -30,23 +30,23 @@ dependencies: flutter: sdk: flutter - sherpa_onnx_android: ^1.10.38 + sherpa_onnx_android: ^1.10.39 # sherpa_onnx_android: # path: ../sherpa_onnx_android - sherpa_onnx_macos: ^1.10.38 + sherpa_onnx_macos: ^1.10.39 # sherpa_onnx_macos: # path: ../sherpa_onnx_macos - sherpa_onnx_linux: ^1.10.38 + sherpa_onnx_linux: ^1.10.39 # sherpa_onnx_linux: # path: ../sherpa_onnx_linux - sherpa_onnx_windows: ^1.10.38 + sherpa_onnx_windows: ^1.10.39 # sherpa_onnx_windows: # path: ../sherpa_onnx_windows - sherpa_onnx_ios: ^1.10.38 + sherpa_onnx_ios: ^1.10.39 # sherpa_onnx_ios: # path: ../sherpa_onnx_ios diff --git a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec index 528c2021e..02f2be1e6 100644 --- a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec +++ b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec @@ -7,7 +7,7 @@ # https://groups.google.com/g/dart-ffi/c/nUATMBy7r0c Pod::Spec.new do |s| s.name = 'sherpa_onnx_ios' - s.version = '1.10.38' + s.version = '1.10.39' s.summary = 'A new Flutter FFI plugin project.' s.description = <<-DESC A new Flutter FFI plugin project. diff --git a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec index 319c6eae4..89d9f7bab 100644 --- a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec +++ b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec @@ -4,7 +4,7 @@ # Pod::Spec.new do |s| s.name = 'sherpa_onnx_macos' - s.version = '1.10.38' + s.version = '1.10.39' s.summary = 'sherpa-onnx Flutter FFI plugin project.' s.description = <<-DESC sherpa-onnx Flutter FFI plugin project. diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md b/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md index a7e3e5317..917401b43 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md @@ -23,7 +23,7 @@ or update your `oh-package.json5` to include the following: ``` "dependencies": { - "sherpa_onnx": "1.10.38", + "sherpa_onnx": "1.10.39", }, ``` diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 index f64630c3b..34af61da2 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 @@ -1,6 +1,6 @@ { "name": "sherpa_onnx", - "version": "1.10.38", + "version": "1.10.39", "description": "On-device speech-to-text, text-to-speech, and speaker diarization using Next-gen Kaldi without Internet connection", "main": "Index.ets", "author": "The next-gen Kaldi team", diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 b/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 index c625a902c..5449d3e81 100644 --- a/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.38" + "sherpa_onnx": "1.10.39" } } diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 b/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 index 802435184..5f2f6b5ff 100644 --- a/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.38", + "sherpa_onnx": "1.10.39", } } diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 index 802435184..5f2f6b5ff 100644 --- a/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.38", + "sherpa_onnx": "1.10.39", } } diff --git a/harmony-os/SherpaOnnxTts/entry/oh-package.json5 b/harmony-os/SherpaOnnxTts/entry/oh-package.json5 index 802435184..5f2f6b5ff 100644 --- a/harmony-os/SherpaOnnxTts/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxTts/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.38", + "sherpa_onnx": "1.10.39", } } diff --git a/harmony-os/SherpaOnnxVadAsr/entry/README.md b/harmony-os/SherpaOnnxVadAsr/entry/README.md index 59053f756..467c21391 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/README.md +++ b/harmony-os/SherpaOnnxVadAsr/entry/README.md @@ -1,6 +1,6 @@ # Introduction -Please download ./sherpa_onnx-v1.10.38.har +Please download ./sherpa_onnx-v1.10.39.har from Hint: For users who have no access to huggingface, please use diff --git a/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 b/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 index 55d35f104..fae5caf53 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 @@ -7,7 +7,7 @@ "license": "", "dependencies": { // please see https://ohpm.openharmony.cn/#/cn/detail/sherpa_onnx - "sherpa_onnx": "1.10.38", + "sherpa_onnx": "1.10.39", } } diff --git a/jitpack.yml b/jitpack.yml index 6c1ecb597..7f283a09f 100644 --- a/jitpack.yml +++ b/jitpack.yml @@ -2,8 +2,8 @@ jdk: - openjdk17 before_install: - - wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.38/sherpa-onnx-1.10.38.aar + - wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.39/sherpa-onnx-1.10.39.aar install: - - FILE="-Dfile=sherpa-onnx-1.10.38.aar" - - mvn install:install-file $FILE -DgroupId=com.k2fsa.sherpa.onnx -DartifactId=sherpa-onnx -Dversion=1.10.38 -Dpackaging=aar -DgeneratePom=true + - FILE="-Dfile=sherpa-onnx-1.10.39.aar" + - mvn install:install-file $FILE -DgroupId=com.k2fsa.sherpa.onnx -DartifactId=sherpa-onnx -Dversion=1.10.39 -Dpackaging=aar -DgeneratePom=true diff --git a/new-release.sh b/new-release.sh index 1a6993071..e52bf65b6 100755 --- a/new-release.sh +++ b/new-release.sh @@ -2,18 +2,18 @@ set -ex -sed -i.bak 's/1\.10\.37/1\.10\.38/g' ./build-ios-shared.sh -sed -i.bak 's/1\.10\.37/1\.10\.38/g' ./pom.xml -sed -i.bak 's/1\.10\.37/1\.10\.38/g' ./jitpack.yml -sed -i.bak 's/1\.10\.37/1\.10\.38/g' ./android/SherpaOnnxAar/README.md +sed -i.bak 's/1\.10\.38/1\.10\.39/g' ./build-ios-shared.sh +sed -i.bak 's/1\.10\.38/1\.10\.39/g' ./pom.xml +sed -i.bak 's/1\.10\.38/1\.10\.39/g' ./jitpack.yml +sed -i.bak 's/1\.10\.38/1\.10\.39/g' ./android/SherpaOnnxAar/README.md -find android -name build.gradle -type f -exec sed -i.bak 's/sherpa-onnx:v1\.10\.37/sherpa-onnx:v1\.10\.38/g' {} \; +find android -name build.gradle -type f -exec sed -i.bak 's/sherpa-onnx:v1\.10\.38/sherpa-onnx:v1\.10\.39/g' {} \; -find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.37/1\.10\.38/g' {} \; -find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.37/1\.10\.38/g' {} \; -find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.37/1\.10\.38/g' {} \; -find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.37/1\.10\.38/g' {} \; -find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.37/1\.10\.38/g' {} \; +find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.38/1\.10\.39/g' {} \; +find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.38/1\.10\.39/g' {} \; +find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.38/1\.10\.39/g' {} \; +find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.38/1\.10\.39/g' {} \; +find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.38/1\.10\.39/g' {} \; -find harmony-os -name "README.md" -type f -exec sed -i.bak 's/1\.10\.37/1\.10\.38/g' {} \; -find harmony-os -name oh-package.json5 -type f -exec sed -i.bak 's/1\.10\.37/1\.10\.38/g' {} \; +find harmony-os -name "README.md" -type f -exec sed -i.bak 's/1\.10\.38/1\.10\.39/g' {} \; +find harmony-os -name oh-package.json5 -type f -exec sed -i.bak 's/1\.10\.38/1\.10\.39/g' {} \; diff --git a/nodejs-addon-examples/package.json b/nodejs-addon-examples/package.json index 0cdfc5912..debe3ac96 100644 --- a/nodejs-addon-examples/package.json +++ b/nodejs-addon-examples/package.json @@ -1,5 +1,5 @@ { "dependencies": { - "sherpa-onnx-node": "^1.10.38" + "sherpa-onnx-node": "^1.10.39" } } diff --git a/pom.xml b/pom.xml index f412806ec..7f9c217b6 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.k2fsa.sherpa.onnx sherpa-onnx-android - 1.10.38 + 1.10.39 https://github.com/k2-fsa/sherpa-onnx pom First Android Library diff --git a/sherpa-onnx/csrc/offline-whisper-model-config.cc b/sherpa-onnx/csrc/offline-whisper-model-config.cc index 5d36b82c4..708531f73 100644 --- a/sherpa-onnx/csrc/offline-whisper-model-config.cc +++ b/sherpa-onnx/csrc/offline-whisper-model-config.cc @@ -20,7 +20,7 @@ void OfflineWhisperModelConfig::Register(ParseOptions *po) { po->Register( "whisper-language", &language, - "The spoke language in the input audio file. Example values: " + "The spoken language in the input audio file. Example values: " "en, de, fr, zh, jp. If it is not given for a multilingual model, we will" " infer the language from the input audio file. " "Please refer to " From ce71b6327a7f5e3c08517eee4abc8b4e1daf0418 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 13 Jan 2025 12:00:45 +0800 Subject: [PATCH 153/183] Fix building wheels (#1703) --- .github/workflows/build-wheels-aarch64-cuda.yaml | 4 +--- .github/workflows/build-wheels-aarch64.yaml | 4 +--- .github/workflows/build-wheels-linux.yaml | 2 +- flutter/sherpa_onnx_ios/README.md | 2 +- flutter/sherpa_onnx_macos/README.md | 2 +- flutter/sherpa_onnx_windows/README.md | 2 +- 6 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-wheels-aarch64-cuda.yaml b/.github/workflows/build-wheels-aarch64-cuda.yaml index 9c226c431..a221553a4 100644 --- a/.github/workflows/build-wheels-aarch64-cuda.yaml +++ b/.github/workflows/build-wheels-aarch64-cuda.yaml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest] + os: [ubuntu-20.04] python-version: ["cp37", "cp38", "cp39", "cp310", "cp311", "cp312", "cp313"] manylinux: [manylinux2014] #, manylinux_2_28] @@ -60,7 +60,6 @@ jobs: ls -lh ./wheelhouse/ - name: Install patchelf - if: matrix.os == 'ubuntu-latest' shell: bash run: | sudo apt-get update -q @@ -69,7 +68,6 @@ jobs: - name: Patch wheels shell: bash - if: matrix.os == 'ubuntu-latest' run: | mkdir ./wheels sudo ./scripts/wheel/patch_wheel.py --in-dir ./wheelhouse --out-dir ./wheels diff --git a/.github/workflows/build-wheels-aarch64.yaml b/.github/workflows/build-wheels-aarch64.yaml index 6b56293bd..d65e867a8 100644 --- a/.github/workflows/build-wheels-aarch64.yaml +++ b/.github/workflows/build-wheels-aarch64.yaml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest] + os: [ubuntu-20.04] python-version: ["cp37", "cp38", "cp39", "cp310", "cp311", "cp312", "cp313"] manylinux: [manylinux2014] #, manylinux_2_28] @@ -60,7 +60,6 @@ jobs: ls -lh ./wheelhouse/ - name: Install patchelf - if: matrix.os == 'ubuntu-latest' shell: bash run: | sudo apt-get update -q @@ -69,7 +68,6 @@ jobs: - name: Patch wheels shell: bash - if: matrix.os == 'ubuntu-latest' run: | mkdir ./wheels sudo ./scripts/wheel/patch_wheel.py --in-dir ./wheelhouse --out-dir ./wheels diff --git a/.github/workflows/build-wheels-linux.yaml b/.github/workflows/build-wheels-linux.yaml index 2abd236f9..db2d77376 100644 --- a/.github/workflows/build-wheels-linux.yaml +++ b/.github/workflows/build-wheels-linux.yaml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest] + os: [ubuntu-20.04] python-version: ["cp37", "cp38", "cp39", "cp310", "cp311", "cp312", "cp313"] manylinux: [manylinux2014] #, manylinux_2_28] diff --git a/flutter/sherpa_onnx_ios/README.md b/flutter/sherpa_onnx_ios/README.md index 1334c2564..974250c30 100644 --- a/flutter/sherpa_onnx_ios/README.md +++ b/flutter/sherpa_onnx_ios/README.md @@ -1,4 +1,4 @@ -# sherpa_onnx_linux +# sherpa_onnx_ios This is a sub project of [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx). diff --git a/flutter/sherpa_onnx_macos/README.md b/flutter/sherpa_onnx_macos/README.md index 1334c2564..171c76752 100644 --- a/flutter/sherpa_onnx_macos/README.md +++ b/flutter/sherpa_onnx_macos/README.md @@ -1,4 +1,4 @@ -# sherpa_onnx_linux +# sherpa_onnx_macos This is a sub project of [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx). diff --git a/flutter/sherpa_onnx_windows/README.md b/flutter/sherpa_onnx_windows/README.md index 1334c2564..71c910982 100644 --- a/flutter/sherpa_onnx_windows/README.md +++ b/flutter/sherpa_onnx_windows/README.md @@ -1,4 +1,4 @@ -# sherpa_onnx_linux +# sherpa_onnx_windows This is a sub project of [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx). From 9efe26a64624ed0ab71cb0dd98e23b53f4874a47 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 15 Jan 2025 16:49:10 +0800 Subject: [PATCH 154/183] Export kokoro to sherpa-onnx (#1713) --- .github/workflows/export-kokoro.yaml | 122 +++++++++++++++ scripts/kokoro/.gitignore | 3 + scripts/kokoro/README.md | 10 ++ scripts/kokoro/add-meta-data.py | 107 +++++++++++++ scripts/kokoro/run.sh | 50 ++++++ scripts/kokoro/test.py | 223 +++++++++++++++++++++++++++ scripts/melo-tts/export-onnx.py | 2 +- 7 files changed, 516 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/export-kokoro.yaml create mode 100644 scripts/kokoro/.gitignore create mode 100644 scripts/kokoro/README.md create mode 100755 scripts/kokoro/add-meta-data.py create mode 100755 scripts/kokoro/run.sh create mode 100755 scripts/kokoro/test.py diff --git a/.github/workflows/export-kokoro.yaml b/.github/workflows/export-kokoro.yaml new file mode 100644 index 000000000..6429c7b0b --- /dev/null +++ b/.github/workflows/export-kokoro.yaml @@ -0,0 +1,122 @@ +name: export-kokoro-to-onnx + +on: + push: + branches: + - export-kokoro + + workflow_dispatch: + +concurrency: + group: export-kokoro-to-onnx-${{ github.ref }} + cancel-in-progress: true + +jobs: + export-kokoro-to-onnx: + if: github.repository_owner == 'k2-fsa' || github.repository_owner == 'csukuangfj' + name: export kokoro + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: ["3.10"] + + steps: + - uses: actions/checkout@v4 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Python dependencies + shell: bash + run: | + pip install -q "numpy<=1.26.4" onnx==1.16.0 onnxruntime==1.17.1 librosa soundfile piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html + + - name: Run + shell: bash + run: | + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/espeak-ng-data.tar.bz2 + tar xf espeak-ng-data.tar.bz2 + rm espeak-ng-data.tar.bz2 + cd scripts/kokoro + ./run.sh + + - name: Collect results + shell: bash + run: | + src=scripts/kokoro + + d=kokoro-en-v0_19 + mkdir $d + cp -a LICENSE $d/LICENSE + cp -a espeak-ng-data $d/ + cp -v $src/kokoro-v0_19_hf.onnx $d/model.onnx + cp -v $src/voices.bin $d/ + cp -v $src/tokens.txt $d/ + cp -v $src/README-new.md $d/README.md + ls -lh $d/ + tar cjfv $d.tar.bz2 $d + rm -rf $d + + ls -h $.tar.bz2 + + - name: Publish to huggingface + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v3 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + + rm -rf huggingface + export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false + + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/kokoro-en-v0_19 huggingface + cd huggingface + rm -rf ./* + git fetch + git pull + + git lfs track "cmn_dict" + git lfs track "ru_dict" + git lfs track "*.wav" + + cp -a ../espeak-ng-data ./ + mkdir -p test_wavs + + cp -v ../scripts/kokoro/kokoro-v0_19_hf.onnx ./model.onnx + + cp -v ../scripts/kokoro/kokoro-v0_19_hf-*.wav ./test_wavs/ + + cp -v ../scripts/kokoro/tokens.txt . + cp -v ../scripts/kokoro/voices.bin . + cp -v ../scripts/kokoro/README-new.md ./README.md + cp -v ../LICENSE ./ + + git lfs track "*.onnx" + git add . + + ls -lh + + git status + + git commit -m "add models" + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/kokoro-en-v0_19 main || true + + - name: Release + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + file: ./*.tar.bz2 + overwrite: true + repo_name: k2-fsa/sherpa-onnx + repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} + tag: tts-models diff --git a/scripts/kokoro/.gitignore b/scripts/kokoro/.gitignore new file mode 100644 index 000000000..3802b5171 --- /dev/null +++ b/scripts/kokoro/.gitignore @@ -0,0 +1,3 @@ +voices.json +voices.bin +README-new.md diff --git a/scripts/kokoro/README.md b/scripts/kokoro/README.md new file mode 100644 index 000000000..5a0e09c29 --- /dev/null +++ b/scripts/kokoro/README.md @@ -0,0 +1,10 @@ +# Introduction + +This folder contains scripts for adding meta data to models +from https://github.com/thewh1teagle/kokoro-onnx/releases/tag/model-files + +See also +https://huggingface.co/hexgrad/Kokoro-82M/tree/main +and +https://huggingface.co/spaces/hexgrad/Kokoro-TTS + diff --git a/scripts/kokoro/add-meta-data.py b/scripts/kokoro/add-meta-data.py new file mode 100755 index 000000000..cf42f13f3 --- /dev/null +++ b/scripts/kokoro/add-meta-data.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# Copyright 2025 Xiaomi Corp. (authors: Fangjun Kuang) + + +import argparse +import json +from pathlib import Path + +import numpy as np +import onnx + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--model", type=str, required=True, help="input and output onnx model" + ) + + parser.add_argument("--voices", type=str, required=True, help="Path to voices.json") + return parser.parse_args() + + +def load_voices(filename): + with open(filename) as f: + voices = json.load(f) + for key in voices: + voices[key] = np.array(voices[key], dtype=np.float32) + return voices + + +def get_vocab(): + _pad = "$" + _punctuation = ';:,.!?¡¿—…"«»“” ' + _letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + _letters_ipa = "ɑɐɒæɓʙβɔɕçɗɖðʤəɘɚɛɜɝɞɟʄɡɠɢʛɦɧħɥʜɨɪʝɭɬɫɮʟɱɯɰŋɳɲɴøɵɸθœɶʘɹɺɾɻʀʁɽʂʃʈʧʉʊʋⱱʌɣɤʍχʎʏʑʐʒʔʡʕʢǀǁǂǃˈˌːˑʼʴʰʱʲʷˠˤ˞↓↑→↗↘'̩'ᵻ" + symbols = [_pad] + list(_punctuation) + list(_letters) + list(_letters_ipa) + dicts = {} + for i in range(len((symbols))): + dicts[symbols[i]] = i + return dicts + + +def generate_tokens(): + token2id = get_vocab() + with open("tokens.txt", "w", encoding="utf-8") as f: + for s, i in token2id.items(): + f.write(f"{s} {i}\n") + + +def main(): + args = get_args() + print(args.model, args.voices) + + model = onnx.load(args.model) + voices = load_voices(args.voices) + + if Path("./tokens.txt").is_file(): + print("./tokens.txt exist, skip generating it") + else: + generate_tokens() + + keys = list(voices.keys()) + print(",".join(keys)) + + if Path("./voices.bin").is_file(): + print("./voices.bin exists, skip generating it") + else: + with open("voices.bin", "wb") as f: + for k in keys: + f.write(voices[k].tobytes()) + + meta_data = { + "model_type": "kokoro", + "language": "English", + "has_espeak": 1, + "sample_rate": 24000, + "version": 1, + "voice": "en-us", + "style_dim": ",".join(map(str, voices[keys[0]].shape)), + "n_speakers": len(keys), + "speaker_names": ",".join(keys), + "model_url": "https://github.com/thewh1teagle/kokoro-onnx/releases/tag/model-files", + "see_also": "https://huggingface.co/spaces/hexgrad/Kokoro-TTS", + "see_also_2": "https://huggingface.co/hexgrad/Kokoro-82M", + "maintainer": "k2-fsa", + } + + print(model.metadata_props) + + while len(model.metadata_props): + model.metadata_props.pop() + + for key, value in meta_data.items(): + meta = model.metadata_props.add() + meta.key = key + meta.value = str(value) + print("--------------------") + + print(model.metadata_props) + + onnx.save(model, args.model) + + print(f"Please see {args.model}, ./voices.bin, and ./tokens.txt") + + +if __name__ == "__main__": + main() diff --git a/scripts/kokoro/run.sh b/scripts/kokoro/run.sh new file mode 100755 index 000000000..422472cae --- /dev/null +++ b/scripts/kokoro/run.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Copyright 2025 Xiaomi Corp. (authors: Fangjun Kuang) + +set -ex + +cat > README-new.md < Dict[str, int]: + ans = dict() + with open(filename, encoding="utf-8") as f: + for line in f: + fields = line.strip().split() + if len(fields) == 2: + token, idx = fields + ans[token] = int(idx) + else: + assert len(fields) == 1, (len(fields), line) + ans[" "] = int(fields[0]) + return ans + + +def load_voices(speaker_names: List[str], dim: List[int], voices_bin: str): + embedding = ( + np.fromfile(voices_bin, dtype="uint8") + .view(np.float32) + .reshape(len(speaker_names), *dim) + ) + print("embedding.shape", embedding.shape) + ans = dict() + for i in range(len(speaker_names)): + ans[speaker_names[i]] = embedding[i] + + return ans + + +class OnnxModel: + def __init__(self, model_filename: str, voices_bin: str, tokens: str): + session_opts = ort.SessionOptions() + session_opts.inter_op_num_threads = 1 + session_opts.intra_op_num_threads = 1 + + self.session_opts = session_opts + self.model = ort.InferenceSession( + model_filename, + sess_options=self.session_opts, + providers=["CPUExecutionProvider"], + ) + self.token2id = load_tokens(tokens) + + meta = self.model.get_modelmeta().custom_metadata_map + print(meta) + dim = list(map(int, meta["style_dim"].split(","))) + speaker_names = meta["speaker_names"].split(",") + + self.voices = load_voices( + speaker_names=speaker_names, dim=dim, voices_bin=voices_bin + ) + + self.sample_rate = int(meta["sample_rate"]) + + print(list(self.voices.keys())) + # ['af', 'af_bella', 'af_nicole', 'af_sarah', 'af_sky', 'am_adam', + # 'am_michael', 'bf_emma', 'bf_isabella', 'bm_george', 'bm_lewis'] + # af -> (511, 1, 256) + self.max_len = self.voices[next(iter(self.voices))].shape[0] - 1 + + def __call__(self, text: str, voice): + tokens = phonemize_espeak(text, "en-us") + # tokens is List[List[str]] + # Each sentence is a List[str] + # len(tokens) == number of sentences + + tokens = sum(tokens, []) # flatten + tokens = "".join(tokens) + + tokens = tokens.replace("kəkˈoːɹoʊ", "kˈoʊkəɹoʊ").replace( + "kəkˈɔːɹəʊ", "kˈəʊkəɹəʊ" + ) + + tokens = list(tokens) + + token_ids = [self.token2id[i] for i in tokens] + token_ids = token_ids[: self.max_len] + + style = self.voices[voice][len(token_ids)] + + token_ids = [0, *token_ids, 0] + token_ids = np.array([token_ids], dtype=np.int64) + + speed = np.array([1.0], dtype=np.float32) + + audio = self.model.run( + [ + self.model.get_outputs()[0].name, + ], + { + self.model.get_inputs()[0].name: token_ids, + self.model.get_inputs()[1].name: style, + self.model.get_inputs()[2].name: speed, + }, + )[0] + return audio + + +def test(model, voice, text) -> np.ndarray: + pass + + +def main(): + args = get_args() + print(vars(args)) + show(args.model) + + # tokens = phonemize_espeak("how are you doing?", "en-us") + # [['h', 'ˌ', 'a', 'ʊ', ' ', 'ɑ', 'ː', 'ɹ', ' ', 'j', 'u', 'ː', ' ', 'd', 'ˈ', 'u', 'ː', 'ɪ', 'ŋ', '?']] + m = OnnxModel( + model_filename=args.model, voices_bin=args.voices_bin, tokens=args.tokens + ) + + text = ( + "Today as always, men fall into two groups: slaves and free men." + + " Whoever does not have two-thirds of his day for himself, " + + "is a slave, whatever he may be: a statesman, a businessman, " + + "an official, or a scholar." + ) + + for i, voice in enumerate(m.voices.keys(), 1): + print(f"Testing {i}/{len(m.voices)} - {voice}/{args.model}") + + start = time.time() + audio = m(text, voice=voice) + end = time.time() + + elapsed_seconds = end - start + audio_duration = len(audio) / m.sample_rate + real_time_factor = elapsed_seconds / audio_duration + + filename = f"{Path(args.model).stem}-{voice}.wav" + sf.write( + filename, + audio, + samplerate=m.sample_rate, + subtype="PCM_16", + ) + print(f" Saved to {filename}") + print(f" Elapsed seconds: {elapsed_seconds:.3f}") + print(f" Audio duration in seconds: {audio_duration:.3f}") + print( + f" RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {real_time_factor:.3f}" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/melo-tts/export-onnx.py b/scripts/melo-tts/export-onnx.py index 69d9066ee..84e7eaf1b 100755 --- a/scripts/melo-tts/export-onnx.py +++ b/scripts/melo-tts/export-onnx.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# This script export ZH_EN TTS model, which supports both Chinese and English. +# This script exports ZH_EN TTS model, which supports both Chinese and English. # This model has only 1 speaker. from typing import Any, Dict From ffc6b480a0564fbd5d0a675e4af9a268cb385bc3 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 16 Jan 2025 14:24:51 +0800 Subject: [PATCH 155/183] Add C++ and Python API for Kokoro TTS models. (#1715) --- .github/scripts/test-offline-tts.sh | 25 ++ .github/scripts/test-python.sh | 19 + python-api-examples/offline-tts-play.py | 63 ++- python-api-examples/offline-tts.py | 66 ++- sherpa-onnx/csrc/CMakeLists.txt | 2 + sherpa-onnx/csrc/melo-tts-lexicon.h | 2 +- .../csrc/offline-tts-character-frontend.h | 2 +- sherpa-onnx/csrc/offline-tts-impl.cc | 10 +- sherpa-onnx/csrc/offline-tts-kokoro-impl.h | 376 ++++++++++++++++++ .../csrc/offline-tts-kokoro-model-config.cc | 96 +++++ .../csrc/offline-tts-kokoro-model-config.h | 44 ++ .../csrc/offline-tts-kokoro-model-meta-data.h | 25 ++ sherpa-onnx/csrc/offline-tts-kokoro-model.cc | 251 ++++++++++++ sherpa-onnx/csrc/offline-tts-kokoro-model.h | 39 ++ ...h => offline-tts-matcha-model-meta-data.h} | 8 +- sherpa-onnx/csrc/offline-tts-matcha-model.h | 2 +- sherpa-onnx/csrc/offline-tts-model-config.cc | 8 +- sherpa-onnx/csrc/offline-tts-model-config.h | 4 + ...a.h => offline-tts-vits-model-meta-data.h} | 8 +- sherpa-onnx/csrc/offline-tts-vits-model.h | 2 +- sherpa-onnx/csrc/piper-phonemize-lexicon.cc | 96 +++++ sherpa-onnx/csrc/piper-phonemize-lexicon.h | 18 +- sherpa-onnx/python/csrc/CMakeLists.txt | 1 + .../csrc/offline-tts-kokoro-model-config.cc | 31 ++ .../csrc/offline-tts-kokoro-model-config.h | 16 + .../python/csrc/offline-tts-model-config.cc | 7 +- sherpa-onnx/python/sherpa_onnx/__init__.py | 1 + 27 files changed, 1193 insertions(+), 29 deletions(-) create mode 100644 sherpa-onnx/csrc/offline-tts-kokoro-impl.h create mode 100644 sherpa-onnx/csrc/offline-tts-kokoro-model-config.cc create mode 100644 sherpa-onnx/csrc/offline-tts-kokoro-model-config.h create mode 100644 sherpa-onnx/csrc/offline-tts-kokoro-model-meta-data.h create mode 100644 sherpa-onnx/csrc/offline-tts-kokoro-model.cc create mode 100644 sherpa-onnx/csrc/offline-tts-kokoro-model.h rename sherpa-onnx/csrc/{offline-tts-matcha-model-metadata.h => offline-tts-matcha-model-meta-data.h} (66%) rename sherpa-onnx/csrc/{offline-tts-vits-model-metadata.h => offline-tts-vits-model-meta-data.h} (80%) create mode 100644 sherpa-onnx/python/csrc/offline-tts-kokoro-model-config.cc create mode 100644 sherpa-onnx/python/csrc/offline-tts-kokoro-model-config.h diff --git a/.github/scripts/test-offline-tts.sh b/.github/scripts/test-offline-tts.sh index 70fd2247e..baa2b37bb 100755 --- a/.github/scripts/test-offline-tts.sh +++ b/.github/scripts/test-offline-tts.sh @@ -18,6 +18,31 @@ which $EXE # test waves are saved in ./tts mkdir ./tts +log "------------------------------------------------------------" +log "kokoro-en-v0_19" +log "------------------------------------------------------------" +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 +tar xf kokoro-en-v0_19.tar.bz2 +rm kokoro-en-v0_19.tar.bz2 + +# mapping of sid to voice name +# 0->af, 1->af_bella, 2->af_nicole, 3->af_sarah, 4->af_sky, 5->am_adam +# 6->am_michael, 7->bf_emma, 8->bf_isabella, 9->bm_george, 10->bm_lewis + +for sid in $(seq 0 10); do + $EXE \ + --debug=1 \ + --kokoro-model=./kokoro-en-v0_19/model.onnx \ + --kokoro-voices=./kokoro-en-v0_19/voices.bin \ + --kokoro-tokens=./kokoro-en-v0_19/tokens.txt \ + --kokoro-data-dir=./kokoro-en-v0_19/espeak-ng-data \ + --num-threads=2 \ + --sid=$sid \ + --output-filename="./tts/kokoro-$sid.wav" \ + "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be a statesman, a businessman, an official, or a scholar." +done +rm -rf kokoro-en-v0_19 + log "------------------------------------------------------------" log "matcha-icefall-en_US-ljspeech" log "------------------------------------------------------------" diff --git a/.github/scripts/test-python.sh b/.github/scripts/test-python.sh index 350d9c185..ad037438f 100755 --- a/.github/scripts/test-python.sh +++ b/.github/scripts/test-python.sh @@ -267,6 +267,25 @@ log "Offline TTS test" # test waves are saved in ./tts mkdir ./tts +log "kokoro-en-v0_19 test" + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 +tar xf kokoro-en-v0_19.tar.bz2 +rm kokoro-en-v0_19.tar.bz2 + +python3 ./python-api-examples/offline-tts.py \ + --debug=1 \ + --kokoro-model=./kokoro-en-v0_19/model.onnx \ + --kokoro-voices=./kokoro-en-v0_19/voices.bin \ + --kokoro-tokens=./kokoro-en-v0_19/tokens.txt \ + --kokoro-data-dir=./kokoro-en-v0_19/espeak-ng-data \ + --num-threads=2 \ + --sid=10 \ + --output-filename="./tts/kokoro-10.wav" \ + "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be a statesman, a businessman, an official, or a scholar." + +rm -rf kokoro-en-v0_19 + log "matcha-ljspeech-en test" curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 diff --git a/python-api-examples/offline-tts-play.py b/python-api-examples/offline-tts-play.py index 09d03dae6..5ece997b5 100755 --- a/python-api-examples/offline-tts-play.py +++ b/python-api-examples/offline-tts-play.py @@ -11,7 +11,7 @@ Usage: -Example (1/5) +Example (1/6) wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 tar xf vits-piper-en_US-amy-low.tar.bz2 @@ -23,7 +23,7 @@ --output-filename=./generated.wav \ "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar." -Example (2/5) +Example (2/6) wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-zh-aishell3.tar.bz2 tar xvf vits-zh-aishell3.tar.bz2 @@ -37,7 +37,7 @@ --output-filename=./liubei-21.wav \ "勿以恶小而为之，勿以善小而不为。惟贤惟德，能服于人。122334" -Example (3/5) +Example (3/6) wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-vits-zh-ll.tar.bz2 tar xvf sherpa-onnx-vits-zh-ll.tar.bz2 @@ -53,7 +53,7 @@ --output-filename=./test-2.wav \ "当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转，思念如涟漪荡漾，梦境如画卷展开，我与自然融为一体，沉静在这片宁静的美丽之中，感受着生命的奇迹与温柔。2024年5月11号，拨打110或者18920240511。123456块钱。" -Example (4/5) +Example (4/6) curl -O -SL https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 tar xvf matcha-icefall-zh-baker.tar.bz2 @@ -71,7 +71,7 @@ --output-filename=./test-matcha.wav \ "某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。" -Example (5/5) +Example (5/6) curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 @@ -88,6 +88,22 @@ --num-threads=2 \ "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar." +Example (6/6) + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 +tar xf kokoro-en-v0_19.tar.bz2 +rm kokoro-en-v0_19.tar.bz2 + +python3 ./python-api-examples/offline-tts.py \ + --debug=1 \ + --kokoro-model=./kokoro-en-v0_19/model.onnx \ + --kokoro-voices=./kokoro-en-v0_19/voices.bin \ + --kokoro-tokens=./kokoro-en-v0_19/tokens.txt \ + --kokoro-data-dir=./kokoro-en-v0_19/espeak-ng-data \ + --num-threads=2 \ + --sid=10 \ + --output-filename="./kokoro-10.wav" \ + "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be a statesman, a businessman, an official, or a scholar." You can find more models at https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models @@ -202,6 +218,36 @@ def add_matcha_args(parser): ) +def add_kokoro_args(parser): + parser.add_argument( + "--kokoro-model", + type=str, + default="", + help="Path to model.onnx for kokoro", + ) + + parser.add_argument( + "--kokoro-voices", + type=str, + default="", + help="Path to voices.bin for kokoro", + ) + + parser.add_argument( + "--kokoro-tokens", + type=str, + default="", + help="Path to tokens.txt for kokoro", + ) + + parser.add_argument( + "--kokoro-data-dir", + type=str, + default="", + help="Path to the dict directory of espeak-ng.", + ) + + def get_args(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter @@ -209,6 +255,7 @@ def get_args(): add_vits_args(parser) add_matcha_args(parser) + add_kokoro_args(parser) parser.add_argument( "--tts-rule-fsts", @@ -407,6 +454,12 @@ def main(): data_dir=args.matcha_data_dir, dict_dir=args.matcha_dict_dir, ), + kokoro=sherpa_onnx.OfflineTtsKokoroModelConfig( + model=args.kokoro_model, + voices=args.kokoro_voices, + tokens=args.kokoro_tokens, + data_dir=args.kokoro_data_dir, + ), provider=args.provider, debug=args.debug, num_threads=args.num_threads, diff --git a/python-api-examples/offline-tts.py b/python-api-examples/offline-tts.py index 72bf77959..aace840fa 100755 --- a/python-api-examples/offline-tts.py +++ b/python-api-examples/offline-tts.py @@ -12,7 +12,7 @@ Usage: -Example (1/5) +Example (1/6) wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 tar xf vits-piper-en_US-amy-low.tar.bz2 @@ -24,7 +24,7 @@ --output-filename=./generated.wav \ "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar." -Example (2/5) +Example (2/6) wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-icefall-zh-aishell3.tar.bz2 tar xvf vits-icefall-zh-aishell3.tar.bz2 @@ -38,7 +38,7 @@ --output-filename=./liubei-21.wav \ "勿以恶小而为之，勿以善小而不为。惟贤惟德，能服于人。122334" -Example (3/5) +Example (3/6) wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-vits-zh-ll.tar.bz2 tar xvf sherpa-onnx-vits-zh-ll.tar.bz2 @@ -54,7 +54,7 @@ --output-filename=./test-2.wav \ "当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转，思念如涟漪荡漾，梦境如画卷展开，我与自然融为一体，沉静在这片宁静的美丽之中，感受着生命的奇迹与温柔。2024年5月11号，拨打110或者18920240511。123456块钱。" -Example (4/5) +Example (4/6) curl -O -SL https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 tar xvf matcha-icefall-zh-baker.tar.bz2 @@ -72,7 +72,7 @@ --output-filename=./test-matcha.wav \ "某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。" -Example (5/5) +Example (5/6) curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 @@ -89,6 +89,23 @@ --num-threads=2 \ "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar." +Example (6/6) + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 +tar xf kokoro-en-v0_19.tar.bz2 +rm kokoro-en-v0_19.tar.bz2 + +python3 ./python-api-examples/offline-tts.py \ + --debug=1 \ + --kokoro-model=./kokoro-en-v0_19/model.onnx \ + --kokoro-voices=./kokoro-en-v0_19/voices.bin \ + --kokoro-tokens=./kokoro-en-v0_19/tokens.txt \ + --kokoro-data-dir=./kokoro-en-v0_19/espeak-ng-data \ + --num-threads=2 \ + --sid=10 \ + --output-filename="./kokoro-10.wav" \ + "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be a statesman, a businessman, an official, or a scholar." + You can find more models at https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models @@ -188,6 +205,36 @@ def add_matcha_args(parser): ) +def add_kokoro_args(parser): + parser.add_argument( + "--kokoro-model", + type=str, + default="", + help="Path to model.onnx for kokoro", + ) + + parser.add_argument( + "--kokoro-voices", + type=str, + default="", + help="Path to voices.bin for kokoro", + ) + + parser.add_argument( + "--kokoro-tokens", + type=str, + default="", + help="Path to tokens.txt for kokoro", + ) + + parser.add_argument( + "--kokoro-data-dir", + type=str, + default="", + help="Path to the dict directory of espeak-ng.", + ) + + def get_args(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter @@ -195,6 +242,7 @@ def get_args(): add_vits_args(parser) add_matcha_args(parser) + add_kokoro_args(parser) parser.add_argument( "--tts-rule-fsts", @@ -206,7 +254,7 @@ def get_args(): parser.add_argument( "--max-num-sentences", type=int, - default=2, + default=1, help="""Max number of sentences in a batch to avoid OOM if the input text is very long. Set it to -1 to process all the sentences in a single batch. A smaller value does not mean it is slower compared @@ -289,6 +337,12 @@ def main(): data_dir=args.matcha_data_dir, dict_dir=args.matcha_dict_dir, ), + kokoro=sherpa_onnx.OfflineTtsKokoroModelConfig( + model=args.kokoro_model, + voices=args.kokoro_voices, + tokens=args.kokoro_tokens, + data_dir=args.kokoro_data_dir, + ), provider=args.provider, debug=args.debug, num_threads=args.num_threads, diff --git a/sherpa-onnx/csrc/CMakeLists.txt b/sherpa-onnx/csrc/CMakeLists.txt index f146b09e2..d5303b757 100644 --- a/sherpa-onnx/csrc/CMakeLists.txt +++ b/sherpa-onnx/csrc/CMakeLists.txt @@ -158,6 +158,8 @@ if(SHERPA_ONNX_ENABLE_TTS) offline-tts-character-frontend.cc offline-tts-frontend.cc offline-tts-impl.cc + offline-tts-kokoro-model-config.cc + offline-tts-kokoro-model.cc offline-tts-matcha-model-config.cc offline-tts-matcha-model.cc offline-tts-model-config.cc diff --git a/sherpa-onnx/csrc/melo-tts-lexicon.h b/sherpa-onnx/csrc/melo-tts-lexicon.h index e91cf33f2..ef7dd0298 100644 --- a/sherpa-onnx/csrc/melo-tts-lexicon.h +++ b/sherpa-onnx/csrc/melo-tts-lexicon.h @@ -11,7 +11,7 @@ #include #include "sherpa-onnx/csrc/offline-tts-frontend.h" -#include "sherpa-onnx/csrc/offline-tts-vits-model-metadata.h" +#include "sherpa-onnx/csrc/offline-tts-vits-model-meta-data.h" namespace sherpa_onnx { diff --git a/sherpa-onnx/csrc/offline-tts-character-frontend.h b/sherpa-onnx/csrc/offline-tts-character-frontend.h index fcd2f6dd5..55bf6a70f 100644 --- a/sherpa-onnx/csrc/offline-tts-character-frontend.h +++ b/sherpa-onnx/csrc/offline-tts-character-frontend.h @@ -10,7 +10,7 @@ #include #include "sherpa-onnx/csrc/offline-tts-frontend.h" -#include "sherpa-onnx/csrc/offline-tts-vits-model-metadata.h" +#include "sherpa-onnx/csrc/offline-tts-vits-model-meta-data.h" namespace sherpa_onnx { diff --git a/sherpa-onnx/csrc/offline-tts-impl.cc b/sherpa-onnx/csrc/offline-tts-impl.cc index 92ccb7fdb..199b0f792 100644 --- a/sherpa-onnx/csrc/offline-tts-impl.cc +++ b/sherpa-onnx/csrc/offline-tts-impl.cc @@ -16,6 +16,7 @@ #include "rawfile/raw_file_manager.h" #endif +#include "sherpa-onnx/csrc/offline-tts-kokoro-impl.h" #include "sherpa-onnx/csrc/offline-tts-matcha-impl.h" #include "sherpa-onnx/csrc/offline-tts-vits-impl.h" @@ -37,8 +38,11 @@ std::unique_ptr OfflineTtsImpl::Create( const OfflineTtsConfig &config) { if (!config.model.vits.model.empty()) { return std::make_unique(config); + } else if (!config.model.matcha.acoustic_model.empty()) { + return std::make_unique(config); } - return std::make_unique(config); + + return std::make_unique(config); } template @@ -46,9 +50,11 @@ std::unique_ptr OfflineTtsImpl::Create( Manager *mgr, const OfflineTtsConfig &config) { if (!config.model.vits.model.empty()) { return std::make_unique(mgr, config); + } else if (!config.model.matcha.acoustic_model.empty()) { + return std::make_unique(mgr, config); } - return std::make_unique(mgr, config); + return std::make_unique(mgr, config); } #if __ANDROID_API__ >= 9 diff --git a/sherpa-onnx/csrc/offline-tts-kokoro-impl.h b/sherpa-onnx/csrc/offline-tts-kokoro-impl.h new file mode 100644 index 000000000..4c3efbf65 --- /dev/null +++ b/sherpa-onnx/csrc/offline-tts-kokoro-impl.h @@ -0,0 +1,376 @@ +// sherpa-onnx/csrc/offline-tts-kokoro-impl.h +// +// Copyright (c) 2025 Xiaomi Corporation +#ifndef SHERPA_ONNX_CSRC_OFFLINE_TTS_KOKORO_IMPL_H_ +#define SHERPA_ONNX_CSRC_OFFLINE_TTS_KOKORO_IMPL_H_ + +#include +#include +#include +#include +#include + +#include "fst/extensions/far/far.h" +#include "kaldifst/csrc/kaldi-fst-io.h" +#include "kaldifst/csrc/text-normalizer.h" +#include "sherpa-onnx/csrc/lexicon.h" +#include "sherpa-onnx/csrc/macros.h" +#include "sherpa-onnx/csrc/offline-tts-frontend.h" +#include "sherpa-onnx/csrc/offline-tts-impl.h" +#include "sherpa-onnx/csrc/offline-tts-kokoro-model.h" +#include "sherpa-onnx/csrc/onnx-utils.h" +#include "sherpa-onnx/csrc/piper-phonemize-lexicon.h" +#include "sherpa-onnx/csrc/text-utils.h" + +namespace sherpa_onnx { + +class OfflineTtsKokoroImpl : public OfflineTtsImpl { + public: + explicit OfflineTtsKokoroImpl(const OfflineTtsConfig &config) + : config_(config), + model_(std::make_unique(config.model)) { + InitFrontend(); + + if (!config.rule_fsts.empty()) { + std::vector files; + SplitStringToVector(config.rule_fsts, ",", false, &files); + tn_list_.reserve(files.size()); + for (const auto &f : files) { + if (config.model.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("rule fst: %{public}s", f.c_str()); +#else + SHERPA_ONNX_LOGE("rule fst: %s", f.c_str()); +#endif + } + tn_list_.push_back(std::make_unique(f)); + } + } + + if (!config.rule_fars.empty()) { + if (config.model.debug) { + SHERPA_ONNX_LOGE("Loading FST archives"); + } + std::vector files; + SplitStringToVector(config.rule_fars, ",", false, &files); + + tn_list_.reserve(files.size() + tn_list_.size()); + + for (const auto &f : files) { + if (config.model.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("rule far: %{public}s", f.c_str()); +#else + SHERPA_ONNX_LOGE("rule far: %s", f.c_str()); +#endif + } + std::unique_ptr> reader( + fst::FarReader::Open(f)); + for (; !reader->Done(); reader->Next()) { + std::unique_ptr r( + fst::CastOrConvertToConstFst(reader->GetFst()->Copy())); + + tn_list_.push_back( + std::make_unique(std::move(r))); + } + } + + if (config.model.debug) { + SHERPA_ONNX_LOGE("FST archives loaded!"); + } + } + } + + template + OfflineTtsKokoroImpl(Manager *mgr, const OfflineTtsConfig &config) + : config_(config), + model_(std::make_unique(mgr, config.model)) { + InitFrontend(mgr); + + if (!config.rule_fsts.empty()) { + std::vector files; + SplitStringToVector(config.rule_fsts, ",", false, &files); + tn_list_.reserve(files.size()); + for (const auto &f : files) { + if (config.model.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("rule fst: %{public}s", f.c_str()); +#else + SHERPA_ONNX_LOGE("rule fst: %s", f.c_str()); +#endif + } + auto buf = ReadFile(mgr, f); + std::istrstream is(buf.data(), buf.size()); + tn_list_.push_back(std::make_unique(is)); + } + } + + if (!config.rule_fars.empty()) { + std::vector files; + SplitStringToVector(config.rule_fars, ",", false, &files); + tn_list_.reserve(files.size() + tn_list_.size()); + + for (const auto &f : files) { + if (config.model.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("rule far: %{public}s", f.c_str()); +#else + SHERPA_ONNX_LOGE("rule far: %s", f.c_str()); +#endif + } + + auto buf = ReadFile(mgr, f); + + std::unique_ptr s( + new std::istrstream(buf.data(), buf.size())); + + std::unique_ptr> reader( + fst::FarReader::Open(std::move(s))); + + for (; !reader->Done(); reader->Next()) { + std::unique_ptr r( + fst::CastOrConvertToConstFst(reader->GetFst()->Copy())); + + tn_list_.push_back( + std::make_unique(std::move(r))); + } // for (; !reader->Done(); reader->Next()) + } // for (const auto &f : files) + } // if (!config.rule_fars.empty()) + } + + int32_t SampleRate() const override { + return model_->GetMetaData().sample_rate; + } + + int32_t NumSpeakers() const override { + return model_->GetMetaData().num_speakers; + } + + GeneratedAudio Generate( + const std::string &_text, int64_t sid = 0, float speed = 1.0, + GeneratedAudioCallback callback = nullptr) const override { + const auto &meta_data = model_->GetMetaData(); + int32_t num_speakers = meta_data.num_speakers; + + if (num_speakers == 0 && sid != 0) { +#if __OHOS__ + SHERPA_ONNX_LOGE( + "This is a single-speaker model and supports only sid 0. Given sid: " + "%{public}d. sid is ignored", + static_cast(sid)); +#else + SHERPA_ONNX_LOGE( + "This is a single-speaker model and supports only sid 0. Given sid: " + "%d. sid is ignored", + static_cast(sid)); +#endif + } + + if (num_speakers != 0 && (sid >= num_speakers || sid < 0)) { +#if __OHOS__ + SHERPA_ONNX_LOGE( + "This model contains only %{public}d speakers. sid should be in the " + "range [%{public}d, %{public}d]. Given: %{public}d. Use sid=0", + num_speakers, 0, num_speakers - 1, static_cast(sid)); +#else + SHERPA_ONNX_LOGE( + "This model contains only %d speakers. sid should be in the range " + "[%d, %d]. Given: %d. Use sid=0", + num_speakers, 0, num_speakers - 1, static_cast(sid)); +#endif + sid = 0; + } + + std::string text = _text; + if (config_.model.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("Raw text: %{public}s", text.c_str()); +#else + SHERPA_ONNX_LOGE("Raw text: %s", text.c_str()); +#endif + } + + if (!tn_list_.empty()) { + for (const auto &tn : tn_list_) { + text = tn->Normalize(text); + if (config_.model.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("After normalizing: %{public}s", text.c_str()); +#else + SHERPA_ONNX_LOGE("After normalizing: %s", text.c_str()); +#endif + } + } + } + + std::vector token_ids = + frontend_->ConvertTextToTokenIds(text, "en-us"); + + if (token_ids.empty() || + (token_ids.size() == 1 && token_ids[0].tokens.empty())) { +#if __OHOS__ + SHERPA_ONNX_LOGE("Failed to convert '%{public}s' to token IDs", + text.c_str()); +#else + SHERPA_ONNX_LOGE("Failed to convert '%s' to token IDs", text.c_str()); +#endif + return {}; + } + + std::vector> x; + + x.reserve(token_ids.size()); + + for (auto &i : token_ids) { + x.push_back(std::move(i.tokens)); + } + + int32_t x_size = static_cast(x.size()); + + if (config_.max_num_sentences != 1) { +#if __OHOS__ + SHERPA_ONNX_LOGE( + "max_num_sentences (%{public}d) != 1 is ignored for Kokoro TTS " + "models", + config_.max_num_sentences); +#else + SHERPA_ONNX_LOGE( + "max_num_sentences (%d) != 1 is ignored for Kokoro TTS models", + config_.max_num_sentences); +#endif + } + + // the input text is too long, we process sentences within it in batches + // to avoid OOM. Batch size is config_.max_num_sentences + std::vector> batch_x; + + int32_t batch_size = 1; + batch_x.reserve(config_.max_num_sentences); + int32_t num_batches = x_size / batch_size; + + if (config_.model.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE( + "Split it into %{public}d batches. batch size: " + "%{public}d. Number of sentences: %{public}d", + num_batches, batch_size, x_size); +#else + SHERPA_ONNX_LOGE( + "Split it into %d batches. batch size: %d. Number " + "of sentences: %d", + num_batches, batch_size, x_size); +#endif + } + + GeneratedAudio ans; + + int32_t should_continue = 1; + + int32_t k = 0; + + for (int32_t b = 0; b != num_batches && should_continue; ++b) { + batch_x.clear(); + for (int32_t i = 0; i != batch_size; ++i, ++k) { + batch_x.push_back(std::move(x[k])); + } + + auto audio = Process(batch_x, sid, speed); + ans.sample_rate = audio.sample_rate; + ans.samples.insert(ans.samples.end(), audio.samples.begin(), + audio.samples.end()); + if (callback) { + should_continue = callback(audio.samples.data(), audio.samples.size(), + (b + 1) * 1.0 / num_batches); + // Caution(fangjun): audio is freed when the callback returns, so users + // should copy the data if they want to access the data after + // the callback returns to avoid segmentation fault. + } + } + + batch_x.clear(); + while (k < static_cast(x.size()) && should_continue) { + batch_x.push_back(std::move(x[k])); + + ++k; + } + + if (!batch_x.empty()) { + auto audio = Process(batch_x, sid, speed); + ans.sample_rate = audio.sample_rate; + ans.samples.insert(ans.samples.end(), audio.samples.begin(), + audio.samples.end()); + if (callback) { + callback(audio.samples.data(), audio.samples.size(), 1.0); + // Caution(fangjun): audio is freed when the callback returns, so users + // should copy the data if they want to access the data after + // the callback returns to avoid segmentation fault. + } + } + + return ans; + } + + private: + template + void InitFrontend(Manager *mgr) { + const auto &meta_data = model_->GetMetaData(); + frontend_ = std::make_unique( + mgr, config_.model.kokoro.tokens, config_.model.kokoro.data_dir, + meta_data); + } + + void InitFrontend() { + const auto &meta_data = model_->GetMetaData(); + + frontend_ = std::make_unique( + config_.model.kokoro.tokens, config_.model.kokoro.data_dir, meta_data); + } + + GeneratedAudio Process(const std::vector> &tokens, + int32_t sid, float speed) const { + int32_t num_tokens = 0; + for (const auto &k : tokens) { + num_tokens += k.size(); + } + + std::vector x; + x.reserve(num_tokens); + for (const auto &k : tokens) { + x.insert(x.end(), k.begin(), k.end()); + } + + auto memory_info = + Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault); + + std::array x_shape = {1, static_cast(x.size())}; + Ort::Value x_tensor = Ort::Value::CreateTensor( + memory_info, x.data(), x.size(), x_shape.data(), x_shape.size()); + + Ort::Value audio = model_->Run(std::move(x_tensor), sid, speed); + + std::vector audio_shape = + audio.GetTensorTypeAndShapeInfo().GetShape(); + + int64_t total = 1; + // The output shape may be (1, 1, total) or (1, total) or (total,) + for (auto i : audio_shape) { + total *= i; + } + + const float *p = audio.GetTensorData(); + + GeneratedAudio ans; + ans.sample_rate = model_->GetMetaData().sample_rate; + ans.samples = std::vector(p, p + total); + return ans; + } + + private: + OfflineTtsConfig config_; + std::unique_ptr model_; + std::vector> tn_list_; + std::unique_ptr frontend_; +}; + +} // namespace sherpa_onnx +#endif // SHERPA_ONNX_CSRC_OFFLINE_TTS_KOKORO_IMPL_H_ diff --git a/sherpa-onnx/csrc/offline-tts-kokoro-model-config.cc b/sherpa-onnx/csrc/offline-tts-kokoro-model-config.cc new file mode 100644 index 000000000..3eb5ad7e0 --- /dev/null +++ b/sherpa-onnx/csrc/offline-tts-kokoro-model-config.cc @@ -0,0 +1,96 @@ +// sherpa-onnx/csrc/offline-tts-kokoro-model-config.cc +// +// Copyright (c) 2025 Xiaomi Corporation + +#include "sherpa-onnx/csrc/offline-tts-kokoro-model-config.h" + +#include + +#include "sherpa-onnx/csrc/file-utils.h" +#include "sherpa-onnx/csrc/macros.h" + +namespace sherpa_onnx { + +void OfflineTtsKokoroModelConfig::Register(ParseOptions *po) { + po->Register("kokoro-model", &model, "Path to Kokoro model"); + po->Register("kokoro-voices", &voices, + "Path to voices.bin for Kokoro models"); + po->Register("kokoro-tokens", &tokens, + "Path to tokens.txt for Kokoro models"); + po->Register("kokoro-data-dir", &data_dir, + "Path to the directory containing dict for espeak-ng."); + po->Register("kokoro-length-scale", &length_scale, + "Speech speed. Larger->Slower; Smaller->faster."); +} + +bool OfflineTtsKokoroModelConfig::Validate() const { + if (model.empty()) { + SHERPA_ONNX_LOGE("Please provide --kokoro-model"); + return false; + } + + if (!FileExists(model)) { + SHERPA_ONNX_LOGE("--kokoro-model: '%s' does not exist", model.c_str()); + return false; + } + + if (tokens.empty()) { + SHERPA_ONNX_LOGE("Please provide --kokoro-tokens"); + return false; + } + + if (!FileExists(tokens)) { + SHERPA_ONNX_LOGE("--kokoro-tokens: '%s' does not exist", tokens.c_str()); + return false; + } + + if (data_dir.empty()) { + SHERPA_ONNX_LOGE("Please provide --kokoro-data-dir"); + return false; + } + + if (!FileExists(data_dir + "/phontab")) { + SHERPA_ONNX_LOGE( + "'%s/phontab' does not exist. Please check --kokoro-data-dir", + data_dir.c_str()); + return false; + } + + if (!FileExists(data_dir + "/phonindex")) { + SHERPA_ONNX_LOGE( + "'%s/phonindex' does not exist. Please check --kokoro-data-dir", + data_dir.c_str()); + return false; + } + + if (!FileExists(data_dir + "/phondata")) { + SHERPA_ONNX_LOGE( + "'%s/phondata' does not exist. Please check --kokoro-data-dir", + data_dir.c_str()); + return false; + } + + if (!FileExists(data_dir + "/intonations")) { + SHERPA_ONNX_LOGE( + "'%s/intonations' does not exist. Please check --kokoro-data-dir", + data_dir.c_str()); + return false; + } + + return true; +} + +std::string OfflineTtsKokoroModelConfig::ToString() const { + std::ostringstream os; + + os << "OfflineTtsKokoroModelConfig("; + os << "model=\"" << model << "\", "; + os << "voices=\"" << voices << "\", "; + os << "tokens=\"" << tokens << "\", "; + os << "data_dir=\"" << data_dir << "\", "; + os << "length_scale=" << length_scale << ")"; + + return os.str(); +} + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-tts-kokoro-model-config.h b/sherpa-onnx/csrc/offline-tts-kokoro-model-config.h new file mode 100644 index 000000000..a4a68aca8 --- /dev/null +++ b/sherpa-onnx/csrc/offline-tts-kokoro-model-config.h @@ -0,0 +1,44 @@ +// sherpa-onnx/csrc/offline-tts-kokoro-model-config.h +// +// Copyright (c) 2025 Xiaomi Corporation + +#ifndef SHERPA_ONNX_CSRC_OFFLINE_TTS_KOKORO_MODEL_CONFIG_H_ +#define SHERPA_ONNX_CSRC_OFFLINE_TTS_KOKORO_MODEL_CONFIG_H_ + +#include + +#include "sherpa-onnx/csrc/parse-options.h" + +namespace sherpa_onnx { + +struct OfflineTtsKokoroModelConfig { + std::string model; + std::string voices; + std::string tokens; + + std::string data_dir; + + // speed = 1 / length_scale + float length_scale = 1.0; + + OfflineTtsKokoroModelConfig() = default; + + OfflineTtsKokoroModelConfig(const std::string &model, + const std::string &voices, + const std::string &tokens, + const std::string &data_dir, float length_scale) + : model(model), + voices(voices), + tokens(tokens), + data_dir(data_dir), + length_scale(length_scale) {} + + void Register(ParseOptions *po); + bool Validate() const; + + std::string ToString() const; +}; + +} // namespace sherpa_onnx + +#endif // SHERPA_ONNX_CSRC_OFFLINE_TTS_KOKORO_MODEL_CONFIG_H_ diff --git a/sherpa-onnx/csrc/offline-tts-kokoro-model-meta-data.h b/sherpa-onnx/csrc/offline-tts-kokoro-model-meta-data.h new file mode 100644 index 000000000..64b70851c --- /dev/null +++ b/sherpa-onnx/csrc/offline-tts-kokoro-model-meta-data.h @@ -0,0 +1,25 @@ +// sherpa-onnx/csrc/offline-tts-kokoro-model-metadata.h +// +// Copyright (c) 2025 Xiaomi Corporation + +#ifndef SHERPA_ONNX_CSRC_OFFLINE_TTS_KOKORO_MODEL_META_DATA_H_ +#define SHERPA_ONNX_CSRC_OFFLINE_TTS_KOKORO_MODEL_META_DATA_H_ + +#include +#include + +namespace sherpa_onnx { + +// please refer to +// https://github.com/k2-fsa/sherpa-onnx/blob/master/scripts/kokoro/add-meta-data.py +struct OfflineTtsKokoroModelMetaData { + int32_t sample_rate = 0; + int32_t num_speakers = 0; + int32_t version = 1; + int32_t has_espeak = 1; + int32_t max_token_len = 0; +}; + +} // namespace sherpa_onnx + +#endif // SHERPA_ONNX_CSRC_OFFLINE_TTS_KOKORO_MODEL_META_DATA_H_ diff --git a/sherpa-onnx/csrc/offline-tts-kokoro-model.cc b/sherpa-onnx/csrc/offline-tts-kokoro-model.cc new file mode 100644 index 000000000..7f7c90131 --- /dev/null +++ b/sherpa-onnx/csrc/offline-tts-kokoro-model.cc @@ -0,0 +1,251 @@ +// sherpa-onnx/csrc/offline-tts-kokoro-model.cc +// +// Copyright (c) 2025 Xiaomi Corporation + +#include "sherpa-onnx/csrc/offline-tts-kokoro-model.h" + +#include +#include +#include +#include + +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + +#include "sherpa-onnx/csrc/macros.h" +#include "sherpa-onnx/csrc/onnx-utils.h" +#include "sherpa-onnx/csrc/session.h" +#include "sherpa-onnx/csrc/text-utils.h" + +namespace sherpa_onnx { + +class OfflineTtsKokoroModel::Impl { + public: + explicit Impl(const OfflineTtsModelConfig &config) + : config_(config), + env_(ORT_LOGGING_LEVEL_ERROR), + sess_opts_(GetSessionOptions(config)), + allocator_{} { + auto model_buf = ReadFile(config.kokoro.model); + auto voices_buf = ReadFile(config.kokoro.voices); + Init(model_buf.data(), model_buf.size(), voices_buf.data(), + voices_buf.size()); + } + + template + Impl(Manager *mgr, const OfflineTtsModelConfig &config) + : config_(config), + env_(ORT_LOGGING_LEVEL_ERROR), + sess_opts_(GetSessionOptions(config)), + allocator_{} { + auto model_buf = ReadFile(mgr, config.kokoro.model); + auto voices_buf = ReadFile(mgr, config.kokoro.voices); + Init(model_buf.data(), model_buf.size(), voices_buf.data(), + voices_buf.size()); + } + + const OfflineTtsKokoroModelMetaData &GetMetaData() const { + return meta_data_; + } + + Ort::Value Run(Ort::Value x, int32_t sid, float speed) { + auto memory_info = + Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault); + + std::vector x_shape = x.GetTensorTypeAndShapeInfo().GetShape(); + if (x_shape[0] != 1) { + SHERPA_ONNX_LOGE("Support only batch_size == 1. Given: %d", + static_cast(x_shape[0])); + exit(-1); + } + + // there is a 0 at the front and end of x + int32_t len = static_cast(x_shape[1]) - 2; + int32_t num_speakers = meta_data_.num_speakers; + int32_t dim0 = style_dim_[0]; + int32_t dim1 = style_dim_[2]; + if (len >= dim0) { + SHERPA_ONNX_LOGE("Bad things happened! %d vs %d", len, dim0); + SHERPA_ONNX_EXIT(-1); + } + + /*const*/ float *p = styles_.data() + sid * dim0 * dim1 + len * dim1; + + std::array style_embedding_shape = {1, dim1}; + Ort::Value style_embedding = Ort::Value::CreateTensor( + memory_info, p, dim1, style_embedding_shape.data(), + style_embedding_shape.size()); + + int64_t speed_shape = 1; + + Ort::Value speed_tensor = + Ort::Value::CreateTensor(memory_info, &speed, 1, &speed_shape, 1); + + std::array inputs = { + std::move(x), std::move(style_embedding), std::move(speed_tensor)}; + + auto out = + sess_->Run({}, input_names_ptr_.data(), inputs.data(), inputs.size(), + output_names_ptr_.data(), output_names_ptr_.size()); + + return std::move(out[0]); + } + + private: + void Init(void *model_data, size_t model_data_length, const char *voices_data, + size_t voices_data_length) { + sess_ = std::make_unique(env_, model_data, model_data_length, + sess_opts_); + + GetInputNames(sess_.get(), &input_names_, &input_names_ptr_); + + GetOutputNames(sess_.get(), &output_names_, &output_names_ptr_); + // get meta data + Ort::ModelMetadata meta_data = sess_->GetModelMetadata(); + if (config_.debug) { + std::ostringstream os; + os << "---kokoro model---\n"; + PrintModelMetadata(os, meta_data); + + os << "----------input names----------\n"; + int32_t i = 0; + for (const auto &s : input_names_) { + os << i << " " << s << "\n"; + ++i; + } + os << "----------output names----------\n"; + i = 0; + for (const auto &s : output_names_) { + os << i << " " << s << "\n"; + ++i; + } + +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s\n", os.str().c_str()); +#else + SHERPA_ONNX_LOGE("%s\n", os.str().c_str()); +#endif + } + + Ort::AllocatorWithDefaultOptions allocator; // used in the macro below + SHERPA_ONNX_READ_META_DATA(meta_data_.sample_rate, "sample_rate"); + SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(meta_data_.version, "version", 1); + SHERPA_ONNX_READ_META_DATA(meta_data_.num_speakers, "n_speakers"); + SHERPA_ONNX_READ_META_DATA(meta_data_.has_espeak, "has_espeak"); + + if (config_.debug) { + std::vector speaker_names; + SHERPA_ONNX_READ_META_DATA_VEC_STRING(speaker_names, "speaker_names"); + std::ostringstream os; + os << "\n"; + for (int32_t i = 0; i != speaker_names.size(); ++i) { + os << i << "->" << speaker_names[i] << ", "; + } + os << "\n"; + +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s\n", os.str().c_str()); +#else + SHERPA_ONNX_LOGE("%s\n", os.str().c_str()); +#endif + } + + SHERPA_ONNX_READ_META_DATA_VEC(style_dim_, "style_dim"); + if (style_dim_.size() != 3) { + SHERPA_ONNX_LOGE("style_dim should be 3-d, given: %d", + static_cast(style_dim_.size())); + SHERPA_ONNX_EXIT(-1); + } + + if (style_dim_[1] != 1) { + SHERPA_ONNX_LOGE("style_dim[0] should be 1, given: %d", style_dim_[1]); + SHERPA_ONNX_EXIT(-1); + } + + int32_t actual_num_floats = voices_data_length / sizeof(float); + int32_t expected_num_floats = + style_dim_[0] * style_dim_[2] * meta_data_.num_speakers; + + if (actual_num_floats != expected_num_floats) { +#if __OHOS__ + SHERPA_ONNX_LOGE( + "Corrupted --kokoro-voices '%{public}s'. Expected #floats: " + "%{public}d, actual: %{public}d", + config_.kokoro.voices.c_str(), expected_num_floats, + actual_num_floats); +#else + SHERPA_ONNX_LOGE( + "Corrupted --kokoro-voices '%s'. Expected #floats: %d, actual: %d", + config_.kokoro.voices.c_str(), expected_num_floats, + actual_num_floats); +#endif + + SHERPA_ONNX_EXIT(-1); + } + + styles_ = std::vector( + reinterpret_cast(voices_data), + reinterpret_cast(voices_data) + expected_num_floats); + + meta_data_.max_token_len = style_dim_[0]; + } + + private: + OfflineTtsModelConfig config_; + Ort::Env env_; + Ort::SessionOptions sess_opts_; + Ort::AllocatorWithDefaultOptions allocator_; + + std::unique_ptr sess_; + + std::vector input_names_; + std::vector input_names_ptr_; + + std::vector output_names_; + std::vector output_names_ptr_; + + OfflineTtsKokoroModelMetaData meta_data_; + std::vector style_dim_; + + // (num_speakers, style_dim_[0], style_dim_[2]) + std::vector styles_; +}; + +OfflineTtsKokoroModel::OfflineTtsKokoroModel( + const OfflineTtsModelConfig &config) + : impl_(std::make_unique(config)) {} + +template +OfflineTtsKokoroModel::OfflineTtsKokoroModel( + Manager *mgr, const OfflineTtsModelConfig &config) + : impl_(std::make_unique(mgr, config)) {} + +OfflineTtsKokoroModel::~OfflineTtsKokoroModel() = default; + +const OfflineTtsKokoroModelMetaData &OfflineTtsKokoroModel::GetMetaData() + const { + return impl_->GetMetaData(); +} + +Ort::Value OfflineTtsKokoroModel::Run(Ort::Value x, int64_t sid /*= 0*/, + float speed /*= 1.0*/) const { + return impl_->Run(std::move(x), sid, speed); +} + +#if __ANDROID_API__ >= 9 +template OfflineTtsKokoroModel::OfflineTtsKokoroModel( + AAssetManager *mgr, const OfflineTtsModelConfig &config); +#endif + +#if __OHOS__ +template OfflineTtsKokoroModel::OfflineTtsKokoroModel( + NativeResourceManager *mgr, const OfflineTtsModelConfig &config); +#endif + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-tts-kokoro-model.h b/sherpa-onnx/csrc/offline-tts-kokoro-model.h new file mode 100644 index 000000000..694f27f77 --- /dev/null +++ b/sherpa-onnx/csrc/offline-tts-kokoro-model.h @@ -0,0 +1,39 @@ +// sherpa-onnx/csrc/offline-tts-kokoro-model.h +// +// Copyright (c) 2025 Xiaomi Corporation + +#ifndef SHERPA_ONNX_CSRC_OFFLINE_TTS_KOKORO_MODEL_H_ +#define SHERPA_ONNX_CSRC_OFFLINE_TTS_KOKORO_MODEL_H_ + +#include +#include + +#include "onnxruntime_cxx_api.h" // NOLINT +#include "sherpa-onnx/csrc/offline-tts-kokoro-model-meta-data.h" +#include "sherpa-onnx/csrc/offline-tts-model-config.h" + +namespace sherpa_onnx { + +class OfflineTtsKokoroModel { + public: + ~OfflineTtsKokoroModel(); + + explicit OfflineTtsKokoroModel(const OfflineTtsModelConfig &config); + + template + OfflineTtsKokoroModel(Manager *mgr, const OfflineTtsModelConfig &config); + + // Return a float32 tensor containing the mel + // of shape (batch_size, mel_dim, num_frames) + Ort::Value Run(Ort::Value x, int64_t sid = 0, float speed = 1.0) const; + + const OfflineTtsKokoroModelMetaData &GetMetaData() const; + + private: + class Impl; + std::unique_ptr impl_; +}; + +} // namespace sherpa_onnx + +#endif // SHERPA_ONNX_CSRC_OFFLINE_TTS_KOKORO_MODEL_H_ diff --git a/sherpa-onnx/csrc/offline-tts-matcha-model-metadata.h b/sherpa-onnx/csrc/offline-tts-matcha-model-meta-data.h similarity index 66% rename from sherpa-onnx/csrc/offline-tts-matcha-model-metadata.h rename to sherpa-onnx/csrc/offline-tts-matcha-model-meta-data.h index c5cee9465..06e91011c 100644 --- a/sherpa-onnx/csrc/offline-tts-matcha-model-metadata.h +++ b/sherpa-onnx/csrc/offline-tts-matcha-model-meta-data.h @@ -1,9 +1,9 @@ -// sherpa-onnx/csrc/offline-tts-matcha-model-metadata.h +// sherpa-onnx/csrc/offline-tts-matcha-model-meta-data.h // // Copyright (c) 2023 Xiaomi Corporation -#ifndef SHERPA_ONNX_CSRC_OFFLINE_TTS_MATCHA_MODEL_METADATA_H_ -#define SHERPA_ONNX_CSRC_OFFLINE_TTS_MATCHA_MODEL_METADATA_H_ +#ifndef SHERPA_ONNX_CSRC_OFFLINE_TTS_MATCHA_MODEL_META_DATA_H_ +#define SHERPA_ONNX_CSRC_OFFLINE_TTS_MATCHA_MODEL_META_DATA_H_ #include #include @@ -25,4 +25,4 @@ struct OfflineTtsMatchaModelMetaData { } // namespace sherpa_onnx -#endif // SHERPA_ONNX_CSRC_OFFLINE_TTS_MATCHA_MODEL_METADATA_H_ +#endif // SHERPA_ONNX_CSRC_OFFLINE_TTS_MATCHA_MODEL_META_DATA_H_ diff --git a/sherpa-onnx/csrc/offline-tts-matcha-model.h b/sherpa-onnx/csrc/offline-tts-matcha-model.h index 5b02ec9b3..27ddaffdb 100644 --- a/sherpa-onnx/csrc/offline-tts-matcha-model.h +++ b/sherpa-onnx/csrc/offline-tts-matcha-model.h @@ -9,7 +9,7 @@ #include #include "onnxruntime_cxx_api.h" // NOLINT -#include "sherpa-onnx/csrc/offline-tts-matcha-model-metadata.h" +#include "sherpa-onnx/csrc/offline-tts-matcha-model-meta-data.h" #include "sherpa-onnx/csrc/offline-tts-model-config.h" namespace sherpa_onnx { diff --git a/sherpa-onnx/csrc/offline-tts-model-config.cc b/sherpa-onnx/csrc/offline-tts-model-config.cc index 4af179a4b..d2153b7e8 100644 --- a/sherpa-onnx/csrc/offline-tts-model-config.cc +++ b/sherpa-onnx/csrc/offline-tts-model-config.cc @@ -11,6 +11,7 @@ namespace sherpa_onnx { void OfflineTtsModelConfig::Register(ParseOptions *po) { vits.Register(po); matcha.Register(po); + kokoro.Register(po); po->Register("num-threads", &num_threads, "Number of threads to run the neural network"); @@ -32,7 +33,11 @@ bool OfflineTtsModelConfig::Validate() const { return vits.Validate(); } - return matcha.Validate(); + if (!matcha.acoustic_model.empty()) { + return matcha.Validate(); + } + + return kokoro.Validate(); } std::string OfflineTtsModelConfig::ToString() const { @@ -41,6 +46,7 @@ std::string OfflineTtsModelConfig::ToString() const { os << "OfflineTtsModelConfig("; os << "vits=" << vits.ToString() << ", "; os << "matcha=" << matcha.ToString() << ", "; + os << "kokoro=" << kokoro.ToString() << ", "; os << "num_threads=" << num_threads << ", "; os << "debug=" << (debug ? "True" : "False") << ", "; os << "provider=\"" << provider << "\")"; diff --git a/sherpa-onnx/csrc/offline-tts-model-config.h b/sherpa-onnx/csrc/offline-tts-model-config.h index 232686960..ce07cbd91 100644 --- a/sherpa-onnx/csrc/offline-tts-model-config.h +++ b/sherpa-onnx/csrc/offline-tts-model-config.h @@ -7,6 +7,7 @@ #include +#include "sherpa-onnx/csrc/offline-tts-kokoro-model-config.h" #include "sherpa-onnx/csrc/offline-tts-matcha-model-config.h" #include "sherpa-onnx/csrc/offline-tts-vits-model-config.h" #include "sherpa-onnx/csrc/parse-options.h" @@ -16,6 +17,7 @@ namespace sherpa_onnx { struct OfflineTtsModelConfig { OfflineTtsVitsModelConfig vits; OfflineTtsMatchaModelConfig matcha; + OfflineTtsKokoroModelConfig kokoro; int32_t num_threads = 1; bool debug = false; @@ -25,10 +27,12 @@ struct OfflineTtsModelConfig { OfflineTtsModelConfig(const OfflineTtsVitsModelConfig &vits, const OfflineTtsMatchaModelConfig &matcha, + const OfflineTtsKokoroModelConfig &kokoro, int32_t num_threads, bool debug, const std::string &provider) : vits(vits), matcha(matcha), + kokoro(kokoro), num_threads(num_threads), debug(debug), provider(provider) {} diff --git a/sherpa-onnx/csrc/offline-tts-vits-model-metadata.h b/sherpa-onnx/csrc/offline-tts-vits-model-meta-data.h similarity index 80% rename from sherpa-onnx/csrc/offline-tts-vits-model-metadata.h rename to sherpa-onnx/csrc/offline-tts-vits-model-meta-data.h index 5ce00d745..3019d17d4 100644 --- a/sherpa-onnx/csrc/offline-tts-vits-model-metadata.h +++ b/sherpa-onnx/csrc/offline-tts-vits-model-meta-data.h @@ -1,9 +1,9 @@ -// sherpa-onnx/csrc/offline-tts-vits-model-metadata.h +// sherpa-onnx/csrc/offline-tts-vits-model-meta-data.h // // Copyright (c) 2023 Xiaomi Corporation -#ifndef SHERPA_ONNX_CSRC_OFFLINE_TTS_VITS_MODEL_METADATA_H_ -#define SHERPA_ONNX_CSRC_OFFLINE_TTS_VITS_MODEL_METADATA_H_ +#ifndef SHERPA_ONNX_CSRC_OFFLINE_TTS_VITS_MODEL_META_DATA_H_ +#define SHERPA_ONNX_CSRC_OFFLINE_TTS_VITS_MODEL_META_DATA_H_ #include #include @@ -46,4 +46,4 @@ struct OfflineTtsVitsModelMetaData { } // namespace sherpa_onnx -#endif // SHERPA_ONNX_CSRC_OFFLINE_TTS_VITS_MODEL_METADATA_H_ +#endif // SHERPA_ONNX_CSRC_OFFLINE_TTS_VITS_MODEL_META_DATA_H_ diff --git a/sherpa-onnx/csrc/offline-tts-vits-model.h b/sherpa-onnx/csrc/offline-tts-vits-model.h index 30e4205dc..90803005f 100644 --- a/sherpa-onnx/csrc/offline-tts-vits-model.h +++ b/sherpa-onnx/csrc/offline-tts-vits-model.h @@ -10,7 +10,7 @@ #include "onnxruntime_cxx_api.h" // NOLINT #include "sherpa-onnx/csrc/offline-tts-model-config.h" -#include "sherpa-onnx/csrc/offline-tts-vits-model-metadata.h" +#include "sherpa-onnx/csrc/offline-tts-vits-model-meta-data.h" namespace sherpa_onnx { diff --git a/sherpa-onnx/csrc/piper-phonemize-lexicon.cc b/sherpa-onnx/csrc/piper-phonemize-lexicon.cc index 982260003..ec312d8da 100644 --- a/sherpa-onnx/csrc/piper-phonemize-lexicon.cc +++ b/sherpa-onnx/csrc/piper-phonemize-lexicon.cc @@ -155,6 +155,36 @@ static std::vector PiperPhonemesToIdsMatcha( return ans; } +static std::vector> PiperPhonemesToIdsKokoro( + const std::unordered_map &token2id, + const std::vector &phonemes, int32_t max_len) { + std::vector> ans; + + std::vector current; + current.reserve(phonemes.size()); + + for (auto p : phonemes) { + if (token2id.count(p)) { + if (current.size() > max_len - 1) { + current.push_back(0); + ans.push_back(std::move(current)); + + current.reserve(phonemes.size()); + current.push_back(0); + } + + current.push_back(token2id.at(p)); + } else { + SHERPA_ONNX_LOGE("Skip unknown phonemes. Unicode codepoint: \\U+%04x.", + static_cast(p)); + } + } + + current.push_back(0); + ans.push_back(std::move(current)); + return ans; +} + static std::vector CoquiPhonemesToIds( const std::unordered_map &token2id, const std::vector &phonemes, @@ -269,6 +299,18 @@ PiperPhonemizeLexicon::PiperPhonemizeLexicon( InitEspeak(data_dir); } +PiperPhonemizeLexicon::PiperPhonemizeLexicon( + const std::string &tokens, const std::string &data_dir, + const OfflineTtsKokoroModelMetaData &kokoro_meta_data) + : kokoro_meta_data_(kokoro_meta_data), is_kokoro_(true) { + { + std::ifstream is(tokens); + token2id_ = ReadTokens(is); + } + + InitEspeak(data_dir); +} + template PiperPhonemizeLexicon::PiperPhonemizeLexicon( Manager *mgr, const std::string &tokens, const std::string &data_dir, @@ -286,10 +328,29 @@ PiperPhonemizeLexicon::PiperPhonemizeLexicon( InitEspeak(data_dir); } +template +PiperPhonemizeLexicon::PiperPhonemizeLexicon( + Manager *mgr, const std::string &tokens, const std::string &data_dir, + const OfflineTtsKokoroModelMetaData &kokoro_meta_data) + : kokoro_meta_data_(kokoro_meta_data), is_kokoro_(true) { + { + auto buf = ReadFile(mgr, tokens); + std::istrstream is(buf.data(), buf.size()); + token2id_ = ReadTokens(is); + } + + // We should copy the directory of espeak-ng-data from the asset to + // some internal or external storage and then pass the directory to + // data_dir. + InitEspeak(data_dir); +} + std::vector PiperPhonemizeLexicon::ConvertTextToTokenIds( const std::string &text, const std::string &voice /*= ""*/) const { if (is_matcha_) { return ConvertTextToTokenIdsMatcha(text, voice); + } else if (is_kokoro_) { + return ConvertTextToTokenIdsKokoro(text, voice); } else { return ConvertTextToTokenIdsVits(text, voice); } @@ -320,6 +381,32 @@ std::vector PiperPhonemizeLexicon::ConvertTextToTokenIdsMatcha( return ans; } +std::vector PiperPhonemizeLexicon::ConvertTextToTokenIdsKokoro( + const std::string &text, const std::string &voice /*= ""*/) const { + piper::eSpeakPhonemeConfig config; + + // ./bin/espeak-ng-bin --path ./install/share/espeak-ng-data/ --voices + // to list available voices + config.voice = voice; // e.g., voice is en-us + + std::vector> phonemes; + + CallPhonemizeEspeak(text, config, &phonemes); + + std::vector ans; + + for (const auto &p : phonemes) { + auto phoneme_ids = + PiperPhonemesToIdsKokoro(token2id_, p, kokoro_meta_data_.max_token_len); + + for (auto &ids : phoneme_ids) { + ans.emplace_back(std::move(ids)); + } + } + + return ans; +} + std::vector PiperPhonemizeLexicon::ConvertTextToTokenIdsVits( const std::string &text, const std::string &voice /*= ""*/) const { piper::eSpeakPhonemeConfig config; @@ -363,6 +450,10 @@ template PiperPhonemizeLexicon::PiperPhonemizeLexicon( template PiperPhonemizeLexicon::PiperPhonemizeLexicon( AAssetManager *mgr, const std::string &tokens, const std::string &data_dir, const OfflineTtsMatchaModelMetaData &matcha_meta_data); + +template PiperPhonemizeLexicon::PiperPhonemizeLexicon( + AAssetManager *mgr, const std::string &tokens, const std::string &data_dir, + const OfflineTtsKokoroModelMetaData &kokoro_meta_data); #endif #if __OHOS__ @@ -375,6 +466,11 @@ template PiperPhonemizeLexicon::PiperPhonemizeLexicon( NativeResourceManager *mgr, const std::string &tokens, const std::string &data_dir, const OfflineTtsMatchaModelMetaData &matcha_meta_data); + +template PiperPhonemizeLexicon::PiperPhonemizeLexicon( + NativeResourceManager *mgr, const std::string &tokens, + const std::string &data_dir, + const OfflineTtsKokoroModelMetaData &kokoro_meta_data); #endif } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/piper-phonemize-lexicon.h b/sherpa-onnx/csrc/piper-phonemize-lexicon.h index f703f0b87..bb8c6e30c 100644 --- a/sherpa-onnx/csrc/piper-phonemize-lexicon.h +++ b/sherpa-onnx/csrc/piper-phonemize-lexicon.h @@ -10,8 +10,9 @@ #include #include "sherpa-onnx/csrc/offline-tts-frontend.h" -#include "sherpa-onnx/csrc/offline-tts-matcha-model-metadata.h" -#include "sherpa-onnx/csrc/offline-tts-vits-model-metadata.h" +#include "sherpa-onnx/csrc/offline-tts-kokoro-model-meta-data.h" +#include "sherpa-onnx/csrc/offline-tts-matcha-model-meta-data.h" +#include "sherpa-onnx/csrc/offline-tts-vits-model-meta-data.h" namespace sherpa_onnx { @@ -23,6 +24,9 @@ class PiperPhonemizeLexicon : public OfflineTtsFrontend { PiperPhonemizeLexicon(const std::string &tokens, const std::string &data_dir, const OfflineTtsMatchaModelMetaData &matcha_meta_data); + PiperPhonemizeLexicon(const std::string &tokens, const std::string &data_dir, + const OfflineTtsKokoroModelMetaData &kokoro_meta_data); + template PiperPhonemizeLexicon(Manager *mgr, const std::string &tokens, const std::string &data_dir, @@ -33,6 +37,11 @@ class PiperPhonemizeLexicon : public OfflineTtsFrontend { const std::string &data_dir, const OfflineTtsMatchaModelMetaData &matcha_meta_data); + template + PiperPhonemizeLexicon(Manager *mgr, const std::string &tokens, + const std::string &data_dir, + const OfflineTtsKokoroModelMetaData &kokoro_meta_data); + std::vector ConvertTextToTokenIds( const std::string &text, const std::string &voice = "") const override; @@ -43,12 +52,17 @@ class PiperPhonemizeLexicon : public OfflineTtsFrontend { std::vector ConvertTextToTokenIdsMatcha( const std::string &text, const std::string &voice = "") const; + std::vector ConvertTextToTokenIdsKokoro( + const std::string &text, const std::string &voice = "") const; + private: // map unicode codepoint to an integer ID std::unordered_map token2id_; OfflineTtsVitsModelMetaData vits_meta_data_; OfflineTtsMatchaModelMetaData matcha_meta_data_; + OfflineTtsKokoroModelMetaData kokoro_meta_data_; bool is_matcha_ = false; + bool is_kokoro_ = false; }; } // namespace sherpa_onnx diff --git a/sherpa-onnx/python/csrc/CMakeLists.txt b/sherpa-onnx/python/csrc/CMakeLists.txt index 38d32de50..a4c15713c 100644 --- a/sherpa-onnx/python/csrc/CMakeLists.txt +++ b/sherpa-onnx/python/csrc/CMakeLists.txt @@ -54,6 +54,7 @@ endif() if(SHERPA_ONNX_ENABLE_TTS) list(APPEND srcs + offline-tts-kokoro-model-config.cc offline-tts-matcha-model-config.cc offline-tts-model-config.cc offline-tts-vits-model-config.cc diff --git a/sherpa-onnx/python/csrc/offline-tts-kokoro-model-config.cc b/sherpa-onnx/python/csrc/offline-tts-kokoro-model-config.cc new file mode 100644 index 000000000..fbb24db5f --- /dev/null +++ b/sherpa-onnx/python/csrc/offline-tts-kokoro-model-config.cc @@ -0,0 +1,31 @@ +// sherpa-onnx/python/csrc/offline-tts-kokoro-model-config.cc +// +// Copyright (c) 2025 Xiaomi Corporation + +#include "sherpa-onnx/python/csrc/offline-tts-kokoro-model-config.h" + +#include + +#include "sherpa-onnx/csrc/offline-tts-kokoro-model-config.h" + +namespace sherpa_onnx { + +void PybindOfflineTtsKokoroModelConfig(py::module *m) { + using PyClass = OfflineTtsKokoroModelConfig; + + py::class_(*m, "OfflineTtsKokoroModelConfig") + .def(py::init<>()) + .def(py::init(), + py::arg("model"), py::arg("voices"), py::arg("tokens"), + py::arg("data_dir"), py::arg("length_scale") = 1.0) + .def_readwrite("model", &PyClass::model) + .def_readwrite("voices", &PyClass::voices) + .def_readwrite("tokens", &PyClass::tokens) + .def_readwrite("data_dir", &PyClass::data_dir) + .def_readwrite("length_scale", &PyClass::length_scale) + .def("__str__", &PyClass::ToString) + .def("validate", &PyClass::Validate); +} + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/python/csrc/offline-tts-kokoro-model-config.h b/sherpa-onnx/python/csrc/offline-tts-kokoro-model-config.h new file mode 100644 index 000000000..cc5f517ab --- /dev/null +++ b/sherpa-onnx/python/csrc/offline-tts-kokoro-model-config.h @@ -0,0 +1,16 @@ +// sherpa-onnx/python/csrc/offline-tts-kokoro-model-config.h +// +// Copyright (c) 2025 Xiaomi Corporation + +#ifndef SHERPA_ONNX_PYTHON_CSRC_OFFLINE_TTS_KOKORO_MODEL_CONFIG_H_ +#define SHERPA_ONNX_PYTHON_CSRC_OFFLINE_TTS_KOKORO_MODEL_CONFIG_H_ + +#include "sherpa-onnx/python/csrc/sherpa-onnx.h" + +namespace sherpa_onnx { + +void PybindOfflineTtsKokoroModelConfig(py::module *m); + +} + +#endif // SHERPA_ONNX_PYTHON_CSRC_OFFLINE_TTS_KOKORO_MODEL_CONFIG_H_ diff --git a/sherpa-onnx/python/csrc/offline-tts-model-config.cc b/sherpa-onnx/python/csrc/offline-tts-model-config.cc index ed6a6e090..99769957d 100644 --- a/sherpa-onnx/python/csrc/offline-tts-model-config.cc +++ b/sherpa-onnx/python/csrc/offline-tts-model-config.cc @@ -7,6 +7,7 @@ #include #include "sherpa-onnx/csrc/offline-tts-model-config.h" +#include "sherpa-onnx/python/csrc/offline-tts-kokoro-model-config.h" #include "sherpa-onnx/python/csrc/offline-tts-matcha-model-config.h" #include "sherpa-onnx/python/csrc/offline-tts-vits-model-config.h" @@ -15,20 +16,24 @@ namespace sherpa_onnx { void PybindOfflineTtsModelConfig(py::module *m) { PybindOfflineTtsVitsModelConfig(m); PybindOfflineTtsMatchaModelConfig(m); + PybindOfflineTtsKokoroModelConfig(m); using PyClass = OfflineTtsModelConfig; py::class_(*m, "OfflineTtsModelConfig") .def(py::init<>()) .def(py::init(), py::arg("vits") = OfflineTtsVitsModelConfig{}, py::arg("matcha") = OfflineTtsMatchaModelConfig{}, + py::arg("kokoro") = OfflineTtsKokoroModelConfig{}, py::arg("num_threads") = 1, py::arg("debug") = false, py::arg("provider") = "cpu") .def_readwrite("vits", &PyClass::vits) .def_readwrite("matcha", &PyClass::matcha) + .def_readwrite("kokoro", &PyClass::kokoro) .def_readwrite("num_threads", &PyClass::num_threads) .def_readwrite("debug", &PyClass::debug) .def_readwrite("provider", &PyClass::provider) diff --git a/sherpa-onnx/python/sherpa_onnx/__init__.py b/sherpa-onnx/python/sherpa_onnx/__init__.py index 330c8d2df..5eeeffa56 100644 --- a/sherpa-onnx/python/sherpa_onnx/__init__.py +++ b/sherpa-onnx/python/sherpa_onnx/__init__.py @@ -20,6 +20,7 @@ OfflineStream, OfflineTts, OfflineTtsConfig, + OfflineTtsKokoroModelConfig, OfflineTtsMatchaModelConfig, OfflineTtsModelConfig, OfflineTtsVitsModelConfig, From af671e2b630929766672d6f34b256c829736882a Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 16 Jan 2025 15:07:26 +0800 Subject: [PATCH 156/183] Add C API for Kokoro TTS models (#1717) --- .github/workflows/c-api.yaml | 26 +++++++ .github/workflows/cxx-api.yaml | 27 ++++++++ .gitignore | 1 + c-api-examples/CMakeLists.txt | 3 + c-api-examples/kokoro-tts-en-c-api.c | 84 +++++++++++++++++++++++ cxx-api-examples/CMakeLists.txt | 3 + cxx-api-examples/kokoro-tts-en-cxx-api.cc | 73 ++++++++++++++++++++ sherpa-onnx/c-api/c-api.cc | 12 ++++ sherpa-onnx/c-api/c-api.h | 10 +++ sherpa-onnx/c-api/cxx-api.cc | 6 ++ sherpa-onnx/c-api/cxx-api.h | 10 +++ 11 files changed, 255 insertions(+) create mode 100644 c-api-examples/kokoro-tts-en-c-api.c create mode 100644 cxx-api-examples/kokoro-tts-en-cxx-api.cc diff --git a/.github/workflows/c-api.yaml b/.github/workflows/c-api.yaml index 3f1776412..58820dc06 100644 --- a/.github/workflows/c-api.yaml +++ b/.github/workflows/c-api.yaml @@ -79,6 +79,32 @@ jobs: otool -L ./install/lib/libsherpa-onnx-c-api.dylib fi + - name: Test Kokoro TTS (en) + shell: bash + run: | + gcc -o kokoro-tts-en-c-api ./c-api-examples/kokoro-tts-en-c-api.c \ + -I ./build/install/include \ + -L ./build/install/lib/ \ + -l sherpa-onnx-c-api \ + -l onnxruntime + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 + tar xf kokoro-en-v0_19.tar.bz2 + rm kokoro-en-v0_19.tar.bz2 + + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH + export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH + + ./kokoro-tts-en-c-api + + rm ./kokoro-tts-en-c-api + rm -rf kokoro-en-* + + - uses: actions/upload-artifact@v4 + with: + name: kokoro-tts-${{ matrix.os }} + path: ./generated-kokoro-*.wav + - name: Test Matcha TTS (zh) shell: bash run: | diff --git a/.github/workflows/cxx-api.yaml b/.github/workflows/cxx-api.yaml index e2108ed03..2f6a3b2e6 100644 --- a/.github/workflows/cxx-api.yaml +++ b/.github/workflows/cxx-api.yaml @@ -81,6 +81,33 @@ jobs: otool -L ./install/lib/libsherpa-onnx-cxx-api.dylib fi + - name: Test Kokoro TTS (en) + shell: bash + run: | + g++ -std=c++17 -o kokoro-tts-en-cxx-api ./cxx-api-examples/kokoro-tts-en-cxx-api.cc \ + -I ./build/install/include \ + -L ./build/install/lib/ \ + -l sherpa-onnx-cxx-api \ + -l sherpa-onnx-c-api \ + -l onnxruntime + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 + tar xf kokoro-en-v0_19.tar.bz2 + rm kokoro-en-v0_19.tar.bz2 + + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH + export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH + + ./kokoro-tts-en-cxx-api + + rm kokoro-tts-en-cxx-api + rm -rf kokoro-en-* + + - uses: actions/upload-artifact@v4 + with: + name: kokoro-tts-${{ matrix.os }} + path: ./generated-kokoro-*.wav + - name: Test Matcha TTS (zh) shell: bash run: | diff --git a/.gitignore b/.gitignore index e5226f0e1..da3399c5e 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,4 @@ harmony-os/SherpaOnnxHar/sherpa_onnx/LICENSE harmony-os/SherpaOnnxHar/sherpa_onnx/CHANGELOG.md matcha-icefall-zh-baker matcha-icefall-en_US-ljspeech +kokoro-en-v0_19 diff --git a/c-api-examples/CMakeLists.txt b/c-api-examples/CMakeLists.txt index 36d7b4e42..a2bfb6fdd 100644 --- a/c-api-examples/CMakeLists.txt +++ b/c-api-examples/CMakeLists.txt @@ -13,6 +13,9 @@ if(SHERPA_ONNX_ENABLE_TTS) add_executable(matcha-tts-en-c-api matcha-tts-en-c-api.c) target_link_libraries(matcha-tts-en-c-api sherpa-onnx-c-api) + + add_executable(kokoro-tts-en-c-api kokoro-tts-en-c-api.c) + target_link_libraries(kokoro-tts-en-c-api sherpa-onnx-c-api) endif() if(SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION) diff --git a/c-api-examples/kokoro-tts-en-c-api.c b/c-api-examples/kokoro-tts-en-c-api.c new file mode 100644 index 000000000..44e6c28d8 --- /dev/null +++ b/c-api-examples/kokoro-tts-en-c-api.c @@ -0,0 +1,84 @@ +// c-api-examples/kokoro-tts-en-c-api.c +// +// Copyright (c) 2025 Xiaomi Corporation + +// This file shows how to use sherpa-onnx C API +// for English TTS with Kokoro. +// +// clang-format off +/* +Usage + + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 +tar xf kokoro-en-v0_19.tar.bz2 +rm kokoro-en-v0_19.tar.bz2 + +./kokoro-tts-en-c-api + + */ +// clang-format on + +#include +#include +#include + +#include "sherpa-onnx/c-api/c-api.h" + +static int32_t ProgressCallback(const float *samples, int32_t num_samples, + float progress) { + fprintf(stderr, "Progress: %.3f%%\n", progress * 100); + // return 1 to continue generating + // return 0 to stop generating + return 1; +} + +int32_t main(int32_t argc, char *argv[]) { + SherpaOnnxOfflineTtsConfig config; + memset(&config, 0, sizeof(config)); + config.model.kokoro.model = "./kokoro-en-v0_19/model.onnx"; + config.model.kokoro.voices = "./kokoro-en-v0_19/voices.bin"; + config.model.kokoro.tokens = "./kokoro-en-v0_19/tokens.txt"; + config.model.kokoro.data_dir = "./kokoro-en-v0_19/espeak-ng-data"; + + config.model.num_threads = 2; + + // If you don't want to see debug messages, please set it to 0 + config.model.debug = 1; + + const char *filename = "./generated-kokoro-en.wav"; + const char *text = + "Today as always, men fall into two groups: slaves and free men. Whoever " + "does not have two-thirds of his day for himself, is a slave, whatever " + "he may be: a statesman, a businessman, an official, or a scholar. " + "Friends fell out often because life was changing so fast. The easiest " + "thing in the world was to lose touch with someone."; + + const SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTts(&config); + // mapping of sid to voice name + // 0->af, 1->af_bella, 2->af_nicole, 3->af_sarah, 4->af_sky, 5->am_adam + // 6->am_michael, 7->bf_emma, 8->bf_isabella, 9->bm_george, 10->bm_lewis + int32_t sid = 0; + float speed = 1.0; // larger -> faster in speech speed + +#if 0 + // If you don't want to use a callback, then please enable this branch + const SherpaOnnxGeneratedAudio *audio = + SherpaOnnxOfflineTtsGenerate(tts, text, sid, speed); +#else + const SherpaOnnxGeneratedAudio *audio = + SherpaOnnxOfflineTtsGenerateWithProgressCallback(tts, text, sid, speed, + ProgressCallback); +#endif + + SherpaOnnxWriteWave(audio->samples, audio->n, audio->sample_rate, filename); + + SherpaOnnxDestroyOfflineTtsGeneratedAudio(audio); + SherpaOnnxDestroyOfflineTts(tts); + + fprintf(stderr, "Input text is: %s\n", text); + fprintf(stderr, "Speaker ID is is: %d\n", sid); + fprintf(stderr, "Saved to: %s\n", filename); + + return 0; +} diff --git a/cxx-api-examples/CMakeLists.txt b/cxx-api-examples/CMakeLists.txt index dd61d3294..040925c29 100644 --- a/cxx-api-examples/CMakeLists.txt +++ b/cxx-api-examples/CMakeLists.txt @@ -21,4 +21,7 @@ if(SHERPA_ONNX_ENABLE_TTS) add_executable(matcha-tts-en-cxx-api ./matcha-tts-en-cxx-api.cc) target_link_libraries(matcha-tts-en-cxx-api sherpa-onnx-cxx-api) + + add_executable(kokoro-tts-en-cxx-api ./kokoro-tts-en-cxx-api.cc) + target_link_libraries(kokoro-tts-en-cxx-api sherpa-onnx-cxx-api) endif() diff --git a/cxx-api-examples/kokoro-tts-en-cxx-api.cc b/cxx-api-examples/kokoro-tts-en-cxx-api.cc new file mode 100644 index 000000000..ccd5bb1db --- /dev/null +++ b/cxx-api-examples/kokoro-tts-en-cxx-api.cc @@ -0,0 +1,73 @@ +// cxx-api-examples/kokoro-tts-en-cxx-api.c +// +// Copyright (c) 2025 Xiaomi Corporation + +// This file shows how to use sherpa-onnx CXX API +// for Chinese TTS with Kokoro. +// +// clang-format off +/* +Usage + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 +tar xf kokoro-en-v0_19.tar.bz2 +rm kokoro-en-v0_19.tar.bz2 + +./kokoro-tts-en-cxx-api + + */ +// clang-format on + +#include + +#include "sherpa-onnx/c-api/cxx-api.h" + +static int32_t ProgressCallback(const float *samples, int32_t num_samples, + float progress, void *arg) { + fprintf(stderr, "Progress: %.3f%%\n", progress * 100); + // return 1 to continue generating + // return 0 to stop generating + return 1; +} + +int32_t main(int32_t argc, char *argv[]) { + using namespace sherpa_onnx::cxx; // NOLINT + OfflineTtsConfig config; + + config.model.kokoro.model = "./kokoro-en-v0_19/model.onnx"; + config.model.kokoro.voices = "./kokoro-en-v0_19/voices.bin"; + config.model.kokoro.tokens = "./kokoro-en-v0_19/tokens.txt"; + config.model.kokoro.data_dir = "./kokoro-en-v0_19/espeak-ng-data"; + + config.model.num_threads = 2; + + // If you don't want to see debug messages, please set it to 0 + config.model.debug = 1; + + std::string filename = "./generated-kokoro-en-cxx.wav"; + std::string text = + "Today as always, men fall into two groups: slaves and free men. Whoever " + "does not have two-thirds of his day for himself, is a slave, whatever " + "he may be: a statesman, a businessman, an official, or a scholar. " + "Friends fell out often because life was changing so fast. The easiest " + "thing in the world was to lose touch with someone."; + + auto tts = OfflineTts::Create(config); + int32_t sid = 0; + float speed = 1.0; // larger -> faster in speech speed + +#if 0 + // If you don't want to use a callback, then please enable this branch + GeneratedAudio audio = tts.Generate(text, sid, speed); +#else + GeneratedAudio audio = tts.Generate(text, sid, speed, ProgressCallback); +#endif + + WriteWave(filename, {audio.samples, audio.sample_rate}); + + fprintf(stderr, "Input text is: %s\n", text.c_str()); + fprintf(stderr, "Speaker ID is is: %d\n", sid); + fprintf(stderr, "Saved to: %s\n", filename.c_str()); + + return 0; +} diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index 625f608a9..da7c73178 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -1092,6 +1092,18 @@ static sherpa_onnx::OfflineTtsConfig GetOfflineTtsConfig( tts_config.model.matcha.dict_dir = SHERPA_ONNX_OR(config->model.matcha.dict_dir, ""); + // kokoro + tts_config.model.kokoro.model = + SHERPA_ONNX_OR(config->model.kokoro.model, ""); + tts_config.model.kokoro.voices = + SHERPA_ONNX_OR(config->model.kokoro.voices, ""); + tts_config.model.kokoro.tokens = + SHERPA_ONNX_OR(config->model.kokoro.tokens, ""); + tts_config.model.kokoro.data_dir = + SHERPA_ONNX_OR(config->model.kokoro.data_dir, ""); + tts_config.model.kokoro.length_scale = + SHERPA_ONNX_OR(config->model.kokoro.length_scale, 1.0); + tts_config.model.num_threads = SHERPA_ONNX_OR(config->model.num_threads, 1); tts_config.model.debug = config->model.debug; tts_config.model.provider = SHERPA_ONNX_OR(config->model.provider, "cpu"); diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index a669b50dc..db8321781 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -910,12 +910,22 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOfflineTtsMatchaModelConfig { const char *dict_dir; } SherpaOnnxOfflineTtsMatchaModelConfig; +SHERPA_ONNX_API typedef struct SherpaOnnxOfflineTtsKokoroModelConfig { + const char *model; + const char *voices; + const char *tokens; + const char *data_dir; + + float length_scale; // < 1, faster in speech speed; > 1, slower in speed +} SherpaOnnxOfflineTtsKokoroModelConfig; + SHERPA_ONNX_API typedef struct SherpaOnnxOfflineTtsModelConfig { SherpaOnnxOfflineTtsVitsModelConfig vits; int32_t num_threads; int32_t debug; const char *provider; SherpaOnnxOfflineTtsMatchaModelConfig matcha; + SherpaOnnxOfflineTtsKokoroModelConfig kokoro; } SherpaOnnxOfflineTtsModelConfig; SHERPA_ONNX_API typedef struct SherpaOnnxOfflineTtsConfig { diff --git a/sherpa-onnx/c-api/cxx-api.cc b/sherpa-onnx/c-api/cxx-api.cc index 0b19e112e..50a1f4e1b 100644 --- a/sherpa-onnx/c-api/cxx-api.cc +++ b/sherpa-onnx/c-api/cxx-api.cc @@ -338,6 +338,12 @@ OfflineTts OfflineTts::Create(const OfflineTtsConfig &config) { c.model.matcha.length_scale = config.model.matcha.length_scale; c.model.matcha.dict_dir = config.model.matcha.dict_dir.c_str(); + c.model.kokoro.model = config.model.kokoro.model.c_str(); + c.model.kokoro.voices = config.model.kokoro.voices.c_str(); + c.model.kokoro.tokens = config.model.kokoro.tokens.c_str(); + c.model.kokoro.data_dir = config.model.kokoro.data_dir.c_str(); + c.model.kokoro.length_scale = config.model.kokoro.length_scale; + c.model.num_threads = config.model.num_threads; c.model.debug = config.model.debug; c.model.provider = config.model.provider.c_str(); diff --git a/sherpa-onnx/c-api/cxx-api.h b/sherpa-onnx/c-api/cxx-api.h index 12932f3f2..ce65a9ec7 100644 --- a/sherpa-onnx/c-api/cxx-api.h +++ b/sherpa-onnx/c-api/cxx-api.h @@ -338,9 +338,19 @@ struct OfflineTtsMatchaModelConfig { float length_scale = 1.0; // < 1, faster in speed; > 1, slower in speed }; +struct OfflineTtsKokoroModelConfig { + std::string model; + std::string voices; + std::string tokens; + std::string data_dir; + + float length_scale = 1.0; // < 1, faster in speed; > 1, slower in speed +}; + struct OfflineTtsModelConfig { OfflineTtsVitsModelConfig vits; OfflineTtsMatchaModelConfig matcha; + OfflineTtsKokoroModelConfig kokoro; int32_t num_threads = 1; bool debug = false; std::string provider = "cpu"; From 2d0869c7091c8621ba738e8fd0aaa04e66499eb0 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 16 Jan 2025 15:43:51 +0800 Subject: [PATCH 157/183] Fix style issues (#1718) --- sherpa-onnx/csrc/provider-config.h | 52 ++++++++++++++---------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/sherpa-onnx/csrc/provider-config.h b/sherpa-onnx/csrc/provider-config.h index fdc875e0a..4c9c0db01 100644 --- a/sherpa-onnx/csrc/provider-config.h +++ b/sherpa-onnx/csrc/provider-config.h @@ -7,9 +7,9 @@ #include -#include "sherpa-onnx/csrc/parse-options.h" -#include "sherpa-onnx/csrc/macros.h" #include "onnxruntime_cxx_api.h" // NOLINT +#include "sherpa-onnx/csrc/macros.h" +#include "sherpa-onnx/csrc/parse-options.h" namespace sherpa_onnx { @@ -40,25 +40,23 @@ struct TensorrtConfig { TensorrtConfig() = default; TensorrtConfig(int64_t trt_max_workspace_size, - int32_t trt_max_partition_iterations, - int32_t trt_min_subgraph_size, - bool trt_fp16_enable, - bool trt_detailed_build_log, - bool trt_engine_cache_enable, - bool trt_timing_cache_enable, - const std::string &trt_engine_cache_path, - const std::string &trt_timing_cache_path, - bool trt_dump_subgraphs) + int32_t trt_max_partition_iterations, + int32_t trt_min_subgraph_size, bool trt_fp16_enable, + bool trt_detailed_build_log, bool trt_engine_cache_enable, + bool trt_timing_cache_enable, + const std::string &trt_engine_cache_path, + const std::string &trt_timing_cache_path, + bool trt_dump_subgraphs) : trt_max_workspace_size(trt_max_workspace_size), - trt_max_partition_iterations(trt_max_partition_iterations), - trt_min_subgraph_size(trt_min_subgraph_size), - trt_fp16_enable(trt_fp16_enable), - trt_detailed_build_log(trt_detailed_build_log), - trt_engine_cache_enable(trt_engine_cache_enable), - trt_timing_cache_enable(trt_timing_cache_enable), - trt_engine_cache_path(trt_engine_cache_path), - trt_timing_cache_path(trt_timing_cache_path), - trt_dump_subgraphs(trt_dump_subgraphs) {} + trt_max_partition_iterations(trt_max_partition_iterations), + trt_min_subgraph_size(trt_min_subgraph_size), + trt_fp16_enable(trt_fp16_enable), + trt_detailed_build_log(trt_detailed_build_log), + trt_engine_cache_enable(trt_engine_cache_enable), + trt_timing_cache_enable(trt_timing_cache_enable), + trt_engine_cache_path(trt_engine_cache_path), + trt_timing_cache_path(trt_timing_cache_path), + trt_dump_subgraphs(trt_dump_subgraphs) {} void Register(ParseOptions *po); bool Validate() const; @@ -74,15 +72,15 @@ struct ProviderConfig { // device only used for cuda and trt ProviderConfig() = default; - ProviderConfig(const std::string &provider, - int32_t device) + ProviderConfig(const std::string &provider, int32_t device) : provider(provider), device(device) {} ProviderConfig(const TensorrtConfig &trt_config, - const CudaConfig &cuda_config, - const std::string &provider, - int32_t device) - : trt_config(trt_config), cuda_config(cuda_config), - provider(provider), device(device) {} + const CudaConfig &cuda_config, const std::string &provider, + int32_t device) + : trt_config(trt_config), + cuda_config(cuda_config), + provider(provider), + device(device) {} void Register(ParseOptions *po); bool Validate() const; From cc812e6237095da5f1376765a63bb099ced54e0f Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 16 Jan 2025 16:30:10 +0800 Subject: [PATCH 158/183] Add C# API for Kokoro TTS models (#1720) --- .github/scripts/test-dot-net.sh | 9 +- dotnet-examples/kokoro-tts-play/Program.cs | 189 ++++++++++++++++++ .../kokoro-tts-play/kokoro-tts-play.csproj | 19 ++ .../kokoro-tts-play/run-kokoro-en.sh | 10 + dotnet-examples/kokoro-tts/Program.cs | 70 +++++++ dotnet-examples/kokoro-tts/kokoro-tts.csproj | 15 ++ dotnet-examples/kokoro-tts/run-kokoro-en.sh | 10 + dotnet-examples/sherpa-onnx.sln | 12 ++ scripts/dotnet/OfflineTts.cs | 14 ++ scripts/dotnet/OfflineTtsKokoroModelConfig.cs | 33 +++ scripts/dotnet/OfflineTtsModelConfig.cs | 2 + 11 files changed, 381 insertions(+), 2 deletions(-) create mode 100644 dotnet-examples/kokoro-tts-play/Program.cs create mode 100644 dotnet-examples/kokoro-tts-play/kokoro-tts-play.csproj create mode 100755 dotnet-examples/kokoro-tts-play/run-kokoro-en.sh create mode 100644 dotnet-examples/kokoro-tts/Program.cs create mode 100644 dotnet-examples/kokoro-tts/kokoro-tts.csproj create mode 100755 dotnet-examples/kokoro-tts/run-kokoro-en.sh create mode 100644 scripts/dotnet/OfflineTtsKokoroModelConfig.cs diff --git a/.github/scripts/test-dot-net.sh b/.github/scripts/test-dot-net.sh index 7c339e157..f120653c6 100755 --- a/.github/scripts/test-dot-net.sh +++ b/.github/scripts/test-dot-net.sh @@ -2,7 +2,11 @@ cd dotnet-examples/ -cd ./offline-tts +cd ./kokoro-tts +./run-kokoro-en.sh +ls -lh + +cd ../offline-tts ./run-matcha-zh.sh ls -lh *.wav ./run-matcha-en.sh @@ -19,7 +23,8 @@ pushd ../.. mkdir tts -cp dotnet-examples/offline-tts/*.wav ./tts +cp -v dotnet-examples/kokoro-tts/*.wav ./tts +cp -v dotnet-examples/offline-tts/*.wav ./tts popd cd ../offline-speaker-diarization diff --git a/dotnet-examples/kokoro-tts-play/Program.cs b/dotnet-examples/kokoro-tts-play/Program.cs new file mode 100644 index 000000000..eea22cc2f --- /dev/null +++ b/dotnet-examples/kokoro-tts-play/Program.cs @@ -0,0 +1,189 @@ +// Copyright (c) 2025 Xiaomi Corporation +// +// This file shows how to use a non-streaming Kokoro TTS model +// for text-to-speech +// Please refer to +// https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html +// and +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models +// to download pre-trained models +using PortAudioSharp; +using SherpaOnnx; +using System.Collections.Concurrent; +using System.Runtime.InteropServices; + +class OfflineTtsDemo +{ + static void Main(string[] args) + { + var config = new OfflineTtsConfig(); + config.Model.Kokoro.Model = "./kokoro-en-v0_19/model.onnx"; + config.Model.Kokoro.Voices = "./kokoro-en-v0_19/voices.bin"; + config.Model.Kokoro.Tokens = "./kokoro-en-v0_19/tokens.txt"; + config.Model.Kokoro.DataDir = "./kokoro-en-v0_19/espeak-ng-data"; + + config.Model.NumThreads = 2; + config.Model.Debug = 1; + config.Model.Provider = "cpu"; + + var tts = new OfflineTts(config); + var speed = 1.0f; + var text = "Today as always, men fall into two groups: slaves and free men. Whoever " + + "does not have two-thirds of his day for himself, is a slave, whatever " + + "he may be: a statesman, a businessman, an official, or a scholar. " + + "Friends fell out often because life was changing so fast. The easiest " + + "thing in the world was to lose touch with someone."; + + // mapping of sid to voice name + // 0->af, 1->af_bella, 2->af_nicole, 3->af_sarah, 4->af_sky, 5->am_adam + // 6->am_michael, 7->bf_emma, 8->bf_isabella, 9->bm_george, 10->bm_lewis + var sid = 0; + + + Console.WriteLine(PortAudio.VersionInfo.versionText); + PortAudio.Initialize(); + Console.WriteLine($"Number of devices: {PortAudio.DeviceCount}"); + + for (int i = 0; i != PortAudio.DeviceCount; ++i) + { + Console.WriteLine($" Device {i}"); + DeviceInfo deviceInfo = PortAudio.GetDeviceInfo(i); + Console.WriteLine($" Name: {deviceInfo.name}"); + Console.WriteLine($" Max output channels: {deviceInfo.maxOutputChannels}"); + Console.WriteLine($" Default sample rate: {deviceInfo.defaultSampleRate}"); + } + int deviceIndex = PortAudio.DefaultOutputDevice; + if (deviceIndex == PortAudio.NoDevice) + { + Console.WriteLine("No default output device found. Please use ../offline-tts instead"); + Environment.Exit(1); + } + + var info = PortAudio.GetDeviceInfo(deviceIndex); + Console.WriteLine(); + Console.WriteLine($"Use output default device {deviceIndex} ({info.name})"); + + var param = new StreamParameters(); + param.device = deviceIndex; + param.channelCount = 1; + param.sampleFormat = SampleFormat.Float32; + param.suggestedLatency = info.defaultLowOutputLatency; + param.hostApiSpecificStreamInfo = IntPtr.Zero; + + // https://learn.microsoft.com/en-us/dotnet/standard/collections/thread-safe/blockingcollection-overview + var dataItems = new BlockingCollection(); + + var MyCallback = (IntPtr samples, int n, float progress) => + { + Console.WriteLine($"Progress {progress*100}%"); + + float[] data = new float[n]; + + Marshal.Copy(samples, data, 0, n); + + dataItems.Add(data); + + // 1 means to keep generating + // 0 means to stop generating + return 1; + }; + + var playFinished = false; + + float[]? lastSampleArray = null; + int lastIndex = 0; // not played + + PortAudioSharp.Stream.Callback playCallback = (IntPtr input, IntPtr output, + UInt32 frameCount, + ref StreamCallbackTimeInfo timeInfo, + StreamCallbackFlags statusFlags, + IntPtr userData + ) => + { + if (dataItems.IsCompleted && lastSampleArray == null && lastIndex == 0) + { + Console.WriteLine($"Finished playing"); + playFinished = true; + return StreamCallbackResult.Complete; + } + + int expected = Convert.ToInt32(frameCount); + int i = 0; + + while ((lastSampleArray != null || dataItems.Count != 0) && (i < expected)) + { + int needed = expected - i; + + if (lastSampleArray != null) + { + int remaining = lastSampleArray.Length - lastIndex; + if (remaining >= needed) + { + float[] this_block = lastSampleArray.Skip(lastIndex).Take(needed).ToArray(); + lastIndex += needed; + if (lastIndex == lastSampleArray.Length) + { + lastSampleArray = null; + lastIndex = 0; + } + + Marshal.Copy(this_block, 0, IntPtr.Add(output, i * sizeof(float)), needed); + return StreamCallbackResult.Continue; + } + + float[] this_block2 = lastSampleArray.Skip(lastIndex).Take(remaining).ToArray(); + lastIndex = 0; + lastSampleArray = null; + + Marshal.Copy(this_block2, 0, IntPtr.Add(output, i * sizeof(float)), remaining); + i += remaining; + continue; + } + + if (dataItems.Count != 0) + { + lastSampleArray = dataItems.Take(); + lastIndex = 0; + } + } + + if (i < expected) + { + int sizeInBytes = (expected - i) * 4; + Marshal.Copy(new byte[sizeInBytes], 0, IntPtr.Add(output, i * sizeof(float)), sizeInBytes); + } + + return StreamCallbackResult.Continue; + }; + + PortAudioSharp.Stream stream = new PortAudioSharp.Stream(inParams: null, outParams: param, sampleRate: tts.SampleRate, + framesPerBuffer: 0, + streamFlags: StreamFlags.ClipOff, + callback: playCallback, + userData: IntPtr.Zero + ); + + stream.Start(); + + var callback = new OfflineTtsCallbackProgress(MyCallback); + + var audio = tts.GenerateWithCallbackProgress(text, speed, sid, callback); + var outputFilename = "./generated-kokoro-0.wav"; + var ok = audio.SaveToWaveFile(outputFilename); + + if (ok) + { + Console.WriteLine($"Wrote to {outputFilename} succeeded!"); + } + else + { + Console.WriteLine($"Failed to write {outputFilename}"); + } + dataItems.CompleteAdding(); + + while (!playFinished) + { + Thread.Sleep(100); // 100ms + } + } +} diff --git a/dotnet-examples/kokoro-tts-play/kokoro-tts-play.csproj b/dotnet-examples/kokoro-tts-play/kokoro-tts-play.csproj new file mode 100644 index 000000000..6c725686c --- /dev/null +++ b/dotnet-examples/kokoro-tts-play/kokoro-tts-play.csproj @@ -0,0 +1,19 @@ + + + + Exe + net8.0 + kokoro_tts_play + enable + enable + + + + + + + + + + + diff --git a/dotnet-examples/kokoro-tts-play/run-kokoro-en.sh b/dotnet-examples/kokoro-tts-play/run-kokoro-en.sh new file mode 100755 index 000000000..08bdc693a --- /dev/null +++ b/dotnet-examples/kokoro-tts-play/run-kokoro-en.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -ex + +if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 + tar xf kokoro-en-v0_19.tar.bz2 + rm kokoro-en-v0_19.tar.bz2 +fi + +dotnet run diff --git a/dotnet-examples/kokoro-tts/Program.cs b/dotnet-examples/kokoro-tts/Program.cs new file mode 100644 index 000000000..61792683a --- /dev/null +++ b/dotnet-examples/kokoro-tts/Program.cs @@ -0,0 +1,70 @@ +// Copyright (c) 2025 Xiaomi Corporation +// +// This file shows how to use a non-streaming Kokoro TTS model +// for text-to-speech +// Please refer to +// https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html +// and +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models +// to download pre-trained models +using SherpaOnnx; +using System.Runtime.InteropServices; + +class OfflineTtsDemo +{ + static void Main(string[] args) + { + var config = new OfflineTtsConfig(); + config.Model.Kokoro.Model = "./kokoro-en-v0_19/model.onnx"; + config.Model.Kokoro.Voices = "./kokoro-en-v0_19/voices.bin"; + config.Model.Kokoro.Tokens = "./kokoro-en-v0_19/tokens.txt"; + config.Model.Kokoro.DataDir = "./kokoro-en-v0_19/espeak-ng-data"; + + config.Model.NumThreads = 2; + config.Model.Debug = 1; + config.Model.Provider = "cpu"; + + var tts = new OfflineTts(config); + var speed = 1.0f; + var text = "Today as always, men fall into two groups: slaves and free men. Whoever " + + "does not have two-thirds of his day for himself, is a slave, whatever " + + "he may be: a statesman, a businessman, an official, or a scholar. " + + "Friends fell out often because life was changing so fast. The easiest " + + "thing in the world was to lose touch with someone."; + + // mapping of sid to voice name + // 0->af, 1->af_bella, 2->af_nicole, 3->af_sarah, 4->af_sky, 5->am_adam + // 6->am_michael, 7->bf_emma, 8->bf_isabella, 9->bm_george, 10->bm_lewis + var sid = 0; + + var MyCallback = (IntPtr samples, int n, float progress) => + { + float[] data = new float[n]; + Marshal.Copy(samples, data, 0, n); + // You can process samples here, e.g., play them. + // See ../kokoro-tts-playback for how to play them + Console.WriteLine($"Progress {progress*100}%"); + + // 1 means to keep generating + // 0 means to stop generating + return 1; + }; + + var callback = new OfflineTtsCallbackProgress(MyCallback); + + var audio = tts.GenerateWithCallbackProgress(text, speed, sid, callback); + + var outputFilename = "./generated-kokoro-0.wav"; + var ok = audio.SaveToWaveFile(outputFilename); + + if (ok) + { + Console.WriteLine($"Wrote to {outputFilename} succeeded!"); + } + else + { + Console.WriteLine($"Failed to write {outputFilename}"); + } + } +} + diff --git a/dotnet-examples/kokoro-tts/kokoro-tts.csproj b/dotnet-examples/kokoro-tts/kokoro-tts.csproj new file mode 100644 index 000000000..132819c6f --- /dev/null +++ b/dotnet-examples/kokoro-tts/kokoro-tts.csproj @@ -0,0 +1,15 @@ + + + + Exe + net8.0 + kokoro_tts + enable + enable + + + + + + + diff --git a/dotnet-examples/kokoro-tts/run-kokoro-en.sh b/dotnet-examples/kokoro-tts/run-kokoro-en.sh new file mode 100755 index 000000000..08bdc693a --- /dev/null +++ b/dotnet-examples/kokoro-tts/run-kokoro-en.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -ex + +if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 + tar xf kokoro-en-v0_19.tar.bz2 + rm kokoro-en-v0_19.tar.bz2 +fi + +dotnet run diff --git a/dotnet-examples/sherpa-onnx.sln b/dotnet-examples/sherpa-onnx.sln index 1ebcdf464..404c49762 100644 --- a/dotnet-examples/sherpa-onnx.sln +++ b/dotnet-examples/sherpa-onnx.sln @@ -31,6 +31,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "keyword-spotting-from-micro EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "offline-speaker-diarization", "offline-speaker-diarization\offline-speaker-diarization.csproj", "{D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "kokoro-tts", "kokoro-tts\kokoro-tts.csproj", "{9C0ABE6C-1F54-42B5-804E-C3FED6668F52}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "kokoro-tts-play", "kokoro-tts-play\kokoro-tts-play.csproj", "{EC0BCEAB-1B4E-4129-82CE-9880426AFA0B}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -93,6 +97,14 @@ Global {D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}.Debug|Any CPU.Build.0 = Debug|Any CPU {D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}.Release|Any CPU.ActiveCfg = Release|Any CPU {D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}.Release|Any CPU.Build.0 = Release|Any CPU + {9C0ABE6C-1F54-42B5-804E-C3FED6668F52}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9C0ABE6C-1F54-42B5-804E-C3FED6668F52}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9C0ABE6C-1F54-42B5-804E-C3FED6668F52}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9C0ABE6C-1F54-42B5-804E-C3FED6668F52}.Release|Any CPU.Build.0 = Release|Any CPU + {EC0BCEAB-1B4E-4129-82CE-9880426AFA0B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EC0BCEAB-1B4E-4129-82CE-9880426AFA0B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EC0BCEAB-1B4E-4129-82CE-9880426AFA0B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EC0BCEAB-1B4E-4129-82CE-9880426AFA0B}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/scripts/dotnet/OfflineTts.cs b/scripts/dotnet/OfflineTts.cs index e4d29733c..334ba2287 100644 --- a/scripts/dotnet/OfflineTts.cs +++ b/scripts/dotnet/OfflineTts.cs @@ -7,6 +7,7 @@ namespace SherpaOnnx { // IntPtr is actually a `const float*` from C++ public delegate int OfflineTtsCallback(IntPtr samples, int n); + public delegate int OfflineTtsCallbackProgress(IntPtr samples, int n, float progress); public class OfflineTts : IDisposable { @@ -36,6 +37,16 @@ public OfflineTtsGeneratedAudio GenerateWithCallback(String text, float speed, i return new OfflineTtsGeneratedAudio(p); } + public OfflineTtsGeneratedAudio GenerateWithCallbackProgress(String text, float speed, int speakerId, OfflineTtsCallbackProgress callback) + { + byte[] utf8Bytes = Encoding.UTF8.GetBytes(text); + byte[] utf8BytesWithNull = new byte[utf8Bytes.Length + 1]; // +1 for null terminator + Array.Copy(utf8Bytes, utf8BytesWithNull, utf8Bytes.Length); + utf8BytesWithNull[utf8Bytes.Length] = 0; // Null terminator + IntPtr p = SherpaOnnxOfflineTtsGenerateWithProgressCallback(_handle.Handle, utf8BytesWithNull, speakerId, speed, callback); + return new OfflineTtsGeneratedAudio(p); + } + public void Dispose() { Cleanup(); @@ -92,5 +103,8 @@ public int NumSpeakers [DllImport(Dll.Filename, CallingConvention = CallingConvention.Cdecl)] private static extern IntPtr SherpaOnnxOfflineTtsGenerateWithCallback(IntPtr handle, [MarshalAs(UnmanagedType.LPArray, ArraySubType = UnmanagedType.I1)] byte[] utf8Text, int sid, float speed, OfflineTtsCallback callback); + + [DllImport(Dll.Filename, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr SherpaOnnxOfflineTtsGenerateWithProgressCallback(IntPtr handle, [MarshalAs(UnmanagedType.LPArray, ArraySubType = UnmanagedType.I1)] byte[] utf8Text, int sid, float speed, OfflineTtsCallbackProgress callback); } } diff --git a/scripts/dotnet/OfflineTtsKokoroModelConfig.cs b/scripts/dotnet/OfflineTtsKokoroModelConfig.cs new file mode 100644 index 000000000..18fd60daa --- /dev/null +++ b/scripts/dotnet/OfflineTtsKokoroModelConfig.cs @@ -0,0 +1,33 @@ +/// Copyright (c) 2025 Xiaomi Corporation (authors: Fangjun Kuang) + +using System.Runtime.InteropServices; + +namespace SherpaOnnx +{ + [StructLayout(LayoutKind.Sequential)] + public struct OfflineTtsKokoroModelConfig + { + public OfflineTtsKokoroModelConfig() + { + Model = ""; + Voices = ""; + Tokens = ""; + DataDir = ""; + + LengthScale = 1.0F; + } + [MarshalAs(UnmanagedType.LPStr)] + public string Model; + + [MarshalAs(UnmanagedType.LPStr)] + public string Voices; + + [MarshalAs(UnmanagedType.LPStr)] + public string Tokens; + + [MarshalAs(UnmanagedType.LPStr)] + public string DataDir; + + public float LengthScale; + } +} diff --git a/scripts/dotnet/OfflineTtsModelConfig.cs b/scripts/dotnet/OfflineTtsModelConfig.cs index e5caa1173..9b1ec5506 100644 --- a/scripts/dotnet/OfflineTtsModelConfig.cs +++ b/scripts/dotnet/OfflineTtsModelConfig.cs @@ -12,6 +12,7 @@ public OfflineTtsModelConfig() { Vits = new OfflineTtsVitsModelConfig(); Matcha = new OfflineTtsMatchaModelConfig(); + Kokoro = new OfflineTtsKokoroModelConfig(); NumThreads = 1; Debug = 0; Provider = "cpu"; @@ -24,5 +25,6 @@ public OfflineTtsModelConfig() public string Provider; public OfflineTtsMatchaModelConfig Matcha; + public OfflineTtsKokoroModelConfig Kokoro; } } From ad61ad6ff52b8a0ad220d7d2a6959a7d9db341e1 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 16 Jan 2025 16:47:37 +0800 Subject: [PATCH 159/183] Add Swift API for Kokoro TTS models (#1721) --- .github/scripts/test-swift.sh | 4 ++ swift-api-examples/.gitignore | 1 + swift-api-examples/SherpaOnnx.swift | 28 +++++++++-- swift-api-examples/run-tts-kokoro-en.sh | 37 ++++++++++++++ swift-api-examples/run-tts-matcha-en.sh | 2 +- swift-api-examples/run-tts-matcha-zh.sh | 2 +- swift-api-examples/run-tts-vits.sh | 2 +- swift-api-examples/tts-kokoro-en.swift | 65 +++++++++++++++++++++++++ 8 files changed, 134 insertions(+), 7 deletions(-) create mode 100755 swift-api-examples/run-tts-kokoro-en.sh create mode 100644 swift-api-examples/tts-kokoro-en.swift diff --git a/.github/scripts/test-swift.sh b/.github/scripts/test-swift.sh index f333bc0a9..9ab682d20 100755 --- a/.github/scripts/test-swift.sh +++ b/.github/scripts/test-swift.sh @@ -11,6 +11,10 @@ ls -lh ls -lh rm -rf vits-piper-* +./run-tts-kokoro-en.sh +ls -lh +rm -rf kokoro-en-* + ./run-tts-matcha-zh.sh ls -lh rm -rf matcha-icefall-* diff --git a/swift-api-examples/.gitignore b/swift-api-examples/.gitignore index 1a90488aa..5bcf31489 100644 --- a/swift-api-examples/.gitignore +++ b/swift-api-examples/.gitignore @@ -12,3 +12,4 @@ keyword-spotting-from-file add-punctuations tts-matcha-zh tts-matcha-en +tts-kokoro-en diff --git a/swift-api-examples/SherpaOnnx.swift b/swift-api-examples/SherpaOnnx.swift index 6d11a0011..d4c39645f 100644 --- a/swift-api-examples/SherpaOnnx.swift +++ b/swift-api-examples/SherpaOnnx.swift @@ -736,7 +736,8 @@ func sherpaOnnxOfflineTtsVitsModelConfig( noise_scale: noiseScale, noise_scale_w: noiseScaleW, length_scale: lengthScale, - dict_dir: toCPointer(dictDir)) + dict_dir: toCPointer(dictDir) + ) } func sherpaOnnxOfflineTtsMatchaModelConfig( @@ -757,12 +758,30 @@ func sherpaOnnxOfflineTtsMatchaModelConfig( data_dir: toCPointer(dataDir), noise_scale: noiseScale, length_scale: lengthScale, - dict_dir: toCPointer(dictDir)) + dict_dir: toCPointer(dictDir) + ) +} + +func sherpaOnnxOfflineTtsKokoroModelConfig( + model: String = "", + voices: String = "", + tokens: String = "", + dataDir: String = "", + lengthScale: Float = 1.0 +) -> SherpaOnnxOfflineTtsKokoroModelConfig { + return SherpaOnnxOfflineTtsKokoroModelConfig( + model: toCPointer(model), + voices: toCPointer(voices), + tokens: toCPointer(tokens), + data_dir: toCPointer(dataDir), + length_scale: lengthScale + ) } func sherpaOnnxOfflineTtsModelConfig( vits: SherpaOnnxOfflineTtsVitsModelConfig = sherpaOnnxOfflineTtsVitsModelConfig(), matcha: SherpaOnnxOfflineTtsMatchaModelConfig = sherpaOnnxOfflineTtsMatchaModelConfig(), + kokoro: SherpaOnnxOfflineTtsKokoroModelConfig = sherpaOnnxOfflineTtsKokoroModelConfig(), numThreads: Int = 1, debug: Int = 0, provider: String = "cpu" @@ -772,7 +791,8 @@ func sherpaOnnxOfflineTtsModelConfig( num_threads: Int32(numThreads), debug: Int32(debug), provider: toCPointer(provider), - matcha: matcha + matcha: matcha, + kokoro: kokoro ) } @@ -780,7 +800,7 @@ func sherpaOnnxOfflineTtsConfig( model: SherpaOnnxOfflineTtsModelConfig, ruleFsts: String = "", ruleFars: String = "", - maxNumSentences: Int = 2 + maxNumSentences: Int = 1 ) -> SherpaOnnxOfflineTtsConfig { return SherpaOnnxOfflineTtsConfig( model: model, diff --git a/swift-api-examples/run-tts-kokoro-en.sh b/swift-api-examples/run-tts-kokoro-en.sh new file mode 100755 index 000000000..2c7408d8b --- /dev/null +++ b/swift-api-examples/run-tts-kokoro-en.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -ex + +if [ ! -d ../build-swift-macos ]; then + echo "Please run ../build-swift-macos.sh first!" + exit 1 +fi + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html +# to download more models +if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 + tar xf kokoro-en-v0_19.tar.bz2 + rm kokoro-en-v0_19.tar.bz2 +fi + +if [ ! -e ./tts-kokoro-en ]; then + # Note: We use -lc++ to link against libc++ instead of libstdc++ + swiftc \ + -lc++ \ + -I ../build-swift-macos/install/include \ + -import-objc-header ./SherpaOnnx-Bridging-Header.h \ + ./tts-kokoro-en.swift ./SherpaOnnx.swift \ + -L ../build-swift-macos/install/lib/ \ + -l sherpa-onnx \ + -l onnxruntime \ + -o tts-kokoro-en + + strip tts-kokoro-en +else + echo "./tts-kokoro-en exists - skip building" +fi + +export DYLD_LIBRARY_PATH=$PWD/../build-swift-macos/install/lib:$DYLD_LIBRARY_PATH +./tts-kokoro-en diff --git a/swift-api-examples/run-tts-matcha-en.sh b/swift-api-examples/run-tts-matcha-en.sh index 1f23f56ec..f472b090b 100755 --- a/swift-api-examples/run-tts-matcha-en.sh +++ b/swift-api-examples/run-tts-matcha-en.sh @@ -21,7 +21,7 @@ if [ ! -f ./hifigan_v2.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx fi -if [ ! -e ./tts ]; then +if [ ! -e ./tts-matcha-en ]; then # Note: We use -lc++ to link against libc++ instead of libstdc++ swiftc \ -lc++ \ diff --git a/swift-api-examples/run-tts-matcha-zh.sh b/swift-api-examples/run-tts-matcha-zh.sh index decbbde4a..5d4f75c1e 100755 --- a/swift-api-examples/run-tts-matcha-zh.sh +++ b/swift-api-examples/run-tts-matcha-zh.sh @@ -20,7 +20,7 @@ if [ ! -f ./hifigan_v2.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx fi -if [ ! -e ./tts ]; then +if [ ! -e ./tts-matcha-zh ]; then # Note: We use -lc++ to link against libc++ instead of libstdc++ swiftc \ -lc++ \ diff --git a/swift-api-examples/run-tts-vits.sh b/swift-api-examples/run-tts-vits.sh index 4f385bd72..b4722c0ac 100755 --- a/swift-api-examples/run-tts-vits.sh +++ b/swift-api-examples/run-tts-vits.sh @@ -15,7 +15,7 @@ if [ ! -d ./vits-piper-en_US-amy-low ]; then rm vits-piper-en_US-amy-low.tar.bz2 fi -if [ ! -e ./tts ]; then +if [ ! -e ./tts-vits ]; then # Note: We use -lc++ to link against libc++ instead of libstdc++ swiftc \ -lc++ \ diff --git a/swift-api-examples/tts-kokoro-en.swift b/swift-api-examples/tts-kokoro-en.swift new file mode 100644 index 000000000..a0459cf8d --- /dev/null +++ b/swift-api-examples/tts-kokoro-en.swift @@ -0,0 +1,65 @@ +class MyClass { + func playSamples(samples: [Float]) { + print("Play \(samples.count) samples") + } +} + +func run() { + let model = "./kokoro-en-v0_19/model.onnx" + let voices = "./kokoro-en-v0_19/voices.bin" + let tokens = "./kokoro-en-v0_19/tokens.txt" + let dataDir = "./kokoro-en-v0_19/espeak-ng-data" + let kokoro = sherpaOnnxOfflineTtsKokoroModelConfig( + model: model, + voices: voices, + tokens: tokens, + dataDir: dataDir + ) + let modelConfig = sherpaOnnxOfflineTtsModelConfig(kokoro: kokoro, debug: 0) + var ttsConfig = sherpaOnnxOfflineTtsConfig(model: modelConfig) + + let myClass = MyClass() + + // We use Unretained here so myClass must be kept alive as the callback is invoked + // + // See also + // https://medium.com/codex/swift-c-callback-interoperability-6d57da6c8ee6 + let arg = Unmanaged.passUnretained(myClass).toOpaque() + + let callback: TtsCallbackWithArg = { samples, n, arg in + let o = Unmanaged.fromOpaque(arg!).takeUnretainedValue() + var savedSamples: [Float] = [] + for index in 0.. Date: Thu, 16 Jan 2025 17:35:31 +0800 Subject: [PATCH 160/183] Add Go API for Kokoro TTS models (#1722) --- .github/workflows/test-go-package.yaml | 5 ++++ .github/workflows/test-go.yaml | 5 ++++ go-api-examples/non-streaming-tts/main.go | 6 +++++ .../non-streaming-tts/run-kokoro-en.sh | 21 ++++++++++++++++ .../non-streaming-tts/run-kokoro-en.sh | 1 + scripts/go/sherpa_onnx.go | 24 +++++++++++++++++++ 6 files changed, 62 insertions(+) create mode 100755 go-api-examples/non-streaming-tts/run-kokoro-en.sh create mode 120000 scripts/go/_internal/non-streaming-tts/run-kokoro-en.sh diff --git a/.github/workflows/test-go-package.yaml b/.github/workflows/test-go-package.yaml index 95c868b8f..f13d7eb74 100644 --- a/.github/workflows/test-go-package.yaml +++ b/.github/workflows/test-go-package.yaml @@ -209,6 +209,11 @@ jobs: go build ls -lh + echo "Test kokoro en" + ./run-kokoro-en.sh + rm -rf kokoro-en-* + ls -lh + echo "Test matcha zh" ./run-matcha-zh.sh rm -rf matcha-icefall-* diff --git a/.github/workflows/test-go.yaml b/.github/workflows/test-go.yaml index 083f01d6b..c3df3d75e 100644 --- a/.github/workflows/test-go.yaml +++ b/.github/workflows/test-go.yaml @@ -224,6 +224,11 @@ jobs: go build ls -lh + echo "Test kokoro en" + ./run-kokoro-en.sh + rm -rf kokoro-en-* + ls -lh + echo "Test matcha zh" ./run-matcha-zh.sh rm -rf matcha-icefall-* diff --git a/go-api-examples/non-streaming-tts/main.go b/go-api-examples/non-streaming-tts/main.go index 73638d8f4..f3df7f105 100644 --- a/go-api-examples/non-streaming-tts/main.go +++ b/go-api-examples/non-streaming-tts/main.go @@ -33,6 +33,12 @@ func main() { flag.Float32Var(&config.Model.Matcha.NoiseScale, "matcha-noise-scale", 0.667, "noise_scale for Matcha") flag.Float32Var(&config.Model.Matcha.LengthScale, "matcha-length-scale", 1.0, "length_scale for Matcha. small -> faster in speech speed; large -> slower") + flag.StringVar(&config.Model.Kokoro.Model, "kokoro-model", "", "Path to the Kokoro ONNX model") + flag.StringVar(&config.Model.Kokoro.Voices, "kokoro-voices", "", "Path to voices.bin for Kokoro") + flag.StringVar(&config.Model.Kokoro.Tokens, "kokoro-tokens", "", "Path to tokens.txt for Kokoro") + flag.StringVar(&config.Model.Kokoro.DataDir, "kokoro-data-dir", "", "Path to espeak-ng-data for Kokoro") + flag.Float32Var(&config.Model.Kokoro.LengthScale, "kokoro-length-scale", 1.0, "length_scale for Kokoro. small -> faster in speech speed; large -> slower") + flag.IntVar(&config.Model.NumThreads, "num-threads", 1, "Number of threads for computing") flag.IntVar(&config.Model.Debug, "debug", 0, "Whether to show debug message") flag.StringVar(&config.Model.Provider, "provider", "cpu", "Provider to use") diff --git a/go-api-examples/non-streaming-tts/run-kokoro-en.sh b/go-api-examples/non-streaming-tts/run-kokoro-en.sh new file mode 100755 index 000000000..a7d356d1c --- /dev/null +++ b/go-api-examples/non-streaming-tts/run-kokoro-en.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -ex + +if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 + tar xf kokoro-en-v0_19.tar.bz2 + rm kokoro-en-v0_19.tar.bz2 +fi + +go mod tidy +go build + +./non-streaming-tts \ + --kokoro-model=./kokoro-en-v0_19/model.onnx \ + --kokoro-voices=./kokoro-en-v0_19/voices.bin \ + --kokoro-tokens=./kokoro-en-v0_19/tokens.txt \ + --kokoro-data-dir=./kokoro-en-v0_19/espeak-ng-data \ + --debug=1 \ + --output-filename=./test-kokoro-en.wav \ + "Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone." diff --git a/scripts/go/_internal/non-streaming-tts/run-kokoro-en.sh b/scripts/go/_internal/non-streaming-tts/run-kokoro-en.sh new file mode 120000 index 000000000..43687f274 --- /dev/null +++ b/scripts/go/_internal/non-streaming-tts/run-kokoro-en.sh @@ -0,0 +1 @@ +../../../../go-api-examples/non-streaming-tts/run-kokoro-en.sh \ No newline at end of file diff --git a/scripts/go/sherpa_onnx.go b/scripts/go/sherpa_onnx.go index 763752840..4da12cfb4 100644 --- a/scripts/go/sherpa_onnx.go +++ b/scripts/go/sherpa_onnx.go @@ -682,9 +682,18 @@ type OfflineTtsMatchaModelConfig struct { DictDir string // Path to dict directory for jieba (used only in Chinese tts) } +type OfflineTtsKokoroModelConfig struct { + Model string // Path to the model for kokoro + Voices string // Path to the voices.bin for kokoro + Tokens string // Path to tokens.txt + DataDir string // Path to espeak-ng-data directory + LengthScale float32 // Please use 1.0 in general. Smaller -> Faster speech speed. Larger -> Slower speech speed +} + type OfflineTtsModelConfig struct { Vits OfflineTtsVitsModelConfig Matcha OfflineTtsMatchaModelConfig + Kokoro OfflineTtsKokoroModelConfig // Number of threads to use for neural network computation NumThreads int @@ -776,6 +785,21 @@ func NewOfflineTts(config *OfflineTtsConfig) *OfflineTts { c.model.matcha.dict_dir = C.CString(config.Model.Matcha.DictDir) defer C.free(unsafe.Pointer(c.model.matcha.dict_dir)) + // kokoro + c.model.kokoro.model = C.CString(config.Model.Kokoro.Model) + defer C.free(unsafe.Pointer(c.model.kokoro.model)) + + c.model.kokoro.voices = C.CString(config.Model.Kokoro.Voices) + defer C.free(unsafe.Pointer(c.model.kokoro.voices)) + + c.model.kokoro.tokens = C.CString(config.Model.Kokoro.Tokens) + defer C.free(unsafe.Pointer(c.model.kokoro.tokens)) + + c.model.kokoro.data_dir = C.CString(config.Model.Kokoro.DataDir) + defer C.free(unsafe.Pointer(c.model.kokoro.data_dir)) + + c.model.kokoro.length_scale = C.float(config.Model.Kokoro.LengthScale) + c.model.num_threads = C.int(config.Model.NumThreads) c.model.debug = C.int(config.Model.Debug) From 4335e2accdbde0fded9443591ff81853cb6d3544 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 16 Jan 2025 17:58:19 +0800 Subject: [PATCH 161/183] Add Dart API for Kokoro TTS models (#1723) --- .github/scripts/test-dart.sh | 1 + dart-api-examples/tts/bin/kokoro-en.dart | 86 +++++++++++++++++++ dart-api-examples/tts/run-kokoro-en.sh | 27 ++++++ .../lib/src/sherpa_onnx_bindings.dart | 11 +++ flutter/sherpa_onnx/lib/src/tts.dart | 38 +++++++- 5 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 dart-api-examples/tts/bin/kokoro-en.dart create mode 100755 dart-api-examples/tts/run-kokoro-en.sh diff --git a/.github/scripts/test-dart.sh b/.github/scripts/test-dart.sh index 6ba747652..2392ca00c 100755 --- a/.github/scripts/test-dart.sh +++ b/.github/scripts/test-dart.sh @@ -7,6 +7,7 @@ cd dart-api-examples pushd tts echo '----------matcha tts----------' +./run-kokoro-en.sh ./run-matcha-zh.sh ./run-matcha-en.sh ls -lh *.wav diff --git a/dart-api-examples/tts/bin/kokoro-en.dart b/dart-api-examples/tts/bin/kokoro-en.dart new file mode 100644 index 000000000..b92d92883 --- /dev/null +++ b/dart-api-examples/tts/bin/kokoro-en.dart @@ -0,0 +1,86 @@ +// Copyright (c) 2025 Xiaomi Corporation +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; + +import './init.dart'; + +void main(List arguments) async { + await initSherpaOnnx(); + + final parser = ArgParser() + ..addOption('model', help: 'Path to the onnx model') + ..addOption('voices', help: 'Path to the voices.bin') + ..addOption('tokens', help: 'Path to tokens.txt') + ..addOption( + 'data-dir', + help: 'Path to espeak-ng-data directory', + defaultsTo: '', + ) + ..addOption('rule-fsts', help: 'Path to rule fsts', defaultsTo: '') + ..addOption('rule-fars', help: 'Path to rule fars', defaultsTo: '') + ..addOption('text', help: 'Text to generate TTS for') + ..addOption('output-wav', help: 'Filename to save the generated audio') + ..addOption('speed', help: 'Speech speed', defaultsTo: '1.0') + ..addOption( + 'sid', + help: 'Speaker ID to select. Used only for multi-speaker TTS', + defaultsTo: '0', + ); + final res = parser.parse(arguments); + if (res['model'] == null || + res['voices'] == null || + res['tokens'] == null || + res['data-dir'] == null || + res['output-wav'] == null || + res['text'] == null) { + print(parser.usage); + exit(1); + } + final model = res['model'] as String; + final voices = res['voices'] as String; + final tokens = res['tokens'] as String; + final dataDir = res['data-dir'] as String; + final ruleFsts = res['rule-fsts'] as String; + final ruleFars = res['rule-fars'] as String; + final text = res['text'] as String; + final outputWav = res['output-wav'] as String; + var speed = double.tryParse(res['speed'] as String) ?? 1.0; + final sid = int.tryParse(res['sid'] as String) ?? 0; + + if (speed == 0) { + speed = 1.0; + } + + final kokoro = sherpa_onnx.OfflineTtsKokoroModelConfig( + model: model, + voices: voices, + tokens: tokens, + dataDir: dataDir, + lengthScale: 1 / speed, + ); + + final modelConfig = sherpa_onnx.OfflineTtsModelConfig( + kokoro: kokoro, + numThreads: 1, + debug: true, + ); + final config = sherpa_onnx.OfflineTtsConfig( + model: modelConfig, + maxNumSenetences: 1, + ruleFsts: ruleFsts, + ruleFars: ruleFars, + ); + + final tts = sherpa_onnx.OfflineTts(config); + final audio = tts.generate(text: text, sid: sid, speed: speed); + tts.free(); + + sherpa_onnx.writeWave( + filename: outputWav, + samples: audio.samples, + sampleRate: audio.sampleRate, + ); + print('Saved to $outputWav'); +} diff --git a/dart-api-examples/tts/run-kokoro-en.sh b/dart-api-examples/tts/run-kokoro-en.sh new file mode 100755 index 000000000..7db694e63 --- /dev/null +++ b/dart-api-examples/tts/run-kokoro-en.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -ex + +dart pub get + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html +# to download more models +if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 + tar xf kokoro-en-v0_19.tar.bz2 + rm kokoro-en-v0_19.tar.bz2 +fi + +dart run \ + ./bin/kokoro-en.dart \ + --model ./kokoro-en-v0_19/model.onnx \ + --voices ./kokoro-en-v0_19/voices.bin \ + --tokens ./kokoro-en-v0_19/tokens.txt \ + --data-dir ./kokoro-en-v0_19/espeak-ng-data \ + --sid 9 \ + --speed 1.0 \ + --output-wav kokoro-en-9.wav \ + --text "Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone." \ + +ls -lh *.wav diff --git a/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart b/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart index 7baf53f26..1e41d0918 100644 --- a/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart +++ b/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart @@ -147,6 +147,16 @@ final class SherpaOnnxOfflineTtsMatchaModelConfig extends Struct { external Pointer dictDir; } +final class SherpaOnnxOfflineTtsKokoroModelConfig extends Struct { + external Pointer model; + external Pointer voices; + external Pointer tokens; + external Pointer dataDir; + + @Float() + external double lengthScale; +} + final class SherpaOnnxOfflineTtsModelConfig extends Struct { external SherpaOnnxOfflineTtsVitsModelConfig vits; @Int32() @@ -157,6 +167,7 @@ final class SherpaOnnxOfflineTtsModelConfig extends Struct { external Pointer provider; external SherpaOnnxOfflineTtsMatchaModelConfig matcha; + external SherpaOnnxOfflineTtsKokoroModelConfig kokoro; } final class SherpaOnnxOfflineTtsConfig extends Struct { diff --git a/flutter/sherpa_onnx/lib/src/tts.dart b/flutter/sherpa_onnx/lib/src/tts.dart index b5dcda48d..fdaf8edcd 100644 --- a/flutter/sherpa_onnx/lib/src/tts.dart +++ b/flutter/sherpa_onnx/lib/src/tts.dart @@ -60,10 +60,32 @@ class OfflineTtsMatchaModelConfig { final String dictDir; } +class OfflineTtsKokoroModelConfig { + const OfflineTtsKokoroModelConfig({ + this.model = '', + this.voices = '', + this.tokens = '', + this.dataDir = '', + this.lengthScale = 1.0, + }); + + @override + String toString() { + return 'OfflineTtsKokoroModelConfig(model: $model, voices: $voices, tokens: $tokens, dataDir: $dataDir, lengthScale: $lengthScale)'; + } + + final String model; + final String voices; + final String tokens; + final String dataDir; + final double lengthScale; +} + class OfflineTtsModelConfig { const OfflineTtsModelConfig({ this.vits = const OfflineTtsVitsModelConfig(), this.matcha = const OfflineTtsMatchaModelConfig(), + this.kokoro = const OfflineTtsKokoroModelConfig(), this.numThreads = 1, this.debug = true, this.provider = 'cpu', @@ -71,11 +93,12 @@ class OfflineTtsModelConfig { @override String toString() { - return 'OfflineTtsModelConfig(vits: $vits, matcha: $matcha, numThreads: $numThreads, debug: $debug, provider: $provider)'; + return 'OfflineTtsModelConfig(vits: $vits, matcha: $matcha, kokoro: $kokoro, numThreads: $numThreads, debug: $debug, provider: $provider)'; } final OfflineTtsVitsModelConfig vits; final OfflineTtsMatchaModelConfig matcha; + final OfflineTtsKokoroModelConfig kokoro; final int numThreads; final bool debug; final String provider; @@ -138,6 +161,12 @@ class OfflineTts { c.ref.model.matcha.lengthScale = config.model.matcha.lengthScale; c.ref.model.matcha.dictDir = config.model.matcha.dictDir.toNativeUtf8(); + c.ref.model.kokoro.model = config.model.kokoro.model.toNativeUtf8(); + c.ref.model.kokoro.voices = config.model.kokoro.voices.toNativeUtf8(); + c.ref.model.kokoro.tokens = config.model.kokoro.tokens.toNativeUtf8(); + c.ref.model.kokoro.dataDir = config.model.kokoro.dataDir.toNativeUtf8(); + c.ref.model.kokoro.lengthScale = config.model.kokoro.lengthScale; + c.ref.model.numThreads = config.model.numThreads; c.ref.model.debug = config.model.debug ? 1 : 0; c.ref.model.provider = config.model.provider.toNativeUtf8(); @@ -151,12 +180,19 @@ class OfflineTts { calloc.free(c.ref.ruleFars); calloc.free(c.ref.ruleFsts); calloc.free(c.ref.model.provider); + + calloc.free(c.ref.model.kokoro.dataDir); + calloc.free(c.ref.model.kokoro.tokens); + calloc.free(c.ref.model.kokoro.voices); + calloc.free(c.ref.model.kokoro.model); + calloc.free(c.ref.model.matcha.dictDir); calloc.free(c.ref.model.matcha.dataDir); calloc.free(c.ref.model.matcha.tokens); calloc.free(c.ref.model.matcha.lexicon); calloc.free(c.ref.model.matcha.vocoder); calloc.free(c.ref.model.matcha.acousticModel); + calloc.free(c.ref.model.vits.dictDir); calloc.free(c.ref.model.vits.dataDir); calloc.free(c.ref.model.vits.tokens); From 46f2e32e8a2d34bdb42e0e02f7b0e1d0209ec97b Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 16 Jan 2025 18:20:21 +0800 Subject: [PATCH 162/183] Add Pascal API for Kokoro TTS models (#1724) --- .github/workflows/pascal.yaml | 6 + pascal-api-examples/tts/.gitignore | 2 + .../tts/kokoro-en-playback.pas | 239 ++++++++++++++++++ pascal-api-examples/tts/kokoro-en.pas | 55 ++++ .../tts/matcha-en-playback.pas | 6 +- pascal-api-examples/tts/matcha-en.pas | 2 +- .../tts/matcha-zh-playback.pas | 2 +- .../tts/run-kokoro-en-playback.sh | 47 ++++ pascal-api-examples/tts/run-kokoro-en.sh | 43 ++++ sherpa-onnx/pascal-api/sherpa_onnx.pas | 49 +++- 10 files changed, 444 insertions(+), 7 deletions(-) create mode 100644 pascal-api-examples/tts/kokoro-en-playback.pas create mode 100644 pascal-api-examples/tts/kokoro-en.pas create mode 100755 pascal-api-examples/tts/run-kokoro-en-playback.sh create mode 100755 pascal-api-examples/tts/run-kokoro-en.sh diff --git a/.github/workflows/pascal.yaml b/.github/workflows/pascal.yaml index bb04c98b5..45378141b 100644 --- a/.github/workflows/pascal.yaml +++ b/.github/workflows/pascal.yaml @@ -154,6 +154,12 @@ jobs: ls -lh echo "---" + ./run-kokoro-en.sh + rm -rf kokoro-en-* + rm kokoro-en + ls -lh + echo "---" + ./run-matcha-zh.sh rm -rf matcha-icefall-* rm matcha-zh diff --git a/pascal-api-examples/tts/.gitignore b/pascal-api-examples/tts/.gitignore index c7d282825..429005958 100644 --- a/pascal-api-examples/tts/.gitignore +++ b/pascal-api-examples/tts/.gitignore @@ -6,3 +6,5 @@ matcha-zh matcha-en matcha-zh-playback matcha-en-playback +kokoro-en +kokoro-en-playback diff --git a/pascal-api-examples/tts/kokoro-en-playback.pas b/pascal-api-examples/tts/kokoro-en-playback.pas new file mode 100644 index 000000000..7796a6fee --- /dev/null +++ b/pascal-api-examples/tts/kokoro-en-playback.pas @@ -0,0 +1,239 @@ +{ Copyright (c) 2025 Xiaomi Corporation } +program kokoro_en_playback; +{ +This file shows how to use the text to speech API of sherpa-onnx +with Kokoro models. + +It generates speech from text and saves it to a wave file. + +Note that it plays the audio back as it is still generating. +} + +{$mode objfpc} + +uses + {$ifdef unix} + cthreads, + {$endif} + SysUtils, + dos, + ctypes, + portaudio, + sherpa_onnx; + +var + CriticalSection: TRTLCriticalSection; + + Tts: TSherpaOnnxOfflineTts; + Audio: TSherpaOnnxGeneratedAudio; + Resampler: TSherpaOnnxLinearResampler; + + Text: AnsiString; + Speed: Single = 1.0; {Use a larger value to speak faster} + SpeakerId: Integer = 7; + Buffer: TSherpaOnnxCircularBuffer; + FinishedGeneration: Boolean = False; + FinishedPlaying: Boolean = False; + + Version: String; + EnvStr: String; + Status: Integer; + NumDevices: Integer; + DeviceIndex: Integer; + DeviceInfo: PPaDeviceInfo; + + { If you get EDivByZero: Division by zero error, please change the sample rate + to the one supported by your microphone. + } + DeviceSampleRate: Integer = 48000; + I: Integer; + Param: TPaStreamParameters; + Stream: PPaStream; + Wave: TSherpaOnnxWave; + +function GenerateCallback( + Samples: pcfloat; N: cint32; + Arg: Pointer): cint; cdecl; +begin + EnterCriticalSection(CriticalSection); + try + if Resampler <> nil then + Buffer.Push(Resampler.Resample(Samples, N, False)) + else + Buffer.Push(Samples, N); + finally + LeaveCriticalSection(CriticalSection); + end; + + { 1 means to continue generating; 0 means to stop generating. } + Result := 1; +end; + +function PlayCallback( + input: Pointer; output: Pointer; + frameCount: culong; + timeInfo: PPaStreamCallbackTimeInfo; + statusFlags: TPaStreamCallbackFlags; + userData: Pointer ): cint; cdecl; +var + Samples: TSherpaOnnxSamplesArray; + I: Integer; +begin + EnterCriticalSection(CriticalSection); + try + if Buffer.Size >= frameCount then + begin + Samples := Buffer.Get(Buffer.Head, FrameCount); + Buffer.Pop(FrameCount); + end + else if Buffer.Size > 0 then + begin + Samples := Buffer.Get(Buffer.Head, Buffer.Size); + Buffer.Pop(Buffer.Size); + SetLength(Samples, frameCount); + end + else + SetLength(Samples, frameCount); + + for I := 0 to frameCount - 1 do + pcfloat(output)[I] := Samples[I]; + + if (Buffer.Size > 0) or (not FinishedGeneration) then + Result := paContinue + else + begin + Result := paComplete; + FinishedPlaying := True; + end; + finally + LeaveCriticalSection(CriticalSection); + end; +end; + +function GetOfflineTts: TSherpaOnnxOfflineTts; +var + Config: TSherpaOnnxOfflineTtsConfig; +begin + Config.Model.Kokoro.Model := './kokoro-en-v0_19/model.onnx'; + Config.Model.Kokoro.Voices := './kokoro-en-v0_19/voices.bin'; + Config.Model.Kokoro.Tokens := './kokoro-en-v0_19/tokens.txt'; + Config.Model.Kokoro.DataDir := './kokoro-en-v0_19/espeak-ng-data'; + Config.Model.NumThreads := 2; + Config.Model.Debug := False; + Config.MaxNumSentences := 1; + + Result := TSherpaOnnxOfflineTts.Create(Config); +end; + +begin + Tts := GetOfflineTts; + if Tts.GetSampleRate <> DeviceSampleRate then + Resampler := TSherpaOnnxLinearResampler.Create(Tts.GetSampleRate, DeviceSampleRate); + + Version := String(Pa_GetVersionText); + WriteLn('Version is ', Version); + Status := Pa_Initialize; + if Status <> paNoError then + begin + WriteLn('Failed to initialize portaudio, ', Pa_GetErrorText(Status)); + Exit; + end; + + NumDevices := Pa_GetDeviceCount; + WriteLn('Num devices: ', NumDevices); + + DeviceIndex := Pa_GetDefaultOutputDevice; + + if DeviceIndex = paNoDevice then + begin + WriteLn('No default output device found'); + Pa_Terminate; + Exit; + end; + + EnvStr := GetEnv('SHERPA_ONNX_MIC_DEVICE'); + if EnvStr <> '' then + begin + DeviceIndex := StrToIntDef(EnvStr, DeviceIndex); + WriteLn('Use device index from environment variable SHERPA_ONNX_MIC_DEVICE: ', EnvStr); + end; + + for I := 0 to (NumDevices - 1) do + begin + DeviceInfo := Pa_GetDeviceInfo(I); + if I = DeviceIndex then + { WriteLn(Format(' * %d %s', [I, DeviceInfo^.Name])) } + WriteLn(Format(' * %d %s', [I, AnsiString(DeviceInfo^.Name)])) + else + WriteLn(Format(' %d %s', [I, AnsiString(DeviceInfo^.Name)])); + end; + + WriteLn('Use device ', DeviceIndex); + WriteLn(' Name ', Pa_GetDeviceInfo(DeviceIndex)^.Name); + WriteLn(' Max output channels ', Pa_GetDeviceInfo(DeviceIndex)^.MaxOutputChannels); + + Initialize(Param); + Param.Device := DeviceIndex; + Param.ChannelCount := 1; + Param.SampleFormat := paFloat32; + param.SuggestedLatency := Pa_GetDeviceInfo(DeviceIndex)^.DefaultHighOutputLatency; + param.HostApiSpecificStreamInfo := nil; + + Buffer := TSherpaOnnxCircularBuffer.Create(30 * DeviceSampleRate); + + + { Note(fangjun): PortAudio invokes PlayCallback in a separate thread. } + Status := Pa_OpenStream(stream, nil, @Param, DeviceSampleRate, paFramesPerBufferUnspecified, paNoFlag, + PPaStreamCallback(@PlayCallback), nil); + + if Status <> paNoError then + begin + WriteLn('Failed to open stream, ', Pa_GetErrorText(Status)); + Pa_Terminate; + Exit; + end; + + InitCriticalSection(CriticalSection); + + Status := Pa_StartStream(stream); + if Status <> paNoError then + begin + WriteLn('Failed to start stream, ', Pa_GetErrorText(Status)); + Pa_Terminate; + Exit; + end; + + WriteLn('There are ', Tts.GetNumSpeakers, ' speakers'); + + Text := 'Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone.'; + + Audio := Tts.Generate(Text, SpeakerId, Speed, + PSherpaOnnxGeneratedAudioCallbackWithArg(@GenerateCallback), nil); + FinishedGeneration := True; + SherpaOnnxWriteWave('./kokoro-en-playback-7.wav', Audio.Samples, Audio.SampleRate); + WriteLn('Saved to ./kokoro-en-playback-7.wav'); + + while not FinishedPlaying do + Pa_Sleep(100); {sleep for 0.1 second } + {TODO(fangjun): Use an event to indicate the play is finished} + + DoneCriticalSection(CriticalSection); + + FreeAndNil(Tts); + FreeAndNil(Resampler); + + Status := Pa_CloseStream(stream); + if Status <> paNoError then + begin + WriteLn('Failed to close stream, ', Pa_GetErrorText(Status)); + Exit; + end; + + Status := Pa_Terminate; + if Status <> paNoError then + begin + WriteLn('Failed to deinitialize portaudio, ', Pa_GetErrorText(Status)); + Exit; + end; +end. + diff --git a/pascal-api-examples/tts/kokoro-en.pas b/pascal-api-examples/tts/kokoro-en.pas new file mode 100644 index 000000000..5e186b24d --- /dev/null +++ b/pascal-api-examples/tts/kokoro-en.pas @@ -0,0 +1,55 @@ +{ Copyright (c) 2025 Xiaomi Corporation } +program kokoro_en; +{ +This file shows how to use the text to speech API of sherpa-onnx +with Kokoro TTS models. + +It generates speech from text and saves it to a wave file. + +If you want to play it while it is generating, please see +./kokoro-en-playback.pas +} + +{$mode objfpc} + +uses + SysUtils, + sherpa_onnx; + +function GetOfflineTts: TSherpaOnnxOfflineTts; +var + Config: TSherpaOnnxOfflineTtsConfig; +begin + Config.Model.Kokoro.Model := './kokoro-en-v0_19/model.onnx'; + Config.Model.Kokoro.Voices := './kokoro-en-v0_19/voices.bin'; + Config.Model.Kokoro.Tokens := './kokoro-en-v0_19/tokens.txt'; + Config.Model.Kokoro.DataDir := './kokoro-en-v0_19/espeak-ng-data'; + Config.Model.NumThreads := 2; + Config.Model.Debug := False; + Config.MaxNumSentences := 1; + + Result := TSherpaOnnxOfflineTts.Create(Config); +end; + +var + Tts: TSherpaOnnxOfflineTts; + Audio: TSherpaOnnxGeneratedAudio; + + Text: AnsiString; + Speed: Single = 1.0; {Use a larger value to speak faster} + SpeakerId: Integer = 8; + +begin + Tts := GetOfflineTts; + + WriteLn('There are ', Tts.GetNumSpeakers, ' speakers'); + + Text := 'Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone.'; + + Audio := Tts.Generate(Text, SpeakerId, Speed); + SherpaOnnxWriteWave('./kokoro-en-8.wav', Audio.Samples, Audio.SampleRate); + WriteLn('Saved to ./kokoro-en-8.wav'); + + FreeAndNil(Tts); +end. + diff --git a/pascal-api-examples/tts/matcha-en-playback.pas b/pascal-api-examples/tts/matcha-en-playback.pas index e750099cb..7a6e8c758 100644 --- a/pascal-api-examples/tts/matcha-en-playback.pas +++ b/pascal-api-examples/tts/matcha-en-playback.pas @@ -2,7 +2,7 @@ program matcha_en_playback; { This file shows how to use the text to speech API of sherpa-onnx -with Piper models. +with MatchaTTS models. It generates speech from text and saves it to a wave file. @@ -210,8 +210,8 @@ function GetOfflineTts: TSherpaOnnxOfflineTts; Audio := Tts.Generate(Text, SpeakerId, Speed, PSherpaOnnxGeneratedAudioCallbackWithArg(@GenerateCallback), nil); FinishedGeneration := True; - SherpaOnnxWriteWave('./matcha-zh-playback.wav', Audio.Samples, Audio.SampleRate); - WriteLn('Saved to ./matcha-zh-playback.wav'); + SherpaOnnxWriteWave('./matcha-en-playback.wav', Audio.Samples, Audio.SampleRate); + WriteLn('Saved to ./matcha-en-playback.wav'); while not FinishedPlaying do Pa_Sleep(100); {sleep for 0.1 second } diff --git a/pascal-api-examples/tts/matcha-en.pas b/pascal-api-examples/tts/matcha-en.pas index 7ef34b703..f818d53e6 100644 --- a/pascal-api-examples/tts/matcha-en.pas +++ b/pascal-api-examples/tts/matcha-en.pas @@ -7,7 +7,7 @@ It generates speech from text and saves it to a wave file. If you want to play it while it is generating, please see -./matcha-zh-playback.pas +./matcha-en-playback.pas } {$mode objfpc} diff --git a/pascal-api-examples/tts/matcha-zh-playback.pas b/pascal-api-examples/tts/matcha-zh-playback.pas index 08b2bbe2d..05f94ba9c 100644 --- a/pascal-api-examples/tts/matcha-zh-playback.pas +++ b/pascal-api-examples/tts/matcha-zh-playback.pas @@ -2,7 +2,7 @@ program matcha_zh_playback; { This file shows how to use the text to speech API of sherpa-onnx -with Piper models. +with MatchaTTS models. It generates speech from text and saves it to a wave file. diff --git a/pascal-api-examples/tts/run-kokoro-en-playback.sh b/pascal-api-examples/tts/run-kokoro-en-playback.sh new file mode 100755 index 000000000..49e280d7a --- /dev/null +++ b/pascal-api-examples/tts/run-kokoro-en-playback.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +set -ex + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +SHERPA_ONNX_DIR=$(cd $SCRIPT_DIR/../.. && pwd) + +echo "SHERPA_ONNX_DIR: $SHERPA_ONNX_DIR" + +if [[ ! -f ../../build/install/lib/libsherpa-onnx-c-api.dylib && ! -f ../../build/install/lib/libsherpa-onnx-c-api.so && ! -f ../../build/install/lib/sherpa-onnx-c-api.dll ]]; then + mkdir -p ../../build + pushd ../../build + cmake \ + -DCMAKE_INSTALL_PREFIX=./install \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + .. + + cmake --build . --target install --config Release + popd +fi + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html +if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 + tar xf kokoro-en-v0_19.tar.bz2 + rm kokoro-en-v0_19.tar.bz2 +fi + +fpc \ + -dSHERPA_ONNX_USE_SHARED_LIBS \ + -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \ + -Fl$SHERPA_ONNX_DIR/build/install/lib \ + -Fl/usr/local/Cellar/portaudio/19.7.0/lib \ + ./kokoro-en-playback.pas + +# Please see ../portaudio-test/README.md +# for how to install portaudio on macOS + +export LD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$LD_LIBRARY_PATH +export DYLD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$DYLD_LIBRARY_PATH + +./kokoro-en-playback diff --git a/pascal-api-examples/tts/run-kokoro-en.sh b/pascal-api-examples/tts/run-kokoro-en.sh new file mode 100755 index 000000000..f26fb335d --- /dev/null +++ b/pascal-api-examples/tts/run-kokoro-en.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +set -ex + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +SHERPA_ONNX_DIR=$(cd $SCRIPT_DIR/../.. && pwd) + +echo "SHERPA_ONNX_DIR: $SHERPA_ONNX_DIR" + +if [[ ! -f ../../build/install/lib/libsherpa-onnx-c-api.dylib && ! -f ../../build/install/lib/libsherpa-onnx-c-api.so && ! -f ../../build/install/lib/sherpa-onnx-c-api.dll ]]; then + mkdir -p ../../build + pushd ../../build + cmake \ + -DCMAKE_INSTALL_PREFIX=./install \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + .. + + cmake --build . --target install --config Release + popd +fi + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html +if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 + tar xf kokoro-en-v0_19.tar.bz2 + rm kokoro-en-v0_19.tar.bz2 +fi + +fpc \ + -dSHERPA_ONNX_USE_SHARED_LIBS \ + -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \ + -Fl$SHERPA_ONNX_DIR/build/install/lib \ + ./kokoro-en.pas + +export LD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$LD_LIBRARY_PATH +export DYLD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$DYLD_LIBRARY_PATH + +./kokoro-en diff --git a/sherpa-onnx/pascal-api/sherpa_onnx.pas b/sherpa-onnx/pascal-api/sherpa_onnx.pas index 442c8a504..182d440ab 100644 --- a/sherpa-onnx/pascal-api/sherpa_onnx.pas +++ b/sherpa-onnx/pascal-api/sherpa_onnx.pas @@ -76,12 +76,24 @@ TSherpaOnnxOfflineTtsMatchaModelConfig = record class operator Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineTtsMatchaModelConfig); end; + TSherpaOnnxOfflineTtsKokoroModelConfig = record + Model: AnsiString; + Voices: AnsiString; + Tokens: AnsiString; + DataDir: AnsiString; + LengthScale: Single; + + function ToString: AnsiString; + class operator Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineTtsKokoroModelConfig); + end; + TSherpaOnnxOfflineTtsModelConfig = record Vits: TSherpaOnnxOfflineTtsVitsModelConfig; NumThreads: Integer; Debug: Boolean; Provider: AnsiString; Matcha: TSherpaOnnxOfflineTtsMatchaModelConfig; + Kokoro: TSherpaOnnxOfflineTtsKokoroModelConfig; function ToString: AnsiString; class operator Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineTtsModelConfig); @@ -739,12 +751,21 @@ SherpaOnnxOfflineTtsMatchaModelConfig = record DictDir: PAnsiChar; end; + SherpaOnnxOfflineTtsKokoroModelConfig = record + Model: PAnsiChar; + Voices: PAnsiChar; + Tokens: PAnsiChar; + DataDir: PAnsiChar; + LengthScale: cfloat; + end; + SherpaOnnxOfflineTtsModelConfig = record Vits: SherpaOnnxOfflineTtsVitsModelConfig; NumThreads: cint32; Debug: cint32; Provider: PAnsiChar; Matcha: SherpaOnnxOfflineTtsMatchaModelConfig; + Kokoro: SherpaOnnxOfflineTtsKokoroModelConfig; end; SherpaOnnxOfflineTtsConfig = record @@ -1903,6 +1924,23 @@ function TSherpaOnnxOfflineTtsMatchaModelConfig.ToString: AnsiString; Dest.LengthScale := 1.0; end; +function TSherpaOnnxOfflineTtsKokoroModelConfig.ToString: AnsiString; +begin + Result := Format('TSherpaOnnxOfflineTtsKokoroModelConfig(' + + 'Model := %s, ' + + 'Voices := %s, ' + + 'Tokens := %s, ' + + 'DataDir := %s, ' + + 'LengthScale := %.2f' + + ')', + [Self.Model, Self.Voices, Self.Tokens, Self.DataDir, Self.LengthScale]); +end; + +class operator TSherpaOnnxOfflineTtsKokoroModelConfig.Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineTtsKokoroModelConfig); +begin + Dest.LengthScale := 1.0; +end; + function TSherpaOnnxOfflineTtsModelConfig.ToString: AnsiString; begin Result := Format('TSherpaOnnxOfflineTtsModelConfig(' + @@ -1910,10 +1948,11 @@ function TSherpaOnnxOfflineTtsModelConfig.ToString: AnsiString; 'NumThreads := %d, ' + 'Debug := %s, ' + 'Provider := %s, ' + - 'Matcha := %s' + + 'Matcha := %s, ' + + 'Kokoro := %s' + ')', [Self.Vits.ToString, Self.NumThreads, Self.Debug.ToString, Self.Provider, - Self.Matcha.ToString + Self.Matcha.ToString, Self.Kokoro.ToString ]); end; @@ -1966,6 +2005,12 @@ constructor TSherpaOnnxOfflineTts.Create(Config: TSherpaOnnxOfflineTtsConfig); C.Model.Matcha.LengthScale := Config.Model.Matcha.LengthScale; C.Model.Matcha.DictDir := PAnsiChar(Config.Model.Matcha.DictDir); + C.Model.Kokoro.Model := PAnsiChar(Config.Model.Kokoro.Model); + C.Model.Kokoro.Voices := PAnsiChar(Config.Model.Kokoro.Voices); + C.Model.Kokoro.Tokens := PAnsiChar(Config.Model.Kokoro.Tokens); + C.Model.Kokoro.DataDir := PAnsiChar(Config.Model.Kokoro.DataDir); + C.Model.Kokoro.LengthScale := Config.Model.Kokoro.LengthScale; + C.Model.NumThreads := Config.Model.NumThreads; C.Model.Provider := PAnsiChar(Config.Model.Provider); C.Model.Debug := Ord(Config.Model.Debug); From e8d499d21882d19769f4588b5c38c5bdbf33b5e2 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 16 Jan 2025 18:33:47 +0800 Subject: [PATCH 163/183] Add JavaScript API (node-addon) for Kokoro TTS models (#1725) --- .github/scripts/test-nodejs-addon-npm.sh | 7 +++ .../src/main/cpp/non-streaming-tts.cc | 36 ++++++++++++++ nodejs-addon-examples/README.md | 11 +++++ .../test_tts_non_streaming_kokoro_en.js | 47 +++++++++++++++++++ 4 files changed, 101 insertions(+) create mode 100644 nodejs-addon-examples/test_tts_non_streaming_kokoro_en.js diff --git a/.github/scripts/test-nodejs-addon-npm.sh b/.github/scripts/test-nodejs-addon-npm.sh index e2d8487be..3d755f88c 100755 --- a/.github/scripts/test-nodejs-addon-npm.sh +++ b/.github/scripts/test-nodejs-addon-npm.sh @@ -85,6 +85,13 @@ fi echo "----------tts----------" +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 +tar xf kokoro-en-v0_19.tar.bz2 +rm kokoro-en-v0_19.tar.bz2 + +node ./test_tts_non_streaming_kokoro_en.js +ls -lh *.wav + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 rm matcha-icefall-en_US-ljspeech.tar.bz2 diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc index 7baf3ce8b..55d4adb2b 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc @@ -53,6 +53,25 @@ static SherpaOnnxOfflineTtsMatchaModelConfig GetOfflineTtsMatchaModelConfig( return c; } +static SherpaOnnxOfflineTtsKokoroModelConfig GetOfflineTtsKokoroModelConfig( + Napi::Object obj) { + SherpaOnnxOfflineTtsKokoroModelConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("kokoro") || !obj.Get("kokoro").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("kokoro").As(); + SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); + SHERPA_ONNX_ASSIGN_ATTR_STR(voices, voices); + SHERPA_ONNX_ASSIGN_ATTR_STR(tokens, tokens); + SHERPA_ONNX_ASSIGN_ATTR_STR(data_dir, dataDir); + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(length_scale, lengthScale); + + return c; +} + static SherpaOnnxOfflineTtsModelConfig GetOfflineTtsModelConfig( Napi::Object obj) { SherpaOnnxOfflineTtsModelConfig c; @@ -66,6 +85,7 @@ static SherpaOnnxOfflineTtsModelConfig GetOfflineTtsModelConfig( c.vits = GetOfflineTtsVitsModelConfig(o); c.matcha = GetOfflineTtsMatchaModelConfig(o); + c.kokoro = GetOfflineTtsKokoroModelConfig(o); SHERPA_ONNX_ASSIGN_ATTR_INT32(num_threads, numThreads); @@ -180,6 +200,22 @@ static Napi::External CreateOfflineTtsWrapper( delete[] c.model.matcha.dict_dir; } + if (c.model.kokoro.model) { + delete[] c.model.kokoro.model; + } + + if (c.model.kokoro.voices) { + delete[] c.model.kokoro.voices; + } + + if (c.model.kokoro.tokens) { + delete[] c.model.kokoro.tokens; + } + + if (c.model.kokoro.data_dir) { + delete[] c.model.kokoro.data_dir; + } + if (c.model.provider) { delete[] c.model.provider; } diff --git a/nodejs-addon-examples/README.md b/nodejs-addon-examples/README.md index 2de8a2143..ccb87a7a6 100644 --- a/nodejs-addon-examples/README.md +++ b/nodejs-addon-examples/README.md @@ -133,6 +133,7 @@ The following tables list the examples in this folder. |File| Description| |---|---| +|[./test_tts_non_streaming_kokoro_en.js](./test_tts_non_streaming_kokoro_en.js)| Text-to-speech with a Kokoro English Model| |[./test_tts_non_streaming_matcha_icefall_en.js](./test_tts_non_streaming_matcha_icefall_en.js)| Text-to-speech with a [MatchaTTS English Model](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker)| |[./test_tts_non_streaming_matcha_icefall_zhjs](./test_tts_non_streaming_matcha_icefall_zh.js)| Text-to-speech with a [MatchaTTS Chinese Model](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker)| |[./test_tts_non_streaming_vits_piper_en.js](./test_tts_non_streaming_vits_piper_en.js)| Text-to-speech with a [piper](https://github.com/rhasspy/piper) English model| @@ -347,6 +348,16 @@ npm install naudiodon2 node ./test_vad_asr_non_streaming_sense_voice_microphone.js ``` +### Text-to-speech with Kokoro TTS models (English TTS) + +```bash +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 +tar xf kokoro-en-v0_19.tar.bz2 +rm kokoro-en-v0_19.tar.bz2 + +node ./test_tts_non_streaming_kokoro_en.js +``` + ### Text-to-speech with MatchaTTS models (English TTS) ```bash wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 diff --git a/nodejs-addon-examples/test_tts_non_streaming_kokoro_en.js b/nodejs-addon-examples/test_tts_non_streaming_kokoro_en.js new file mode 100644 index 000000000..84b03982b --- /dev/null +++ b/nodejs-addon-examples/test_tts_non_streaming_kokoro_en.js @@ -0,0 +1,47 @@ +// Copyright (c) 2025 Xiaomi Corporation +const sherpa_onnx = require('sherpa-onnx-node'); + +// please refer to +// https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html +// to download model files +function createOfflineTts() { + const config = { + model: { + kokoro: { + model: './kokoro-en-v0_19/model.onnx', + voices: './kokoro-en-v0_19/voices.bin', + tokens: './kokoro-en-v0_19/tokens.txt', + dataDir: './kokoro-en-v0_19/espeak-ng-data', + }, + debug: true, + numThreads: 1, + provider: 'cpu', + }, + maxNumSentences: 1, + }; + return new sherpa_onnx.OfflineTts(config); +} + +const tts = createOfflineTts(); + +const text = + 'Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.' + + +let start = Date.now(); +const audio = tts.generate({text: text, sid: 6, speed: 1.0}); +let stop = Date.now(); +const elapsed_seconds = (stop - start) / 1000; +const duration = audio.samples.length / audio.sampleRate; +const real_time_factor = elapsed_seconds / duration; +console.log('Wave duration', duration.toFixed(3), 'secodns') +console.log('Elapsed', elapsed_seconds.toFixed(3), 'secodns') +console.log( + `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, + real_time_factor.toFixed(3)) + +const filename = 'test-kokoro-en-6.wav'; +sherpa_onnx.writeWave( + filename, {samples: audio.samples, sampleRate: audio.sampleRate}); + +console.log(`Saved to ${filename}`); From 3a1de0bfc16e282e60ec264cf3de4811355a86af Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 17 Jan 2025 11:17:18 +0800 Subject: [PATCH 164/183] Add JavaScript (WebAssembly) API for Kokoro TTS models. (#1726) --- .github/scripts/test-nodejs-npm.sh | 10 ++- nodejs-examples/README.md | 16 ++++ nodejs-examples/test-offline-tts-kokoro-en.js | 37 ++++++++ wasm/tts/sherpa-onnx-tts.js | 85 ++++++++++++++++++- wasm/tts/sherpa-onnx-wasm-main-tts.cc | 12 ++- 5 files changed, 154 insertions(+), 6 deletions(-) create mode 100644 nodejs-examples/test-offline-tts-kokoro-en.js diff --git a/.github/scripts/test-nodejs-npm.sh b/.github/scripts/test-nodejs-npm.sh index 967944843..8d17eae48 100755 --- a/.github/scripts/test-nodejs-npm.sh +++ b/.github/scripts/test-nodejs-npm.sh @@ -10,7 +10,15 @@ ls -lh ls -lh node_modules # offline tts -# + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 +tar xf kokoro-en-v0_19.tar.bz2 +rm kokoro-en-v0_19.tar.bz2 + +node ./test-offline-tts-kokoro-en.js + +ls -lh + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 tar xvf matcha-icefall-zh-baker.tar.bz2 rm matcha-icefall-zh-baker.tar.bz2 diff --git a/nodejs-examples/README.md b/nodejs-examples/README.md index 3db3a2952..c544303a4 100644 --- a/nodejs-examples/README.md +++ b/nodejs-examples/README.md @@ -42,6 +42,22 @@ node ./test-offline-speaker-diarization.js In the following, we demonstrate how to run text-to-speech. +## ./test-offline-tts-kokoro-en.js + +[./test-offline-tts-kokoro-en.js](./test-offline-tts-kokoro-en.js) shows how to use +[kokoro-en-v0_19](https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2) +for text-to-speech. + +You can use the following command to run it: + +```bash +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 +tar xf kokoro-en-v0_19.tar.bz2 +rm kokoro-en-v0_19.tar.bz2 + +node ./test-offline-tts-kokoro-en.js +``` + ## ./test-offline-tts-matcha-zh.js [./test-offline-tts-matcha-zh.js](./test-offline-tts-matcha-zh.js) shows how to use diff --git a/nodejs-examples/test-offline-tts-kokoro-en.js b/nodejs-examples/test-offline-tts-kokoro-en.js new file mode 100644 index 000000000..1c011d581 --- /dev/null +++ b/nodejs-examples/test-offline-tts-kokoro-en.js @@ -0,0 +1,37 @@ +// Copyright (c) 2025 Xiaomi Corporation (authors: Fangjun Kuang) + +const sherpa_onnx = require('sherpa-onnx'); + +function createOfflineTts() { + let offlineTtsKokoroModelConfig = { + model: './kokoro-en-v0_19/model.onnx', + voices: './kokoro-en-v0_19/voices.bin', + tokens: './kokoro-en-v0_19/tokens.txt', + dataDir: './kokoro-en-v0_19/espeak-ng-data', + lengthScale: 1.0, + }; + let offlineTtsModelConfig = { + offlineTtsKokoroModelConfig: offlineTtsKokoroModelConfig, + numThreads: 1, + debug: 1, + provider: 'cpu', + }; + + let offlineTtsConfig = { + offlineTtsModelConfig: offlineTtsModelConfig, + maxNumSentences: 1, + }; + + return sherpa_onnx.createOfflineTts(offlineTtsConfig); +} + +const tts = createOfflineTts(); +const speakerId = 0; +const speed = 1.0; +const text = + 'Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.' + +const audio = tts.generate({text: text, sid: speakerId, speed: speed}); +tts.save('./test-kokoro-en.wav', audio); +console.log('Saved to test-kokoro-en.wav successfully.'); +tts.free(); diff --git a/wasm/tts/sherpa-onnx-tts.js b/wasm/tts/sherpa-onnx-tts.js index 59158ae76..833ee936a 100644 --- a/wasm/tts/sherpa-onnx-tts.js +++ b/wasm/tts/sherpa-onnx-tts.js @@ -8,8 +8,12 @@ function freeConfig(config, Module) { freeConfig(config.config, Module) } - if ('config2' in config) { - freeConfig(config.config2, Module) + if ('matcha' in config) { + freeConfig(config.matcha, Module) + } + + if ('kokoro' in config) { + freeConfig(config.kokoro, Module) } Module._free(config.ptr); @@ -132,6 +136,52 @@ function initSherpaOnnxOfflineTtsMatchaModelConfig(config, Module) { } } +function initSherpaOnnxOfflineTtsKokoroModelConfig(config, Module) { + const modelLen = Module.lengthBytesUTF8(config.model) + 1; + const voicesLen = Module.lengthBytesUTF8(config.voices) + 1; + const tokensLen = Module.lengthBytesUTF8(config.tokens || '') + 1; + const dataDirLen = Module.lengthBytesUTF8(config.dataDir || '') + 1; + + const n = modelLen + voicesLen + tokensLen + dataDirLen; + + const buffer = Module._malloc(n); + + const len = 5 * 4; + const ptr = Module._malloc(len); + + let offset = 0; + Module.stringToUTF8(config.model || '', buffer + offset, modelLen); + offset += modelLen; + + Module.stringToUTF8(config.voices || '', buffer + offset, voicesLen); + offset += voicesLen; + + Module.stringToUTF8(config.tokens || '', buffer + offset, tokensLen); + offset += tokensLen; + + Module.stringToUTF8(config.dataDir || '', buffer + offset, dataDirLen); + offset += dataDirLen; + + offset = 0; + Module.setValue(ptr, buffer + offset, 'i8*'); + offset += modelLen; + + Module.setValue(ptr + 4, buffer + offset, 'i8*'); + offset += voicesLen; + + Module.setValue(ptr + 8, buffer + offset, 'i8*'); + offset += tokensLen; + + Module.setValue(ptr + 12, buffer + offset, 'i8*'); + offset += dataDirLen; + + Module.setValue(ptr + 16, config.lengthScale || 1.0, 'float'); + + return { + buffer: buffer, ptr: ptr, len: len, + } +} + function initSherpaOnnxOfflineTtsModelConfig(config, Module) { if (!('offlineTtsVitsModelConfig' in config)) { config.offlineTtsVitsModelConfig = { @@ -159,6 +209,16 @@ function initSherpaOnnxOfflineTtsModelConfig(config, Module) { }; } + if (!('offlineTtsKokoroModelConfig' in config)) { + config.offlineTtsKokoroModelConfig = { + model: '', + voices: '', + tokens: '', + lengthScale: 1.0, + dataDir: '', + }; + } + const vitsModelConfig = initSherpaOnnxOfflineTtsVitsModelConfig( config.offlineTtsVitsModelConfig, Module); @@ -166,7 +226,12 @@ function initSherpaOnnxOfflineTtsModelConfig(config, Module) { const matchaModelConfig = initSherpaOnnxOfflineTtsMatchaModelConfig( config.offlineTtsMatchaModelConfig, Module); - const len = vitsModelConfig.len + matchaModelConfig.len + 3 * 4; + const kokoroModelConfig = initSherpaOnnxOfflineTtsKokoroModelConfig( + config.offlineTtsKokoroModelConfig, Module); + + const len = vitsModelConfig.len + matchaModelConfig.len + + kokoroModelConfig.len + 3 * 4; + const ptr = Module._malloc(len); let offset = 0; @@ -188,9 +253,12 @@ function initSherpaOnnxOfflineTtsModelConfig(config, Module) { Module._CopyHeap(matchaModelConfig.ptr, matchaModelConfig.len, ptr + offset); offset += matchaModelConfig.len; + Module._CopyHeap(kokoroModelConfig.ptr, kokoroModelConfig.len, ptr + offset); + offset += kokoroModelConfig.len; + return { buffer: buffer, ptr: ptr, len: len, config: vitsModelConfig, - config2: matchaModelConfig + matcha: matchaModelConfig, kokoro: kokoroModelConfig, } } @@ -308,9 +376,18 @@ function createOfflineTts(Module, myConfig) { lengthScale: 1.0, }; + const offlineTtsKokoroModelConfig = { + model: '', + voices: '', + tokens: '', + dataDir: '', + lengthScale: 1.0, + }; + const offlineTtsModelConfig = { offlineTtsVitsModelConfig: offlineTtsVitsModelConfig, offlineTtsMatchaModelConfig: offlineTtsMatchaModelConfig, + offlineTtsKokoroModelConfig: offlineTtsKokoroModelConfig, numThreads: 1, debug: 1, provider: 'cpu', diff --git a/wasm/tts/sherpa-onnx-wasm-main-tts.cc b/wasm/tts/sherpa-onnx-wasm-main-tts.cc index 3508b860d..f2cd42c55 100644 --- a/wasm/tts/sherpa-onnx-wasm-main-tts.cc +++ b/wasm/tts/sherpa-onnx-wasm-main-tts.cc @@ -15,9 +15,11 @@ extern "C" { static_assert(sizeof(SherpaOnnxOfflineTtsVitsModelConfig) == 8 * 4, ""); static_assert(sizeof(SherpaOnnxOfflineTtsMatchaModelConfig) == 8 * 4, ""); +static_assert(sizeof(SherpaOnnxOfflineTtsKokoroModelConfig) == 5 * 4, ""); static_assert(sizeof(SherpaOnnxOfflineTtsModelConfig) == sizeof(SherpaOnnxOfflineTtsVitsModelConfig) + - sizeof(SherpaOnnxOfflineTtsMatchaModelConfig) + 3 * 4, + sizeof(SherpaOnnxOfflineTtsMatchaModelConfig) + + sizeof(SherpaOnnxOfflineTtsKokoroModelConfig) + 3 * 4, ""); static_assert(sizeof(SherpaOnnxOfflineTtsConfig) == sizeof(SherpaOnnxOfflineTtsModelConfig) + 3 * 4, @@ -27,6 +29,7 @@ void MyPrint(SherpaOnnxOfflineTtsConfig *tts_config) { auto tts_model_config = &tts_config->model; auto vits_model_config = &tts_model_config->vits; auto matcha_model_config = &tts_model_config->matcha; + auto kokoro = &tts_model_config->kokoro; fprintf(stdout, "----------vits model config----------\n"); fprintf(stdout, "model: %s\n", vits_model_config->model); fprintf(stdout, "lexicon: %s\n", vits_model_config->lexicon); @@ -47,6 +50,13 @@ void MyPrint(SherpaOnnxOfflineTtsConfig *tts_config) { fprintf(stdout, "length scale: %.3f\n", matcha_model_config->length_scale); fprintf(stdout, "dict_dir: %s\n", matcha_model_config->dict_dir); + fprintf(stdout, "----------kokoro model config----------\n"); + fprintf(stdout, "model: %s\n", kokoro->model); + fprintf(stdout, "voices: %s\n", kokoro->voices); + fprintf(stdout, "tokens: %s\n", kokoro->tokens); + fprintf(stdout, "data_dir: %s\n", kokoro->data_dir); + fprintf(stdout, "length scale: %.3f\n", kokoro->length_scale); + fprintf(stdout, "----------tts model config----------\n"); fprintf(stdout, "num threads: %d\n", tts_model_config->num_threads); fprintf(stdout, "debug: %d\n", tts_model_config->debug); From 99cef4198b826e38caccc745e12909e8096264ed Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 17 Jan 2025 17:36:13 +0800 Subject: [PATCH 165/183] Add Koltin and Java API for Kokoro TTS models (#1728) --- .github/workflows/run-java-test.yaml | 4 + .../com/k2fsa/sherpa/onnx/MainActivity.kt | 13 ++ .../sherpa/onnx/tts/engine/GetSampleText.kt | 2 +- .../sherpa/onnx/tts/engine/MainActivity.kt | 214 +++++++++++++++--- .../k2fsa/sherpa/onnx/tts/engine/TtsEngine.kt | 18 +- .../app/src/main/res/values/strings.xml | 2 +- .../NonStreamingTtsKokoroEn.java | 60 +++++ .../run-non-streaming-tts-kokoro-en.sh | 40 ++++ kotlin-api-examples/run.sh | 6 + kotlin-api-examples/test_tts.kt | 26 ++- scripts/apk/build-apk-tts-engine.sh.in | 5 + scripts/apk/build-apk-tts.sh.in | 4 + scripts/apk/generate-tts-apk-script.py | 17 ++ sherpa-onnx/java-api/Makefile | 1 + .../onnx/OfflineTtsKokoroModelConfig.java | 80 +++++++ .../sherpa/onnx/OfflineTtsModelConfig.java | 12 + sherpa-onnx/jni/offline-tts.cc | 37 ++- sherpa-onnx/kotlin-api/Tts.kt | 48 +++- 18 files changed, 549 insertions(+), 40 deletions(-) create mode 100644 java-api-examples/NonStreamingTtsKokoroEn.java create mode 100755 java-api-examples/run-non-streaming-tts-kokoro-en.sh create mode 100644 sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsKokoroModelConfig.java diff --git a/.github/workflows/run-java-test.yaml b/.github/workflows/run-java-test.yaml index 0976f89e2..4ede6514a 100644 --- a/.github/workflows/run-java-test.yaml +++ b/.github/workflows/run-java-test.yaml @@ -234,8 +234,12 @@ jobs: run: | cd ./java-api-examples + ./run-non-streaming-tts-kokoro-en.sh ./run-non-streaming-tts-matcha-zh.sh ./run-non-streaming-tts-matcha-en.sh + ls -lh + + rm -rf kokoro-en-* rm -rf matcha-icefall-* rm hifigan_v2.onnx diff --git a/android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt b/android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt index 5119a50b2..5aa5b9ad8 100644 --- a/android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt +++ b/android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt @@ -185,6 +185,7 @@ class MainActivity : AppCompatActivity() { var modelName: String? var acousticModelName: String? var vocoder: String? + var voices: String? var ruleFsts: String? var ruleFars: String? var lexicon: String? @@ -205,6 +206,10 @@ class MainActivity : AppCompatActivity() { vocoder = null // Matcha -- end + // For Kokoro -- begin + voices = null + // For Kokoro -- end + modelDir = null ruleFsts = null @@ -269,6 +274,13 @@ class MainActivity : AppCompatActivity() { // vocoder = "hifigan_v2.onnx" // dataDir = "matcha-icefall-en_US-ljspeech/espeak-ng-data" + // Example 9 + // kokoro-en-v0_19 + // modelDir = "kokoro-en-v0_19" + // modelName = "model.onnx" + // voices = "voices.bin" + // dataDir = "kokoro-en-v0_19/espeak-ng-data" + if (dataDir != null) { val newDir = copyDataDir(dataDir!!) dataDir = "$newDir/$dataDir" @@ -285,6 +297,7 @@ class MainActivity : AppCompatActivity() { modelName = modelName ?: "", acousticModelName = acousticModelName ?: "", vocoder = vocoder ?: "", + voices = voices ?: "", lexicon = lexicon ?: "", dataDir = dataDir ?: "", dictDir = dictDir ?: "", diff --git a/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/GetSampleText.kt b/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/GetSampleText.kt index a01e0a7b6..e372be432 100644 --- a/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/GetSampleText.kt +++ b/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/GetSampleText.kt @@ -47,7 +47,7 @@ fun getSampleText(lang: String): String { } "eng" -> { - text = "This is a text-to-speech engine using next generation Kaldi" + text = "How are you doing today? This is a text-to-speech engine using next generation Kaldi" } "est" -> { diff --git a/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/MainActivity.kt b/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/MainActivity.kt index f7e34c5dd..18a89f93c 100644 --- a/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/MainActivity.kt +++ b/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/MainActivity.kt @@ -3,6 +3,10 @@ package com.k2fsa.sherpa.onnx.tts.engine import PreferenceHelper +import android.media.AudioAttributes +import android.media.AudioFormat +import android.media.AudioManager +import android.media.AudioTrack import android.media.MediaPlayer import android.net.Uri import android.os.Bundle @@ -36,7 +40,13 @@ import androidx.compose.ui.Modifier import androidx.compose.ui.text.input.KeyboardType import androidx.compose.ui.unit.dp import com.k2fsa.sherpa.onnx.tts.engine.ui.theme.SherpaOnnxTtsEngineTheme +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext import java.io.File +import kotlin.time.TimeSource const val TAG = "sherpa-onnx-tts-engine" @@ -45,9 +55,26 @@ class MainActivity : ComponentActivity() { private val ttsViewModel: TtsViewModel by viewModels() private var mediaPlayer: MediaPlayer? = null + + // see + // https://developer.android.com/reference/kotlin/android/media/AudioTrack + private lateinit var track: AudioTrack + + private var stopped: Boolean = false + + private var samplesChannel = Channel() + override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) + + Log.i(TAG, "Start to initialize TTS") TtsEngine.createTts(this) + Log.i(TAG, "Finish initializing TTS") + + Log.i(TAG, "Start to initialize AudioTrack") + initAudioTrack() + Log.i(TAG, "Finish initializing AudioTrack") + val preferenceHelper = PreferenceHelper(this) setContent { SherpaOnnxTtsEngineTheme { @@ -77,6 +104,11 @@ class MainActivity : ComponentActivity() { val testTextContent = getSampleText(TtsEngine.lang ?: "") var testText by remember { mutableStateOf(testTextContent) } + var startEnabled by remember { mutableStateOf(true) } + var playEnabled by remember { mutableStateOf(false) } + var rtfText by remember { + mutableStateOf("") + } val numSpeakers = TtsEngine.tts!!.numSpeakers() if (numSpeakers > 1) { @@ -119,52 +151,117 @@ class MainActivity : ComponentActivity() { Row { Button( - modifier = Modifier.padding(20.dp), + enabled = startEnabled, + modifier = Modifier.padding(5.dp), onClick = { Log.i(TAG, "Clicked, text: $testText") if (testText.isBlank() || testText.isEmpty()) { Toast.makeText( applicationContext, - "Please input a test sentence", + "Please input some text to generate", Toast.LENGTH_SHORT ).show() } else { - val audio = TtsEngine.tts!!.generate( - text = testText, - sid = TtsEngine.speakerId, - speed = TtsEngine.speed, - ) - - val filename = - application.filesDir.absolutePath + "/generated.wav" - val ok = - audio.samples.isNotEmpty() && audio.save( - filename - ) + startEnabled = false + playEnabled = false + stopped = false - if (ok) { - stopMediaPlayer() - mediaPlayer = MediaPlayer.create( - applicationContext, - Uri.fromFile(File(filename)) - ) - mediaPlayer?.start() - } else { - Log.i(TAG, "Failed to generate or save audio") + track.pause() + track.flush() + track.play() + rtfText = "" + Log.i(TAG, "Started with text $testText") + + samplesChannel = Channel() + + CoroutineScope(Dispatchers.IO).launch { + for (samples in samplesChannel) { + track.write( + samples, + 0, + samples.size, + AudioTrack.WRITE_BLOCKING + ) + if (stopped) { + break + } + } } + + CoroutineScope(Dispatchers.Default).launch { + val timeSource = TimeSource.Monotonic + val startTime = timeSource.markNow() + + val audio = + TtsEngine.tts!!.generateWithCallback( + text = testText, + sid = TtsEngine.speakerId, + speed = TtsEngine.speed, + callback = ::callback, + ) + + val elapsed = + startTime.elapsedNow().inWholeMilliseconds.toFloat() / 1000; + val audioDuration = + audio.samples.size / TtsEngine.tts!!.sampleRate() + .toFloat() + val RTF = String.format( + "Number of threads: %d\nElapsed: %.3f s\nAudio duration: %.3f s\nRTF: %.3f/%.3f = %.3f", + TtsEngine.tts!!.config.model.numThreads, + audioDuration, + elapsed, + elapsed, + audioDuration, + elapsed / audioDuration + ) + samplesChannel.close() + + val filename = + application.filesDir.absolutePath + "/generated.wav" + + + val ok = + audio.samples.isNotEmpty() && audio.save( + filename + ) + + if (ok) { + withContext(Dispatchers.Main) { + startEnabled = true + playEnabled = true + rtfText = RTF + } + } + }.start() } }) { - Text("Test") + Text("Start") } Button( - modifier = Modifier.padding(20.dp), + modifier = Modifier.padding(5.dp), + enabled = playEnabled, onClick = { - TtsEngine.speakerId = 0 - TtsEngine.speed = 1.0f - testText = "" + stopped = true + track.pause() + track.flush() + onClickPlay() }) { - Text("Reset") + Text("Play") + } + + Button( + modifier = Modifier.padding(5.dp), + onClick = { + onClickStop() + startEnabled = true + }) { + Text("Stop") + } + } + if (rtfText.isNotEmpty()) { + Row { + Text(rtfText) } } } @@ -185,4 +282,63 @@ class MainActivity : ComponentActivity() { mediaPlayer?.release() mediaPlayer = null } + + private fun onClickPlay() { + val filename = application.filesDir.absolutePath + "/generated.wav" + stopMediaPlayer() + mediaPlayer = MediaPlayer.create( + applicationContext, + Uri.fromFile(File(filename)) + ) + mediaPlayer?.start() + } + + private fun onClickStop() { + stopped = true + track.pause() + track.flush() + + stopMediaPlayer() + } + + // this function is called from C++ + private fun callback(samples: FloatArray): Int { + if (!stopped) { + val samplesCopy = samples.copyOf() + CoroutineScope(Dispatchers.IO).launch { + samplesChannel.send(samplesCopy) + } + return 1 + } else { + track.stop() + Log.i(TAG, " return 0") + return 0 + } + } + + private fun initAudioTrack() { + val sampleRate = TtsEngine.tts!!.sampleRate() + val bufLength = AudioTrack.getMinBufferSize( + sampleRate, + AudioFormat.CHANNEL_OUT_MONO, + AudioFormat.ENCODING_PCM_FLOAT + ) + Log.i(TAG, "sampleRate: $sampleRate, buffLength: $bufLength") + + val attr = AudioAttributes.Builder().setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .setUsage(AudioAttributes.USAGE_MEDIA) + .build() + + val format = AudioFormat.Builder() + .setEncoding(AudioFormat.ENCODING_PCM_FLOAT) + .setChannelMask(AudioFormat.CHANNEL_OUT_MONO) + .setSampleRate(sampleRate) + .build() + + track = AudioTrack( + attr, format, bufLength, AudioTrack.MODE_STREAM, + AudioManager.AUDIO_SESSION_ID_GENERATE + ) + track.play() + } } diff --git a/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsEngine.kt b/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsEngine.kt index cec07ffd5..69792a560 100644 --- a/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsEngine.kt +++ b/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsEngine.kt @@ -41,8 +41,9 @@ object TtsEngine { private var modelDir: String? = null private var modelName: String? = null - private var acousticModelName: String? = null - private var vocoder: String? = null + private var acousticModelName: String? = null // for matcha tts + private var vocoder: String? = null // for matcha tts + private var voices: String? = null // for kokoro private var ruleFsts: String? = null private var ruleFars: String? = null private var lexicon: String? = null @@ -64,6 +65,10 @@ object TtsEngine { vocoder = null // For Matcha -- end + // For Kokoro -- begin + voices = null + // For Kokoro -- end + modelDir = null ruleFsts = null ruleFars = null @@ -139,6 +144,14 @@ object TtsEngine { // vocoder = "hifigan_v2.onnx" // dataDir = "matcha-icefall-en_US-ljspeech/espeak-ng-data" // lang = "eng" + + // Example 9 + // kokoro-en-v0_19 + // modelDir = "kokoro-en-v0_19" + // modelName = "model.onnx" + // voices = "voices.bin" + // dataDir = "kokoro-en-v0_19/espeak-ng-data" + // lang = "eng" } fun createTts(context: Context) { @@ -167,6 +180,7 @@ object TtsEngine { modelName = modelName ?: "", acousticModelName = acousticModelName ?: "", vocoder = vocoder ?: "", + voices = voices ?: "", lexicon = lexicon ?: "", dataDir = dataDir ?: "", dictDir = dictDir ?: "", diff --git a/android/SherpaOnnxTtsEngine/app/src/main/res/values/strings.xml b/android/SherpaOnnxTtsEngine/app/src/main/res/values/strings.xml index ac2847314..67518e0a3 100755 --- a/android/SherpaOnnxTtsEngine/app/src/main/res/values/strings.xml +++ b/android/SherpaOnnxTtsEngine/app/src/main/res/values/strings.xml @@ -1,3 +1,3 @@ - TTS Engine + TTS Engine: Next-gen Kaldi \ No newline at end of file diff --git a/java-api-examples/NonStreamingTtsKokoroEn.java b/java-api-examples/NonStreamingTtsKokoroEn.java new file mode 100644 index 000000000..e36ba663c --- /dev/null +++ b/java-api-examples/NonStreamingTtsKokoroEn.java @@ -0,0 +1,60 @@ +// Copyright 2025 Xiaomi Corporation + +// This file shows how to use a Kokoro English model +// to convert text to speech +import com.k2fsa.sherpa.onnx.*; + +public class NonStreamingTtsKokoroEn { + public static void main(String[] args) { + // please visit + // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html + // to download model files + String model = "./kokoro-en-v0_19/model.onnx"; + String voices = "./kokoro-en-v0_19/voices.bin"; + String tokens = "./kokoro-en-v0_19/tokens.txt"; + String dataDir = "./kokoro-en-v0_19/espeak-ng-data"; + String text = + "Today as always, men fall into two groups: slaves and free men. Whoever does not have" + + " two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a" + + " businessman, an official, or a scholar."; + + OfflineTtsKokoroModelConfig kokoroModelConfig = + OfflineTtsKokoroModelConfig.builder() + .setModel(model) + .setVoices(voices) + .setTokens(tokens) + .setDataDir(dataDir) + .build(); + + OfflineTtsModelConfig modelConfig = + OfflineTtsModelConfig.builder() + .setKokoro(kokoroModelConfig) + .setNumThreads(2) + .setDebug(true) + .build(); + + OfflineTtsConfig config = OfflineTtsConfig.builder().setModel(modelConfig).build(); + OfflineTts tts = new OfflineTts(config); + + int sid = 0; + float speed = 1.0f; + long start = System.currentTimeMillis(); + GeneratedAudio audio = tts.generate(text, sid, speed); + long stop = System.currentTimeMillis(); + + float timeElapsedSeconds = (stop - start) / 1000.0f; + + float audioDuration = audio.getSamples().length / (float) audio.getSampleRate(); + float real_time_factor = timeElapsedSeconds / audioDuration; + + String waveFilename = "tts-kokoro-en.wav"; + audio.save(waveFilename); + System.out.printf("-- elapsed : %.3f seconds\n", timeElapsedSeconds); + System.out.printf("-- audio duration: %.3f seconds\n", timeElapsedSeconds); + System.out.printf("-- real-time factor (RTF): %.3f\n", real_time_factor); + System.out.printf("-- text: %s\n", text); + System.out.printf("-- Saved to %s\n", waveFilename); + + tts.release(); + } +} diff --git a/java-api-examples/run-non-streaming-tts-kokoro-en.sh b/java-api-examples/run-non-streaming-tts-kokoro-en.sh new file mode 100755 index 000000000..cf8101cd8 --- /dev/null +++ b/java-api-examples/run-non-streaming-tts-kokoro-en.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -ex + +if [[ ! -f ../build/lib/libsherpa-onnx-jni.dylib && ! -f ../build/lib/libsherpa-onnx-jni.so ]]; then + mkdir -p ../build + pushd ../build + cmake \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + -DSHERPA_ONNX_ENABLE_JNI=ON \ + .. + + make -j4 + ls -lh lib + popd +fi + +if [ ! -f ../sherpa-onnx/java-api/build/sherpa-onnx.jar ]; then + pushd ../sherpa-onnx/java-api + make + popd +fi + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html +# to download more models +if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 + tar xf kokoro-en-v0_19.tar.bz2 + rm kokoro-en-v0_19.tar.bz2 +fi + +java \ + -Djava.library.path=$PWD/../build/lib \ + -cp ../sherpa-onnx/java-api/build/sherpa-onnx.jar \ + NonStreamingTtsKokoroEn.java diff --git a/kotlin-api-examples/run.sh b/kotlin-api-examples/run.sh index 63ea224d1..02339f952 100755 --- a/kotlin-api-examples/run.sh +++ b/kotlin-api-examples/run.sh @@ -115,6 +115,12 @@ function testTts() { curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx fi + if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 + tar xf kokoro-en-v0_19.tar.bz2 + rm kokoro-en-v0_19.tar.bz2 + fi + out_filename=test_tts.jar kotlinc-jvm -include-runtime -d $out_filename \ test_tts.kt \ diff --git a/kotlin-api-examples/test_tts.kt b/kotlin-api-examples/test_tts.kt index 3865c33e3..d9637873d 100644 --- a/kotlin-api-examples/test_tts.kt +++ b/kotlin-api-examples/test_tts.kt @@ -3,6 +3,28 @@ package com.k2fsa.sherpa.onnx fun main() { testVits() testMatcha() + testKokoro() +} + +fun testKokoro() { + // see https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models + var config = OfflineTtsConfig( + model=OfflineTtsModelConfig( + kokoro=OfflineTtsKokoroModelConfig( + model="./kokoro-en-v0_19/model.onnx", + voices="./kokoro-en-v0_19/voices.bin", + tokens="./kokoro-en-v0_19/tokens.txt", + dataDir="./kokoro-en-v0_19/espeak-ng-data", + ), + numThreads=2, + debug=true, + ), + ) + val tts = OfflineTts(config=config) + val audio = tts.generateWithCallback(text="How are you doing today?", callback=::callback) + audio.save(filename="test-kokoro-en.wav") + tts.release() + println("Saved to test-kokoro-en.wav") } fun testMatcha() { @@ -24,9 +46,9 @@ fun testMatcha() { ) val tts = OfflineTts(config=config) val audio = tts.generateWithCallback(text="某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。", callback=::callback) - audio.save(filename="test-zh.wav") + audio.save(filename="test-matcha-zh.wav") tts.release() - println("Saved to test-zh.wav") + println("Saved to test-matcha-zh.wav") } fun testVits() { diff --git a/scripts/apk/build-apk-tts-engine.sh.in b/scripts/apk/build-apk-tts-engine.sh.in index 69933d2fc..dd00b72e9 100644 --- a/scripts/apk/build-apk-tts-engine.sh.in +++ b/scripts/apk/build-apk-tts-engine.sh.in @@ -39,6 +39,7 @@ model_dir={{ tts_model.model_dir }} model_name={{ tts_model.model_name }} acoustic_model_name={{ tts_model.acoustic_model_name }} vocoder={{ tts_model.vocoder }} +voices={{ tts_model.voices }} lang={{ tts_model.lang }} lang_iso_639_3={{ tts_model.lang_iso_639_3 }} @@ -70,6 +71,10 @@ sed -i.bak s/"lang = null"/"lang = \"$lang_iso_639_3\""/ ./TtsEngine.kt sed -i.bak s/"vocoder = null"/"vocoder = \"$vocoder\""/ ./TtsEngine.kt {% endif %} +{% if tts_model.voices %} + sed -i.bak s/"voices = null"/"voices = \"$voices\""/ ./TtsEngine.kt +{% endif %} + {% if tts_model.rule_fsts %} rule_fsts={{ tts_model.rule_fsts }} sed -i.bak s%"ruleFsts = null"%"ruleFsts = \"$rule_fsts\""% ./TtsEngine.kt diff --git a/scripts/apk/build-apk-tts.sh.in b/scripts/apk/build-apk-tts.sh.in index 34135f1a1..f972007e2 100644 --- a/scripts/apk/build-apk-tts.sh.in +++ b/scripts/apk/build-apk-tts.sh.in @@ -39,6 +39,7 @@ model_dir={{ tts_model.model_dir }} model_name={{ tts_model.model_name }} acoustic_model_name={{ tts_model.acoustic_model_name }} vocoder={{ tts_model.vocoder }} +voices={{ tts_model.voices }} lang={{ tts_model.lang }} wget -qq https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/$model_dir.tar.bz2 @@ -69,6 +70,9 @@ sed -i.bak s/"modelDir = null"/"modelDir = \"$model_dir\""/ ./MainActivity.kt sed -i.bak s/"vocoder = null"/"vocoder = \"$vocoder\""/ ./MainActivity.kt {% endif %} +{% if tts_model.voices %} + sed -i.bak s/"voices = null"/"voices = \"$voices\""/ ./MainActivity.kt +{% endif %} {% if tts_model.rule_fsts %} rule_fsts={{ tts_model.rule_fsts }} diff --git a/scripts/apk/generate-tts-apk-script.py b/scripts/apk/generate-tts-apk-script.py index 1d804ecf9..f43dc644b 100755 --- a/scripts/apk/generate-tts-apk-script.py +++ b/scripts/apk/generate-tts-apk-script.py @@ -33,6 +33,7 @@ class TtsModel: model_name: str = "" # for vits acoustic_model_name: str = "" # for matcha vocoder: str = "" # for matcha + voices: str = "" # for kokoro lang: str = "" # en, zh, fr, de, etc. rule_fsts: Optional[List[str]] = None rule_fars: Optional[List[str]] = None @@ -409,6 +410,21 @@ def get_matcha_models() -> List[TtsModel]: return chinese_models + english_models +def get_kokoro_models() -> List[TtsModel]: + english_models = [ + TtsModel( + model_dir="kokoro-en-v0_19", + model_name="model.onnx", + lang="en", + ) + ] + for m in english_models: + m.data_dir = f"{m.model_dir}/espeak-ng-data" + m.voices = "voices.bin" + + return english_models + + def main(): args = get_args() index = args.index @@ -421,6 +437,7 @@ def main(): all_model_list += get_mimic3_models() all_model_list += get_coqui_models() all_model_list += get_matcha_models() + all_model_list += get_kokoro_models() convert_lang_to_iso_639_3(all_model_list) print(all_model_list) diff --git a/sherpa-onnx/java-api/Makefile b/sherpa-onnx/java-api/Makefile index 816d2139f..0721daf16 100644 --- a/sherpa-onnx/java-api/Makefile +++ b/sherpa-onnx/java-api/Makefile @@ -35,6 +35,7 @@ java_files += OfflineRecognizerResult.java java_files += OfflineStream.java java_files += OfflineRecognizer.java +java_files += OfflineTtsKokoroModelConfig.java java_files += OfflineTtsMatchaModelConfig.java java_files += OfflineTtsVitsModelConfig.java java_files += OfflineTtsModelConfig.java diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsKokoroModelConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsKokoroModelConfig.java new file mode 100644 index 000000000..4088acfd3 --- /dev/null +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsKokoroModelConfig.java @@ -0,0 +1,80 @@ +// Copyright 2025 Xiaomi Corporation +package com.k2fsa.sherpa.onnx; + +public class OfflineTtsKokoroModelConfig { + private final String model; + private final String voices; + private final String tokens; + private final String dataDir; + private final float lengthScale; + + private OfflineTtsKokoroModelConfig(Builder builder) { + this.model = builder.model; + this.voices = builder.voices; + this.tokens = builder.tokens; + this.dataDir = builder.dataDir; + this.lengthScale = builder.lengthScale; + } + + public static Builder builder() { + return new Builder(); + } + + public String getModel() { + return model; + } + + public String getVoices() { + return voices; + } + + public String getTokens() { + return tokens; + } + + public String getDataDir() { + return dataDir; + } + + public float getLengthScale() { + return lengthScale; + } + + + public static class Builder { + private String model = ""; + private String voices = ""; + private String tokens = ""; + private String dataDir = ""; + private float lengthScale = 1.0f; + + public OfflineTtsKokoroModelConfig build() { + return new OfflineTtsKokoroModelConfig(this); + } + + public Builder setModel(String model) { + this.model = model; + return this; + } + + public Builder setVoices(String voices) { + this.voices = voices; + return this; + } + + public Builder setTokens(String tokens) { + this.tokens = tokens; + return this; + } + + public Builder setDataDir(String dataDir) { + this.dataDir = dataDir; + return this; + } + + public Builder setLengthScale(float lengthScale) { + this.lengthScale = lengthScale; + return this; + } + } +} diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsModelConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsModelConfig.java index ff3589b13..24df8a5d3 100644 --- a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsModelConfig.java +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineTtsModelConfig.java @@ -5,6 +5,7 @@ public class OfflineTtsModelConfig { private final OfflineTtsVitsModelConfig vits; private final OfflineTtsMatchaModelConfig matcha; + private final OfflineTtsKokoroModelConfig kokoro; private final int numThreads; private final boolean debug; private final String provider; @@ -12,6 +13,7 @@ public class OfflineTtsModelConfig { private OfflineTtsModelConfig(Builder builder) { this.vits = builder.vits; this.matcha = builder.matcha; + this.kokoro = builder.kokoro; this.numThreads = builder.numThreads; this.debug = builder.debug; this.provider = builder.provider; @@ -29,9 +31,14 @@ public OfflineTtsMatchaModelConfig getMatcha() { return matcha; } + public OfflineTtsKokoroModelConfig getKokoro() { + return kokoro; + } + public static class Builder { private OfflineTtsVitsModelConfig vits = OfflineTtsVitsModelConfig.builder().build(); private OfflineTtsMatchaModelConfig matcha = OfflineTtsMatchaModelConfig.builder().build(); + private OfflineTtsKokoroModelConfig kokoro = OfflineTtsKokoroModelConfig.builder().build(); private int numThreads = 1; private boolean debug = true; private String provider = "cpu"; @@ -50,6 +57,11 @@ public Builder setMatcha(OfflineTtsMatchaModelConfig matcha) { return this; } + public Builder setKokoro(OfflineTtsKokoroModelConfig kokoro) { + this.kokoro = kokoro; + return this; + } + public Builder setNumThreads(int numThreads) { this.numThreads = numThreads; return this; diff --git a/sherpa-onnx/jni/offline-tts.cc b/sherpa-onnx/jni/offline-tts.cc index 985d581ef..6af10788a 100644 --- a/sherpa-onnx/jni/offline-tts.cc +++ b/sherpa-onnx/jni/offline-tts.cc @@ -113,6 +113,39 @@ static OfflineTtsConfig GetOfflineTtsConfig(JNIEnv *env, jobject config) { fid = env->GetFieldID(matcha_cls, "lengthScale", "F"); ans.model.matcha.length_scale = env->GetFloatField(matcha, fid); + // kokoro + fid = env->GetFieldID(model_config_cls, "kokoro", + "Lcom/k2fsa/sherpa/onnx/OfflineTtsKokoroModelConfig;"); + jobject kokoro = env->GetObjectField(model, fid); + jclass kokoro_cls = env->GetObjectClass(kokoro); + + fid = env->GetFieldID(kokoro_cls, "model", "Ljava/lang/String;"); + s = (jstring)env->GetObjectField(kokoro, fid); + p = env->GetStringUTFChars(s, nullptr); + ans.model.kokoro.model = p; + env->ReleaseStringUTFChars(s, p); + + fid = env->GetFieldID(kokoro_cls, "voices", "Ljava/lang/String;"); + s = (jstring)env->GetObjectField(kokoro, fid); + p = env->GetStringUTFChars(s, nullptr); + ans.model.kokoro.voices = p; + env->ReleaseStringUTFChars(s, p); + + fid = env->GetFieldID(kokoro_cls, "tokens", "Ljava/lang/String;"); + s = (jstring)env->GetObjectField(kokoro, fid); + p = env->GetStringUTFChars(s, nullptr); + ans.model.kokoro.tokens = p; + env->ReleaseStringUTFChars(s, p); + + fid = env->GetFieldID(kokoro_cls, "dataDir", "Ljava/lang/String;"); + s = (jstring)env->GetObjectField(kokoro, fid); + p = env->GetStringUTFChars(s, nullptr); + ans.model.kokoro.data_dir = p; + env->ReleaseStringUTFChars(s, p); + + fid = env->GetFieldID(kokoro_cls, "lengthScale", "F"); + ans.model.kokoro.length_scale = env->GetFloatField(kokoro, fid); + fid = env->GetFieldID(model_config_cls, "numThreads", "I"); ans.model.num_threads = env->GetIntField(model, fid); @@ -273,8 +306,8 @@ Java_com_k2fsa_sherpa_onnx_OfflineTts_generateWithCallbackImpl( return env->CallIntMethod(should_continue, int_value_mid); }; - auto audio = reinterpret_cast(ptr)->Generate( - p_text, sid, speed, callback_wrapper); + auto tts = reinterpret_cast(ptr); + auto audio = tts->Generate(p_text, sid, speed, callback_wrapper); jfloatArray samples_arr = env->NewFloatArray(audio.samples.size()); env->SetFloatArrayRegion(samples_arr, 0, audio.samples.size(), diff --git a/sherpa-onnx/kotlin-api/Tts.kt b/sherpa-onnx/kotlin-api/Tts.kt index 98efe6644..ce85a04da 100644 --- a/sherpa-onnx/kotlin-api/Tts.kt +++ b/sherpa-onnx/kotlin-api/Tts.kt @@ -25,9 +25,18 @@ data class OfflineTtsMatchaModelConfig( var lengthScale: Float = 1.0f, ) +data class OfflineTtsKokoroModelConfig( + var model: String = "", + var voices: String = "", + var tokens: String = "", + var dataDir: String = "", + var lengthScale: Float = 1.0f, +) + data class OfflineTtsModelConfig( var vits: OfflineTtsVitsModelConfig = OfflineTtsVitsModelConfig(), var matcha: OfflineTtsMatchaModelConfig = OfflineTtsMatchaModelConfig(), + var kokoro: OfflineTtsKokoroModelConfig = OfflineTtsKokoroModelConfig(), var numThreads: Int = 1, var debug: Boolean = false, var provider: String = "cpu", @@ -176,12 +185,32 @@ fun getOfflineTtsConfig( modelName: String, // for VITS acousticModelName: String, // for Matcha vocoder: String, // for Matcha + voices: String, // for Kokoro lexicon: String, dataDir: String, dictDir: String, ruleFsts: String, - ruleFars: String + ruleFars: String, + numThreads: Int? = null ): OfflineTtsConfig { + // For Matcha TTS, please set + // acousticModelName, vocoder + + // For Kokoro TTS, please set + // modelName, voices + + // For VITS, please set + // modelName + + val numberOfThreads = if (numThreads != null) { + numThreads + } else if (voices.isNotEmpty()) { + // for Kokoro TTS models, we use more threads + 4 + } else { + 2 + } + if (modelName.isEmpty() && acousticModelName.isEmpty()) { throw IllegalArgumentException("Please specify a TTS model") } @@ -193,7 +222,8 @@ fun getOfflineTtsConfig( if (acousticModelName.isNotEmpty() && vocoder.isEmpty()) { throw IllegalArgumentException("Please provide vocoder for Matcha TTS") } - val vits = if (modelName.isNotEmpty()) { + + val vits = if (modelName.isNotEmpty() && voices.isEmpty()) { OfflineTtsVitsModelConfig( model = "$modelDir/$modelName", lexicon = "$modelDir/$lexicon", @@ -218,11 +248,23 @@ fun getOfflineTtsConfig( OfflineTtsMatchaModelConfig() } + val kokoro = if (voices.isNotEmpty()) { + OfflineTtsKokoroModelConfig( + model = "$modelDir/$modelName", + voices = "$modelDir/$voices", + tokens = "$modelDir/tokens.txt", + dataDir = dataDir, + ) + } else { + OfflineTtsKokoroModelConfig() + } + return OfflineTtsConfig( model = OfflineTtsModelConfig( vits = vits, matcha = matcha, - numThreads = 2, + kokoro = kokoro, + numThreads = numberOfThreads, debug = true, provider = "cpu", ), From bad82f35cf9dc527f7077c07e3b3165f613891a0 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 17 Jan 2025 17:48:42 +0800 Subject: [PATCH 166/183] Update README.md for KWS to not use `git lfs`. (#1729) --- wasm/kws/assets/README.md | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/wasm/kws/assets/README.md b/wasm/kws/assets/README.md index ac67fb5a0..18f792b49 100644 --- a/wasm/kws/assets/README.md +++ b/wasm/kws/assets/README.md @@ -7,21 +7,34 @@ to download a model. # Kws The following is an example: -``` -cd sherpa-onnx/wasm/kws -git clone https://www.modelscope.cn/pkufool/sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.git assets +```bash +cd sherpa-onnx/wasm/kws/assets +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/kws-models/sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.tar.bz2 +tar xvf sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.tar.bz2 +rm sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.tar.bz2 + +mv sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/encoder-epoch-12-avg-2-chunk-16-left-64.onnx ./ +mv sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/decoder-epoch-12-avg-2-chunk-16-left-64.onnx ./ +mv sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/joiner-epoch-12-avg-2-chunk-16-left-64.onnx ./ +mv sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/tokens.txt ./ +rm -rf sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01 ``` You should have the following files in `assets` before you can run `build-wasm-simd-kws.sh` -``` -├── decoder-epoch-12-avg-2-chunk-16-left-64.onnx -├── encoder-epoch-12-avg-2-chunk-16-left-64.onnx -├── joiner-epoch-12-avg-2-chunk-16-left-64.onnx -├── keywords_raw.txt -├── keywords.txt -├── README.md -└── tokens.txt +```bash +fangjuns-MacBook-Pro:assets fangjun$ pwd +/Users/fangjun/open-source/sherpa-onnx/wasm/kws/assets +fangjuns-MacBook-Pro:assets fangjun$ ls -lh +total 25616 +-rw-r--r-- 1 fangjun staff 692B Oct 29 16:53 README.md +-rw-r--r-- 1 fangjun staff 660K Aug 14 15:21 decoder-epoch-12-avg-2-chunk-16-left-64.onnx +-rw-r--r-- 1 fangjun staff 12M Aug 14 15:21 encoder-epoch-12-avg-2-chunk-16-left-64.onnx +-rw-r--r-- 1 fangjun staff 247K Aug 14 15:21 joiner-epoch-12-avg-2-chunk-16-left-64.onnx +-rw-r--r-- 1 fangjun staff 1.6K Aug 14 15:08 tokens.txt ``` + +**Hint**: Remember to remove extra files from ``assets``. For instance, please remember to remove +the file `sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.tar.bz2`. From 2df43b378827ba36fd5d9f3ef88dc4117139e161 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 17 Jan 2025 18:09:19 +0800 Subject: [PATCH 167/183] Release v1.10.40 (#1731) --- CHANGELOG.md | 20 ++++++++++++++++ CMakeLists.txt | 2 +- android/SherpaOnnxAar/README.md | 6 ++--- android/SherpaOnnxJavaDemo/app/build.gradle | 2 +- build-ios-shared.sh | 2 +- .../add-punctuations/pubspec.yaml | 2 +- dart-api-examples/audio-tagging/pubspec.yaml | 2 +- .../keyword-spotter/pubspec.yaml | 2 +- .../non-streaming-asr/pubspec.yaml | 2 +- .../speaker-diarization/pubspec.yaml | 2 +- .../speaker-identification/pubspec.yaml | 2 +- dart-api-examples/streaming-asr/pubspec.yaml | 2 +- dart-api-examples/tts/pubspec.yaml | 2 +- .../vad-with-non-streaming-asr/pubspec.yaml | 2 +- dart-api-examples/vad/pubspec.yaml | 2 +- flutter-examples/streaming_asr/pubspec.yaml | 4 ++-- flutter-examples/tts/pubspec.yaml | 4 ++-- flutter/sherpa_onnx/pubspec.yaml | 12 +++++----- .../ios/sherpa_onnx_ios.podspec | 2 +- .../macos/sherpa_onnx_macos.podspec | 2 +- .../SherpaOnnxHar/sherpa_onnx/README.md | 2 +- .../sherpa_onnx/oh-package.json5 | 2 +- .../entry/oh-package.json5 | 2 +- .../entry/oh-package.json5 | 2 +- .../entry/oh-package.json5 | 2 +- .../SherpaOnnxTts/entry/oh-package.json5 | 2 +- harmony-os/SherpaOnnxVadAsr/entry/README.md | 2 +- .../SherpaOnnxVadAsr/entry/oh-package.json5 | 2 +- jitpack.yml | 6 ++--- new-release.sh | 24 +++++++++---------- nodejs-addon-examples/package.json | 2 +- pom.xml | 2 +- 32 files changed, 73 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7a29ecb1..dfd4ef92b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,23 @@ +## 1.10.40 + +* Fix building wheels (#1703) +* Export kokoro to sherpa-onnx (#1713) +* Add C++ and Python API for Kokoro TTS models. (#1715) +* Add C API for Kokoro TTS models (#1717) +* Fix style issues (#1718) +* Add C# API for Kokoro TTS models (#1720) +* Add Swift API for Kokoro TTS models (#1721) +* Add Go API for Kokoro TTS models (#1722) +* Add Dart API for Kokoro TTS models (#1723) +* Add Pascal API for Kokoro TTS models (#1724) +* Add JavaScript API (node-addon) for Kokoro TTS models (#1725) +* Add JavaScript (WebAssembly) API for Kokoro TTS models. (#1726) +* Add Koltin and Java API for Kokoro TTS models (#1728) +* Update README.md for KWS to not use git lfs. (#1729) + + + + ## 1.10.39 * Fix building without TTS (#1691) diff --git a/CMakeLists.txt b/CMakeLists.txt index e9f427666..9955bc9f7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ project(sherpa-onnx) # Remember to update # ./CHANGELOG.md # ./new-release.sh -set(SHERPA_ONNX_VERSION "1.10.39") +set(SHERPA_ONNX_VERSION "1.10.40") # Disable warning about # diff --git a/android/SherpaOnnxAar/README.md b/android/SherpaOnnxAar/README.md index ec50ebf40..8f4932072 100644 --- a/android/SherpaOnnxAar/README.md +++ b/android/SherpaOnnxAar/README.md @@ -4,8 +4,8 @@ git clone https://github.com/k2-fsa/sherpa-onnx cd sherpa-onnx -wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.39/sherpa-onnx-v1.10.39-android.tar.bz2 -tar xvf sherpa-onnx-v1.10.39-android.tar.bz2 +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.40/sherpa-onnx-v1.10.40-android.tar.bz2 +tar xvf sherpa-onnx-v1.10.40-android.tar.bz2 cp -v jniLibs/arm64-v8a/* android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/arm64-v8a/ cp -v jniLibs/armeabi-v7a/* android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/armeabi-v7a/ @@ -16,5 +16,5 @@ cd android/SherpaOnnxAar ./gradlew :sherpa_onnx:assembleRelease ls -lh ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar -cp ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar ../../sherpa-onnx-1.10.39.aar +cp ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar ../../sherpa-onnx-1.10.40.aar ``` diff --git a/android/SherpaOnnxJavaDemo/app/build.gradle b/android/SherpaOnnxJavaDemo/app/build.gradle index 8e66f4355..38405d916 100644 --- a/android/SherpaOnnxJavaDemo/app/build.gradle +++ b/android/SherpaOnnxJavaDemo/app/build.gradle @@ -34,5 +34,5 @@ dependencies { implementation 'pub.devrel:easypermissions:3.0.0' implementation 'androidx.core:core-ktx:1.7.0' // implementation files('/Users/fangjun/open-source/sherpa-onnx/android/SherpaOnnxAar/sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar') - implementation 'com.github.k2-fsa:sherpa-onnx:v1.10.39' + implementation 'com.github.k2-fsa:sherpa-onnx:v1.10.40' } diff --git a/build-ios-shared.sh b/build-ios-shared.sh index 39e5c0f2f..54a374352 100755 --- a/build-ios-shared.sh +++ b/build-ios-shared.sh @@ -242,7 +242,7 @@ for d in ios-arm64_x86_64-simulator ios-arm64; do CFBundlePackageType FMWK CFBundleShortVersionString - 1.10.39 + 1.10.40 CFBundleSupportedPlatforms iPhoneOS diff --git a/dart-api-examples/add-punctuations/pubspec.yaml b/dart-api-examples/add-punctuations/pubspec.yaml index ca01d32df..4ac2868b5 100644 --- a/dart-api-examples/add-punctuations/pubspec.yaml +++ b/dart-api-examples/add-punctuations/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.39 + sherpa_onnx: ^1.10.40 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/audio-tagging/pubspec.yaml b/dart-api-examples/audio-tagging/pubspec.yaml index d47d93bfb..15ecf8286 100644 --- a/dart-api-examples/audio-tagging/pubspec.yaml +++ b/dart-api-examples/audio-tagging/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.39 + sherpa_onnx: ^1.10.40 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/keyword-spotter/pubspec.yaml b/dart-api-examples/keyword-spotter/pubspec.yaml index 4210cbfb4..02a8c1eb9 100644 --- a/dart-api-examples/keyword-spotter/pubspec.yaml +++ b/dart-api-examples/keyword-spotter/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.39 + sherpa_onnx: ^1.10.40 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/non-streaming-asr/pubspec.yaml b/dart-api-examples/non-streaming-asr/pubspec.yaml index 93a0ad3a2..02f5b7927 100644 --- a/dart-api-examples/non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.39 + sherpa_onnx: ^1.10.40 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/speaker-diarization/pubspec.yaml b/dart-api-examples/speaker-diarization/pubspec.yaml index df44568d3..ab74d425f 100644 --- a/dart-api-examples/speaker-diarization/pubspec.yaml +++ b/dart-api-examples/speaker-diarization/pubspec.yaml @@ -8,7 +8,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.39 + sherpa_onnx: ^1.10.40 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/speaker-identification/pubspec.yaml b/dart-api-examples/speaker-identification/pubspec.yaml index bc70976d9..5725fb434 100644 --- a/dart-api-examples/speaker-identification/pubspec.yaml +++ b/dart-api-examples/speaker-identification/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.39 + sherpa_onnx: ^1.10.40 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/streaming-asr/pubspec.yaml b/dart-api-examples/streaming-asr/pubspec.yaml index ee79c40dd..092506fcc 100644 --- a/dart-api-examples/streaming-asr/pubspec.yaml +++ b/dart-api-examples/streaming-asr/pubspec.yaml @@ -11,7 +11,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.39 + sherpa_onnx: ^1.10.40 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/tts/pubspec.yaml b/dart-api-examples/tts/pubspec.yaml index 718c4201a..34309474a 100644 --- a/dart-api-examples/tts/pubspec.yaml +++ b/dart-api-examples/tts/pubspec.yaml @@ -8,7 +8,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.39 + sherpa_onnx: ^1.10.40 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml index 250c45d9a..eff21233b 100644 --- a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.39 + sherpa_onnx: ^1.10.40 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad/pubspec.yaml b/dart-api-examples/vad/pubspec.yaml index 66ad030ac..a6b1b626d 100644 --- a/dart-api-examples/vad/pubspec.yaml +++ b/dart-api-examples/vad/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.39 + sherpa_onnx: ^1.10.40 path: ^1.9.0 args: ^2.5.0 diff --git a/flutter-examples/streaming_asr/pubspec.yaml b/flutter-examples/streaming_asr/pubspec.yaml index 12116dd6e..0f6734375 100644 --- a/flutter-examples/streaming_asr/pubspec.yaml +++ b/flutter-examples/streaming_asr/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' -version: 1.10.39 +version: 1.10.40 topics: - speech-recognition @@ -31,7 +31,7 @@ dependencies: record: ^5.1.0 url_launcher: ^6.2.6 - sherpa_onnx: ^1.10.39 + sherpa_onnx: ^1.10.40 # sherpa_onnx: # path: ../../flutter/sherpa_onnx diff --git a/flutter-examples/tts/pubspec.yaml b/flutter-examples/tts/pubspec.yaml index 976a03e5d..d0a2e3f97 100644 --- a/flutter-examples/tts/pubspec.yaml +++ b/flutter-examples/tts/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' # Remove this line if you wish to publish to pub.dev -version: 1.10.39 +version: 1.10.40 environment: sdk: ">=2.17.0 <4.0.0" @@ -18,7 +18,7 @@ dependencies: cupertino_icons: ^1.0.6 path_provider: ^2.1.3 path: ^1.9.0 - sherpa_onnx: ^1.10.39 + sherpa_onnx: ^1.10.40 # sherpa_onnx: # path: ../../flutter/sherpa_onnx url_launcher: 6.2.6 diff --git a/flutter/sherpa_onnx/pubspec.yaml b/flutter/sherpa_onnx/pubspec.yaml index 8dc0f43b4..b0ed0e556 100644 --- a/flutter/sherpa_onnx/pubspec.yaml +++ b/flutter/sherpa_onnx/pubspec.yaml @@ -17,7 +17,7 @@ topics: - voice-activity-detection # remember to change the version in ../sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec -version: 1.10.39 +version: 1.10.40 homepage: https://github.com/k2-fsa/sherpa-onnx @@ -30,23 +30,23 @@ dependencies: flutter: sdk: flutter - sherpa_onnx_android: ^1.10.39 + sherpa_onnx_android: ^1.10.40 # sherpa_onnx_android: # path: ../sherpa_onnx_android - sherpa_onnx_macos: ^1.10.39 + sherpa_onnx_macos: ^1.10.40 # sherpa_onnx_macos: # path: ../sherpa_onnx_macos - sherpa_onnx_linux: ^1.10.39 + sherpa_onnx_linux: ^1.10.40 # sherpa_onnx_linux: # path: ../sherpa_onnx_linux - sherpa_onnx_windows: ^1.10.39 + sherpa_onnx_windows: ^1.10.40 # sherpa_onnx_windows: # path: ../sherpa_onnx_windows - sherpa_onnx_ios: ^1.10.39 + sherpa_onnx_ios: ^1.10.40 # sherpa_onnx_ios: # path: ../sherpa_onnx_ios diff --git a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec index 02f2be1e6..8b2106c43 100644 --- a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec +++ b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec @@ -7,7 +7,7 @@ # https://groups.google.com/g/dart-ffi/c/nUATMBy7r0c Pod::Spec.new do |s| s.name = 'sherpa_onnx_ios' - s.version = '1.10.39' + s.version = '1.10.40' s.summary = 'A new Flutter FFI plugin project.' s.description = <<-DESC A new Flutter FFI plugin project. diff --git a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec index 89d9f7bab..0090f84e0 100644 --- a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec +++ b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec @@ -4,7 +4,7 @@ # Pod::Spec.new do |s| s.name = 'sherpa_onnx_macos' - s.version = '1.10.39' + s.version = '1.10.40' s.summary = 'sherpa-onnx Flutter FFI plugin project.' s.description = <<-DESC sherpa-onnx Flutter FFI plugin project. diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md b/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md index 917401b43..358a2d107 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md @@ -23,7 +23,7 @@ or update your `oh-package.json5` to include the following: ``` "dependencies": { - "sherpa_onnx": "1.10.39", + "sherpa_onnx": "1.10.40", }, ``` diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 index 34af61da2..676e14092 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 @@ -1,6 +1,6 @@ { "name": "sherpa_onnx", - "version": "1.10.39", + "version": "1.10.40", "description": "On-device speech-to-text, text-to-speech, and speaker diarization using Next-gen Kaldi without Internet connection", "main": "Index.ets", "author": "The next-gen Kaldi team", diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 b/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 index 5449d3e81..faf4a788a 100644 --- a/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.39" + "sherpa_onnx": "1.10.40" } } diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 b/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 index 5f2f6b5ff..e50200119 100644 --- a/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.39", + "sherpa_onnx": "1.10.40", } } diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 index 5f2f6b5ff..e50200119 100644 --- a/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.39", + "sherpa_onnx": "1.10.40", } } diff --git a/harmony-os/SherpaOnnxTts/entry/oh-package.json5 b/harmony-os/SherpaOnnxTts/entry/oh-package.json5 index 5f2f6b5ff..e50200119 100644 --- a/harmony-os/SherpaOnnxTts/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxTts/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.39", + "sherpa_onnx": "1.10.40", } } diff --git a/harmony-os/SherpaOnnxVadAsr/entry/README.md b/harmony-os/SherpaOnnxVadAsr/entry/README.md index 467c21391..948dc7ffb 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/README.md +++ b/harmony-os/SherpaOnnxVadAsr/entry/README.md @@ -1,6 +1,6 @@ # Introduction -Please download ./sherpa_onnx-v1.10.39.har +Please download ./sherpa_onnx-v1.10.40.har from Hint: For users who have no access to huggingface, please use diff --git a/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 b/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 index fae5caf53..ca1e2eaf1 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 @@ -7,7 +7,7 @@ "license": "", "dependencies": { // please see https://ohpm.openharmony.cn/#/cn/detail/sherpa_onnx - "sherpa_onnx": "1.10.39", + "sherpa_onnx": "1.10.40", } } diff --git a/jitpack.yml b/jitpack.yml index 7f283a09f..ba9c64759 100644 --- a/jitpack.yml +++ b/jitpack.yml @@ -2,8 +2,8 @@ jdk: - openjdk17 before_install: - - wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.39/sherpa-onnx-1.10.39.aar + - wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.40/sherpa-onnx-1.10.40.aar install: - - FILE="-Dfile=sherpa-onnx-1.10.39.aar" - - mvn install:install-file $FILE -DgroupId=com.k2fsa.sherpa.onnx -DartifactId=sherpa-onnx -Dversion=1.10.39 -Dpackaging=aar -DgeneratePom=true + - FILE="-Dfile=sherpa-onnx-1.10.40.aar" + - mvn install:install-file $FILE -DgroupId=com.k2fsa.sherpa.onnx -DartifactId=sherpa-onnx -Dversion=1.10.40 -Dpackaging=aar -DgeneratePom=true diff --git a/new-release.sh b/new-release.sh index e52bf65b6..dfddf6b92 100755 --- a/new-release.sh +++ b/new-release.sh @@ -2,18 +2,18 @@ set -ex -sed -i.bak 's/1\.10\.38/1\.10\.39/g' ./build-ios-shared.sh -sed -i.bak 's/1\.10\.38/1\.10\.39/g' ./pom.xml -sed -i.bak 's/1\.10\.38/1\.10\.39/g' ./jitpack.yml -sed -i.bak 's/1\.10\.38/1\.10\.39/g' ./android/SherpaOnnxAar/README.md +sed -i.bak 's/1\.10\.39/1\.10\.40/g' ./build-ios-shared.sh +sed -i.bak 's/1\.10\.39/1\.10\.40/g' ./pom.xml +sed -i.bak 's/1\.10\.39/1\.10\.40/g' ./jitpack.yml +sed -i.bak 's/1\.10\.39/1\.10\.40/g' ./android/SherpaOnnxAar/README.md -find android -name build.gradle -type f -exec sed -i.bak 's/sherpa-onnx:v1\.10\.38/sherpa-onnx:v1\.10\.39/g' {} \; +find android -name build.gradle -type f -exec sed -i.bak 's/sherpa-onnx:v1\.10\.39/sherpa-onnx:v1\.10\.40/g' {} \; -find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.38/1\.10\.39/g' {} \; -find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.38/1\.10\.39/g' {} \; -find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.38/1\.10\.39/g' {} \; -find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.38/1\.10\.39/g' {} \; -find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.38/1\.10\.39/g' {} \; +find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.39/1\.10\.40/g' {} \; +find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.39/1\.10\.40/g' {} \; +find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.39/1\.10\.40/g' {} \; +find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.39/1\.10\.40/g' {} \; +find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.39/1\.10\.40/g' {} \; -find harmony-os -name "README.md" -type f -exec sed -i.bak 's/1\.10\.38/1\.10\.39/g' {} \; -find harmony-os -name oh-package.json5 -type f -exec sed -i.bak 's/1\.10\.38/1\.10\.39/g' {} \; +find harmony-os -name "README.md" -type f -exec sed -i.bak 's/1\.10\.39/1\.10\.40/g' {} \; +find harmony-os -name oh-package.json5 -type f -exec sed -i.bak 's/1\.10\.39/1\.10\.40/g' {} \; diff --git a/nodejs-addon-examples/package.json b/nodejs-addon-examples/package.json index debe3ac96..6a7779165 100644 --- a/nodejs-addon-examples/package.json +++ b/nodejs-addon-examples/package.json @@ -1,5 +1,5 @@ { "dependencies": { - "sherpa-onnx-node": "^1.10.39" + "sherpa-onnx-node": "^1.10.40" } } diff --git a/pom.xml b/pom.xml index 7f9c217b6..1063994c3 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.k2fsa.sherpa.onnx sherpa-onnx-android - 1.10.39 + 1.10.40 https://github.com/k2-fsa/sherpa-onnx pom First Android Library From 9d6c0e55b57004fec181d2efcec982e4edbc786a Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 20 Jan 2025 10:08:34 +0800 Subject: [PATCH 168/183] Fix UI for Android TTS Engine. (#1735) Set max number of lines of the input text field so that the buttons are still visible when the input text is long. --- .../java/com/k2fsa/sherpa/onnx/tts/engine/MainActivity.kt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/MainActivity.kt b/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/MainActivity.kt index 18a89f93c..c96f9f0ef 100644 --- a/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/MainActivity.kt +++ b/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/MainActivity.kt @@ -22,7 +22,9 @@ import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.wrapContentHeight +import androidx.compose.foundation.rememberScrollState import androidx.compose.foundation.text.KeyboardOptions +import androidx.compose.foundation.verticalScroll import androidx.compose.material3.Button import androidx.compose.material3.ExperimentalMaterial3Api import androidx.compose.material3.MaterialTheme @@ -109,6 +111,7 @@ class MainActivity : ComponentActivity() { var rtfText by remember { mutableStateOf("") } + val scrollState = rememberScrollState(0) val numSpeakers = TtsEngine.tts!!.numSpeakers() if (numSpeakers > 1) { @@ -142,9 +145,11 @@ class MainActivity : ComponentActivity() { value = testText, onValueChange = { testText = it }, label = { Text("Please input your text here") }, + maxLines = 10, modifier = Modifier .fillMaxWidth() .padding(bottom = 16.dp) + .verticalScroll(scrollState) .wrapContentHeight(), singleLine = false, ) From e2f096bf3cf535f2ad75b4617810abb92663a2b1 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 20 Jan 2025 11:41:59 +0800 Subject: [PATCH 169/183] Add iOS TTS example for MatchaTTS (#1736) --- .../SherpaOnnxTts/ViewModel.swift | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/ViewModel.swift b/ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/ViewModel.swift index 45db6ec17..880313f6f 100644 --- a/ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/ViewModel.swift +++ b/ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/ViewModel.swift @@ -127,9 +127,44 @@ func getTtsFor_zh_en_melo_tts() -> SherpaOnnxOfflineTtsWrapper { return SherpaOnnxOfflineTtsWrapper(config: &config) } +func getTtsFor_matcha_icefall_zh_baker() -> SherpaOnnxOfflineTtsWrapper { + // please see https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker + + let acousticModel = getResource("model-steps-3", "onnx") + let vocoder = getResource("hifigan_v2", "onnx") + + let tokens = getResource("tokens", "txt") + let lexicon = getResource("lexicon", "txt") + + let dictDir = resourceURL(to: "dict") + + let numFst = getResource("number", "fst") + let dateFst = getResource("date", "fst") + let phoneFst = getResource("phone", "fst") + let ruleFsts = "\(dateFst),\(phoneFst),\(numFst)" + + let matcha = sherpaOnnxOfflineTtsMatchaModelConfig( + acousticModel: acousticModel, + vocoder: vocoder, + lexicon: lexicon, + tokens: tokens, + dictDir: dictDir + ) + + let modelConfig = sherpaOnnxOfflineTtsModelConfig(matcha: matcha) + var config = sherpaOnnxOfflineTtsConfig( + model: modelConfig, + ruleFsts: ruleFsts + ) + + return SherpaOnnxOfflineTtsWrapper(config: &config) +} + func createOfflineTts() -> SherpaOnnxOfflineTtsWrapper { // Please enable only one of them + // return getTtsFor_matcha_icefall_zh_baker() + return getTtsFor_en_US_amy_low() // return getTtsForVCTK() From a2650b7ddd30cd70b8a88ab5a7976305716517a2 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 20 Jan 2025 13:12:43 +0800 Subject: [PATCH 170/183] Add iOS example for Kokoro TTS (#1737) --- .../SherpaOnnxTts/ViewModel.swift | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/ViewModel.swift b/ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/ViewModel.swift index 880313f6f..4d3289036 100644 --- a/ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/ViewModel.swift +++ b/ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/ViewModel.swift @@ -160,12 +160,34 @@ func getTtsFor_matcha_icefall_zh_baker() -> SherpaOnnxOfflineTtsWrapper { return SherpaOnnxOfflineTtsWrapper(config: &config) } +func getTtsFor_kokoro_en_v0_19() -> SherpaOnnxOfflineTtsWrapper { + // please see https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html#kokoro-en-v0-19-english-11-speakers + + let model = getResource("model", "onnx") + let voices = getResource("voices", "bin") + + // tokens.txt + let tokens = getResource("tokens", "txt") + + // in this case, we don't need lexicon.txt + let dataDir = resourceURL(to: "espeak-ng-data") + + let kokoro = sherpaOnnxOfflineTtsKokoroModelConfig( + model: model, voices: voices, tokens: tokens, dataDir: dataDir) + let modelConfig = sherpaOnnxOfflineTtsModelConfig(kokoro: kokoro) + var config = sherpaOnnxOfflineTtsConfig(model: modelConfig) + + return SherpaOnnxOfflineTtsWrapper(config: &config) +} + func createOfflineTts() -> SherpaOnnxOfflineTtsWrapper { // Please enable only one of them + return getTtsFor_kokoro_en_v0_19() + // return getTtsFor_matcha_icefall_zh_baker() - return getTtsFor_en_US_amy_low() + // return getTtsFor_en_US_amy_low() // return getTtsForVCTK() From b943341fb1d059196d91adfbe4bc27f5fc024044 Mon Sep 17 00:00:00 2001 From: Jacklyn <149936290+jacklynblack@users.noreply.github.com> Date: Mon, 20 Jan 2025 10:29:36 +0200 Subject: [PATCH 171/183] Fix `dither` binding in Pybind11 to ensure independence from `high_freq` in `FeatureExtractorConfig` (#1739) --- sherpa-onnx/python/csrc/features.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sherpa-onnx/python/csrc/features.cc b/sherpa-onnx/python/csrc/features.cc index 4f179999a..63c0143c6 100644 --- a/sherpa-onnx/python/csrc/features.cc +++ b/sherpa-onnx/python/csrc/features.cc @@ -19,7 +19,7 @@ static void PybindFeatureExtractorConfig(py::module *m) { .def_readwrite("feature_dim", &PyClass::feature_dim) .def_readwrite("low_freq", &PyClass::low_freq) .def_readwrite("high_freq", &PyClass::high_freq) - .def_readwrite("dither", &PyClass::high_freq) + .def_readwrite("dither", &PyClass::dither) .def("__str__", &PyClass::ToString); } From 8b989a851cbb759976d1f8d40cae91dd9362f816 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 20 Jan 2025 16:41:10 +0800 Subject: [PATCH 172/183] Fix keyword spotting. (#1689) Reset the stream right after detecting a keyword --- .github/scripts/test-python.sh | 33 +-- .github/workflows/c-api.yaml | 21 ++ .github/workflows/cxx-api.yaml | 22 ++ .../com/k2fsa/sherpa/onnx/MainActivity.kt | 29 +- c-api-examples/CMakeLists.txt | 3 + c-api-examples/kws-c-api.c | 150 +++++++++++ cxx-api-examples/CMakeLists.txt | 3 + cxx-api-examples/kws-cxx-api.cc | 141 ++++++++++ .../bin/zipformer-transducer.dart | 2 + .../keyword-spotting-from-files/Program.cs | 6 + .../Program.cs | 13 +- .../sherpa_onnx/lib/src/keyword_spotter.dart | 4 + .../lib/src/sherpa_onnx_bindings.dart | 12 + .../keyword-spotting-from-file/main.go | 2 + .../src/main/cpp/keyword-spotting.cc | 61 ++++- java-api-examples/KeywordSpotterFromFile.java | 2 + .../test-keyword-spotter-transducer.js | 3 + .../keyword-spotter-from-microphone.py | 11 +- python-api-examples/keyword-spotter.py | 255 +++++------------- scripts/dotnet/KeywordSpotter.cs | 8 + scripts/go/sherpa_onnx.go | 5 + scripts/node-addon-api/lib/keyword-spotter.js | 4 + sherpa-onnx/c-api/c-api.cc | 37 +-- sherpa-onnx/c-api/c-api.h | 33 ++- sherpa-onnx/c-api/cxx-api.cc | 108 ++++++++ sherpa-onnx/c-api/cxx-api.h | 47 ++++ sherpa-onnx/csrc/keyword-spotter-impl.h | 2 + .../csrc/keyword-spotter-transducer-impl.h | 16 ++ sherpa-onnx/csrc/keyword-spotter.cc | 2 + sherpa-onnx/csrc/keyword-spotter.h | 3 + .../csrc/sherpa-onnx-keyword-spotter-alsa.cc | 14 +- .../sherpa-onnx-keyword-spotter-microphone.cc | 14 +- .../com/k2fsa/sherpa/onnx/KeywordSpotter.java | 6 + sherpa-onnx/jni/keyword-spotter.cc | 9 + sherpa-onnx/kotlin-api/KeywordSpotter.kt | 2 + sherpa-onnx/python/csrc/keyword-spotter.cc | 1 + .../python/sherpa_onnx/keyword_spotter.py | 7 +- .../python/tests/test_keyword_spotter.py | 6 + swift-api-examples/SherpaOnnx.swift | 4 + .../keyword-spotting-from-file.swift | 3 + wasm/kws/CMakeLists.txt | 1 + wasm/kws/app.js | 14 +- wasm/kws/sherpa-onnx-kws.js | 7 +- 43 files changed, 823 insertions(+), 303 deletions(-) create mode 100644 c-api-examples/kws-c-api.c create mode 100644 cxx-api-examples/kws-cxx-api.cc diff --git a/.github/scripts/test-python.sh b/.github/scripts/test-python.sh index ad037438f..39e6577a2 100755 --- a/.github/scripts/test-python.sh +++ b/.github/scripts/test-python.sh @@ -574,29 +574,6 @@ echo "sherpa_onnx version: $sherpa_onnx_version" pwd ls -lh -repo=sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01 -log "Start testing ${repo}" - -pushd $dir -curl -LS -O https://github.com/pkufool/keyword-spotting-models/releases/download/v0.1/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.tar.bz -tar xf sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.tar.bz -rm sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.tar.bz -popd - -repo=$dir/$repo -ls -lh $repo - -python3 ./python-api-examples/keyword-spotter.py \ - --tokens=$repo/tokens.txt \ - --encoder=$repo/encoder-epoch-12-avg-2-chunk-16-left-64.onnx \ - --decoder=$repo/decoder-epoch-12-avg-2-chunk-16-left-64.onnx \ - --joiner=$repo/joiner-epoch-12-avg-2-chunk-16-left-64.onnx \ - --keywords-file=$repo/test_wavs/test_keywords.txt \ - $repo/test_wavs/0.wav \ - $repo/test_wavs/1.wav - -rm -rf $repo - if [[ x$OS != x'windows-latest' ]]; then echo "OS: $OS" @@ -612,15 +589,7 @@ if [[ x$OS != x'windows-latest' ]]; then repo=$dir/$repo ls -lh $repo - python3 ./python-api-examples/keyword-spotter.py \ - --tokens=$repo/tokens.txt \ - --encoder=$repo/encoder-epoch-12-avg-2-chunk-16-left-64.onnx \ - --decoder=$repo/decoder-epoch-12-avg-2-chunk-16-left-64.onnx \ - --joiner=$repo/joiner-epoch-12-avg-2-chunk-16-left-64.onnx \ - --keywords-file=$repo/test_wavs/test_keywords.txt \ - $repo/test_wavs/3.wav \ - $repo/test_wavs/4.wav \ - $repo/test_wavs/5.wav + python3 ./python-api-examples/keyword-spotter.py python3 sherpa-onnx/python/tests/test_keyword_spotter.py --verbose diff --git a/.github/workflows/c-api.yaml b/.github/workflows/c-api.yaml index 58820dc06..443797693 100644 --- a/.github/workflows/c-api.yaml +++ b/.github/workflows/c-api.yaml @@ -79,6 +79,27 @@ jobs: otool -L ./install/lib/libsherpa-onnx-c-api.dylib fi + - name: Test kws (zh) + shell: bash + run: | + gcc -o kws-c-api ./c-api-examples/kws-c-api.c \ + -I ./build/install/include \ + -L ./build/install/lib/ \ + -l sherpa-onnx-c-api \ + -l onnxruntime + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/kws-models/sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile.tar.bz2 + tar xvf sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile.tar.bz2 + rm sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile.tar.bz2 + + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH + export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH + + ./kws-c-api + + rm ./kws-c-api + rm -rf sherpa-onnx-kws-* + - name: Test Kokoro TTS (en) shell: bash run: | diff --git a/.github/workflows/cxx-api.yaml b/.github/workflows/cxx-api.yaml index 2f6a3b2e6..7227dd427 100644 --- a/.github/workflows/cxx-api.yaml +++ b/.github/workflows/cxx-api.yaml @@ -81,6 +81,28 @@ jobs: otool -L ./install/lib/libsherpa-onnx-cxx-api.dylib fi + - name: Test KWS (zh) + shell: bash + run: | + g++ -std=c++17 -o kws-cxx-api ./cxx-api-examples/kws-cxx-api.cc \ + -I ./build/install/include \ + -L ./build/install/lib/ \ + -l sherpa-onnx-cxx-api \ + -l sherpa-onnx-c-api \ + -l onnxruntime + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/kws-models/sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile.tar.bz2 + tar xvf sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile.tar.bz2 + rm sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile.tar.bz2 + + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH + export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH + + ./kws-cxx-api + + rm kws-cxx-api + rm -rf sherpa-onnx-kws-* + - name: Test Kokoro TTS (en) shell: bash run: | diff --git a/android/SherpaOnnxKws/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt b/android/SherpaOnnxKws/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt index b17a6ea6c..b42937ad3 100644 --- a/android/SherpaOnnxKws/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt +++ b/android/SherpaOnnxKws/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt @@ -151,24 +151,27 @@ class MainActivity : AppCompatActivity() { stream.acceptWaveform(samples, sampleRate = sampleRateInHz) while (kws.isReady(stream)) { kws.decode(stream) - } - val text = kws.getResult(stream).keyword + val text = kws.getResult(stream).keyword + + var textToDisplay = lastText - var textToDisplay = lastText + if (text.isNotBlank()) { + // Remember to reset the stream right after detecting a keyword - if (text.isNotBlank()) { - if (lastText.isBlank()) { - textToDisplay = "$idx: $text" - } else { - textToDisplay = "$idx: $text\n$lastText" + kws.reset(stream) + if (lastText.isBlank()) { + textToDisplay = "$idx: $text" + } else { + textToDisplay = "$idx: $text\n$lastText" + } + lastText = "$idx: $text\n$lastText" + idx += 1 } - lastText = "$idx: $text\n$lastText" - idx += 1 - } - runOnUiThread { - textView.text = textToDisplay + runOnUiThread { + textView.text = textToDisplay + } } } } diff --git a/c-api-examples/CMakeLists.txt b/c-api-examples/CMakeLists.txt index a2bfb6fdd..3db3f2539 100644 --- a/c-api-examples/CMakeLists.txt +++ b/c-api-examples/CMakeLists.txt @@ -4,6 +4,9 @@ include_directories(${CMAKE_SOURCE_DIR}) add_executable(decode-file-c-api decode-file-c-api.c) target_link_libraries(decode-file-c-api sherpa-onnx-c-api cargs) +add_executable(kws-c-api kws-c-api.c) +target_link_libraries(kws-c-api sherpa-onnx-c-api) + if(SHERPA_ONNX_ENABLE_TTS) add_executable(offline-tts-c-api offline-tts-c-api.c) target_link_libraries(offline-tts-c-api sherpa-onnx-c-api cargs) diff --git a/c-api-examples/kws-c-api.c b/c-api-examples/kws-c-api.c new file mode 100644 index 000000000..3ac427585 --- /dev/null +++ b/c-api-examples/kws-c-api.c @@ -0,0 +1,150 @@ +// c-api-examples/kws-c-api.c +// +// Copyright (c) 2025 Xiaomi Corporation +// +// This file demonstrates how to use keywords spotter with sherpa-onnx's C +// clang-format off +// +// Usage +// +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/kws-models/sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile.tar.bz2 +// tar xvf sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile.tar.bz2 +// rm sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile.tar.bz2 +// +// ./kws-c-api +// +// clang-format on +#include +#include // exit +#include // memset + +#include "sherpa-onnx/c-api/c-api.h" + +int32_t main() { + SherpaOnnxKeywordSpotterConfig config; + + memset(&config, 0, sizeof(config)); + config.model_config.transducer.encoder = + "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/" + "encoder-epoch-12-avg-2-chunk-16-left-64.onnx"; + + config.model_config.transducer.decoder = + "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/" + "decoder-epoch-12-avg-2-chunk-16-left-64.onnx"; + + config.model_config.transducer.joiner = + "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/" + "joiner-epoch-12-avg-2-chunk-16-left-64.onnx"; + + config.model_config.tokens = + "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/tokens.txt"; + + config.model_config.provider = "cpu"; + config.model_config.num_threads = 1; + config.model_config.debug = 1; + + config.keywords_file = + "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/test_wavs/" + "test_keywords.txt"; + + const SherpaOnnxKeywordSpotter *kws = SherpaOnnxCreateKeywordSpotter(&config); + if (!kws) { + fprintf(stderr, "Please check your config"); + exit(-1); + } + + fprintf(stderr, + "--Test pre-defined keywords from test_wavs/test_keywords.txt--\n"); + + const char *wav_filename = + "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/test_wavs/3.wav"; + + float tail_paddings[8000] = {0}; // 0.5 seconds + + const SherpaOnnxWave *wave = SherpaOnnxReadWave(wav_filename); + if (wave == NULL) { + fprintf(stderr, "Failed to read %s\n", wav_filename); + exit(-1); + } + + const SherpaOnnxOnlineStream *stream = SherpaOnnxCreateKeywordStream(kws); + if (!stream) { + fprintf(stderr, "Failed to create stream\n"); + exit(-1); + } + + SherpaOnnxOnlineStreamAcceptWaveform(stream, wave->sample_rate, wave->samples, + wave->num_samples); + + SherpaOnnxOnlineStreamAcceptWaveform(stream, wave->sample_rate, tail_paddings, + sizeof(tail_paddings) / sizeof(float)); + SherpaOnnxOnlineStreamInputFinished(stream); + while (SherpaOnnxIsKeywordStreamReady(kws, stream)) { + SherpaOnnxDecodeKeywordStream(kws, stream); + const SherpaOnnxKeywordResult *r = SherpaOnnxGetKeywordResult(kws, stream); + if (r && r->json && strlen(r->keyword)) { + fprintf(stderr, "Detected keyword: %s\n", r->json); + + // Remember to reset the keyword stream right after a keyword is detected + SherpaOnnxResetKeywordStream(kws, stream); + } + SherpaOnnxDestroyKeywordResult(r); + } + SherpaOnnxDestroyOnlineStream(stream); + + // -------------------------------------------------------------------------- + + fprintf(stderr, "--Use pre-defined keywords + add a new keyword--\n"); + + stream = SherpaOnnxCreateKeywordStreamWithKeywords(kws, "y ǎn y uán @演员"); + + SherpaOnnxOnlineStreamAcceptWaveform(stream, wave->sample_rate, wave->samples, + wave->num_samples); + + SherpaOnnxOnlineStreamAcceptWaveform(stream, wave->sample_rate, tail_paddings, + sizeof(tail_paddings) / sizeof(float)); + SherpaOnnxOnlineStreamInputFinished(stream); + while (SherpaOnnxIsKeywordStreamReady(kws, stream)) { + SherpaOnnxDecodeKeywordStream(kws, stream); + const SherpaOnnxKeywordResult *r = SherpaOnnxGetKeywordResult(kws, stream); + if (r && r->json && strlen(r->keyword)) { + fprintf(stderr, "Detected keyword: %s\n", r->json); + + // Remember to reset the keyword stream + SherpaOnnxResetKeywordStream(kws, stream); + } + SherpaOnnxDestroyKeywordResult(r); + } + SherpaOnnxDestroyOnlineStream(stream); + + // -------------------------------------------------------------------------- + + fprintf(stderr, "--Use pre-defined keywords + add two new keywords--\n"); + + stream = SherpaOnnxCreateKeywordStreamWithKeywords( + kws, "y ǎn y uán @演员/zh ī m íng @知名"); + + SherpaOnnxOnlineStreamAcceptWaveform(stream, wave->sample_rate, wave->samples, + wave->num_samples); + + SherpaOnnxOnlineStreamAcceptWaveform(stream, wave->sample_rate, tail_paddings, + sizeof(tail_paddings) / sizeof(float)); + SherpaOnnxOnlineStreamInputFinished(stream); + while (SherpaOnnxIsKeywordStreamReady(kws, stream)) { + SherpaOnnxDecodeKeywordStream(kws, stream); + const SherpaOnnxKeywordResult *r = SherpaOnnxGetKeywordResult(kws, stream); + if (r && r->json && strlen(r->keyword)) { + fprintf(stderr, "Detected keyword: %s\n", r->json); + + // Remember to reset the keyword stream + SherpaOnnxResetKeywordStream(kws, stream); + } + SherpaOnnxDestroyKeywordResult(r); + } + SherpaOnnxDestroyOnlineStream(stream); + + SherpaOnnxFreeWave(wave); + SherpaOnnxDestroyKeywordSpotter(kws); + + return 0; +} diff --git a/cxx-api-examples/CMakeLists.txt b/cxx-api-examples/CMakeLists.txt index 040925c29..2250736d7 100644 --- a/cxx-api-examples/CMakeLists.txt +++ b/cxx-api-examples/CMakeLists.txt @@ -3,6 +3,9 @@ include_directories(${CMAKE_SOURCE_DIR}) add_executable(streaming-zipformer-cxx-api ./streaming-zipformer-cxx-api.cc) target_link_libraries(streaming-zipformer-cxx-api sherpa-onnx-cxx-api) +add_executable(kws-cxx-api ./kws-cxx-api.cc) +target_link_libraries(kws-cxx-api sherpa-onnx-cxx-api) + add_executable(streaming-zipformer-rtf-cxx-api ./streaming-zipformer-rtf-cxx-api.cc) target_link_libraries(streaming-zipformer-rtf-cxx-api sherpa-onnx-cxx-api) diff --git a/cxx-api-examples/kws-cxx-api.cc b/cxx-api-examples/kws-cxx-api.cc new file mode 100644 index 000000000..cdcb86bae --- /dev/null +++ b/cxx-api-examples/kws-cxx-api.cc @@ -0,0 +1,141 @@ +// cxx-api-examples/kws-cxx-api.cc +// +// Copyright (c) 2025 Xiaomi Corporation +// +// This file demonstrates how to use keywords spotter with sherpa-onnx's C +// clang-format off +// +// Usage +// +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/kws-models/sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile.tar.bz2 +// tar xvf sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile.tar.bz2 +// rm sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile.tar.bz2 +// +// ./kws-cxx-api +// +// clang-format on +#include +#include + +#include "sherpa-onnx/c-api/cxx-api.h" + +int32_t main() { + using namespace sherpa_onnx::cxx; // NOLINT + + KeywordSpotterConfig config; + config.model_config.transducer.encoder = + "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/" + "encoder-epoch-12-avg-2-chunk-16-left-64.onnx"; + + config.model_config.transducer.decoder = + "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/" + "decoder-epoch-12-avg-2-chunk-16-left-64.onnx"; + + config.model_config.transducer.joiner = + "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/" + "joiner-epoch-12-avg-2-chunk-16-left-64.onnx"; + + config.model_config.tokens = + "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/tokens.txt"; + + config.model_config.provider = "cpu"; + config.model_config.num_threads = 1; + config.model_config.debug = 1; + + config.keywords_file = + "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/test_wavs/" + "test_keywords.txt"; + + KeywordSpotter kws = KeywordSpotter::Create(config); + if (!kws.Get()) { + std::cerr << "Please check your config\n"; + return -1; + } + + std::cout + << "--Test pre-defined keywords from test_wavs/test_keywords.txt--\n"; + + std::string wave_filename = + "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/test_wavs/3.wav"; + + std::array tail_paddings = {0}; // 0.5 seconds + + Wave wave = ReadWave(wave_filename); + if (wave.samples.empty()) { + std::cerr << "Failed to read: '" << wave_filename << "'\n"; + return -1; + } + + OnlineStream stream = kws.CreateStream(); + if (!stream.Get()) { + std::cerr << "Failed to create stream\n"; + return -1; + } + + stream.AcceptWaveform(wave.sample_rate, wave.samples.data(), + wave.samples.size()); + + stream.AcceptWaveform(wave.sample_rate, tail_paddings.data(), + tail_paddings.size()); + stream.InputFinished(); + + while (kws.IsReady(&stream)) { + kws.Decode(&stream); + auto r = kws.GetResult(&stream); + if (!r.keyword.empty()) { + std::cout << "Detected keyword: " << r.json << "\n"; + + // Remember to reset the keyword stream right after a keyword is detected + kws.Reset(&stream); + } + } + + // -------------------------------------------------------------------------- + + std::cout << "--Use pre-defined keywords + add a new keyword--\n"; + + stream = kws.CreateStream("y ǎn y uán @演员"); + + stream.AcceptWaveform(wave.sample_rate, wave.samples.data(), + wave.samples.size()); + + stream.AcceptWaveform(wave.sample_rate, tail_paddings.data(), + tail_paddings.size()); + stream.InputFinished(); + + while (kws.IsReady(&stream)) { + kws.Decode(&stream); + auto r = kws.GetResult(&stream); + if (!r.keyword.empty()) { + std::cout << "Detected keyword: " << r.json << "\n"; + + // Remember to reset the keyword stream right after a keyword is detected + kws.Reset(&stream); + } + } + + // -------------------------------------------------------------------------- + + std::cout << "--Use pre-defined keywords + add two new keywords--\n"; + + stream = kws.CreateStream("y ǎn y uán @演员/zh ī m íng @知名"); + + stream.AcceptWaveform(wave.sample_rate, wave.samples.data(), + wave.samples.size()); + + stream.AcceptWaveform(wave.sample_rate, tail_paddings.data(), + tail_paddings.size()); + stream.InputFinished(); + + while (kws.IsReady(&stream)) { + kws.Decode(&stream); + auto r = kws.GetResult(&stream); + if (!r.keyword.empty()) { + std::cout << "Detected keyword: " << r.json << "\n"; + + // Remember to reset the keyword stream right after a keyword is detected + kws.Reset(&stream); + } + } + return 0; +} diff --git a/dart-api-examples/keyword-spotter/bin/zipformer-transducer.dart b/dart-api-examples/keyword-spotter/bin/zipformer-transducer.dart index ebef1fd7c..47d587989 100644 --- a/dart-api-examples/keyword-spotter/bin/zipformer-transducer.dart +++ b/dart-api-examples/keyword-spotter/bin/zipformer-transducer.dart @@ -73,6 +73,8 @@ void main(List arguments) async { spotter.decode(stream); final result = spotter.getResult(stream); if (result.keyword != '') { + // Remember to reset the stream right after detecting a keyword + spotter.reset(stream); print('Detected: ${result.keyword}'); } } diff --git a/dotnet-examples/keyword-spotting-from-files/Program.cs b/dotnet-examples/keyword-spotting-from-files/Program.cs index 00ba3777a..7ab0da2fa 100644 --- a/dotnet-examples/keyword-spotting-from-files/Program.cs +++ b/dotnet-examples/keyword-spotting-from-files/Program.cs @@ -53,6 +53,8 @@ static void Main(string[] args) var result = kws.GetResult(s); if (result.Keyword != string.Empty) { + // Remember to call Reset() right after detecting a keyword + kws.Reset(s); Console.WriteLine("Detected: {0}", result.Keyword); } } @@ -70,6 +72,8 @@ static void Main(string[] args) var result = kws.GetResult(s); if (result.Keyword != string.Empty) { + // Remember to call Reset() right after detecting a keyword + kws.Reset(s); Console.WriteLine("Detected: {0}", result.Keyword); } } @@ -89,6 +93,8 @@ static void Main(string[] args) var result = kws.GetResult(s); if (result.Keyword != string.Empty) { + // Remember to call Reset() right after detecting a keyword + kws.Reset(s); Console.WriteLine("Detected: {0}", result.Keyword); } } diff --git a/dotnet-examples/keyword-spotting-from-microphone/Program.cs b/dotnet-examples/keyword-spotting-from-microphone/Program.cs index 05d22aee0..140e6a40e 100644 --- a/dotnet-examples/keyword-spotting-from-microphone/Program.cs +++ b/dotnet-examples/keyword-spotting-from-microphone/Program.cs @@ -107,12 +107,15 @@ IntPtr userData while (kws.IsReady(s)) { kws.Decode(s); - } - var result = kws.GetResult(s); - if (result.Keyword != string.Empty) - { - Console.WriteLine("Detected: {0}", result.Keyword); + var result = kws.GetResult(s); + if (result.Keyword != string.Empty) + { + // Remember to call Reset() right after detecting a keyword + kws.Reset(s); + + Console.WriteLine("Detected: {0}", result.Keyword); + } } Thread.Sleep(200); // ms diff --git a/flutter/sherpa_onnx/lib/src/keyword_spotter.dart b/flutter/sherpa_onnx/lib/src/keyword_spotter.dart index 6e2c669ae..310657d1a 100644 --- a/flutter/sherpa_onnx/lib/src/keyword_spotter.dart +++ b/flutter/sherpa_onnx/lib/src/keyword_spotter.dart @@ -168,6 +168,10 @@ class KeywordSpotter { SherpaOnnxBindings.decodeKeywordStream?.call(ptr, stream.ptr); } + void reset(OnlineStream stream) { + SherpaOnnxBindings.resetKeywordStream?.call(ptr, stream.ptr); + } + Pointer ptr; KeywordSpotterConfig config; } diff --git a/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart b/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart index 1e41d0918..e544da957 100644 --- a/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart +++ b/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart @@ -667,6 +667,12 @@ typedef DecodeKeywordStreamNative = Void Function( typedef DecodeKeywordStream = void Function( Pointer, Pointer); +typedef ResetKeywordStreamNative = Void Function( + Pointer, Pointer); + +typedef ResetKeywordStream = void Function( + Pointer, Pointer); + typedef GetKeywordResultAsJsonNative = Pointer Function( Pointer, Pointer); @@ -1157,6 +1163,7 @@ class SherpaOnnxBindings { static CreateKeywordStreamWithKeywords? createKeywordStreamWithKeywords; static IsKeywordStreamReady? isKeywordStreamReady; static DecodeKeywordStream? decodeKeywordStream; + static ResetKeywordStream? resetKeywordStream; static GetKeywordResultAsJson? getKeywordResultAsJson; static FreeKeywordResultJson? freeKeywordResultJson; @@ -1459,6 +1466,11 @@ class SherpaOnnxBindings { 'SherpaOnnxDecodeKeywordStream') .asFunction(); + resetKeywordStream ??= dynamicLibrary + .lookup>( + 'SherpaOnnxResetKeywordStream') + .asFunction(); + getKeywordResultAsJson ??= dynamicLibrary .lookup>( 'SherpaOnnxGetKeywordResultAsJson') diff --git a/go-api-examples/keyword-spotting-from-file/main.go b/go-api-examples/keyword-spotting-from-file/main.go index cf6ffa84e..697f9f4d7 100644 --- a/go-api-examples/keyword-spotting-from-file/main.go +++ b/go-api-examples/keyword-spotting-from-file/main.go @@ -43,6 +43,8 @@ func main() { spotter.Decode(stream) result := spotter.GetResult(stream) if result.Keyword != "" { + // You have to reset the stream right after detecting a keyword + spotter.Reset(stream) log.Printf("Detected %v\n", result.Keyword) } } diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/keyword-spotting.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/keyword-spotting.cc index 2b5a24100..08f1b5179 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/keyword-spotting.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/keyword-spotting.cc @@ -46,7 +46,7 @@ static Napi::External CreateKeywordSpotterWrapper( SHERPA_ONNX_ASSIGN_ATTR_STR(keywords_buf, keywordsBuf); SHERPA_ONNX_ASSIGN_ATTR_INT32(keywords_buf_size, keywordsBufSize); - SherpaOnnxKeywordSpotter *kws = SherpaOnnxCreateKeywordSpotter(&c); + const SherpaOnnxKeywordSpotter *kws = SherpaOnnxCreateKeywordSpotter(&c); if (c.model_config.transducer.encoder) { delete[] c.model_config.transducer.encoder; @@ -100,7 +100,8 @@ static Napi::External CreateKeywordSpotterWrapper( } return Napi::External::New( - env, kws, [](Napi::Env env, SherpaOnnxKeywordSpotter *kws) { + env, const_cast(kws), + [](Napi::Env env, SherpaOnnxKeywordSpotter *kws) { SherpaOnnxDestroyKeywordSpotter(kws); }); } @@ -125,13 +126,14 @@ static Napi::External CreateKeywordStreamWrapper( return {}; } - SherpaOnnxKeywordSpotter *kws = + const SherpaOnnxKeywordSpotter *kws = info[0].As>().Data(); - SherpaOnnxOnlineStream *stream = SherpaOnnxCreateKeywordStream(kws); + const SherpaOnnxOnlineStream *stream = SherpaOnnxCreateKeywordStream(kws); return Napi::External::New( - env, stream, [](Napi::Env env, SherpaOnnxOnlineStream *stream) { + env, const_cast(stream), + [](Napi::Env env, SherpaOnnxOnlineStream *stream) { SherpaOnnxDestroyOnlineStream(stream); }); } @@ -162,10 +164,10 @@ static Napi::Boolean IsKeywordStreamReadyWrapper( return {}; } - SherpaOnnxKeywordSpotter *kws = + const SherpaOnnxKeywordSpotter *kws = info[0].As>().Data(); - SherpaOnnxOnlineStream *stream = + const SherpaOnnxOnlineStream *stream = info[1].As>().Data(); int32_t is_ready = SherpaOnnxIsKeywordStreamReady(kws, stream); @@ -198,15 +200,49 @@ static void DecodeKeywordStreamWrapper(const Napi::CallbackInfo &info) { return; } - SherpaOnnxKeywordSpotter *kws = + const SherpaOnnxKeywordSpotter *kws = info[0].As>().Data(); - SherpaOnnxOnlineStream *stream = + const SherpaOnnxOnlineStream *stream = info[1].As>().Data(); SherpaOnnxDecodeKeywordStream(kws, stream); } +static void ResetKeywordStreamWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be a keyword spotter pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + if (!info[1].IsExternal()) { + Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.") + .ThrowAsJavaScriptException(); + + return; + } + + const SherpaOnnxKeywordSpotter *kws = + info[0].As>().Data(); + + const SherpaOnnxOnlineStream *stream = + info[1].As>().Data(); + + SherpaOnnxResetKeywordStream(kws, stream); +} + static Napi::String GetKeywordResultAsJsonWrapper( const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); @@ -233,10 +269,10 @@ static Napi::String GetKeywordResultAsJsonWrapper( return {}; } - SherpaOnnxKeywordSpotter *kws = + const SherpaOnnxKeywordSpotter *kws = info[0].As>().Data(); - SherpaOnnxOnlineStream *stream = + const SherpaOnnxOnlineStream *stream = info[1].As>().Data(); const char *json = SherpaOnnxGetKeywordResultAsJson(kws, stream); @@ -261,6 +297,9 @@ void InitKeywordSpotting(Napi::Env env, Napi::Object exports) { exports.Set(Napi::String::New(env, "decodeKeywordStream"), Napi::Function::New(env, DecodeKeywordStreamWrapper)); + exports.Set(Napi::String::New(env, "resetKeywordStream"), + Napi::Function::New(env, ResetKeywordStreamWrapper)); + exports.Set(Napi::String::New(env, "getKeywordResultAsJson"), Napi::Function::New(env, GetKeywordResultAsJsonWrapper)); } diff --git a/java-api-examples/KeywordSpotterFromFile.java b/java-api-examples/KeywordSpotterFromFile.java index 1b7a739a2..9634800a1 100644 --- a/java-api-examples/KeywordSpotterFromFile.java +++ b/java-api-examples/KeywordSpotterFromFile.java @@ -56,6 +56,8 @@ public static void main(String[] args) { String keyword = kws.getResult(stream).getKeyword(); if (!keyword.isEmpty()) { + // Remember to reset the stream right after detecting a keyword + kws.reset(stream); System.out.printf("Detected keyword: %s\n", keyword); } } diff --git a/nodejs-examples/test-keyword-spotter-transducer.js b/nodejs-examples/test-keyword-spotter-transducer.js index 9ead2b191..746d9995e 100644 --- a/nodejs-examples/test-keyword-spotter-transducer.js +++ b/nodejs-examples/test-keyword-spotter-transducer.js @@ -41,6 +41,9 @@ while (kws.isReady(stream)) { const keyword = kws.getResult(stream).keyword; if (keyword != '') { detectedKeywords.push(keyword); + + // remember to reset the stream right after detecting a keyword + kws.reset(stream); } } console.log(detectedKeywords); diff --git a/python-api-examples/keyword-spotter-from-microphone.py b/python-api-examples/keyword-spotter-from-microphone.py index 65a59fca6..b634c9070 100755 --- a/python-api-examples/keyword-spotter-from-microphone.py +++ b/python-api-examples/keyword-spotter-from-microphone.py @@ -169,6 +169,8 @@ def main(): print("Started! Please speak") + idx = 0 + sample_rate = 16000 samples_per_read = int(0.1 * sample_rate) # 0.1 second = 100 ms stream = keyword_spotter.create_stream() @@ -179,9 +181,12 @@ def main(): stream.accept_waveform(sample_rate, samples) while keyword_spotter.is_ready(stream): keyword_spotter.decode_stream(stream) - result = keyword_spotter.get_result(stream) - if result: - print("\r{}".format(result), end="", flush=True) + result = keyword_spotter.get_result(stream) + if result: + print(f"{idx}: {result }") + idx += 1 + # Remember to reset stream right after detecting a keyword + keyword_spotter.reset_stream(stream) if __name__ == "__main__": diff --git a/python-api-examples/keyword-spotter.py b/python-api-examples/keyword-spotter.py index 1b1de77e3..f3f76420b 100755 --- a/python-api-examples/keyword-spotter.py +++ b/python-api-examples/keyword-spotter.py @@ -18,122 +18,6 @@ import sherpa_onnx -def get_args(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - - parser.add_argument( - "--tokens", - type=str, - help="Path to tokens.txt", - ) - - parser.add_argument( - "--encoder", - type=str, - help="Path to the transducer encoder model", - ) - - parser.add_argument( - "--decoder", - type=str, - help="Path to the transducer decoder model", - ) - - parser.add_argument( - "--joiner", - type=str, - help="Path to the transducer joiner model", - ) - - parser.add_argument( - "--num-threads", - type=int, - default=1, - help="Number of threads for neural network computation", - ) - - parser.add_argument( - "--provider", - type=str, - default="cpu", - help="Valid values: cpu, cuda, coreml", - ) - - parser.add_argument( - "--max-active-paths", - type=int, - default=4, - help=""" - It specifies number of active paths to keep during decoding. - """, - ) - - parser.add_argument( - "--num-trailing-blanks", - type=int, - default=1, - help="""The number of trailing blanks a keyword should be followed. Setting - to a larger value (e.g. 8) when your keywords has overlapping tokens - between each other. - """, - ) - - parser.add_argument( - "--keywords-file", - type=str, - help=""" - The file containing keywords, one words/phrases per line, and for each - phrase the bpe/cjkchar/pinyin are separated by a space. For example: - - ▁HE LL O ▁WORLD - x iǎo ài t óng x ué - """, - ) - - parser.add_argument( - "--keywords-score", - type=float, - default=1.0, - help=""" - The boosting score of each token for keywords. The larger the easier to - survive beam search. - """, - ) - - parser.add_argument( - "--keywords-threshold", - type=float, - default=0.25, - help=""" - The trigger threshold (i.e. probability) of the keyword. The larger the - harder to trigger. - """, - ) - - parser.add_argument( - "sound_files", - type=str, - nargs="+", - help="The input sound file(s) to decode. Each file must be of WAVE" - "format with a single channel, and each sample has 16-bit, " - "i.e., int16_t. " - "The sample rate of the file can be arbitrary and does not need to " - "be 16 kHz", - ) - - return parser.parse_args() - - -def assert_file_exists(filename: str): - assert Path(filename).is_file(), ( - f"{filename} does not exist!\n" - "Please refer to " - "https://k2-fsa.github.io/sherpa/onnx/kws/pretrained_models/index.html to download it" - ) - - def read_wave(wave_filename: str) -> Tuple[np.ndarray, int]: """ Args: @@ -159,83 +43,74 @@ def read_wave(wave_filename: str) -> Tuple[np.ndarray, int]: return samples_float32, f.getframerate() -def main(): - args = get_args() - assert_file_exists(args.tokens) - assert_file_exists(args.encoder) - assert_file_exists(args.decoder) - assert_file_exists(args.joiner) - - assert Path( - args.keywords_file - ).is_file(), ( - f"keywords_file : {args.keywords_file} not exist, please provide a valid path." +def create_keyword_spotter(): + kws = sherpa_onnx.KeywordSpotter( + tokens="./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/tokens.txt", + encoder="./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/encoder-epoch-12-avg-2-chunk-16-left-64.onnx", + decoder="./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/decoder-epoch-12-avg-2-chunk-16-left-64.onnx", + joiner="./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/joiner-epoch-12-avg-2-chunk-16-left-64.onnx", + num_threads=2, + keywords_file="./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/test_wavs/test_keywords.txt", + provider="cpu", ) - keyword_spotter = sherpa_onnx.KeywordSpotter( - tokens=args.tokens, - encoder=args.encoder, - decoder=args.decoder, - joiner=args.joiner, - num_threads=args.num_threads, - max_active_paths=args.max_active_paths, - keywords_file=args.keywords_file, - keywords_score=args.keywords_score, - keywords_threshold=args.keywords_threshold, - num_trailing_blanks=args.num_trailing_blanks, - provider=args.provider, - ) + return kws - print("Started!") - start_time = time.time() - - streams = [] - total_duration = 0 - for wave_filename in args.sound_files: - assert_file_exists(wave_filename) - samples, sample_rate = read_wave(wave_filename) - duration = len(samples) / sample_rate - total_duration += duration - - s = keyword_spotter.create_stream() - - s.accept_waveform(sample_rate, samples) - - tail_paddings = np.zeros(int(0.66 * sample_rate), dtype=np.float32) - s.accept_waveform(sample_rate, tail_paddings) - - s.input_finished() - - streams.append(s) - - results = [""] * len(streams) - while True: - ready_list = [] - for i, s in enumerate(streams): - if keyword_spotter.is_ready(s): - ready_list.append(s) - r = keyword_spotter.get_result(s) - if r: - results[i] += f"{r}/" - print(f"{r} is detected.") - if len(ready_list) == 0: - break - keyword_spotter.decode_streams(ready_list) - end_time = time.time() - print("Done!") - - for wave_filename, result in zip(args.sound_files, results): - print(f"{wave_filename}\n{result}") - print("-" * 10) - - elapsed_seconds = end_time - start_time - rtf = elapsed_seconds / total_duration - print(f"num_threads: {args.num_threads}") - print(f"Wave duration: {total_duration:.3f} s") - print(f"Elapsed time: {elapsed_seconds:.3f} s") - print( - f"Real time factor (RTF): {elapsed_seconds:.3f}/{total_duration:.3f} = {rtf:.3f}" - ) + +def main(): + kws = create_keyword_spotter() + + wave_filename = ( + "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/test_wavs/3.wav" + ) + + samples, sample_rate = read_wave(wave_filename) + + tail_paddings = np.zeros(int(0.66 * sample_rate), dtype=np.float32) + + print("----------Use pre-defined keywords----------") + s = kws.create_stream() + s.accept_waveform(sample_rate, samples) + s.accept_waveform(sample_rate, tail_paddings) + s.input_finished() + while kws.is_ready(s): + kws.decode_stream(s) + r = kws.get_result(s) + if r != "": + # Remember to call reset right after detected a keyword + kws.reset_stream(s) + + print(f"Detected {r}") + + print("----------Use pre-defined keywords + add a new keyword----------") + + s = kws.create_stream("y ǎn y uán @演员") + s.accept_waveform(sample_rate, samples) + s.accept_waveform(sample_rate, tail_paddings) + s.input_finished() + while kws.is_ready(s): + kws.decode_stream(s) + r = kws.get_result(s) + if r != "": + # Remember to call reset right after detected a keyword + kws.reset_stream(s) + + print(f"Detected {r}") + + print("----------Use pre-defined keywords + add 2 new keywords----------") + + s = kws.create_stream("y ǎn y uán @演员/zh ī m íng @知名") + s.accept_waveform(sample_rate, samples) + s.accept_waveform(sample_rate, tail_paddings) + s.input_finished() + while kws.is_ready(s): + kws.decode_stream(s) + r = kws.get_result(s) + if r != "": + # Remember to call reset right after detected a keyword + kws.reset_stream(s) + + print(f"Detected {r}") if __name__ == "__main__": diff --git a/scripts/dotnet/KeywordSpotter.cs b/scripts/dotnet/KeywordSpotter.cs index 6f19fc945..d71d8924d 100644 --- a/scripts/dotnet/KeywordSpotter.cs +++ b/scripts/dotnet/KeywordSpotter.cs @@ -46,6 +46,11 @@ public void Decode(OnlineStream stream) Decode(_handle.Handle, stream.Handle); } + public void Reset(OnlineStream stream) + { + Reset(_handle.Handle, stream.Handle); + } + // The caller should ensure all passed streams are ready for decoding. public void Decode(IEnumerable streams) { @@ -110,6 +115,9 @@ private void Cleanup() [DllImport(Dll.Filename, EntryPoint = "SherpaOnnxDecodeKeywordStream")] private static extern void Decode(IntPtr handle, IntPtr stream); + [DllImport(Dll.Filename, EntryPoint = "SherpaOnnxResetKeywordStream")] + private static extern void Reset(IntPtr handle, IntPtr stream); + [DllImport(Dll.Filename, EntryPoint = "SherpaOnnxDecodeMultipleKeywordStreams")] private static extern void Decode(IntPtr handle, IntPtr[] streams, int n); diff --git a/scripts/go/sherpa_onnx.go b/scripts/go/sherpa_onnx.go index 4da12cfb4..37f596627 100644 --- a/scripts/go/sherpa_onnx.go +++ b/scripts/go/sherpa_onnx.go @@ -1584,6 +1584,11 @@ func (spotter *KeywordSpotter) Decode(s *OnlineStream) { C.SherpaOnnxDecodeKeywordStream(spotter.impl, s.impl) } +// You MUST call it right after detecting a keyword +func (spotter *KeywordSpotter) Reset(s *OnlineStream) { + C.SherpaOnnxResetKeywordStream(spotter.impl, s.impl) +} + // Get the current result of stream since the last invoke of Reset() func (spotter *KeywordSpotter) GetResult(s *OnlineStream) *KeywordSpotterResult { p := C.SherpaOnnxGetKeywordResult(spotter.impl, s.impl) diff --git a/scripts/node-addon-api/lib/keyword-spotter.js b/scripts/node-addon-api/lib/keyword-spotter.js index 9fbadef4c..d06764e81 100644 --- a/scripts/node-addon-api/lib/keyword-spotter.js +++ b/scripts/node-addon-api/lib/keyword-spotter.js @@ -20,6 +20,10 @@ class KeywordSpotter { addon.decodeKeywordStream(this.handle, stream.handle); } + reset(stream) { + addon.resetKeywordStream(this.handle, stream.handle); + } + getResult(stream) { const jsonStr = addon.getKeywordResultAsJson(this.handle, stream.handle); diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index da7c73178..ce6d55634 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -678,7 +678,7 @@ struct SherpaOnnxKeywordSpotter { std::unique_ptr impl; }; -SherpaOnnxKeywordSpotter *SherpaOnnxCreateKeywordSpotter( +const SherpaOnnxKeywordSpotter *SherpaOnnxCreateKeywordSpotter( const SherpaOnnxKeywordSpotterConfig *config) { sherpa_onnx::KeywordSpotterConfig spotter_config; @@ -755,37 +755,42 @@ SherpaOnnxKeywordSpotter *SherpaOnnxCreateKeywordSpotter( return spotter; } -void SherpaOnnxDestroyKeywordSpotter(SherpaOnnxKeywordSpotter *spotter) { +void SherpaOnnxDestroyKeywordSpotter(const SherpaOnnxKeywordSpotter *spotter) { delete spotter; } -SherpaOnnxOnlineStream *SherpaOnnxCreateKeywordStream( +const SherpaOnnxOnlineStream *SherpaOnnxCreateKeywordStream( const SherpaOnnxKeywordSpotter *spotter) { SherpaOnnxOnlineStream *stream = new SherpaOnnxOnlineStream(spotter->impl->CreateStream()); return stream; } -SherpaOnnxOnlineStream *SherpaOnnxCreateKeywordStreamWithKeywords( +const SherpaOnnxOnlineStream *SherpaOnnxCreateKeywordStreamWithKeywords( const SherpaOnnxKeywordSpotter *spotter, const char *keywords) { SherpaOnnxOnlineStream *stream = new SherpaOnnxOnlineStream(spotter->impl->CreateStream(keywords)); return stream; } -int32_t SherpaOnnxIsKeywordStreamReady(SherpaOnnxKeywordSpotter *spotter, - SherpaOnnxOnlineStream *stream) { +int32_t SherpaOnnxIsKeywordStreamReady(const SherpaOnnxKeywordSpotter *spotter, + const SherpaOnnxOnlineStream *stream) { return spotter->impl->IsReady(stream->impl.get()); } -void SherpaOnnxDecodeKeywordStream(SherpaOnnxKeywordSpotter *spotter, - SherpaOnnxOnlineStream *stream) { - return spotter->impl->DecodeStream(stream->impl.get()); +void SherpaOnnxDecodeKeywordStream(const SherpaOnnxKeywordSpotter *spotter, + const SherpaOnnxOnlineStream *stream) { + spotter->impl->DecodeStream(stream->impl.get()); } -void SherpaOnnxDecodeMultipleKeywordStreams(SherpaOnnxKeywordSpotter *spotter, - SherpaOnnxOnlineStream **streams, - int32_t n) { +void SherpaOnnxResetKeywordStream(const SherpaOnnxKeywordSpotter *spotter, + const SherpaOnnxOnlineStream *stream) { + spotter->impl->Reset(stream->impl.get()); +} + +void SherpaOnnxDecodeMultipleKeywordStreams( + const SherpaOnnxKeywordSpotter *spotter, + const SherpaOnnxOnlineStream **streams, int32_t n) { std::vector ss(n); for (int32_t i = 0; i != n; ++i) { ss[i] = streams[i]->impl.get(); @@ -794,7 +799,8 @@ void SherpaOnnxDecodeMultipleKeywordStreams(SherpaOnnxKeywordSpotter *spotter, } const SherpaOnnxKeywordResult *SherpaOnnxGetKeywordResult( - SherpaOnnxKeywordSpotter *spotter, SherpaOnnxOnlineStream *stream) { + const SherpaOnnxKeywordSpotter *spotter, + const SherpaOnnxOnlineStream *stream) { const sherpa_onnx::KeywordResult &result = spotter->impl->GetResult(stream->impl.get()); const auto &keyword = result.keyword; @@ -869,8 +875,9 @@ void SherpaOnnxDestroyKeywordResult(const SherpaOnnxKeywordResult *r) { } } -const char *SherpaOnnxGetKeywordResultAsJson(SherpaOnnxKeywordSpotter *spotter, - SherpaOnnxOnlineStream *stream) { +const char *SherpaOnnxGetKeywordResultAsJson( + const SherpaOnnxKeywordSpotter *spotter, + const SherpaOnnxOnlineStream *stream) { const sherpa_onnx::KeywordResult &result = spotter->impl->GetResult(stream->impl.get()); diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index db8321781..10bd101fd 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -600,7 +600,7 @@ SHERPA_ONNX_API const char *SherpaOnnxGetOfflineStreamResultAsJson( SHERPA_ONNX_API void SherpaOnnxDestroyOfflineStreamResultJson(const char *s); // ============================================================ -// For Keyword Spot +// For Keyword Spotter // ============================================================ SHERPA_ONNX_API typedef struct SherpaOnnxKeywordResult { /// The triggered keyword. @@ -660,21 +660,21 @@ SHERPA_ONNX_API typedef struct SherpaOnnxKeywordSpotter /// @param config Config for the keyword spotter. /// @return Return a pointer to the spotter. The user has to invoke /// SherpaOnnxDestroyKeywordSpotter() to free it to avoid memory leak. -SHERPA_ONNX_API SherpaOnnxKeywordSpotter *SherpaOnnxCreateKeywordSpotter( +SHERPA_ONNX_API const SherpaOnnxKeywordSpotter *SherpaOnnxCreateKeywordSpotter( const SherpaOnnxKeywordSpotterConfig *config); /// Free a pointer returned by SherpaOnnxCreateKeywordSpotter() /// /// @param p A pointer returned by SherpaOnnxCreateKeywordSpotter() SHERPA_ONNX_API void SherpaOnnxDestroyKeywordSpotter( - SherpaOnnxKeywordSpotter *spotter); + const SherpaOnnxKeywordSpotter *spotter); /// Create an online stream for accepting wave samples. /// /// @param spotter A pointer returned by SherpaOnnxCreateKeywordSpotter() /// @return Return a pointer to an OnlineStream. The user has to invoke /// SherpaOnnxDestroyOnlineStream() to free it to avoid memory leak. -SHERPA_ONNX_API SherpaOnnxOnlineStream *SherpaOnnxCreateKeywordStream( +SHERPA_ONNX_API const SherpaOnnxOnlineStream *SherpaOnnxCreateKeywordStream( const SherpaOnnxKeywordSpotter *spotter); /// Create an online stream for accepting wave samples with the specified hot @@ -684,7 +684,7 @@ SHERPA_ONNX_API SherpaOnnxOnlineStream *SherpaOnnxCreateKeywordStream( /// @param keywords A pointer points to the keywords that you set /// @return Return a pointer to an OnlineStream. The user has to invoke /// SherpaOnnxDestroyOnlineStream() to free it to avoid memory leak. -SHERPA_ONNX_API SherpaOnnxOnlineStream * +SHERPA_ONNX_API const SherpaOnnxOnlineStream * SherpaOnnxCreateKeywordStreamWithKeywords( const SherpaOnnxKeywordSpotter *spotter, const char *keywords); @@ -693,15 +693,22 @@ SherpaOnnxCreateKeywordStreamWithKeywords( /// /// @param spotter A pointer returned by SherpaOnnxCreateKeywordSpotter /// @param stream A pointer returned by SherpaOnnxCreateKeywordStream -SHERPA_ONNX_API int32_t SherpaOnnxIsKeywordStreamReady( - SherpaOnnxKeywordSpotter *spotter, SherpaOnnxOnlineStream *stream); +SHERPA_ONNX_API int32_t +SherpaOnnxIsKeywordStreamReady(const SherpaOnnxKeywordSpotter *spotter, + const SherpaOnnxOnlineStream *stream); /// Call this function to run the neural network model and decoding. // /// Precondition for this function: SherpaOnnxIsKeywordStreamReady() MUST /// return 1. SHERPA_ONNX_API void SherpaOnnxDecodeKeywordStream( - SherpaOnnxKeywordSpotter *spotter, SherpaOnnxOnlineStream *stream); + const SherpaOnnxKeywordSpotter *spotter, + const SherpaOnnxOnlineStream *stream); + +/// Please call it right after a keyword is detected +SHERPA_ONNX_API void SherpaOnnxResetKeywordStream( + const SherpaOnnxKeywordSpotter *spotter, + const SherpaOnnxOnlineStream *stream); /// This function is similar to SherpaOnnxDecodeKeywordStream(). It decodes /// multiple OnlineStream in parallel. @@ -714,8 +721,8 @@ SHERPA_ONNX_API void SherpaOnnxDecodeKeywordStream( /// SherpaOnnxCreateKeywordStream() /// @param n Number of elements in the given streams array. SHERPA_ONNX_API void SherpaOnnxDecodeMultipleKeywordStreams( - SherpaOnnxKeywordSpotter *spotter, SherpaOnnxOnlineStream **streams, - int32_t n); + const SherpaOnnxKeywordSpotter *spotter, + const SherpaOnnxOnlineStream **streams, int32_t n); /// Get the decoding results so far for an OnlineStream. /// @@ -725,7 +732,8 @@ SHERPA_ONNX_API void SherpaOnnxDecodeMultipleKeywordStreams( /// SherpaOnnxDestroyKeywordResult() to free the returned pointer to /// avoid memory leak. SHERPA_ONNX_API const SherpaOnnxKeywordResult *SherpaOnnxGetKeywordResult( - SherpaOnnxKeywordSpotter *spotter, SherpaOnnxOnlineStream *stream); + const SherpaOnnxKeywordSpotter *spotter, + const SherpaOnnxOnlineStream *stream); /// Destroy the pointer returned by SherpaOnnxGetKeywordResult(). /// @@ -736,7 +744,8 @@ SHERPA_ONNX_API void SherpaOnnxDestroyKeywordResult( // the user has to call SherpaOnnxFreeKeywordResultJson() to free the returned // pointer to avoid memory leak SHERPA_ONNX_API const char *SherpaOnnxGetKeywordResultAsJson( - SherpaOnnxKeywordSpotter *spotter, SherpaOnnxOnlineStream *stream); + const SherpaOnnxKeywordSpotter *spotter, + const SherpaOnnxOnlineStream *stream); SHERPA_ONNX_API void SherpaOnnxFreeKeywordResultJson(const char *s); diff --git a/sherpa-onnx/c-api/cxx-api.cc b/sherpa-onnx/c-api/cxx-api.cc index 50a1f4e1b..63a47cc6e 100644 --- a/sherpa-onnx/c-api/cxx-api.cc +++ b/sherpa-onnx/c-api/cxx-api.cc @@ -391,4 +391,112 @@ GeneratedAudio OfflineTts::Generate(const std::string &text, return ans; } +KeywordSpotter KeywordSpotter::Create(const KeywordSpotterConfig &config) { + struct SherpaOnnxKeywordSpotterConfig c; + memset(&c, 0, sizeof(c)); + + c.feat_config.sample_rate = config.feat_config.sample_rate; + + c.model_config.transducer.encoder = + config.model_config.transducer.encoder.c_str(); + c.model_config.transducer.decoder = + config.model_config.transducer.decoder.c_str(); + c.model_config.transducer.joiner = + config.model_config.transducer.joiner.c_str(); + c.feat_config.feature_dim = config.feat_config.feature_dim; + + c.model_config.paraformer.encoder = + config.model_config.paraformer.encoder.c_str(); + c.model_config.paraformer.decoder = + config.model_config.paraformer.decoder.c_str(); + + c.model_config.zipformer2_ctc.model = + config.model_config.zipformer2_ctc.model.c_str(); + + c.model_config.tokens = config.model_config.tokens.c_str(); + c.model_config.num_threads = config.model_config.num_threads; + c.model_config.provider = config.model_config.provider.c_str(); + c.model_config.debug = config.model_config.debug; + c.model_config.model_type = config.model_config.model_type.c_str(); + c.model_config.modeling_unit = config.model_config.modeling_unit.c_str(); + c.model_config.bpe_vocab = config.model_config.bpe_vocab.c_str(); + c.model_config.tokens_buf = config.model_config.tokens_buf.c_str(); + c.model_config.tokens_buf_size = config.model_config.tokens_buf.size(); + + c.max_active_paths = config.max_active_paths; + c.num_trailing_blanks = config.num_trailing_blanks; + c.keywords_score = config.keywords_score; + c.keywords_threshold = config.keywords_threshold; + c.keywords_file = config.keywords_file.c_str(); + + auto p = SherpaOnnxCreateKeywordSpotter(&c); + return KeywordSpotter(p); +} + +KeywordSpotter::KeywordSpotter(const SherpaOnnxKeywordSpotter *p) + : MoveOnly(p) {} + +void KeywordSpotter::Destroy(const SherpaOnnxKeywordSpotter *p) const { + SherpaOnnxDestroyKeywordSpotter(p); +} + +OnlineStream KeywordSpotter::CreateStream() const { + auto s = SherpaOnnxCreateKeywordStream(p_); + return OnlineStream{s}; +} + +OnlineStream KeywordSpotter::CreateStream(const std::string &keywords) const { + auto s = SherpaOnnxCreateKeywordStreamWithKeywords(p_, keywords.c_str()); + return OnlineStream{s}; +} + +bool KeywordSpotter::IsReady(const OnlineStream *s) const { + return SherpaOnnxIsKeywordStreamReady(p_, s->Get()); +} + +void KeywordSpotter::Decode(const OnlineStream *s) const { + return SherpaOnnxDecodeKeywordStream(p_, s->Get()); +} + +void KeywordSpotter::Decode(const OnlineStream *ss, int32_t n) const { + if (n <= 0) { + return; + } + + std::vector streams(n); + for (int32_t i = 0; i != n; ++n) { + streams[i] = ss[i].Get(); + } + + SherpaOnnxDecodeMultipleKeywordStreams(p_, streams.data(), n); +} + +KeywordResult KeywordSpotter::GetResult(const OnlineStream *s) const { + auto r = SherpaOnnxGetKeywordResult(p_, s->Get()); + + KeywordResult ans; + ans.keyword = r->keyword; + + ans.tokens.resize(r->count); + for (int32_t i = 0; i < r->count; ++i) { + ans.tokens[i] = r->tokens_arr[i]; + } + + if (r->timestamps) { + ans.timestamps.resize(r->count); + std::copy(r->timestamps, r->timestamps + r->count, ans.timestamps.data()); + } + + ans.start_time = r->start_time; + ans.json = r->json; + + SherpaOnnxDestroyKeywordResult(r); + + return ans; +} + +void KeywordSpotter::Reset(const OnlineStream *s) const { + SherpaOnnxResetKeywordStream(p_, s->Get()); +} + } // namespace sherpa_onnx::cxx diff --git a/sherpa-onnx/c-api/cxx-api.h b/sherpa-onnx/c-api/cxx-api.h index ce65a9ec7..8416c5945 100644 --- a/sherpa-onnx/c-api/cxx-api.h +++ b/sherpa-onnx/c-api/cxx-api.h @@ -406,6 +406,53 @@ class SHERPA_ONNX_API OfflineTts explicit OfflineTts(const SherpaOnnxOfflineTts *p); }; +// ============================================================ +// For Keyword Spotter +// ============================================================ + +struct KeywordResult { + std::string keyword; + std::vector tokens; + std::vector timestamps; + float start_time; + std::string json; +}; + +struct KeywordSpotterConfig { + FeatureConfig feat_config; + OnlineModelConfig model_config; + int32_t max_active_paths = 4; + int32_t num_trailing_blanks = 1; + float keywords_score = 1.0f; + float keywords_threshold = 0.25f; + std::string keywords_file; +}; + +class SHERPA_ONNX_API KeywordSpotter + : public MoveOnly { + public: + static KeywordSpotter Create(const KeywordSpotterConfig &config); + + void Destroy(const SherpaOnnxKeywordSpotter *p) const; + + OnlineStream CreateStream() const; + + OnlineStream CreateStream(const std::string &keywords) const; + + bool IsReady(const OnlineStream *s) const; + + void Decode(const OnlineStream *s) const; + + void Decode(const OnlineStream *ss, int32_t n) const; + + void Reset(const OnlineStream *s) const; + + KeywordResult GetResult(const OnlineStream *s) const; + + private: + explicit KeywordSpotter(const SherpaOnnxKeywordSpotter *p); +}; + } // namespace sherpa_onnx::cxx #endif // SHERPA_ONNX_C_API_CXX_API_H_ diff --git a/sherpa-onnx/csrc/keyword-spotter-impl.h b/sherpa-onnx/csrc/keyword-spotter-impl.h index ded735ff5..6180f9172 100644 --- a/sherpa-onnx/csrc/keyword-spotter-impl.h +++ b/sherpa-onnx/csrc/keyword-spotter-impl.h @@ -38,6 +38,8 @@ class KeywordSpotterImpl { virtual bool IsReady(OnlineStream *s) const = 0; + virtual void Reset(OnlineStream *s) const = 0; + virtual void DecodeStreams(OnlineStream **ss, int32_t n) const = 0; virtual KeywordResult GetResult(OnlineStream *s) const = 0; diff --git a/sherpa-onnx/csrc/keyword-spotter-transducer-impl.h b/sherpa-onnx/csrc/keyword-spotter-transducer-impl.h index 759639184..d29b8b58d 100644 --- a/sherpa-onnx/csrc/keyword-spotter-transducer-impl.h +++ b/sherpa-onnx/csrc/keyword-spotter-transducer-impl.h @@ -195,8 +195,24 @@ class KeywordSpotterTransducerImpl : public KeywordSpotterImpl { return s->GetNumProcessedFrames() + model_->ChunkSize() < s->NumFramesReady(); } + void Reset(OnlineStream *s) const override { InitOnlineStream(s); } void DecodeStreams(OnlineStream **ss, int32_t n) const override { + for (int32_t i = 0; i < n; ++i) { + auto s = ss[i]; + auto r = s->GetKeywordResult(true); + int32_t num_trailing_blanks = r.num_trailing_blanks; + // assume subsampling_factor is 4 + // assume frameshift is 0.01 second + float trailing_slience = num_trailing_blanks * 4 * 0.01; + + // it resets automatically after detecting 1.5 seconds of silence + float threshold = 1.5; + if (trailing_slience > threshold) { + Reset(s); + } + } + int32_t chunk_size = model_->ChunkSize(); int32_t chunk_shift = model_->ChunkShift(); diff --git a/sherpa-onnx/csrc/keyword-spotter.cc b/sherpa-onnx/csrc/keyword-spotter.cc index d1bf6d63b..66d0907ab 100644 --- a/sherpa-onnx/csrc/keyword-spotter.cc +++ b/sherpa-onnx/csrc/keyword-spotter.cc @@ -157,6 +157,8 @@ bool KeywordSpotter::IsReady(OnlineStream *s) const { return impl_->IsReady(s); } +void KeywordSpotter::Reset(OnlineStream *s) const { impl_->Reset(s); } + void KeywordSpotter::DecodeStreams(OnlineStream **ss, int32_t n) const { impl_->DecodeStreams(ss, n); } diff --git a/sherpa-onnx/csrc/keyword-spotter.h b/sherpa-onnx/csrc/keyword-spotter.h index f0c31bdb4..c933f4b23 100644 --- a/sherpa-onnx/csrc/keyword-spotter.h +++ b/sherpa-onnx/csrc/keyword-spotter.h @@ -129,6 +129,9 @@ class KeywordSpotter { */ bool IsReady(OnlineStream *s) const; + // Remember to call it after detecting a keyword + void Reset(OnlineStream *s) const; + /** Decode a single stream. */ void DecodeStream(OnlineStream *s) const { OnlineStream *ss[1] = {s}; diff --git a/sherpa-onnx/csrc/sherpa-onnx-keyword-spotter-alsa.cc b/sherpa-onnx/csrc/sherpa-onnx-keyword-spotter-alsa.cc index a909ff250..cfa46dc91 100644 --- a/sherpa-onnx/csrc/sherpa-onnx-keyword-spotter-alsa.cc +++ b/sherpa-onnx/csrc/sherpa-onnx-keyword-spotter-alsa.cc @@ -106,13 +106,15 @@ as the device_name. while (spotter.IsReady(stream.get())) { spotter.DecodeStream(stream.get()); - } - const auto r = spotter.GetResult(stream.get()); - if (!r.keyword.empty()) { - display.Print(keyword_index, r.AsJsonString()); - fflush(stderr); - keyword_index++; + const auto r = spotter.GetResult(stream.get()); + if (!r.keyword.empty()) { + display.Print(keyword_index, r.AsJsonString()); + fflush(stderr); + keyword_index++; + + spotter.Reset(stream.get()); + } } } diff --git a/sherpa-onnx/csrc/sherpa-onnx-keyword-spotter-microphone.cc b/sherpa-onnx/csrc/sherpa-onnx-keyword-spotter-microphone.cc index 903debea9..4d75f9d49 100644 --- a/sherpa-onnx/csrc/sherpa-onnx-keyword-spotter-microphone.cc +++ b/sherpa-onnx/csrc/sherpa-onnx-keyword-spotter-microphone.cc @@ -150,13 +150,15 @@ for a list of pre-trained models to download. while (!stop) { while (spotter.IsReady(s.get())) { spotter.DecodeStream(s.get()); - } - const auto r = spotter.GetResult(s.get()); - if (!r.keyword.empty()) { - display.Print(keyword_index, r.AsJsonString()); - fflush(stderr); - keyword_index++; + const auto r = spotter.GetResult(s.get()); + if (!r.keyword.empty()) { + display.Print(keyword_index, r.AsJsonString()); + fflush(stderr); + keyword_index++; + + spotter.Reset(s.get()); + } } Pa_Sleep(20); // sleep for 20ms diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/KeywordSpotter.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/KeywordSpotter.java index a1b897b0d..3565c05e4 100644 --- a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/KeywordSpotter.java +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/KeywordSpotter.java @@ -27,6 +27,10 @@ public void decode(OnlineStream s) { decode(ptr, s.getPtr()); } + public void reset(OnlineStream s) { + reset(ptr, s.getPtr()); + } + public boolean isReady(OnlineStream s) { return isReady(ptr, s.getPtr()); } @@ -60,6 +64,8 @@ public void release() { private native void decode(long ptr, long streamPtr); + private native void reset(long ptr, long streamPtr); + private native boolean isReady(long ptr, long streamPtr); private native Object[] getResult(long ptr, long streamPtr); diff --git a/sherpa-onnx/jni/keyword-spotter.cc b/sherpa-onnx/jni/keyword-spotter.cc index 4ac80a294..0dc5685ef 100644 --- a/sherpa-onnx/jni/keyword-spotter.cc +++ b/sherpa-onnx/jni/keyword-spotter.cc @@ -161,6 +161,15 @@ JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_KeywordSpotter_decode( kws->DecodeStream(stream); } +SHERPA_ONNX_EXTERN_C +JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_KeywordSpotter_reset( + JNIEnv * /*env*/, jobject /*obj*/, jlong ptr, jlong stream_ptr) { + auto kws = reinterpret_cast(ptr); + auto stream = reinterpret_cast(stream_ptr); + + kws->Reset(stream); +} + SHERPA_ONNX_EXTERN_C JNIEXPORT jlong JNICALL Java_com_k2fsa_sherpa_onnx_KeywordSpotter_createStream( JNIEnv *env, jobject /*obj*/, jlong ptr, jstring keywords) { diff --git a/sherpa-onnx/kotlin-api/KeywordSpotter.kt b/sherpa-onnx/kotlin-api/KeywordSpotter.kt index 3801d32a8..5b3cdbb78 100644 --- a/sherpa-onnx/kotlin-api/KeywordSpotter.kt +++ b/sherpa-onnx/kotlin-api/KeywordSpotter.kt @@ -49,6 +49,7 @@ class KeywordSpotter( } fun decode(stream: OnlineStream) = decode(ptr, stream.ptr) + fun reset(stream: OnlineStream) = reset(ptr, stream.ptr) fun isReady(stream: OnlineStream) = isReady(ptr, stream.ptr) fun getResult(stream: OnlineStream): KeywordSpotterResult { val objArray = getResult(ptr, stream.ptr) @@ -74,6 +75,7 @@ class KeywordSpotter( private external fun createStream(ptr: Long, keywords: String): Long private external fun isReady(ptr: Long, streamPtr: Long): Boolean private external fun decode(ptr: Long, streamPtr: Long) + private external fun reset(ptr: Long, streamPtr: Long) private external fun getResult(ptr: Long, streamPtr: Long): Array companion object { diff --git a/sherpa-onnx/python/csrc/keyword-spotter.cc b/sherpa-onnx/python/csrc/keyword-spotter.cc index 144992605..4a48ada4f 100644 --- a/sherpa-onnx/python/csrc/keyword-spotter.cc +++ b/sherpa-onnx/python/csrc/keyword-spotter.cc @@ -67,6 +67,7 @@ void PybindKeywordSpotter(py::module *m) { py::arg("keywords"), py::call_guard()) .def("is_ready", &PyClass::IsReady, py::call_guard()) + .def("reset", &PyClass::Reset, py::call_guard()) .def("decode_stream", &PyClass::DecodeStream, py::call_guard()) .def( diff --git a/sherpa-onnx/python/sherpa_onnx/keyword_spotter.py b/sherpa-onnx/python/sherpa_onnx/keyword_spotter.py index 66d716984..a9d8573f8 100644 --- a/sherpa-onnx/python/sherpa_onnx/keyword_spotter.py +++ b/sherpa-onnx/python/sherpa_onnx/keyword_spotter.py @@ -104,8 +104,8 @@ def __init__( ) provider_config = ProviderConfig( - provider=provider, - device = device, + provider=provider, + device=device, ) model_config = OnlineModelConfig( @@ -131,6 +131,9 @@ def __init__( ) self.keyword_spotter = _KeywordSpotter(keywords_spotter_config) + def reset_stream(self, s: OnlineStream): + self.keyword_spotter.reset(s) + def create_stream(self, keywords: Optional[str] = None): if keywords is None: return self.keyword_spotter.create_stream() diff --git a/sherpa-onnx/python/tests/test_keyword_spotter.py b/sherpa-onnx/python/tests/test_keyword_spotter.py index f4d79830a..62691d183 100755 --- a/sherpa-onnx/python/tests/test_keyword_spotter.py +++ b/sherpa-onnx/python/tests/test_keyword_spotter.py @@ -98,6 +98,9 @@ def test_zipformer_transducer_en(self): if r: print(f"{r} is detected.") results[i] += f"{r}/" + + keyword_spotter.reset_stream(s) + if len(ready_list) == 0: break keyword_spotter.decode_streams(ready_list) @@ -158,6 +161,9 @@ def test_zipformer_transducer_cn(self): if r: print(f"{r} is detected.") results[i] += f"{r}/" + + keyword_spotter.reset_stream(s) + if len(ready_list) == 0: break keyword_spotter.decode_streams(ready_list) diff --git a/swift-api-examples/SherpaOnnx.swift b/swift-api-examples/SherpaOnnx.swift index d4c39645f..81a1c9e44 100644 --- a/swift-api-examples/SherpaOnnx.swift +++ b/swift-api-examples/SherpaOnnx.swift @@ -1076,6 +1076,10 @@ class SherpaOnnxKeywordSpotterWrapper { SherpaOnnxDecodeKeywordStream(spotter, stream) } + func reset() { + SherpaOnnxResetKeywordStream(spotter, stream) + } + func getResult() -> SherpaOnnxKeywordResultWrapper { let result: UnsafePointer? = SherpaOnnxGetKeywordResult( spotter, stream) diff --git a/swift-api-examples/keyword-spotting-from-file.swift b/swift-api-examples/keyword-spotting-from-file.swift index 08487eb4a..498852a83 100644 --- a/swift-api-examples/keyword-spotting-from-file.swift +++ b/swift-api-examples/keyword-spotting-from-file.swift @@ -70,6 +70,9 @@ func run() { spotter.decode() let keyword = spotter.getResult().keyword if keyword != "" { + // Remember to call reset() right after detecting a keyword + spotter.reset() + print("Detected: \(keyword)") } } diff --git a/wasm/kws/CMakeLists.txt b/wasm/kws/CMakeLists.txt index 197aa38b1..5620b80db 100644 --- a/wasm/kws/CMakeLists.txt +++ b/wasm/kws/CMakeLists.txt @@ -17,6 +17,7 @@ set(exported_functions SherpaOnnxIsKeywordStreamReady SherpaOnnxOnlineStreamAcceptWaveform SherpaOnnxOnlineStreamInputFinished + SherpaOnnxResetKeywordStream ) set(mangled_exported_functions) foreach(x IN LISTS exported_functions) diff --git a/wasm/kws/app.js b/wasm/kws/app.js index 1e97262a1..6df20d23b 100644 --- a/wasm/kws/app.js +++ b/wasm/kws/app.js @@ -102,15 +102,17 @@ if (navigator.mediaDevices.getUserMedia) { recognizer_stream.acceptWaveform(expectedSampleRate, samples); while (recognizer.isReady(recognizer_stream)) { recognizer.decode(recognizer_stream); - } + let result = recognizer.getResult(recognizer_stream); - let result = recognizer.getResult(recognizer_stream); + if (result.keyword.length > 0) { + console.log(result) + lastResult = result; + resultList.push(JSON.stringify(result)); - if (result.keyword.length > 0) { - console.log(result) - lastResult = result; - resultList.push(JSON.stringify(result)); + // remember to reset the stream right after detecting a keyword + recognizer.reset(recognizer_stream); + } } diff --git a/wasm/kws/sherpa-onnx-kws.js b/wasm/kws/sherpa-onnx-kws.js index b7c023356..57c81e096 100644 --- a/wasm/kws/sherpa-onnx-kws.js +++ b/wasm/kws/sherpa-onnx-kws.js @@ -296,8 +296,11 @@ class Kws { } decode(stream) { - return this.Module._SherpaOnnxDecodeKeywordStream( - this.handle, stream.handle); + this.Module._SherpaOnnxDecodeKeywordStream(this.handle, stream.handle); + } + + reset(stream) { + this.Module._SherpaOnnxResetKeywordStream(this.handle, stream.handle); } getResult(stream) { From e764fa61576772f343fe600f47ac8c4649e0ba54 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 20 Jan 2025 17:28:07 +0800 Subject: [PATCH 173/183] Update readme to include https://github.com/hfyydd/sherpa-onnx-server (#1741) --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index cc31f3054..2d924a0b4 100644 --- a/README.md +++ b/README.md @@ -297,6 +297,10 @@ It uses the JavaScript API of sherpa-onnx along with [Electron](https://electron Video demo in Chinese: [爆了！炫神教你开打字挂！真正影响胜率的英雄联盟工具！英雄联盟的最后一块拼图！和游戏中的每个人无障碍沟通！](https://www.bilibili.com/video/BV142tje9E74) +### [Sherpa-ONNX 语音识别服务器](https://github.com/hfyydd/sherpa-onnx-server) + +A server based on nodejs providing Restful API for speech recognition. + [sherpa-rs]: https://github.com/thewh1teagle/sherpa-rs [silero-vad]: https://github.com/snakers4/silero-vad From 5bcd7e100ad1f2ef0f5205639ac6fd2b452cc4af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=B5=E5=B0=8F=E5=87=A1?= <2672931+whyb@users.noreply.github.com> Date: Tue, 21 Jan 2025 15:14:32 +0800 Subject: [PATCH 174/183] Reduce vad-moonshine-c-api example code. (#1742) --- c-api-examples/vad-moonshine-c-api.c | 47 +++++++--------------------- 1 file changed, 11 insertions(+), 36 deletions(-) diff --git a/c-api-examples/vad-moonshine-c-api.c b/c-api-examples/vad-moonshine-c-api.c index 1b0d03624..2ad6f6d63 100644 --- a/c-api-examples/vad-moonshine-c-api.c +++ b/c-api-examples/vad-moonshine-c-api.c @@ -98,12 +98,16 @@ int32_t main() { int32_t window_size = vadConfig.silero_vad.window_size; int32_t i = 0; - - while (i + window_size < wave->num_samples) { - SherpaOnnxVoiceActivityDetectorAcceptWaveform(vad, wave->samples + i, - window_size); - i += window_size; - + int is_eof = 0; + + while (!is_eof) { + if (i + window_size < wave->num_samples) { + SherpaOnnxVoiceActivityDetectorAcceptWaveform(vad, wave->samples + i, + window_size); + } else { + SherpaOnnxVoiceActivityDetectorFlush(vad); + is_eof = 1; + } while (!SherpaOnnxVoiceActivityDetectorEmpty(vad)) { const SherpaOnnxSpeechSegment *segment = SherpaOnnxVoiceActivityDetectorFront(vad); @@ -131,36 +135,7 @@ int32_t main() { SherpaOnnxDestroySpeechSegment(segment); SherpaOnnxVoiceActivityDetectorPop(vad); } - } - - SherpaOnnxVoiceActivityDetectorFlush(vad); - - while (!SherpaOnnxVoiceActivityDetectorEmpty(vad)) { - const SherpaOnnxSpeechSegment *segment = - SherpaOnnxVoiceActivityDetectorFront(vad); - - const SherpaOnnxOfflineStream *stream = - SherpaOnnxCreateOfflineStream(recognizer); - - SherpaOnnxAcceptWaveformOffline(stream, wave->sample_rate, segment->samples, - segment->n); - - SherpaOnnxDecodeOfflineStream(recognizer, stream); - - const SherpaOnnxOfflineRecognizerResult *result = - SherpaOnnxGetOfflineStreamResult(stream); - - float start = segment->start / 16000.0f; - float duration = segment->n / 16000.0f; - float stop = start + duration; - - fprintf(stderr, "%.3f -- %.3f: %s\n", start, stop, result->text); - - SherpaOnnxDestroyOfflineRecognizerResult(result); - SherpaOnnxDestroyOfflineStream(stream); - - SherpaOnnxDestroySpeechSegment(segment); - SherpaOnnxVoiceActivityDetectorPop(vad); + i += window_size; } SherpaOnnxDestroyOfflineRecognizer(recognizer); From bc3322e5a6750aa2de7184cb2de29795abe27817 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 22 Jan 2025 11:14:42 +0800 Subject: [PATCH 175/183] Support Kokoro TTS for HarmonyOS. (#1743) --- .../sherpa_onnx/BuildProfile.ets | 2 +- .../SherpaOnnxHar/sherpa_onnx/Index.ets | 3 +- .../main/ets/components/NonStreamingTts.ets | 9 ++ .../entry/src/main/ets/pages/Index.ets | 3 + .../ets/workers/NonStreamingTtsWorker.ets | 100 ++++++++++++------ 5 files changed, 82 insertions(+), 35 deletions(-) diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets index 4cc33f4e9..274f60379 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets @@ -1,7 +1,7 @@ /** * Use these variables when you tailor your ArkTS code. They must be of the const type. */ -export const HAR_VERSION = '1.10.37'; +export const HAR_VERSION = '1.10.40'; export const BUILD_MODE_NAME = 'debug'; export const DEBUG = true; export const TARGET_NAME = 'default'; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets index 56deee0ce..7c1e95df4 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets @@ -31,7 +31,8 @@ export { OnlineStream, OnlineRecognizer, } from './src/main/ets/components/StreamingAsr'; -export { OfflineTtsMatchaModelConfig, +export { OfflineTtsKokoroModelConfig, + OfflineTtsMatchaModelConfig, OfflineTtsVitsModelConfig, OfflineTtsModelConfig, OfflineTtsConfig, diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets index 814d6e267..d9e2cbd0f 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets @@ -28,9 +28,18 @@ export class OfflineTtsMatchaModelConfig { public lengthScale: number = 1.0; } +export class OfflineTtsKokoroModelConfig { + public model: string = ''; + public voices: string = ''; + public tokens: string = ''; + public dataDir: string = ''; + public lengthScale: number = 1.0; +} + export class OfflineTtsModelConfig { public vits: OfflineTtsVitsModelConfig = new OfflineTtsVitsModelConfig(); public matcha: OfflineTtsMatchaModelConfig = new OfflineTtsMatchaModelConfig(); + public kokoro: OfflineTtsKokoroModelConfig = new OfflineTtsKokoroModelConfig(); public numThreads: number = 1; public debug: boolean = false; public provider: string = 'cpu'; diff --git a/harmony-os/SherpaOnnxTts/entry/src/main/ets/pages/Index.ets b/harmony-os/SherpaOnnxTts/entry/src/main/ets/pages/Index.ets index 45927b772..a98e669c9 100644 --- a/harmony-os/SherpaOnnxTts/entry/src/main/ets/pages/Index.ets +++ b/harmony-os/SherpaOnnxTts/entry/src/main/ets/pages/Index.ets @@ -66,6 +66,7 @@ struct Index { @State initTtsDone: boolean = false; @State ttsGeneratedDone: boolean = true; @State numSpeakers: number = 1; + @State numThreads: number = 1; @State initAudioDone: boolean = false; private controller: TabsController = new TabsController(); private cancelled: boolean = false; @@ -135,6 +136,7 @@ struct Index { this.info = 'Model initialized!\nPlease enter text and press start.'; this.sampleRate = e.data['sampleRate'] as number; this.numSpeakers = e.data['numSpeakers'] as number; + this.numThreads = e.data['numThreads'] as number; this.initTtsDone = true; } @@ -177,6 +179,7 @@ struct Index { this.info = `Audio duration: ${audioDuration} s Elapsed: ${elapsedSeconds} s RTF = ${elapsedSeconds.toFixed(2)}/${audioDuration.toFixed(2)} = ${RTF.toFixed(3)} +Number of threads: ${this.numThreads} `; if (this.cancelled) { this.info += '\nCancelled.'; diff --git a/harmony-os/SherpaOnnxTts/entry/src/main/ets/workers/NonStreamingTtsWorker.ets b/harmony-os/SherpaOnnxTts/entry/src/main/ets/workers/NonStreamingTtsWorker.ets index cf841cbe1..08c53841e 100644 --- a/harmony-os/SherpaOnnxTts/entry/src/main/ets/workers/NonStreamingTtsWorker.ets +++ b/harmony-os/SherpaOnnxTts/entry/src/main/ets/workers/NonStreamingTtsWorker.ets @@ -2,7 +2,7 @@ import worker, { ThreadWorkerGlobalScope, MessageEvents, ErrorEvent } from '@oho import { fileIo as fs } from '@kit.CoreFileKit'; -import {OfflineTtsConfig, OfflineTts, listRawfileDir, TtsInput, TtsOutput} from 'sherpa_onnx'; +import { OfflineTtsConfig, OfflineTts, listRawfileDir, TtsInput, TtsOutput } from 'sherpa_onnx'; import { buffer } from '@kit.ArkTS'; const workerPort: ThreadWorkerGlobalScope = worker.workerPort; @@ -42,18 +42,22 @@ function copyRawFileDirToSandbox(context: Context, srcDir: string) { } } -function copyRawFileToSandbox(context: Context, src: string, dst: string) { - // see https://blog.csdn.net/weixin_44640245/article/details/142634846 - // https://developer.huawei.com/consumer/cn/doc/harmonyos-guides-V5/rawfile-guidelines-V5 +function copyRawFileToSandbox(context: Context, src: string, + dst: string) { + /* see + https://blog.csdn.net/weixin_44640245/article/details/142634846 + https://developer.huawei.com/consumer/cn/doc/harmonyos-guides-V5/rawfile-guidelines-V5 + */ let uint8Array: Uint8Array = context.resourceManager.getRawFileContentSync(src); // https://developer.huawei.com/consumer/cn/doc/harmonyos-references-V5/js-apis-file-fs-V5#fsmkdir let sandboxPath: string = context.getApplicationContext().filesDir; - let filepath = sandboxPath + '/' + dst; + let filepath = sandboxPath + '/' + dst; if (fs.accessSync(filepath)) { - // if the destination exists and has the expected file size, - // then we skip copying it + /* if the destination exists and has the expected file size + then we skip copying it + */ let stat = fs.statSync(filepath); if (stat.size == uint8Array.length) { return; @@ -66,11 +70,12 @@ function copyRawFileToSandbox(context: Context, src: string, dst: string) { } function initTts(context: Context): OfflineTts { - // Such a design is to make it easier to build flutter APPs with - // github actions for a variety of tts models - // - // See https://github.com/k2-fsa/sherpa-onnx/blob/master/scripts/flutter/generate-tts.py - // for details + /* Such a design is to make it easier to build flutter APPs with + github actions for a variety of tts models + + See https://github.com/k2-fsa/sherpa-onnx/blob/master/scripts/flutter/generate-tts.py + for details + */ let modelDir = ''; @@ -83,13 +88,19 @@ function initTts(context: Context): OfflineTts { let vocoder = ''; // for Matcha end + // for Kokoro begin + let voices = ''; + // for Kokoro end + let ruleFsts = ''; let ruleFars = ''; let lexicon = ''; let dataDir = ''; let dictDir = ''; - // You can select an example below and change it according to match your - // selected tts model + /* + You can select an example below and change it according to match your + selected tts model + */ // ============================================================ // Your change starts here @@ -146,19 +157,26 @@ function initTts(context: Context): OfflineTts { // Example 8 // https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker - // modelDir = 'matcha-icefall-zh-baker' - // acousticModelName = 'model-steps-3.onnx' - // vocoder = 'hifigan_v2.onnx' - // lexicon = 'lexicon.txt' + // modelDir = 'matcha-icefall-zh-baker'; + // acousticModelName = 'model-steps-3.onnx'; + // vocoder = 'hifigan_v2.onnx'; + // lexicon = 'lexicon.txt'; // dictDir = 'dict'; // ruleFsts = `date.fst,phone.fst,number.fst`; // Example 9 // https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker - // modelDir = 'matcha-icefall-en_US-ljspeech' - // acousticModelName = 'model-steps-3.onnx' - // vocoder = 'hifigan_v2.onnx' + // modelDir = 'matcha-icefall-en_US-ljspeech'; + // acousticModelName = 'model-steps-3.onnx'; + // vocoder = 'hifigan_v2.onnx'; + // dataDir = 'espeak-ng-data'; + + // Example 10 + // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html#kokoro-en-v0-19-english-11-speakers + // modelDir = 'kokoro-en-v0_19'; + // modelName = 'model.onnx'; + // voices = 'voices.bin' // dataDir = 'espeak-ng-data'; // ============================================================ @@ -185,6 +203,10 @@ function initTts(context: Context): OfflineTts { acousticModelName = modelDir + '/' + acousticModelName; } + if (voices != '') { + voices = modelDir + '/' + voices; + } + if (ruleFsts != '') { let fsts = ruleFsts.split(',') let tmp: string[] = []; @@ -210,19 +232,24 @@ function initTts(context: Context): OfflineTts { if (dataDir != '') { copyRawFileDirToSandbox(context, modelDir + '/' + dataDir) let sandboxPath: string = context.getApplicationContext().filesDir; - dataDir = sandboxPath + '/' + modelDir + '/' + dataDir; + dataDir = sandboxPath + '/' + modelDir + '/' + dataDir; } if (dictDir != '') { copyRawFileDirToSandbox(context, modelDir + '/' + dictDir) let sandboxPath: string = context.getApplicationContext().filesDir; - dictDir = sandboxPath + '/' + modelDir + '/' + dictDir; + dictDir = sandboxPath + '/' + modelDir + '/' + dictDir; } const tokens = modelDir + '/tokens.txt'; const config: OfflineTtsConfig = new OfflineTtsConfig(); - config.model.vits.model = modelName; + if (voices != '') { + config.model.vits.model = ''; + } else { + config.model.vits.model = modelName; + } + config.model.vits.lexicon = lexicon; config.model.vits.tokens = tokens; config.model.vits.dataDir = dataDir; @@ -235,6 +262,15 @@ function initTts(context: Context): OfflineTts { config.model.matcha.dataDir = dataDir; config.model.matcha.dictDir = dictDir; + if (voices != '') { + config.model.kokoro.model = modelName; + } else { + config.model.kokoro.model = ''; + } + config.model.kokoro.voices = voices; + config.model.kokoro.tokens = tokens; + config.model.kokoro.dataDir = dataDir; + config.model.numThreads = 2; config.model.debug = true; config.ruleFsts = ruleFsts; @@ -250,14 +286,12 @@ interface TtsCallbackData { function callback(data: TtsCallbackData): number { workerPort.postMessage({ - 'msgType': 'tts-generate-partial', - samples: Float32Array.from(data.samples), - progress: data.progress, + 'msgType': 'tts-generate-partial', samples: Float32Array.from(data.samples), progress: data.progress, }); // 0 means to stop generating in C++ // 1 means to continue generating in C++ - return cancelled? 0 : 1; + return cancelled ? 0 : 1; } /** @@ -272,9 +306,11 @@ workerPort.onmessage = (e: MessageEvents) => { if (msgType == 'init-tts' && !tts) { const context = e.data['context'] as Context; tts = initTts(context); - workerPort.postMessage({ 'msgType': 'init-tts-done', + workerPort.postMessage({ + 'msgType': 'init-tts-done', sampleRate: tts.sampleRate, numSpeakers: tts.numSpeakers, + numThreads: tts.config.model.numThreads, }); } @@ -297,16 +333,14 @@ workerPort.onmessage = (e: MessageEvents) => { console.log(`sampleRate: ${ttsOutput.sampleRate}`); workerPort.postMessage({ - 'msgType': 'tts-generate-done', - samples: Float32Array.from(ttsOutput.samples), + 'msgType': 'tts-generate-done', samples: Float32Array.from(ttsOutput.samples), }); }); } else { const ttsOutput: TtsOutput = tts.generate(input); workerPort.postMessage({ - 'msgType': 'tts-generate-done', - samples: Float32Array.from(ttsOutput.samples), + 'msgType': 'tts-generate-done', samples: Float32Array.from(ttsOutput.samples), }); } From 66e02d83815c4f4010e81c479bf690e44b865cae Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 22 Jan 2025 11:21:31 +0800 Subject: [PATCH 176/183] Release v1.10.41 (#1744) --- CHANGELOG.md | 11 ++++++++ CMakeLists.txt | 2 +- android/SherpaOnnxAar/README.md | 6 ++--- android/SherpaOnnxJavaDemo/app/build.gradle | 2 +- build-ios-shared.sh | 2 +- .../add-punctuations/pubspec.yaml | 2 +- dart-api-examples/audio-tagging/pubspec.yaml | 2 +- .../keyword-spotter/pubspec.yaml | 2 +- .../non-streaming-asr/pubspec.yaml | 2 +- .../speaker-diarization/pubspec.yaml | 2 +- .../speaker-identification/pubspec.yaml | 2 +- dart-api-examples/streaming-asr/pubspec.yaml | 2 +- dart-api-examples/tts/pubspec.yaml | 2 +- .../vad-with-non-streaming-asr/pubspec.yaml | 2 +- dart-api-examples/vad/pubspec.yaml | 2 +- flutter-examples/streaming_asr/pubspec.yaml | 4 +-- flutter-examples/tts/pubspec.yaml | 4 +-- flutter/sherpa_onnx/pubspec.yaml | 12 ++++----- .../ios/sherpa_onnx_ios.podspec | 2 +- .../macos/sherpa_onnx_macos.podspec | 2 +- .../SherpaOnnxHar/sherpa_onnx/README.md | 2 +- .../sherpa_onnx/oh-package.json5 | 2 +- .../entry/oh-package.json5 | 2 +- .../entry/oh-package.json5 | 2 +- .../entry/oh-package.json5 | 2 +- .../SherpaOnnxTts/entry/oh-package.json5 | 2 +- harmony-os/SherpaOnnxVadAsr/entry/README.md | 2 +- .../SherpaOnnxVadAsr/entry/oh-package.json5 | 2 +- jitpack.yml | 6 ++--- new-release.sh | 26 ++++++++++--------- nodejs-addon-examples/package.json | 2 +- pom.xml | 2 +- 32 files changed, 66 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfd4ef92b..3d6a3bf92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +## 1.10.41 + +* Fix UI for Android TTS Engine. (#1735) +* Add iOS TTS example for MatchaTTS (#1736) +* Add iOS example for Kokoro TTS (#1737) +* Fix dither binding in Pybind11 to ensure independence from high_freq in FeatureExtractorConfig (#1739) +* Fix keyword spotting. (#1689) +* Update readme to include https://github.com/hfyydd/sherpa-onnx-server (#1741) +* Reduce vad-moonshine-c-api example code. (#1742) +* Support Kokoro TTS for HarmonyOS. (#1743) + ## 1.10.40 * Fix building wheels (#1703) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9955bc9f7..017cbee09 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ project(sherpa-onnx) # Remember to update # ./CHANGELOG.md # ./new-release.sh -set(SHERPA_ONNX_VERSION "1.10.40") +set(SHERPA_ONNX_VERSION "1.10.41") # Disable warning about # diff --git a/android/SherpaOnnxAar/README.md b/android/SherpaOnnxAar/README.md index 8f4932072..864434ad1 100644 --- a/android/SherpaOnnxAar/README.md +++ b/android/SherpaOnnxAar/README.md @@ -4,8 +4,8 @@ git clone https://github.com/k2-fsa/sherpa-onnx cd sherpa-onnx -wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.40/sherpa-onnx-v1.10.40-android.tar.bz2 -tar xvf sherpa-onnx-v1.10.40-android.tar.bz2 +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.41/sherpa-onnx-v1.10.41-android.tar.bz2 +tar xvf sherpa-onnx-v1.10.41-android.tar.bz2 cp -v jniLibs/arm64-v8a/* android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/arm64-v8a/ cp -v jniLibs/armeabi-v7a/* android/SherpaOnnxAar/sherpa_onnx/src/main/jniLibs/armeabi-v7a/ @@ -16,5 +16,5 @@ cd android/SherpaOnnxAar ./gradlew :sherpa_onnx:assembleRelease ls -lh ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar -cp ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar ../../sherpa-onnx-1.10.40.aar +cp ./sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar ../../sherpa-onnx-1.10.41.aar ``` diff --git a/android/SherpaOnnxJavaDemo/app/build.gradle b/android/SherpaOnnxJavaDemo/app/build.gradle index 38405d916..271088b7a 100644 --- a/android/SherpaOnnxJavaDemo/app/build.gradle +++ b/android/SherpaOnnxJavaDemo/app/build.gradle @@ -34,5 +34,5 @@ dependencies { implementation 'pub.devrel:easypermissions:3.0.0' implementation 'androidx.core:core-ktx:1.7.0' // implementation files('/Users/fangjun/open-source/sherpa-onnx/android/SherpaOnnxAar/sherpa_onnx/build/outputs/aar/sherpa_onnx-release.aar') - implementation 'com.github.k2-fsa:sherpa-onnx:v1.10.40' + implementation 'com.github.k2-fsa:sherpa-onnx:v1.10.41' } diff --git a/build-ios-shared.sh b/build-ios-shared.sh index 54a374352..d48be8c55 100755 --- a/build-ios-shared.sh +++ b/build-ios-shared.sh @@ -242,7 +242,7 @@ for d in ios-arm64_x86_64-simulator ios-arm64; do CFBundlePackageType FMWK CFBundleShortVersionString - 1.10.40 + 1.10.41 CFBundleSupportedPlatforms iPhoneOS diff --git a/dart-api-examples/add-punctuations/pubspec.yaml b/dart-api-examples/add-punctuations/pubspec.yaml index 4ac2868b5..1491eac4b 100644 --- a/dart-api-examples/add-punctuations/pubspec.yaml +++ b/dart-api-examples/add-punctuations/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.40 + sherpa_onnx: ^1.10.41 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/audio-tagging/pubspec.yaml b/dart-api-examples/audio-tagging/pubspec.yaml index 15ecf8286..7a6134441 100644 --- a/dart-api-examples/audio-tagging/pubspec.yaml +++ b/dart-api-examples/audio-tagging/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.40 + sherpa_onnx: ^1.10.41 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/keyword-spotter/pubspec.yaml b/dart-api-examples/keyword-spotter/pubspec.yaml index 02a8c1eb9..7330d0fc7 100644 --- a/dart-api-examples/keyword-spotter/pubspec.yaml +++ b/dart-api-examples/keyword-spotter/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.40 + sherpa_onnx: ^1.10.41 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/non-streaming-asr/pubspec.yaml b/dart-api-examples/non-streaming-asr/pubspec.yaml index 02f5b7927..498295363 100644 --- a/dart-api-examples/non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.40 + sherpa_onnx: ^1.10.41 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/speaker-diarization/pubspec.yaml b/dart-api-examples/speaker-diarization/pubspec.yaml index ab74d425f..fe6446b57 100644 --- a/dart-api-examples/speaker-diarization/pubspec.yaml +++ b/dart-api-examples/speaker-diarization/pubspec.yaml @@ -8,7 +8,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.40 + sherpa_onnx: ^1.10.41 # sherpa_onnx: # path: ../../flutter/sherpa_onnx path: ^1.9.0 diff --git a/dart-api-examples/speaker-identification/pubspec.yaml b/dart-api-examples/speaker-identification/pubspec.yaml index 5725fb434..ab88b1ace 100644 --- a/dart-api-examples/speaker-identification/pubspec.yaml +++ b/dart-api-examples/speaker-identification/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.40 + sherpa_onnx: ^1.10.41 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/streaming-asr/pubspec.yaml b/dart-api-examples/streaming-asr/pubspec.yaml index 092506fcc..af2ce7be5 100644 --- a/dart-api-examples/streaming-asr/pubspec.yaml +++ b/dart-api-examples/streaming-asr/pubspec.yaml @@ -11,7 +11,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.40 + sherpa_onnx: ^1.10.41 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/tts/pubspec.yaml b/dart-api-examples/tts/pubspec.yaml index 34309474a..d630e0b8c 100644 --- a/dart-api-examples/tts/pubspec.yaml +++ b/dart-api-examples/tts/pubspec.yaml @@ -8,7 +8,7 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: ^1.10.40 + sherpa_onnx: ^1.10.41 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml index eff21233b..63f89b6e6 100644 --- a/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml +++ b/dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml @@ -10,7 +10,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.40 + sherpa_onnx: ^1.10.41 path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/vad/pubspec.yaml b/dart-api-examples/vad/pubspec.yaml index a6b1b626d..ff537c306 100644 --- a/dart-api-examples/vad/pubspec.yaml +++ b/dart-api-examples/vad/pubspec.yaml @@ -9,7 +9,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: - sherpa_onnx: ^1.10.40 + sherpa_onnx: ^1.10.41 path: ^1.9.0 args: ^2.5.0 diff --git a/flutter-examples/streaming_asr/pubspec.yaml b/flutter-examples/streaming_asr/pubspec.yaml index 0f6734375..6fd3fa118 100644 --- a/flutter-examples/streaming_asr/pubspec.yaml +++ b/flutter-examples/streaming_asr/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' -version: 1.10.40 +version: 1.10.41 topics: - speech-recognition @@ -31,7 +31,7 @@ dependencies: record: ^5.1.0 url_launcher: ^6.2.6 - sherpa_onnx: ^1.10.40 + sherpa_onnx: ^1.10.41 # sherpa_onnx: # path: ../../flutter/sherpa_onnx diff --git a/flutter-examples/tts/pubspec.yaml b/flutter-examples/tts/pubspec.yaml index d0a2e3f97..da04269d7 100644 --- a/flutter-examples/tts/pubspec.yaml +++ b/flutter-examples/tts/pubspec.yaml @@ -5,7 +5,7 @@ description: > publish_to: 'none' # Remove this line if you wish to publish to pub.dev -version: 1.10.40 +version: 1.10.41 environment: sdk: ">=2.17.0 <4.0.0" @@ -18,7 +18,7 @@ dependencies: cupertino_icons: ^1.0.6 path_provider: ^2.1.3 path: ^1.9.0 - sherpa_onnx: ^1.10.40 + sherpa_onnx: ^1.10.41 # sherpa_onnx: # path: ../../flutter/sherpa_onnx url_launcher: 6.2.6 diff --git a/flutter/sherpa_onnx/pubspec.yaml b/flutter/sherpa_onnx/pubspec.yaml index b0ed0e556..c26fa6d28 100644 --- a/flutter/sherpa_onnx/pubspec.yaml +++ b/flutter/sherpa_onnx/pubspec.yaml @@ -17,7 +17,7 @@ topics: - voice-activity-detection # remember to change the version in ../sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec -version: 1.10.40 +version: 1.10.41 homepage: https://github.com/k2-fsa/sherpa-onnx @@ -30,23 +30,23 @@ dependencies: flutter: sdk: flutter - sherpa_onnx_android: ^1.10.40 + sherpa_onnx_android: ^1.10.41 # sherpa_onnx_android: # path: ../sherpa_onnx_android - sherpa_onnx_macos: ^1.10.40 + sherpa_onnx_macos: ^1.10.41 # sherpa_onnx_macos: # path: ../sherpa_onnx_macos - sherpa_onnx_linux: ^1.10.40 + sherpa_onnx_linux: ^1.10.41 # sherpa_onnx_linux: # path: ../sherpa_onnx_linux - sherpa_onnx_windows: ^1.10.40 + sherpa_onnx_windows: ^1.10.41 # sherpa_onnx_windows: # path: ../sherpa_onnx_windows - sherpa_onnx_ios: ^1.10.40 + sherpa_onnx_ios: ^1.10.41 # sherpa_onnx_ios: # path: ../sherpa_onnx_ios diff --git a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec index 8b2106c43..19ab22ed3 100644 --- a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec +++ b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec @@ -7,7 +7,7 @@ # https://groups.google.com/g/dart-ffi/c/nUATMBy7r0c Pod::Spec.new do |s| s.name = 'sherpa_onnx_ios' - s.version = '1.10.40' + s.version = '1.10.41' s.summary = 'A new Flutter FFI plugin project.' s.description = <<-DESC A new Flutter FFI plugin project. diff --git a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec index 0090f84e0..2a04b712b 100644 --- a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec +++ b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec @@ -4,7 +4,7 @@ # Pod::Spec.new do |s| s.name = 'sherpa_onnx_macos' - s.version = '1.10.40' + s.version = '1.10.41' s.summary = 'sherpa-onnx Flutter FFI plugin project.' s.description = <<-DESC sherpa-onnx Flutter FFI plugin project. diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md b/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md index 358a2d107..f3ff99b9c 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/README.md @@ -23,7 +23,7 @@ or update your `oh-package.json5` to include the following: ``` "dependencies": { - "sherpa_onnx": "1.10.40", + "sherpa_onnx": "1.10.41", }, ``` diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 index 676e14092..6b675fce8 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/oh-package.json5 @@ -1,6 +1,6 @@ { "name": "sherpa_onnx", - "version": "1.10.40", + "version": "1.10.41", "description": "On-device speech-to-text, text-to-speech, and speaker diarization using Next-gen Kaldi without Internet connection", "main": "Index.ets", "author": "The next-gen Kaldi team", diff --git a/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 b/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 index faf4a788a..da6e49716 100644 --- a/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxSpeakerDiarization/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.40" + "sherpa_onnx": "1.10.41" } } diff --git a/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 b/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 index e50200119..9c7b5e691 100644 --- a/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxSpeakerIdentification/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.40", + "sherpa_onnx": "1.10.41", } } diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 index e50200119..9c7b5e691 100644 --- a/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.40", + "sherpa_onnx": "1.10.41", } } diff --git a/harmony-os/SherpaOnnxTts/entry/oh-package.json5 b/harmony-os/SherpaOnnxTts/entry/oh-package.json5 index e50200119..9c7b5e691 100644 --- a/harmony-os/SherpaOnnxTts/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxTts/entry/oh-package.json5 @@ -6,7 +6,7 @@ "author": "", "license": "", "dependencies": { - "sherpa_onnx": "1.10.40", + "sherpa_onnx": "1.10.41", } } diff --git a/harmony-os/SherpaOnnxVadAsr/entry/README.md b/harmony-os/SherpaOnnxVadAsr/entry/README.md index 948dc7ffb..87e0cacb3 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/README.md +++ b/harmony-os/SherpaOnnxVadAsr/entry/README.md @@ -1,6 +1,6 @@ # Introduction -Please download ./sherpa_onnx-v1.10.40.har +Please download ./sherpa_onnx-v1.10.41.har from Hint: For users who have no access to huggingface, please use diff --git a/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 b/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 index ca1e2eaf1..f15d06900 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 +++ b/harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5 @@ -7,7 +7,7 @@ "license": "", "dependencies": { // please see https://ohpm.openharmony.cn/#/cn/detail/sherpa_onnx - "sherpa_onnx": "1.10.40", + "sherpa_onnx": "1.10.41", } } diff --git a/jitpack.yml b/jitpack.yml index ba9c64759..1e0635433 100644 --- a/jitpack.yml +++ b/jitpack.yml @@ -2,8 +2,8 @@ jdk: - openjdk17 before_install: - - wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.40/sherpa-onnx-1.10.40.aar + - wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v1.10.41/sherpa-onnx-1.10.41.aar install: - - FILE="-Dfile=sherpa-onnx-1.10.40.aar" - - mvn install:install-file $FILE -DgroupId=com.k2fsa.sherpa.onnx -DartifactId=sherpa-onnx -Dversion=1.10.40 -Dpackaging=aar -DgeneratePom=true + - FILE="-Dfile=sherpa-onnx-1.10.41.aar" + - mvn install:install-file $FILE -DgroupId=com.k2fsa.sherpa.onnx -DartifactId=sherpa-onnx -Dversion=1.10.41 -Dpackaging=aar -DgeneratePom=true diff --git a/new-release.sh b/new-release.sh index dfddf6b92..4d3a12ef0 100755 --- a/new-release.sh +++ b/new-release.sh @@ -2,18 +2,20 @@ set -ex -sed -i.bak 's/1\.10\.39/1\.10\.40/g' ./build-ios-shared.sh -sed -i.bak 's/1\.10\.39/1\.10\.40/g' ./pom.xml -sed -i.bak 's/1\.10\.39/1\.10\.40/g' ./jitpack.yml -sed -i.bak 's/1\.10\.39/1\.10\.40/g' ./android/SherpaOnnxAar/README.md +sed -i.bak 's/1\.10\.40/1\.10\.41/g' ./build-ios-shared.sh +sed -i.bak 's/1\.10\.40/1\.10\.41/g' ./pom.xml +sed -i.bak 's/1\.10\.40/1\.10\.41/g' ./jitpack.yml +sed -i.bak 's/1\.10\.40/1\.10\.41/g' ./android/SherpaOnnxAar/README.md -find android -name build.gradle -type f -exec sed -i.bak 's/sherpa-onnx:v1\.10\.39/sherpa-onnx:v1\.10\.40/g' {} \; +find android -name build.gradle -type f -exec sed -i.bak 's/sherpa-onnx:v1\.10\.40/sherpa-onnx:v1\.10\.41/g' {} \; -find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.39/1\.10\.40/g' {} \; -find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.39/1\.10\.40/g' {} \; -find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.39/1\.10\.40/g' {} \; -find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.39/1\.10\.40/g' {} \; -find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.39/1\.10\.40/g' {} \; +find flutter -name *.yaml -type f -exec sed -i.bak 's/1\.10\.40/1\.10\.41/g' {} \; +find dart-api-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.40/1\.10\.41/g' {} \; +find flutter-examples -name *.yaml -type f -exec sed -i.bak 's/1\.10\.40/1\.10\.41/g' {} \; +find flutter -name *.podspec -type f -exec sed -i.bak 's/1\.10\.40/1\.10\.41/g' {} \; +find nodejs-addon-examples -name package.json -type f -exec sed -i.bak 's/1\.10\.40/1\.10\.41/g' {} \; -find harmony-os -name "README.md" -type f -exec sed -i.bak 's/1\.10\.39/1\.10\.40/g' {} \; -find harmony-os -name oh-package.json5 -type f -exec sed -i.bak 's/1\.10\.39/1\.10\.40/g' {} \; +find harmony-os -name "README.md" -type f -exec sed -i.bak 's/1\.10\.40/1\.10\.41/g' {} \; +find harmony-os -name oh-package.json5 -type f -exec sed -i.bak 's/1\.10\.40/1\.10\.41/g' {} \; + +find . -name "*.bak" -exec rm {} \; diff --git a/nodejs-addon-examples/package.json b/nodejs-addon-examples/package.json index 6a7779165..6a4d2283c 100644 --- a/nodejs-addon-examples/package.json +++ b/nodejs-addon-examples/package.json @@ -1,5 +1,5 @@ { "dependencies": { - "sherpa-onnx-node": "^1.10.40" + "sherpa-onnx-node": "^1.10.41" } } diff --git a/pom.xml b/pom.xml index 1063994c3..e3edcae61 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.k2fsa.sherpa.onnx sherpa-onnx-android - 1.10.40 + 1.10.41 https://github.com/k2-fsa/sherpa-onnx pom First Android Library From 340ebca74f776022fee6d30f8527064f25a85566 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 22 Jan 2025 14:02:33 +0800 Subject: [PATCH 177/183] Fix publishing wheels (#1746) --- .github/workflows/build-wheels-aarch64.yaml | 2 +- .github/workflows/build-wheels-armv7l.yaml | 2 +- .github/workflows/build-wheels-linux-cuda.yaml | 2 +- .github/workflows/build-wheels-linux.yaml | 2 +- .github/workflows/build-wheels-macos-arm64.yaml | 2 +- .github/workflows/build-wheels-macos-universal2.yaml | 2 +- .github/workflows/build-wheels-macos-x64.yaml | 2 +- .github/workflows/build-wheels-win32.yaml | 2 +- .github/workflows/build-wheels-win64.yaml | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-wheels-aarch64.yaml b/.github/workflows/build-wheels-aarch64.yaml index d65e867a8..1ba8ebd68 100644 --- a/.github/workflows/build-wheels-aarch64.yaml +++ b/.github/workflows/build-wheels-aarch64.yaml @@ -123,6 +123,6 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | python3 -m pip install --upgrade pip - python3 -m pip install wheel twine setuptools + python3 -m pip install wheel twine==5.0.0 setuptools twine upload ./wheelhouse/*.whl diff --git a/.github/workflows/build-wheels-armv7l.yaml b/.github/workflows/build-wheels-armv7l.yaml index 584c64687..58a7cc897 100644 --- a/.github/workflows/build-wheels-armv7l.yaml +++ b/.github/workflows/build-wheels-armv7l.yaml @@ -129,6 +129,6 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | python3 -m pip install --upgrade pip - python3 -m pip install wheel twine setuptools + python3 -m pip install wheel twine==5.0.0 setuptools twine upload ./wheelhouse/*.whl diff --git a/.github/workflows/build-wheels-linux-cuda.yaml b/.github/workflows/build-wheels-linux-cuda.yaml index 28195fb24..1801840ab 100644 --- a/.github/workflows/build-wheels-linux-cuda.yaml +++ b/.github/workflows/build-wheels-linux-cuda.yaml @@ -34,7 +34,7 @@ jobs: - name: Install Python dependencies shell: bash run: | - pip install -U pip wheel setuptools twine + pip install -U pip wheel setuptools twine==5.0.0 - name: Build alsa-lib shell: bash diff --git a/.github/workflows/build-wheels-linux.yaml b/.github/workflows/build-wheels-linux.yaml index db2d77376..0380e2a99 100644 --- a/.github/workflows/build-wheels-linux.yaml +++ b/.github/workflows/build-wheels-linux.yaml @@ -118,7 +118,7 @@ jobs: shell: bash run: | python3 -m pip install --upgrade pip - python3 -m pip install wheel twine setuptools + python3 -m pip install wheel twine==5.0.0 setuptools twine upload ./wheelhouse/*.whl diff --git a/.github/workflows/build-wheels-macos-arm64.yaml b/.github/workflows/build-wheels-macos-arm64.yaml index ba8739f5b..fe1d31628 100644 --- a/.github/workflows/build-wheels-macos-arm64.yaml +++ b/.github/workflows/build-wheels-macos-arm64.yaml @@ -95,6 +95,6 @@ jobs: fi python3 -m pip install $opts --upgrade pip - python3 -m pip install $opts wheel twine setuptools + python3 -m pip install $opts wheel twine==5.0.0 setuptools twine upload ./wheelhouse/*.whl diff --git a/.github/workflows/build-wheels-macos-universal2.yaml b/.github/workflows/build-wheels-macos-universal2.yaml index 1d25a2314..0f9dcedc7 100644 --- a/.github/workflows/build-wheels-macos-universal2.yaml +++ b/.github/workflows/build-wheels-macos-universal2.yaml @@ -89,6 +89,6 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | python3 -m pip install --break-system-packages --upgrade pip - python3 -m pip install --break-system-packages wheel twine setuptools + python3 -m pip install --break-system-packages wheel twine==5.0.0 setuptools twine upload ./wheelhouse/*.whl diff --git a/.github/workflows/build-wheels-macos-x64.yaml b/.github/workflows/build-wheels-macos-x64.yaml index 7d0f59062..cbb4792e9 100644 --- a/.github/workflows/build-wheels-macos-x64.yaml +++ b/.github/workflows/build-wheels-macos-x64.yaml @@ -110,6 +110,6 @@ jobs: fi python3 -m pip install $opts --upgrade pip - python3 -m pip install $opts wheel twine setuptools + python3 -m pip install $opts wheel twine==5.0.0 setuptools twine upload ./wheelhouse/*.whl diff --git a/.github/workflows/build-wheels-win32.yaml b/.github/workflows/build-wheels-win32.yaml index 8138b37cf..732a17d7b 100644 --- a/.github/workflows/build-wheels-win32.yaml +++ b/.github/workflows/build-wheels-win32.yaml @@ -88,6 +88,6 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | python3 -m pip install --upgrade pip - python3 -m pip install wheel twine setuptools + python3 -m pip install wheel twine==5.0.0 setuptools twine upload ./wheelhouse/*.whl diff --git a/.github/workflows/build-wheels-win64.yaml b/.github/workflows/build-wheels-win64.yaml index a87e63537..f2cc7c157 100644 --- a/.github/workflows/build-wheels-win64.yaml +++ b/.github/workflows/build-wheels-win64.yaml @@ -94,6 +94,6 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | python3 -m pip install --upgrade pip - python3 -m pip install wheel twine setuptools + python3 -m pip install wheel twine==5.0.0 setuptools twine upload ./wheelhouse/*.whl From e25952913c809460de8fc543c29aaf0320c5f9e6 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 23 Jan 2025 10:48:47 +0800 Subject: [PATCH 178/183] Update README to include https://github.com/xinhecuican/QSmartAssistant (#1755) --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 2d924a0b4..b5fc25115 100644 --- a/README.md +++ b/README.md @@ -301,6 +301,14 @@ Video demo in Chinese: [爆了！炫神教你开打字挂！真正影响胜率 A server based on nodejs providing Restful API for speech recognition. +### [QSmartAssistant](https://github.com/xinhecuican/QSmartAssistant) + +一个模块化，全过程可离线，低占用率的对话机器人/智能音箱 + +It uses QT. Both [ASR](https://github.com/xinhecuican/QSmartAssistant/blob/master/doc/%E5%AE%89%E8%A3%85.md#asr) +and [TTS](https://github.com/xinhecuican/QSmartAssistant/blob/master/doc/%E5%AE%89%E8%A3%85.md#tts) +are used. + [sherpa-rs]: https://github.com/thewh1teagle/sherpa-rs [silero-vad]: https://github.com/snakers4/silero-vad From 030aaa7bb9be64cc3511478cae358a828b43a74e Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 24 Jan 2025 11:46:41 +0800 Subject: [PATCH 179/183] Add Kokoro TTS to MFC examples (#1760) --- .../NonStreamingTextToSpeech.rc | Bin 13026 -> 13026 bytes .../NonStreamingTextToSpeechDlg.cpp | 121 ++++++++++++++---- 2 files changed, 98 insertions(+), 23 deletions(-) diff --git a/mfc-examples/NonStreamingTextToSpeech/NonStreamingTextToSpeech.rc b/mfc-examples/NonStreamingTextToSpeech/NonStreamingTextToSpeech.rc index dace58ea1e555bfe299b9c0de5c7a9e09fc4e0f2..1a547bef458ea2969d999349b63ba9fa037992ca 100644 GIT binary patch delta 50 zcmaEq`Y3fnousG{gAs!ngARi!kTeF;h74wtS4-ti{wKML*^t3}^J*z`2>= Faster in speech speed. Example value: 1.0"); - AppendLineToMultilineEditCtrl(my_hint_, "\r\n\r\nPlease input your text and click the button Generate"); + AppendLineToMultilineEditCtrl(my_hint_, "\r\nPlease input your text and click the button Generate"); } @@ -430,7 +430,7 @@ void CNonStreamingTextToSpeechDlg::Init() { output_filename_.SetWindowText(Utf8ToUtf16("./generated.wav").c_str()); bool ok = true; - std::string error_message = "--------------------"; + std::string error_message = "--------------------\r\n"; if (!Exists("./model.onnx")) { error_message += "Cannot find ./model.onnx\r\n"; ok = false; @@ -447,17 +447,64 @@ void CNonStreamingTextToSpeechDlg::Init() { generate_btn_.EnableWindow(FALSE); error_message += "\r\nPlease refer to\r\n" - "https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models"; + "https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models" + "\r\nor\r\n" + "https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models"; + error_message += "\r\nto download models.\r\n"; - error_message += "\r\nWe give an example below\r\n\r\n"; - error_message += - "1. Download vits-piper-en_US-amy-low.tar.bz2 from the following URL\r\n\r\n" - "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2\r\n\r\n" - "2. Uncompress it and you will get a directory vits-piper-en_US-amy-low \r\n\r\n" - "3. Switch to the directory vits-piper-en_US-amy-low \r\n\r\n" - "4. Rename en_US-amy-low.onnx to model.onnx \r\n\r\n" - "5. Copy the current exe to the directory vits-piper-en_US-amy-low\r\n\r\n" - "6. Done! You can now run the exe in the directory vits-piper-en_US-amy-low\r\n\r\n"; + error_message += "\r\nWe give several examples below\r\n"; + error_message += " 1. Use a Kokoro TTS model\r\n"; + error_message += " 2. Use a VITS Piper TTS model\r\n"; + error_message += " 3. Use a VITS Chinese TTS model\r\n"; + error_message += " 4. Use a Matcha TTS model\r\n"; + error_message += "\r\n"; + error_message += + "----------1. Use a Kokoro TTS model----------\r\n" + "(a) Download the model from \r\n" + " https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2\r\n" + "(b) Uncompress it and you will get a directory kokoro-en-v0_19\r\n" + "(c) Switch to the directory kokoro-en-v0_19\r\n" + "(d) Copy the current exe to the directory kokoro-en-v0_19\r\n" + "(e).Done! You can now run the exe in the directory kokoro-en-v0_19\r\n"; + + error_message += "\r\n"; + + error_message += + "----------2. Use a VITS Piper TTS model----------\r\n" + "(a) Download the model from \r\n" + " https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2\r\n" + "(b) Uncompress it and you will get a directory vits-piper-en_US-amy-low\r\n" + "(c) Switch to the directory vits-piper-en_US-amy-low \r\n" + "(d) Rename en_US-amy-low.onnx to model.onnx\r\n" + "(e) Copy the current exe to the directory vits-piper-en_US-amy-low\r\n" + "(f) Done! You can now run the exe in the directory vits-piper-en_US-amy-low\r\n"; + + error_message += "\r\n"; + + error_message += + "----------3. Use a VITS Chinese TTS model----------\r\n" + "(a) Download the model from \r\n" + " https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-vits-zh-ll.tar.bz2\r\n" + "(b) Uncompress it and you will get a directory sherpa-onnx-vits-zh-ll\r\n" + "(c) Switch to the directory sherpa-onnx-vits-zh-ll\r\n" + "(d) Copy the current exe to the directory sherpa-onnx-vits-zh-ll\r\n" + "(e) Done! You can now run the exe in the directory sherpa-onnx-vits-zh-ll\r\n"; + + error_message += "\r\n"; + + error_message += + "----------4. Use a Matcha TTS model----------\r\n" + "(a) Download the model from \r\n" + " https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2\r\n" + "(b) Uncompress it and you will get a directory matcha-icefall-zh-baker\r\n" + "(c) Switch to the directory matcha-icefall-zh-baker\r\n" + "(d) Rename model-steps-3.onnx to model.onnx\r\n" + "(e) Download a vocoder model from \r\n" + " https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx\r\n" + "(f) Rename hifigan_v2.onnx to hifigan.onnx\r\n" + "(g) Remember to put hifigan.onnx in the directory matcha-icefall-zh-baker\r\n" + "(h) Copy the current exe to the directory matcha-icefall-zh-baker\r\n" + "(i) Done! You can now run the exe in the directory matcha-icefall-zh-baker\r\n"; AppendLineToMultilineEditCtrl(my_hint_, error_message); return; @@ -467,17 +514,47 @@ void CNonStreamingTextToSpeechDlg::Init() { SherpaOnnxOfflineTtsConfig config; memset(&config, 0, sizeof(config)); config.model.debug = 0; - config.model.num_threads = 2; + config.model.num_threads = 4; config.model.provider = "cpu"; - config.model.vits.model = "./model.onnx"; - if (Exists("./espeak-ng-data/phontab")) { - config.model.vits.data_dir = "./espeak-ng-data"; - } else if (Exists("./lexicon.txt")) { - config.model.vits.lexicon = "./lexicon.txt"; - } - if (Exists("./dict/jieba.dict.utf8")) { - config.model.vits.dict_dir = "./dict"; + if (Exists("./voices.bin")) { + // it is a kokoro tts model + config.model.kokoro.model = "./model.onnx"; + config.model.kokoro.voices = "./voices.bin"; + config.model.kokoro.tokens = "./tokens.txt"; + config.model.kokoro.data_dir = "./espeak-ng-data"; + } else if (Exists("./hifigan.onnx")) { + // it is a matcha tts model + config.model.matcha.acoustic_model = "./model.onnx"; + config.model.matcha.vocoder = "./hifigan.onnx"; + config.model.matcha.tokens = "./tokens.txt"; + + if (Exists("./espeak-ng-data/phontab")) { + config.model.matcha.data_dir = "./espeak-ng-data"; + } + + if(Exists("./lexicon.txt")) { + config.model.matcha.lexicon = "./lexicon.txt"; + } + + if (Exists("./dict/jieba.dict.utf8")) { + config.model.matcha.dict_dir = "./dict"; + } + } else { + // it is a vits tts model + config.model.vits.model = "./model.onnx"; + config.model.vits.tokens = "./tokens.txt"; + if (Exists("./espeak-ng-data/phontab")) { + config.model.vits.data_dir = "./espeak-ng-data"; + } + + if (Exists("./lexicon.txt")) { + config.model.vits.lexicon = "./lexicon.txt"; + } + + if (Exists("./dict/jieba.dict.utf8")) { + config.model.vits.dict_dir = "./dict"; + } } if (Exists("./phone.fst") && Exists("./date.fst") && Exists("./number.fst")) { @@ -488,8 +565,6 @@ void CNonStreamingTextToSpeechDlg::Init() { config.rule_fars = "./rule.far"; } - config.model.vits.tokens = "./tokens.txt"; - tts_ = SherpaOnnxCreateOfflineTts(&config); } From 73c3695287c682fee8675d3bb4b874854f80caf4 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sat, 25 Jan 2025 22:23:12 +0800 Subject: [PATCH 180/183] Refactor node-addon C++ code. (#1768) Use macros to delete pointers. --- .../sherpa_onnx/src/main/cpp/audio-tagging.cc | 19 +--- .../src/main/cpp/keyword-spotting.cc | 52 +++------- .../cpp/non-streaming-speaker-diarization.cc | 19 +--- .../src/main/cpp/non-streaming-tts.cc | 94 +++++-------------- .../sherpa_onnx/src/main/cpp/punctuation.cc | 9 +- .../src/main/cpp/speaker-identification.cc | 10 +- .../cpp/spoken-language-identification.cc | 14 +-- .../sherpa_onnx/src/main/cpp/streaming-asr.cc | 92 ++++-------------- .../sherpa_onnx/src/main/cpp/vad.cc | 10 +- 9 files changed, 70 insertions(+), 249 deletions(-) diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/audio-tagging.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/audio-tagging.cc index bed4e48a2..f4d6ac539 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/audio-tagging.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/audio-tagging.cc @@ -82,21 +82,10 @@ static Napi::External CreateAudioTaggingWrapper( const SherpaOnnxAudioTagging *at = SherpaOnnxCreateAudioTagging(&c); - if (c.model.zipformer.model) { - delete[] c.model.zipformer.model; - } - - if (c.model.ced) { - delete[] c.model.ced; - } - - if (c.model.provider) { - delete[] c.model.provider; - } - - if (c.labels) { - delete[] c.labels; - } + SHERPA_ONNX_DELETE_C_STR(c.model.zipformer.model); + SHERPA_ONNX_DELETE_C_STR(c.model.ced); + SHERPA_ONNX_DELETE_C_STR(c.model.provider); + SHERPA_ONNX_DELETE_C_STR(c.labels); if (!at) { Napi::TypeError::New(env, "Please check your config!") diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/keyword-spotting.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/keyword-spotting.cc index 08f1b5179..6980f1ab9 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/keyword-spotting.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/keyword-spotting.cc @@ -47,50 +47,20 @@ static Napi::External CreateKeywordSpotterWrapper( SHERPA_ONNX_ASSIGN_ATTR_INT32(keywords_buf_size, keywordsBufSize); const SherpaOnnxKeywordSpotter *kws = SherpaOnnxCreateKeywordSpotter(&c); + SHERPA_ONNX_DELETE_C_STR(c.model_config.transducer.encoder); + SHERPA_ONNX_DELETE_C_STR(c.model_config.transducer.decoder); + SHERPA_ONNX_DELETE_C_STR(c.model_config.transducer.joiner); - if (c.model_config.transducer.encoder) { - delete[] c.model_config.transducer.encoder; - } - - if (c.model_config.transducer.decoder) { - delete[] c.model_config.transducer.decoder; - } - - if (c.model_config.transducer.joiner) { - delete[] c.model_config.transducer.joiner; - } + SHERPA_ONNX_DELETE_C_STR(c.model_config.paraformer.encoder); + SHERPA_ONNX_DELETE_C_STR(c.model_config.paraformer.decoder); - if (c.model_config.paraformer.encoder) { - delete[] c.model_config.paraformer.encoder; - } - - if (c.model_config.paraformer.decoder) { - delete[] c.model_config.paraformer.decoder; - } - - if (c.model_config.zipformer2_ctc.model) { - delete[] c.model_config.zipformer2_ctc.model; - } + SHERPA_ONNX_DELETE_C_STR(c.model_config.zipformer2_ctc.model); - if (c.model_config.tokens) { - delete[] c.model_config.tokens; - } - - if (c.model_config.provider) { - delete[] c.model_config.provider; - } - - if (c.model_config.model_type) { - delete[] c.model_config.model_type; - } - - if (c.keywords_file) { - delete[] c.keywords_file; - } - - if (c.keywords_buf) { - delete[] c.keywords_buf; - } + SHERPA_ONNX_DELETE_C_STR(c.model_config.tokens); + SHERPA_ONNX_DELETE_C_STR(c.model_config.provider); + SHERPA_ONNX_DELETE_C_STR(c.model_config.model_type); + SHERPA_ONNX_DELETE_C_STR(c.keywords_file); + SHERPA_ONNX_DELETE_C_STR(c.keywords_buf); if (!kws) { Napi::TypeError::New(env, "Please check your config!") diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-speaker-diarization.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-speaker-diarization.cc index a6b1f302e..cf23fa75b 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-speaker-diarization.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-speaker-diarization.cc @@ -154,21 +154,10 @@ CreateOfflineSpeakerDiarizationWrapper(const Napi::CallbackInfo &info) { SherpaOnnxCreateOfflineSpeakerDiarization(&c); #endif - if (c.segmentation.pyannote.model) { - delete[] c.segmentation.pyannote.model; - } - - if (c.segmentation.provider) { - delete[] c.segmentation.provider; - } - - if (c.embedding.model) { - delete[] c.embedding.model; - } - - if (c.embedding.provider) { - delete[] c.embedding.provider; - } + SHERPA_ONNX_DELETE_C_STR(c.segmentation.pyannote.model); + SHERPA_ONNX_DELETE_C_STR(c.segmentation.provider); + SHERPA_ONNX_DELETE_C_STR(c.embedding.model); + SHERPA_ONNX_DELETE_C_STR(c.embedding.provider); if (!sd) { Napi::TypeError::New(env, "Please check your config!") diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc index 55d4adb2b..bfd078042 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc @@ -155,78 +155,28 @@ static Napi::External CreateOfflineTtsWrapper( #else const SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTts(&c); #endif - - if (c.model.vits.model) { - delete[] c.model.vits.model; - } - - if (c.model.vits.lexicon) { - delete[] c.model.vits.lexicon; - } - - if (c.model.vits.tokens) { - delete[] c.model.vits.tokens; - } - - if (c.model.vits.data_dir) { - delete[] c.model.vits.data_dir; - } - - if (c.model.vits.dict_dir) { - delete[] c.model.vits.dict_dir; - } - - if (c.model.matcha.acoustic_model) { - delete[] c.model.matcha.acoustic_model; - } - - if (c.model.matcha.vocoder) { - delete[] c.model.matcha.vocoder; - } - - if (c.model.matcha.lexicon) { - delete[] c.model.matcha.lexicon; - } - - if (c.model.matcha.tokens) { - delete[] c.model.matcha.tokens; - } - - if (c.model.matcha.data_dir) { - delete[] c.model.matcha.data_dir; - } - - if (c.model.matcha.dict_dir) { - delete[] c.model.matcha.dict_dir; - } - - if (c.model.kokoro.model) { - delete[] c.model.kokoro.model; - } - - if (c.model.kokoro.voices) { - delete[] c.model.kokoro.voices; - } - - if (c.model.kokoro.tokens) { - delete[] c.model.kokoro.tokens; - } - - if (c.model.kokoro.data_dir) { - delete[] c.model.kokoro.data_dir; - } - - if (c.model.provider) { - delete[] c.model.provider; - } - - if (c.rule_fsts) { - delete[] c.rule_fsts; - } - - if (c.rule_fars) { - delete[] c.rule_fars; - } + SHERPA_ONNX_DELETE_C_STR(c.model.vits.model); + SHERPA_ONNX_DELETE_C_STR(c.model.vits.lexicon); + SHERPA_ONNX_DELETE_C_STR(c.model.vits.tokens); + SHERPA_ONNX_DELETE_C_STR(c.model.vits.data_dir); + SHERPA_ONNX_DELETE_C_STR(c.model.vits.dict_dir); + + SHERPA_ONNX_DELETE_C_STR(c.model.matcha.acoustic_model); + SHERPA_ONNX_DELETE_C_STR(c.model.matcha.vocoder); + SHERPA_ONNX_DELETE_C_STR(c.model.matcha.lexicon); + SHERPA_ONNX_DELETE_C_STR(c.model.matcha.tokens); + SHERPA_ONNX_DELETE_C_STR(c.model.matcha.data_dir); + SHERPA_ONNX_DELETE_C_STR(c.model.matcha.dict_dir); + + SHERPA_ONNX_DELETE_C_STR(c.model.kokoro.model); + SHERPA_ONNX_DELETE_C_STR(c.model.kokoro.voices); + SHERPA_ONNX_DELETE_C_STR(c.model.kokoro.tokens); + SHERPA_ONNX_DELETE_C_STR(c.model.kokoro.data_dir); + + SHERPA_ONNX_DELETE_C_STR(c.model.provider); + + SHERPA_ONNX_DELETE_C_STR(c.rule_fsts); + SHERPA_ONNX_DELETE_C_STR(c.rule_fars); if (!tts) { Napi::TypeError::New(env, "Please check your config!") diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/punctuation.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/punctuation.cc index df079b96d..df4c920f7 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/punctuation.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/punctuation.cc @@ -63,13 +63,8 @@ CreateOfflinePunctuationWrapper(const Napi::CallbackInfo &info) { const SherpaOnnxOfflinePunctuation *punct = SherpaOnnxCreateOfflinePunctuation(&c); - if (c.model.ct_transformer) { - delete[] c.model.ct_transformer; - } - - if (c.model.provider) { - delete[] c.model.provider; - } + SHERPA_ONNX_DELETE_C_STR(c.model.ct_transformer); + SHERPA_ONNX_DELETE_C_STR(c.model.provider); if (!punct) { Napi::TypeError::New(env, "Please check your config!") diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/speaker-identification.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/speaker-identification.cc index bf49fef9a..5a1ee5582 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/speaker-identification.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/speaker-identification.cc @@ -70,14 +70,8 @@ CreateSpeakerEmbeddingExtractorWrapper(const Napi::CallbackInfo &info) { const SherpaOnnxSpeakerEmbeddingExtractor *extractor = SherpaOnnxCreateSpeakerEmbeddingExtractor(&c); #endif - - if (c.model) { - delete[] c.model; - } - - if (c.provider) { - delete[] c.provider; - } + SHERPA_ONNX_DELETE_C_STR(c.model); + SHERPA_ONNX_DELETE_C_STR(c.provider); if (!extractor) { Napi::TypeError::New(env, "Please check your config!") diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/spoken-language-identification.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/spoken-language-identification.cc index 35ade6541..c40e838b6 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/spoken-language-identification.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/spoken-language-identification.cc @@ -66,17 +66,9 @@ CreateSpokenLanguageIdentificationWrapper(const Napi::CallbackInfo &info) { const SherpaOnnxSpokenLanguageIdentification *slid = SherpaOnnxCreateSpokenLanguageIdentification(&c); - if (c.whisper.encoder) { - delete[] c.whisper.encoder; - } - - if (c.whisper.decoder) { - delete[] c.whisper.decoder; - } - - if (c.provider) { - delete[] c.provider; - } + SHERPA_ONNX_DELETE_C_STR(c.whisper.encoder); + SHERPA_ONNX_DELETE_C_STR(c.whisper.decoder); + SHERPA_ONNX_DELETE_C_STR(c.provider); if (!slid) { Napi::TypeError::New(env, "Please check your config!") diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/streaming-asr.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/streaming-asr.cc index 59ad5ce52..fce28358d 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/streaming-asr.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/streaming-asr.cc @@ -222,78 +222,26 @@ static Napi::External CreateOnlineRecognizerWrapper( const SherpaOnnxOnlineRecognizer *recognizer = SherpaOnnxCreateOnlineRecognizer(&c); #endif - - if (c.model_config.transducer.encoder) { - delete[] c.model_config.transducer.encoder; - } - - if (c.model_config.transducer.decoder) { - delete[] c.model_config.transducer.decoder; - } - - if (c.model_config.transducer.joiner) { - delete[] c.model_config.transducer.joiner; - } - - if (c.model_config.paraformer.encoder) { - delete[] c.model_config.paraformer.encoder; - } - - if (c.model_config.paraformer.decoder) { - delete[] c.model_config.paraformer.decoder; - } - - if (c.model_config.zipformer2_ctc.model) { - delete[] c.model_config.zipformer2_ctc.model; - } - - if (c.model_config.tokens) { - delete[] c.model_config.tokens; - } - - if (c.model_config.provider) { - delete[] c.model_config.provider; - } - - if (c.model_config.model_type) { - delete[] c.model_config.model_type; - } - - if (c.model_config.modeling_unit) { - delete[] c.model_config.modeling_unit; - } - - if (c.model_config.bpe_vocab) { - delete[] c.model_config.bpe_vocab; - } - - if (c.model_config.tokens_buf) { - delete[] c.model_config.tokens_buf; - } - - if (c.decoding_method) { - delete[] c.decoding_method; - } - - if (c.hotwords_file) { - delete[] c.hotwords_file; - } - - if (c.rule_fsts) { - delete[] c.rule_fsts; - } - - if (c.rule_fars) { - delete[] c.rule_fars; - } - - if (c.hotwords_buf) { - delete[] c.hotwords_buf; - } - - if (c.ctc_fst_decoder_config.graph) { - delete[] c.ctc_fst_decoder_config.graph; - } + SHERPA_ONNX_DELETE_C_STR(c.model_config.transducer.encoder); + SHERPA_ONNX_DELETE_C_STR(c.model_config.transducer.decoder); + SHERPA_ONNX_DELETE_C_STR(c.model_config.transducer.joiner); + + SHERPA_ONNX_DELETE_C_STR(c.model_config.paraformer.encoder); + SHERPA_ONNX_DELETE_C_STR(c.model_config.paraformer.decoder); + + SHERPA_ONNX_DELETE_C_STR(c.model_config.zipformer2_ctc.model); + SHERPA_ONNX_DELETE_C_STR(c.model_config.tokens); + SHERPA_ONNX_DELETE_C_STR(c.model_config.provider); + SHERPA_ONNX_DELETE_C_STR(c.model_config.model_type); + SHERPA_ONNX_DELETE_C_STR(c.model_config.modeling_unit); + SHERPA_ONNX_DELETE_C_STR(c.model_config.bpe_vocab); + SHERPA_ONNX_DELETE_C_STR(c.model_config.tokens_buf); + SHERPA_ONNX_DELETE_C_STR(c.decoding_method); + SHERPA_ONNX_DELETE_C_STR(c.hotwords_file); + SHERPA_ONNX_DELETE_C_STR(c.rule_fsts); + SHERPA_ONNX_DELETE_C_STR(c.rule_fars); + SHERPA_ONNX_DELETE_C_STR(c.hotwords_buf); + SHERPA_ONNX_DELETE_C_STR(c.ctc_fst_decoder_config.graph); if (!recognizer) { Napi::TypeError::New(env, "Please check your config!") diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/vad.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/vad.cc index b1defcac0..81eed7c8d 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/vad.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/vad.cc @@ -367,14 +367,8 @@ CreateVoiceActivityDetectorWrapper(const Napi::CallbackInfo &info) { SherpaOnnxVoiceActivityDetector *vad = SherpaOnnxCreateVoiceActivityDetector(&c, buffer_size_in_seconds); #endif - - if (c.silero_vad.model) { - delete[] c.silero_vad.model; - } - - if (c.provider) { - delete[] c.provider; - } + SHERPA_ONNX_DELETE_C_STR(c.silero_vad.model); + SHERPA_ONNX_DELETE_C_STR(c.provider); return Napi::External::New( env, vad, [](Napi::Env env, SherpaOnnxVoiceActivityDetector *p) { From f178e96bf04dd98ff741cdcf7f47a6c887741c47 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sun, 26 Jan 2025 14:12:30 +0800 Subject: [PATCH 181/183] Add keyword spotter C API for HarmonyOS (#1769) --- .../src/main/cpp/keyword-spotting.cc | 43 +++++++++++++++++-- sherpa-onnx/c-api/c-api.cc | 30 ++++++++++++- sherpa-onnx/c-api/c-api.h | 4 ++ sherpa-onnx/csrc/keyword-spotter-impl.cc | 22 +++++++++- sherpa-onnx/csrc/keyword-spotter-impl.h | 10 +---- .../csrc/keyword-spotter-transducer-impl.h | 33 ++++++++------ sherpa-onnx/csrc/keyword-spotter.cc | 25 +++++++++-- sherpa-onnx/csrc/keyword-spotter.h | 10 +---- 8 files changed, 135 insertions(+), 42 deletions(-) diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/keyword-spotting.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/keyword-spotting.cc index 6980f1ab9..6562ef5a1 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/keyword-spotting.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/keyword-spotting.cc @@ -16,6 +16,16 @@ SherpaOnnxOnlineModelConfig GetOnlineModelConfig(Napi::Object obj); static Napi::External CreateKeywordSpotterWrapper( const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); +#if __OHOS__ + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } +#else if (info.Length() != 1) { std::ostringstream os; os << "Expect only 1 argument. Given: " << info.Length(); @@ -24,7 +34,7 @@ static Napi::External CreateKeywordSpotterWrapper( return {}; } - +#endif if (!info[0].IsObject()) { Napi::TypeError::New(env, "Expect an object as the argument") .ThrowAsJavaScriptException(); @@ -46,7 +56,18 @@ static Napi::External CreateKeywordSpotterWrapper( SHERPA_ONNX_ASSIGN_ATTR_STR(keywords_buf, keywordsBuf); SHERPA_ONNX_ASSIGN_ATTR_INT32(keywords_buf_size, keywordsBufSize); +#if __OHOS__ + std::unique_ptr + mgr(OH_ResourceManager_InitNativeResourceManager(env, info[1]), + &OH_ResourceManager_ReleaseNativeResourceManager); + + const SherpaOnnxKeywordSpotter *kws = + SherpaOnnxCreateKeywordSpotterOHOS(&c, mgr.get()); +#else const SherpaOnnxKeywordSpotter *kws = SherpaOnnxCreateKeywordSpotter(&c); +#endif + SHERPA_ONNX_DELETE_C_STR(c.model_config.transducer.encoder); SHERPA_ONNX_DELETE_C_STR(c.model_config.transducer.decoder); SHERPA_ONNX_DELETE_C_STR(c.model_config.transducer.joiner); @@ -79,9 +100,9 @@ static Napi::External CreateKeywordSpotterWrapper( static Napi::External CreateKeywordStreamWrapper( const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); - if (info.Length() != 1) { + if (info.Length() != 1 && info.Length() != 2) { std::ostringstream os; - os << "Expect only 1 argument. Given: " << info.Length(); + os << "Expect only 1 or 2 arguments. Given: " << info.Length(); Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); @@ -96,10 +117,24 @@ static Napi::External CreateKeywordStreamWrapper( return {}; } + if (info.Length() == 2 && !info[1].IsString()) { + std::ostringstream os; + os << "Argument 2 should be a string."; + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + return {}; + } + const SherpaOnnxKeywordSpotter *kws = info[0].As>().Data(); - const SherpaOnnxOnlineStream *stream = SherpaOnnxCreateKeywordStream(kws); + const SherpaOnnxOnlineStream *stream; + if (info.Length() == 1) { + stream = SherpaOnnxCreateKeywordStream(kws); + } else { + Napi::String js_keywords = info[1].As(); + std::string keywords = js_keywords.Utf8Value(); + stream = SherpaOnnxCreateKeywordStreamWithKeywords(kws, keywords.c_str()); + } return Napi::External::New( env, const_cast(stream), diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index ce6d55634..78701cf70 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -678,7 +678,7 @@ struct SherpaOnnxKeywordSpotter { std::unique_ptr impl; }; -const SherpaOnnxKeywordSpotter *SherpaOnnxCreateKeywordSpotter( +static sherpa_onnx::KeywordSpotterConfig GetKeywordSpotterConfig( const SherpaOnnxKeywordSpotterConfig *config) { sherpa_onnx::KeywordSpotterConfig spotter_config; @@ -739,10 +739,20 @@ const SherpaOnnxKeywordSpotter *SherpaOnnxCreateKeywordSpotter( std::string(config->keywords_buf, config->keywords_buf_size); } - if (config->model_config.debug) { + if (spotter_config.model_config.debug) { +#if OHOS + SHERPA_ONNX_LOGE("%{public}s\n", spotter_config.ToString().c_str()); +#else SHERPA_ONNX_LOGE("%s\n", spotter_config.ToString().c_str()); +#endif } + return spotter_config; +} + +const SherpaOnnxKeywordSpotter *SherpaOnnxCreateKeywordSpotter( + const SherpaOnnxKeywordSpotterConfig *config) { + auto spotter_config = GetKeywordSpotterConfig(config); if (!spotter_config.Validate()) { SHERPA_ONNX_LOGE("Errors in config!"); return nullptr; @@ -2272,6 +2282,22 @@ SherpaOnnxCreateSpeakerEmbeddingExtractorOHOS( return p; } +const SherpaOnnxKeywordSpotter *SherpaOnnxCreateKeywordSpotterOHOS( + const SherpaOnnxKeywordSpotterConfig *config, NativeResourceManager *mgr) { + if (!mgr) { + return SherpaOnnxCreateKeywordSpotter(config); + } + + auto spotter_config = GetKeywordSpotterConfig(config); + + SherpaOnnxKeywordSpotter *spotter = new SherpaOnnxKeywordSpotter; + + spotter->impl = + std::make_unique(mgr, spotter_config); + + return spotter; +} + #if SHERPA_ONNX_ENABLE_TTS == 1 const SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTtsOHOS( const SherpaOnnxOfflineTtsConfig *config, NativeResourceManager *mgr) { diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index 10bd101fd..5fe124d45 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -1645,6 +1645,10 @@ SherpaOnnxCreateSpeakerEmbeddingExtractorOHOS( const SherpaOnnxSpeakerEmbeddingExtractorConfig *config, NativeResourceManager *mgr); +SHERPA_ONNX_API const SherpaOnnxKeywordSpotter * +SherpaOnnxCreateKeywordSpotterOHOS(const SherpaOnnxKeywordSpotterConfig *config, + NativeResourceManager *mgr); + SHERPA_ONNX_API const SherpaOnnxOfflineSpeakerDiarization * SherpaOnnxCreateOfflineSpeakerDiarizationOHOS( const SherpaOnnxOfflineSpeakerDiarizationConfig *config, diff --git a/sherpa-onnx/csrc/keyword-spotter-impl.cc b/sherpa-onnx/csrc/keyword-spotter-impl.cc index 1c9d59485..affb212c0 100644 --- a/sherpa-onnx/csrc/keyword-spotter-impl.cc +++ b/sherpa-onnx/csrc/keyword-spotter-impl.cc @@ -6,6 +6,15 @@ #include "sherpa-onnx/csrc/keyword-spotter-transducer-impl.h" +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + namespace sherpa_onnx { std::unique_ptr KeywordSpotterImpl::Create( @@ -18,9 +27,9 @@ std::unique_ptr KeywordSpotterImpl::Create( exit(-1); } -#if __ANDROID_API__ >= 9 +template std::unique_ptr KeywordSpotterImpl::Create( - AAssetManager *mgr, const KeywordSpotterConfig &config) { + Manager *mgr, const KeywordSpotterConfig &config) { if (!config.model_config.transducer.encoder.empty()) { return std::make_unique(mgr, config); } @@ -28,6 +37,15 @@ std::unique_ptr KeywordSpotterImpl::Create( SHERPA_ONNX_LOGE("Please specify a model"); exit(-1); } + +#if __ANDROID_API__ >= 9 +template std::unique_ptr KeywordSpotterImpl::Create( + AAssetManager *mgr, const KeywordSpotterConfig &config); +#endif + +#if __OHOS__ +template std::unique_ptr KeywordSpotterImpl::Create( + NativeResourceManager *mgr, const KeywordSpotterConfig &config); #endif } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/keyword-spotter-impl.h b/sherpa-onnx/csrc/keyword-spotter-impl.h index 6180f9172..dafb0a8bb 100644 --- a/sherpa-onnx/csrc/keyword-spotter-impl.h +++ b/sherpa-onnx/csrc/keyword-spotter-impl.h @@ -9,11 +9,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/keyword-spotter.h" #include "sherpa-onnx/csrc/online-stream.h" @@ -24,10 +19,9 @@ class KeywordSpotterImpl { static std::unique_ptr Create( const KeywordSpotterConfig &config); -#if __ANDROID_API__ >= 9 + template static std::unique_ptr Create( - AAssetManager *mgr, const KeywordSpotterConfig &config); -#endif + Manager *mgr, const KeywordSpotterConfig &config); virtual ~KeywordSpotterImpl() = default; diff --git a/sherpa-onnx/csrc/keyword-spotter-transducer-impl.h b/sherpa-onnx/csrc/keyword-spotter-transducer-impl.h index d29b8b58d..e62b3b2c0 100644 --- a/sherpa-onnx/csrc/keyword-spotter-transducer-impl.h +++ b/sherpa-onnx/csrc/keyword-spotter-transducer-impl.h @@ -9,16 +9,10 @@ #include #include // NOLINT #include +#include #include #include -#if __ANDROID_API__ >= 9 -#include - -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/file-utils.h" #include "sherpa-onnx/csrc/keyword-spotter-impl.h" #include "sherpa-onnx/csrc/keyword-spotter.h" @@ -91,9 +85,8 @@ class KeywordSpotterTransducerImpl : public KeywordSpotterImpl { unk_id_); } -#if __ANDROID_API__ >= 9 - KeywordSpotterTransducerImpl(AAssetManager *mgr, - const KeywordSpotterConfig &config) + template + KeywordSpotterTransducerImpl(Manager *mgr, const KeywordSpotterConfig &config) : config_(config), model_(OnlineTransducerModel::Create(mgr, config.model_config)), sym_(mgr, config.model_config.tokens) { @@ -109,7 +102,6 @@ class KeywordSpotterTransducerImpl : public KeywordSpotterImpl { model_.get(), config_.max_active_paths, config_.num_trailing_blanks, unk_id_); } -#endif std::unique_ptr CreateStream() const override { auto stream = @@ -130,7 +122,11 @@ class KeywordSpotterTransducerImpl : public KeywordSpotterImpl { if (!EncodeKeywords(is, sym_, ¤t_ids, ¤t_kws, ¤t_scores, ¤t_thresholds)) { +#if __OHOS__ + SHERPA_ONNX_LOGE("Encode keywords %{public}s failed.", keywords.c_str()); +#else SHERPA_ONNX_LOGE("Encode keywords %s failed.", keywords.c_str()); +#endif return nullptr; } @@ -306,16 +302,21 @@ class KeywordSpotterTransducerImpl : public KeywordSpotterImpl { // each line in keywords_file contains space-separated words std::ifstream is(config_.keywords_file); if (!is) { +#if __OHOS__ + SHERPA_ONNX_LOGE("Open keywords file failed: %{public}s", + config_.keywords_file.c_str()); +#else SHERPA_ONNX_LOGE("Open keywords file failed: %s", config_.keywords_file.c_str()); +#endif exit(-1); } InitKeywords(is); #endif } -#if __ANDROID_API__ >= 9 - void InitKeywords(AAssetManager *mgr) { + template + void InitKeywords(Manager *mgr) { // each line in keywords_file contains space-separated words auto buf = ReadFile(mgr, config_.keywords_file); @@ -323,13 +324,17 @@ class KeywordSpotterTransducerImpl : public KeywordSpotterImpl { std::istrstream is(buf.data(), buf.size()); if (!is) { +#if __OHOS__ + SHERPA_ONNX_LOGE("Open keywords file failed: %{public}s", + config_.keywords_file.c_str()); +#else SHERPA_ONNX_LOGE("Open keywords file failed: %s", config_.keywords_file.c_str()); +#endif exit(-1); } InitKeywords(is); } -#endif void InitKeywordsFromBufStr() { // keywords_buf's content is supposed to be same as the keywords_file's diff --git a/sherpa-onnx/csrc/keyword-spotter.cc b/sherpa-onnx/csrc/keyword-spotter.cc index 66d0907ab..615aab9cd 100644 --- a/sherpa-onnx/csrc/keyword-spotter.cc +++ b/sherpa-onnx/csrc/keyword-spotter.cc @@ -13,6 +13,15 @@ #include #include +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + #include "sherpa-onnx/csrc/keyword-spotter-impl.h" namespace sherpa_onnx { @@ -136,11 +145,9 @@ std::string KeywordSpotterConfig::ToString() const { KeywordSpotter::KeywordSpotter(const KeywordSpotterConfig &config) : impl_(KeywordSpotterImpl::Create(config)) {} -#if __ANDROID_API__ >= 9 -KeywordSpotter::KeywordSpotter(AAssetManager *mgr, - const KeywordSpotterConfig &config) +template +KeywordSpotter::KeywordSpotter(Manager *mgr, const KeywordSpotterConfig &config) : impl_(KeywordSpotterImpl::Create(mgr, config)) {} -#endif KeywordSpotter::~KeywordSpotter() = default; @@ -167,4 +174,14 @@ KeywordResult KeywordSpotter::GetResult(OnlineStream *s) const { return impl_->GetResult(s); } +#if __ANDROID_API__ >= 9 +template KeywordSpotter::KeywordSpotter(AAssetManager *mgr, + const KeywordSpotterConfig &config); +#endif + +#if __OHOS__ +template KeywordSpotter::KeywordSpotter(NativeResourceManager *mgr, + const KeywordSpotterConfig &config); +#endif + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/keyword-spotter.h b/sherpa-onnx/csrc/keyword-spotter.h index c933f4b23..e494b3e56 100644 --- a/sherpa-onnx/csrc/keyword-spotter.h +++ b/sherpa-onnx/csrc/keyword-spotter.h @@ -9,11 +9,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/features.h" #include "sherpa-onnx/csrc/online-model-config.h" #include "sherpa-onnx/csrc/online-stream.h" @@ -101,9 +96,8 @@ class KeywordSpotter { public: explicit KeywordSpotter(const KeywordSpotterConfig &config); -#if __ANDROID_API__ >= 9 - KeywordSpotter(AAssetManager *mgr, const KeywordSpotterConfig &config); -#endif + template + KeywordSpotter(Manager *mgr, const KeywordSpotterConfig &config); ~KeywordSpotter(); From 884715187e4dea65ea2c8a2a7d91d10975f5c9bc Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 29 Jan 2025 21:34:39 +0800 Subject: [PATCH 182/183] Add ArkTS API for Keyword spotting. (#1775) --- .../sherpa_onnx/BuildProfile.ets | 2 +- .../SherpaOnnxHar/sherpa_onnx/Index.ets | 5 ++ .../main/cpp/types/libsherpa_onnx/Index.d.ts | 7 ++ .../main/ets/components/KeywordSpotting.ets | 78 +++++++++++++++++++ 4 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/KeywordSpotting.ets diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets index 274f60379..ea97166bc 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/BuildProfile.ets @@ -1,7 +1,7 @@ /** * Use these variables when you tailor your ArkTS code. They must be of the const type. */ -export const HAR_VERSION = '1.10.40'; +export const HAR_VERSION = '1.10.41'; export const BUILD_MODE_NAME = 'debug'; export const DEBUG = true; export const TARGET_NAME = 'default'; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets index 7c1e95df4..84286294a 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets @@ -53,3 +53,8 @@ export { OfflineSpeakerSegmentationPyannoteModelConfig, OfflineSpeakerDiarization, FastClusteringConfig, } from './src/main/ets/components/NonStreamingSpeakerDiarization'; + +export { KeywordSpotterConfig, + KeywordSpotterResult, + KeywordSpotter, +} from './src/main/ets/components/KeywordSpotting'; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts index 7db410a4c..a51b9ed23 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts @@ -68,3 +68,10 @@ export const getOfflineSpeakerDiarizationSampleRate: (handle: object) => number; export const offlineSpeakerDiarizationProcess: (handle: object, input: object) => object; export const offlineSpeakerDiarizationProcessAsync: (handle: object, input: object, callback: object) => object; export const offlineSpeakerDiarizationSetConfig: (handle: object, config: object) => void; + +export const createKeywordSpotter: (config: object, mgr?: object) => object; +export const createKeywordStream: (handle: object, keywords?: string) => object; +export const isKeywordStreamReady: (handle: object, stream: object) => boolean; +export const decodeKeywordStream: (handle: object, stream: object) => void; +export const resetKeywordStream: (handle: object, stream: object) => void; +export const getKeywordResultAsJson: (handle: object, stream: object) => string; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/KeywordSpotting.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/KeywordSpotting.ets new file mode 100644 index 000000000..f9c84d9ce --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/KeywordSpotting.ets @@ -0,0 +1,78 @@ +import { + createKeywordSpotter, + createKeywordStream, + isKeywordStreamReady, + decodeKeywordStream, + resetKeywordStream, + getKeywordResultAsJson, +} from 'libsherpa_onnx.so'; + +import { FeatureConfig } from './NonStreamingAsr'; +import { OnlineModelConfig, OnlineStream } from './StreamingAsr'; + +export class KeywordSpotterConfig { + public featConfig: FeatureConfig = new FeatureConfig(); + public modelConfig: OnlineModelConfig = new OnlineModelConfig(); + public maxActivePaths: number = 4; + public numTrailingBlanks: number = 1; + public keywordsScore: number = 1; + public keywordsThreshold: number = 0.25; + public keywordsFile: string = ''; +} + +interface KeywordSpotterResultJson { + keyword: string; + timestamps: number[]; + tokens: string[]; +} + +export class KeywordSpotterResult { + public keyword: string = ''; + public tokens: string[] = []; + public timestamps: number[] = []; + public json: string = ''; +} + +export class KeywordSpotter { + public handle: object; + public config: KeywordSpotterConfig; + + constructor(config: KeywordSpotterConfig, mgr?: object) { + this.handle = createKeywordSpotter(config, mgr); + this.config = config + } + + createStream(keywords?: string): OnlineStream { + if (typeof keywords !== "undefined") { + return new OnlineStream(createKeywordStream(this.handle, keywords)); + } else { + return new OnlineStream(createKeywordStream(this.handle)); + } + } + + isReady(stream: OnlineStream): boolean { + return isKeywordStreamReady(this.handle, stream.handle); + } + + decode(stream: OnlineStream) { + decodeKeywordStream(this.handle, stream.handle); + } + + reset(stream: OnlineStream) { + resetKeywordStream(this.handle, stream.handle); + } + + getResult(stream: OnlineStream): KeywordSpotterResult { + const jsonStr: string = getKeywordResultAsJson(this.handle, stream.handle); + + let o = JSON.parse(jsonStr) as KeywordSpotterResultJson; + + const r = new KeywordSpotterResult() + r.keyword = o.keyword + r.timestamps = o.timestamps; + r.tokens = o.tokens; + r.json = jsonStr; + + return r; + } +} From 59ff854222302c55ee1e25bcf6538b38bdf5bcff Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 29 Jan 2025 22:43:17 +0800 Subject: [PATCH 183/183] Add Flutter example for Kokoro TTS (#1776) --- flutter-examples/tts/lib/model.dart | 43 +++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/flutter-examples/tts/lib/model.dart b/flutter-examples/tts/lib/model.dart index 16ada98c3..5be3ebb91 100644 --- a/flutter-examples/tts/lib/model.dart +++ b/flutter-examples/tts/lib/model.dart @@ -24,13 +24,14 @@ Future createOfflineTts() async { String modelDir = ''; String modelName = ''; + String voices = ''; // for Kokoro only String ruleFsts = ''; String ruleFars = ''; String lexicon = ''; String dataDir = ''; String dictDir = ''; - // You can select an example below and change it according to match your + // You can select an example below and change it accordingly to match your // selected tts model // ============================================================ @@ -84,6 +85,13 @@ Future createOfflineTts() async { // lexicon = 'lexicon.txt'; // dictDir = 'vits-melo-tts-zh_en/dict'; + // Example 8 + // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html#kokoro-en-v0-19-english-11-speakers + // modelDir = 'kokoro-en-v0_19'; + // modelName = 'model.onnx'; + // voices = 'voices.bin'; + // dataDir = 'kokoro-en-v0_19/espeak-ng-data'; + // ============================================================ // Please don't change the remaining part of this function // ============================================================ @@ -126,17 +134,36 @@ Future createOfflineTts() async { } final tokens = p.join(directory.path, modelDir, 'tokens.txt'); + if (voices != '') { + voices = p.join(directory.path, modelDir, voices); + } - final vits = sherpa_onnx.OfflineTtsVitsModelConfig( - model: modelName, - lexicon: lexicon, - tokens: tokens, - dataDir: dataDir, - dictDir: dictDir, - ); + late final sherpa_onnx.OfflineTtsVitsModelConfig vits; + late final sherpa_onnx.OfflineTtsKokoroModelConfig kokoro; + + if (voices != '') { + vits = sherpa_onnx.OfflineTtsVitsModelConfig(); + kokoro = sherpa_onnx.OfflineTtsKokoroModelConfig( + model: modelName, + voices: voices, + tokens: tokens, + dataDir: dataDir, + ); + } else { + vits = sherpa_onnx.OfflineTtsVitsModelConfig( + model: modelName, + lexicon: lexicon, + tokens: tokens, + dataDir: dataDir, + dictDir: dictDir, + ); + + kokoro = sherpa_onnx.OfflineTtsKokoroModelConfig(); + } final modelConfig = sherpa_onnx.OfflineTtsModelConfig( vits: vits, + kokoro: kokoro, numThreads: 2, debug: true, provider: 'cpu',