tts(sherpa): update support for matcha tts

xinhecuican · Jan 25, 2025 · 4a9f5a0 · 4a9f5a0
1 parent 5caa8d8
commit 4a9f5a0
Show file tree

Hide file tree

Showing 12 changed files with 55 additions and 1,498 deletions.
diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
@@ -26,7 +26,7 @@ jobs:
           - qt_version: 5.15.2
             qt_arch: gcc_64
             modules: "qtwebengine qtnetworkauth"
-          - qt_version: 6.8.1
+          - qt_version: 6.2.4
             qt_arch: gcc_64
             modules: "qtwebengine qtnetworkauth qtmultimedia"
 

diff --git a/Conversation/NLU/nlumodel.h b/Conversation/NLU/nlumodel.h
@@ -6,12 +6,14 @@
 class NLUModel : public QObject
 {
 public:
-    NLUModel(QObject* parent=nullptr):QObject(parent){valid=true;}
+    NLUModel(QObject* parent=nullptr):QObject(parent){valid=true; start=false;}
     virtual ~NLUModel(){}
     virtual ParsedIntent parseIntent(const QString& text)=0;
     virtual void stop(){}
+    virtual bool isStart() {return start;}
 protected:
     bool valid;
+    bool start;
 };
 
 #endif // NLUMODEL_H
diff --git a/Conversation/NLU/rasanlu.cpp b/Conversation/NLU/rasanlu.cpp
@@ -142,8 +142,10 @@ ParsedIntent RasaNLU::parseIntent(const QString &text) {
                 qaFile.close();
             }
         }
+        start = true;
     } else {
         qWarning() << "rasa request error";
+        start = false;
     }
     reply->deleteLater();
     return parsedIntent;

diff --git a/Conversation/TTS/sherpatts.cpp b/Conversation/TTS/sherpatts.cpp
@@ -9,24 +9,43 @@ SherpaTTS::SherpaTTS(QObject* parent) : TTSModel(parent) {
     QJsonObject ttsConfig = Config::instance()->getConfig("sherpa_tts");
     speakerid = ttsConfig.value("speakerid").toInt();
     extraVol = ttsConfig.value("extra_volume").toInt();
-    std::string rules = Config::getDataPath(ttsConfig.value("rules").toString()).toStdString();
-    std::string model = Config::getDataPath(ttsConfig.value("model").toString()).toStdString();
+    QString rules = ttsConfig.value("rules").toString();
+    QString model = Config::getDataPath(ttsConfig.value("model").toString());
+    std::string modelS = model.toStdString();
     std::string lexicon = Config::getDataPath(ttsConfig.value("lexicon").toString()).toStdString();
     std::string tokens = Config::getDataPath(ttsConfig.value("tokens").toString()).toStdString();
     std::string dataDir = Config::getDataPath(ttsConfig.value("data_dir").toString()).toStdString();
+    std::string vocoder = Config::getDataPath(ttsConfig.value("vocoder").toString()).toStdString();
+    std::string dict = Config::getDataPath(ttsConfig.value("dict").toString()).toStdString();
     std::string dataPath = Config::getDataPath("").toStdString();
+    speed = ttsConfig.value("speed").toDouble(1);
     config.max_num_sentences = 50;
-    if(rules != dataPath) config.rule_fsts = rules.c_str();
+    QList<QString> rules_split = rules.split(',');
+    for (int i = 0; i < rules_split.size(); i++) {
+        rules_split[i] = Config::getDataPath(rules_split[i]);
+    }
+    rules = rules_split.join(',');
+    std::string rulesS = rules.toStdString();
+    config.rule_fsts = rulesS.c_str();
     config.model.debug = 0;
     config.model.num_threads = 2;
     config.model.provider = "cpu";
-    config.model.vits.model = model.c_str();
-    config.model.vits.length_scale = ttsConfig.value("length").toDouble();
-    config.model.vits.noise_scale = ttsConfig.value("noise").toDouble();
-    config.model.vits.noise_scale_w = ttsConfig.value("noise-w").toDouble();
-    if(lexicon != dataPath) config.model.vits.lexicon = lexicon.c_str();
-    if(dataDir != dataPath) config.model.vits.data_dir = dataDir.c_str();
-    config.model.vits.tokens = tokens.c_str();
+    if (model.contains("vits")) {
+        config.model.vits.model = modelS.c_str();
+        config.model.vits.length_scale = ttsConfig.value("length").toDouble();
+        config.model.vits.noise_scale = ttsConfig.value("noise").toDouble();
+        config.model.vits.noise_scale_w = ttsConfig.value("noise-w").toDouble();
+        if(lexicon != dataPath) config.model.vits.lexicon = lexicon.c_str();
+        if(dataDir != dataPath) config.model.vits.data_dir = dataDir.c_str();
+        if(dict != dataPath) config.model.vits.dict_dir = dict.c_str();
+        config.model.vits.tokens = tokens.c_str();
+    } else if (model.contains("matcha")) {
+        config.model.matcha.acoustic_model = modelS.c_str();
+        config.model.matcha.vocoder = vocoder.c_str();
+        config.model.matcha.lexicon = lexicon.c_str();
+        config.model.matcha.tokens = tokens.c_str();
+        config.model.matcha.dict_dir = dict.c_str();
+    }
     tts = SherpaOnnxCreateOfflineTts(&config);
 }
 
@@ -38,7 +57,7 @@ SherpaTTS::~SherpaTTS(){
 void SherpaTTS::detect(const QString& text, const QString& type, int id){
     std::string textS = text.toStdString();
     const SherpaOnnxGeneratedAudio* audio =
-        SherpaOnnxOfflineTtsGenerate(tts, textS.c_str(), speakerid, 1);
+        SherpaOnnxOfflineTtsGenerate(tts, textS.c_str(), speakerid, speed);
     QByteArray data;
     data.resize(audio->n * 2);
     for(int i=0; i<audio->n; i++){

diff --git a/Conversation/TTS/sherpatts.h b/Conversation/TTS/sherpatts.h
@@ -11,9 +11,10 @@ class SherpaTTS : public TTSModel
     void detect(const QString& text, const QString& type, int id) override;
     void stop() override;
 private:
-    SherpaOnnxOfflineTts* tts;
+    const SherpaOnnxOfflineTts* tts;
     int speakerid;
     int extraVol;
+    float speed;
 };
 
 #endif // SHERPATTS_H
diff --git a/Conversation/conversation.cpp b/Conversation/conversation.cpp
@@ -216,7 +216,11 @@ void Conversation::stopSay(const QString &type,
 }
 
 ParsedIntent Conversation::parse(const QString &text) {
-    return nlu->parseIntent(text);
+    ParsedIntent intent = nlu->parseIntent(text);
+    if (!nlu->isStart()) {
+        say("正在启动");
+    }
+    return intent;
 }
 
 void Conversation::onASRRequest(const QByteArray &data, int id) {

diff --git a/Data/default_config.json b/Data/default_config.json
@@ -42,10 +42,14 @@
 		"lexicon": "vits-zh-aishell3/lexicon.txt",
 		"tokens": "vits-zh-aishell3/tokens.txt",
 		"rules": "vits-zh-aishell3/rule.fst",
+		"data_dir": "",
+		"vocoder": "hifigan_v2.onnx",
+		"dict": "matcha-icefall-zh-baker/dict",
 		"noise": 0.3,
 		"noise-w": 0.2,
 		"length": 1,
 		"speakerid": 66,
+		"speed": 1.0,
 		"extra_volume": 0
 	},
     "sherpa_wakeup": {

diff --git a/Wakeup/Wakeup/duilitewakeup.cpp b/Wakeup/Wakeup/duilitewakeup.cpp
@@ -52,8 +52,12 @@ void DuiliteWakeup::detect(const QByteArray& data){
     // for(int i=0; i<chunkSize; i++){
     //     buf[i] = data[i];
     // }
-    if((ret = feedFunc(wakeup, (char*)data.data(), chunkSize)) != 0){
-        qWarning() << "duilite wakeup feed error" << ret;
+    try {
+        if((ret = feedFunc(wakeup, (char*)data.data(), chunkSize)) != 0){
+            qWarning() << "duilite wakeup feed error" << ret;
+        }
+    } catch (std::exception const &e) {
+        qCritical() << "duilite wakeup exception" << e.what(); 
     }
 }
 

diff --git a/Wakeup/wakeup.h b/Wakeup/wakeup.h
@@ -27,7 +27,7 @@ public slots:
 
 signals:
     void dataArrive(QByteArray data);
-    void detected(bool stop);
+    void detected(bool stop, bool command, QString commandStr);
     void finishResponse();
     void wakeup();
 private:

diff --git a/lib/sherpa_onnx/include/cargs.h b/lib/sherpa_onnx/include/cargs.h